Merge branch 'main' into docs/add-new-committers

Correct yongxiang mapping in the diffusion cache ownership list.

Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
Co-authored-by: Cursor <cursoragent@cursor.com>

.buildkite/nightly-release-pipeline.yaml

@@ -1,20 +0,0 @@
-steps:
-  - label: "Build and upload wheel"
-    key: "build-wheel"
-    agents:
-      queue: cpu_queue_release
-    commands:
-      - "curl -LsSf https://astral.sh/uv/install.sh | sh"
-      - 'export PATH="$HOME/.local/bin:$PATH"'
-      - "uv venv --python=3.12 && source .venv/bin/activate"
-      - "uv pip install --upgrade build"
-      - "python3 -m build"
-      - "bash .buildkite/scripts/upload-nightly-wheels.sh"
-
-  - label: "Generate and upload wheel indices"
-    depends_on: "build-wheel"
-    allow_dependency_failure: true
-    agents:
-      queue: small_cpu_queue_release
-    commands:
-      - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh"

.buildkite/pipeline-intel.yaml

@@ -10,7 +10,7 @@ steps:
           DOCKER_BUILDKIT: "1"
           # Buildkite will automatically replace this with the actual commit hash
           VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.20.0"
+          VLLM_VERSION: "v0.21.0"
         priority: 100
         timeout_in_minutes: 60
         soft_fail: false

.buildkite/rebase-pipeline.yaml

@@ -0,0 +1,38 @@
+steps:
+  - label: ":docker: Build image"
+    key: image-build
+    commands:
+      - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
+      - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ."
+      - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
+      - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L2 Test
+  - label: "Upload Ready Pipeline"
+    depends_on: image-build
+    key: upload-ready-pipeline
+    commands:
+      - buildkite-agent pipeline upload .buildkite/test-ready.yml
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L3 Test
+  - label: "Upload Merge Pipeline"
+    depends_on: image-build
+    key: upload-merge-pipeline
+    commands:
+      - buildkite-agent pipeline upload .buildkite/test-merge.yml
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L4 Test
+  - label: "Upload Nightly Pipeline"
+    depends_on: image-build
+    key: upload-nightly-pipeline
+    if: build.env("NIGHTLY") == "1"
+    commands:
+      - buildkite-agent pipeline upload .buildkite/test-nightly.yml
+    agents:
+      queue: "cpu_queue_premerge"

.buildkite/release-pipeline.yaml

@@ -1,38 +1,110 @@
 steps:
-  - label: ":docker: Build image"
-    key: image-build
-    commands:
-      - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
-      - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ."
-      - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
-      - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
-    agents:
-      queue: "cpu_queue_premerge"
+  - input: "Provide Release version here"
+    id: input-release-version
+    fields:
+      - text: "What is the release version?"
+        key: release-version
 
-  # L2 Test
-  - label: "Upload Ready Pipeline"
-    depends_on: image-build
-    key: upload-ready-pipeline
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-ready.yml
+  - label: "Build and upload wheel"
+    depends_on: ~
+    key: "build-wheel"
     agents:
-      queue: "cpu_queue_premerge"
+      queue: cpu_queue_release
+    commands:
+      - "curl -LsSf https://astral.sh/uv/install.sh | sh"
+      - 'export PATH="$HOME/.local/bin:$PATH"'
+      - "uv venv --python=3.12 && source .venv/bin/activate"
+      - "uv pip install --upgrade build"
+      - "python3 -m build"
+      - "bash .buildkite/scripts/upload-nightly-wheels.sh"
 
-  # L3 Test
-  - label: "Upload Merge Pipeline"
-    depends_on: image-build
-    key: upload-merge-pipeline
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-merge.yml
+  - label: "Generate and upload wheel indices"
+    depends_on: "build-wheel"
+    allow_dependency_failure: true
     agents:
-      queue: "cpu_queue_premerge"
+      queue: small_cpu_queue_release
+    commands:
+      - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh"
 
-  # L4 Test
-  - label: "Upload Nightly Pipeline"
-    depends_on: image-build
-    key: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1"
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-nightly.yml
+  - block: "Upload release wheels to PyPI"
+    depends_on: "build-wheel"
+    key: block-upload-release-wheels
+
+  - label: "Upload release wheels to PyPI"
+    depends_on:
+      - block-upload-release-wheels
+      - input-release-version
+    key: upload-release-wheels
     agents:
-      queue: "cpu_queue_premerge"
+      queue: small_cpu_queue_release
+    commands:
+      - "bash .buildkite/scripts/publish-release-wheels-pypi.sh"
+
+  - block: "Unblock to build release Docker images"
+    depends_on: ~
+    key: block-build-release-images
+
+  - group: "Build release Docker images"
+    key: "build-release-images"
+    depends_on: block-build-release-images
+    allow_dependency_failure: true
+    steps:
+      - label: "Build release image - x86_64"
+        key: build-release-image-x86
+        depends_on: ~
+        agents:
+          queue: cpu_queue_release
+        commands:
+          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
+          - |
+            DOCKER_BUILDKIT=1 docker build \
+              --build-arg max_jobs=16 \
+              --tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \
+              --progress plain \
+              -f docker/Dockerfile.cuda .
+          - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
+          # re-tag to default image tag and push, just in case arm64 build fails
+          - "docker tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
+          - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
+        env:
+          DOCKER_BUILDKIT: "1"
+
+      - label: "Build release image - aarch64"
+        key: build-release-image-arm64
+        depends_on: ~
+        agents:
+          queue: arm64_cpu_queue_release
+        commands:
+          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
+          - |
+            DOCKER_BUILDKIT=1 docker build \
+              --build-arg max_jobs=16 \
+              --tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \
+              --progress plain \
+              -f docker/Dockerfile.cuda .
+          - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
+        env:
+          DOCKER_BUILDKIT: "1"
+
+  - block: "Publish release images to DockerHub"
+    key: block-publish-release-images
+    depends_on:
+      - build-release-image-x86
+      - build-release-image-arm64
+      - input-release-version
+
+  - label: "Publish release images to DockerHub"
+    depends_on:
+      - block-publish-release-images
+    key: publish-release-images-dockerhub
+    agents:
+      queue: small_cpu_queue_release
+    commands:
+      - "bash .buildkite/scripts/publish-release-images.sh"
+    plugins:
+      - docker-login#v3.0.0:
+          username: vllmbot
+          password-env: DOCKERHUB_TOKEN
+    env:
+      DOCKER_BUILDKIT: "1"
+      DOCKERHUB_USERNAME: "vllmbot"

.buildkite/scripts/hardware_ci/run-xpu-test.sh

@@ -52,5 +52,6 @@ time timeout -k 30 30m docker run \
     pip install tblib==3.1.0
     cd /workspace/vllm-omni
     pytest -v -s -m "core_model and xpu and B60"
+    export VLLM_XPU_USE_SAMPLER_KERNEL=0    # NOTE: Remove this after vLLM v0.21.1 is merged. Fixes Qwen2-5 omni-expansion tests.
     pytest -v -s -m "advanced_model and xpu and B60"
 '

.buildkite/scripts/publish-release-images.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+#
+# Publish release Docker images from ECR to DockerHub.
+# Pulls per-arch images, tags with `latest` and versioned tags, pushes them,
+# then creates and pushes a multi-arch manifest.
+
+set -euo pipefail
+
+RELEASE_VERSION=$(buildkite-agent meta-data get release-version --default "" | sed 's/^v//')
+if [ -z "${RELEASE_VERSION}" ]; then
+  echo "ERROR: release-version metadata not set"
+  exit 1
+fi
+
+COMMIT="$BUILDKITE_COMMIT"
+DOCKERHUB_REPO="vllm/vllm-omni"
+ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-omni-release-repo"
+
+echo "========================================"
+echo "Publishing release images v${RELEASE_VERSION}"
+echo "  Commit: ${COMMIT}"
+echo "========================================"
+
+# Login to ECR to pull staging images
+aws ecr-public get-login-password --region us-east-1 | \
+  docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
+
+docker pull "${ECR_REPO}:${COMMIT}-x86_64"
+docker pull "${ECR_REPO}:${COMMIT}-aarch64"
+
+docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:latest-x86_64"
+docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64"
+docker push "${DOCKERHUB_REPO}:latest-x86_64"
+docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64"
+
+docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:latest-aarch64"
+docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
+docker push "${DOCKERHUB_REPO}:latest-aarch64"
+docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
+
+docker manifest rm "${DOCKERHUB_REPO}:latest" || true
+docker manifest rm "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" || true
+docker manifest create "${DOCKERHUB_REPO}:latest" "${DOCKERHUB_REPO}:latest-x86_64" "${DOCKERHUB_REPO}:latest-aarch64"
+docker manifest create "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
+docker manifest push "${DOCKERHUB_REPO}:latest"
+docker manifest push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}"
+
+echo ""
+echo "Successfully published release images for v${RELEASE_VERSION}"

.buildkite/scripts/publish-release-wheels-pypi.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+BUCKET="vllm-wheels"
+SUBPATH="omni/$BUILDKITE_COMMIT"
+S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/"
+
+RELEASE_VERSION=$(buildkite-agent meta-data get release-version | sed 's/^v//')
+if [[ -z "$RELEASE_VERSION" ]]; then
+  echo "[FATAL] release-version metadata not set."
+  exit 1
+fi
+echo "Release version: $RELEASE_VERSION"
+
+if [[ -z "$PYPI_TOKEN" ]]; then
+  echo "[FATAL] PYPI_TOKEN is not set."
+  exit 1
+else
+  export TWINE_USERNAME="__token__"
+  export TWINE_PASSWORD="$PYPI_TOKEN"
+fi
+
+set -x
+
+if ! command -v uv &> /dev/null; then
+  curl -LsSf https://astral.sh/uv/install.sh | sh
+  export PATH="$HOME/.local/bin:$PATH"
+fi
+
+uv venv --python 3.12 /tmp/vllm-omni-release-env
+source /tmp/vllm-omni-release-env/bin/activate
+uv pip install twine
+
+DIST_DIR=/tmp/vllm-omni-release-dist
+mkdir -p "$DIST_DIR"
+
+echo "Downloading wheels from S3:"
+aws s3 ls "$S3_COMMIT_PREFIX"
+aws s3 cp --recursive --exclude "*" --include "vllm_omni-${RELEASE_VERSION}*.whl" --exclude "*dev*" "$S3_COMMIT_PREFIX" "$DIST_DIR"
+ls -la "$DIST_DIR"
+
+PYPI_WHEEL_FILES=$(find "$DIST_DIR" -name "vllm_omni-${RELEASE_VERSION}*.whl" -not -name "*+*")
+if [[ -z "$PYPI_WHEEL_FILES" ]]; then
+  echo "[FATAL] No wheels found for version ${RELEASE_VERSION}"
+  exit 1
+fi
+
+python3 -m twine check $PYPI_WHEEL_FILES
+python3 -m twine upload --non-interactive --verbose $PYPI_WHEEL_FILES
+echo "Wheels uploaded to PyPI"

.buildkite/test-nightly.yml

@@ -660,87 +660,6 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
-      - label: ":full_moon: Diffusion X2I(&A&T) · GEBench Accuracy Test"
-        timeout_in_minutes: 60
-        commands:
-          - pytest -s -v tests/e2e/accuracy/test_gebench_h100_smoke.py --run-level full_model --gebench-model Qwen/Qwen-Image-2512 --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gebench-port 8093 --accuracy-workers 1
-          - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gebench_qwen-image-2512/summary*.json"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: ":full_moon: Diffusion X2I(&A&T) · GEdit-Bench Accuracy Test"
-        timeout_in_minutes: 60
-        commands:
-          - pytest -s -v tests/e2e/accuracy/test_gedit_bench_h100_smoke.py --run-level full_model --gedit-model Qwen/Qwen-Image-Edit --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gedit-port 8093 --gedit-samples-per-group 20 --accuracy-workers 1
-          - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gedit_scores_qwen-image-edit/qwen-image-edit_all_all_vie_score_*.csv"
-          - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gedit_scores_qwen-image-edit/qwen-image-edit_all_all_summary_*.json"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: VLLM_HTTP_TIMEOUT_KEEP_ALIVE
-                        value: "120"
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
       - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test"
         timeout_in_minutes: 180
         commands:

.buildkite/test-weekly.yml

@@ -84,3 +84,65 @@ steps:
                 hostPath:
                   path: /mnt/hf-cache
                   type: DirectoryOrCreate
+
+  # Invalid HTTP / WS payloads vs live Omni servers (Issue #3649 tracks known skips).
+  - group: "Reliability Test - Invalid parameters Test"
+    depends_on: upload-weekly-pipeline
+    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
+    steps:
+      - label: "Invalid parameters Test · H100"
+        timeout_in_minutes: 120
+        commands:
+          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 2
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      # Stable Audio Open, Qwen3-TTS speech/batch/voices paths marked ``L4``.
+      - label: "Invalid parameters Test · L4"
+        timeout_in_minutes: 120
+        commands:
+          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"
+        agents:
+          queue: "gpu_1_queue"
+        plugins:
+          - docker#v5.2.0:
+              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              always-pull: true
+              propagate-environment: true
+              shm-size: "8gb"
+              environment:
+                - "HF_HOME=/fsx/hf_cache"
+                - "HF_TOKEN"
+              volumes:
+                - "/fsx/hf_cache:/fsx/hf_cache"

.claude/skills/diffusion-perf-opt/SKILL.md

@@ -0,0 +1,488 @@
+---
+name: diffusion-perf-opt
+description: Diagnose and optimize vLLM Omni diffusion workloads, especially Wan/Qwen/Flux-style image and video generation. Use when Codex is asked to analyze profiling traces, choose parallel strategies, inspect torch profiler trace.json or trace.json.gz timelines, estimate optimization ROI, investigate GPU idle/free bubbles, compare USP/CFG/HSDP/VAE parallelism, or design operator/host/quantization optimizations for vLLM Omni.
+---
+
+# vLLM Omni Performance Optimization
+
+Use this skill to run a disciplined optimization loop for vLLM Omni diffusion workloads. Keep two ideas separate: real performance baselines are collected with low overhead, while torch profiler traces are diagnostic artifacts and may distort latency.
+
+## First Questions
+
+Before proposing changes, ask for the optimization scene if it is not already known:
+
+- GPU model, card count, topology, and whether NVLink is present.
+- Model and pipeline, for example Wan2.2 I2V A14B.
+- User workload: resolution, frames, steps, batch/concurrency, CFG scales, prompt/image inputs.
+- Three runnable commands for each model/workload, or scripts that generate them:
+  - **Server startup command**: exact `vllm serve` command, environment variables, model path, port, parallelism flags, profiler flags, and precision/compile settings.
+  - **User request command**: exact single-request client command (for example curl or Python client) with fixed prompt/media/seed/size/steps. Use this to validate correctness and collect per-request stage timings.
+  - **User benchmark command**: exact repeatable benchmark command or script with warmup count, measured iteration count, concurrency/batch policy, output directory, and summary format.
+- Current enabled strategies: USP/SP, CFG parallel, HSDP/FSDP, VAE patch parallel, torch.compile, profiler options.
+- Optimization target: latency, throughput, memory, cost, or quality-preserving speed.
+- Precision/quality tolerance: bf16/fp8/quantization/sparsity/approximate attention allowed or not.
+
+## Workflow
+
+1. **Freeze the measurement protocol and commands.**
+   - Before analyzing parallel strategies or traces, establish the three commands for the target model/workload: server startup, single user request, and user benchmark.
+   - Prefer checked-in scripts or one-off shell scripts over free-form commands in chat. The scripts should make workload variables explicit, including model path, port/base URL, prompt/media inputs, resolution, frames, steps, seed, warmup, measured iterations, concurrency, and output path.
+   - Measurement reports must preserve the concrete process, not just the final numbers. For each tested configuration, write down the server startup command, the user request command, the user benchmark command or polling loop, the final server response metrics, and the client-side timing/HTTP result.
+   - Present all duration and latency values in **milliseconds (ms)** in tables and summaries. If an API returns seconds, convert to ms before reporting. Keep raw units only inside quoted raw logs or source snippets.
+   - Keep profiler modes separate in the command set:
+     - Baseline/benchmark commands should avoid torch profiler and stack collection.
+     - Diagnostic commands may enable torch profiler after a baseline identifies a bottleneck.
+   - If any of the three commands is missing, ask for it or create a proposed script before proceeding.
+   - Make clear which command is authoritative for correctness validation and which command is authoritative for performance numbers.
+
+2. **Collect a real baseline.**
+   - Disable PyTorch profiler and stack collection.
+   - Disable or fix torch.compile state so A/B is fair.
+   - Avoid `--enforce-eager` for production-speed baselines unless eager is the target.
+   - Prefer `--log-stats` and `--enable-diffusion-pipeline-profiler` for low-overhead stage timing. PR 3069 has the relevant metrics/log-stats changes; if local code does not include them, fetch or cherry-pick the minimal metrics changes rather than merging unrelated PR drift.
+   - Run warmup requests and exclude first-request lazy init.
+
+3. **Model the parallel strategy before testing.**
+   - Estimate compute and communication for self-attention, cross-attention, FFN, CFG, VAE encode/decode, and HSDP.
+   - Select a small candidate matrix rather than exhaustively testing everything.
+   - Typical 2-card candidates: `USP=2`, `CFG=2`, `USP=1/HSDP on-off`, VAE parallel on-off if memory allows.
+   - Typical 4-card candidates: `USP=4`, `CFG=2 x USP=2`, `USP=2 x HSDP`, VAE parallel on-off.
+   - Typical 8-card candidates for official CFG-enabled video diffusion models:
+     - Primary candidate: `CFG=2 x USP=4` with VAE patch parallel across all 8 ranks.
+     - Compare against: `CFG=1 x USP=8` to test whether larger sequence parallel groups beat CFG branch parallelism.
+     - Isolate VAE parallelism: keep the DiT strategy fixed and compare VAE patch parallel on/off or different VAE patch sizes.
+     - If long-video `diffuse` remains dominant and Ulysses all-to-all is suspected, test a hybrid sequence strategy such as `CFG=2 x USP=2 x Ring=2`.
+     - Test HSDP off only if the model fits without it; treat HSDP primarily as a memory strategy until A/B proves otherwise.
+   - Prefer USP/SP for long video token sequences; prefer CFG parallel when CFG doubles transformer forwards and sequence length is modest.
+   - Convert workload shape to latent/patch token counts before choosing candidates. For video models, record frames, latent frames, latent HxW, patch size, approximate token count, and which stages should scale with the token count.
+
+4. **Search for the best parallel configuration.**
+   - Start with a small matrix that answers one question per comparison:
+     - `CFG parallel vs larger USP`: compare `CFG=2 x USP=world/2` against `CFG=1 x USP=world` for CFG-enabled workloads.
+     - `VAE patch parallel value`: compare the best DiT strategy with VAE patch parallel enabled and disabled.
+     - `HSDP cost`: compare HSDP on/off only when both configurations fit memory.
+     - `Ulysses vs Ring`: test a Ring/Ulysses hybrid only after long-sequence `diffuse` is confirmed dominant.
+   - For each configuration, create or record a stable config id, for example `A_cfg2_usp4_vaepp8_hsdp_tiling`.
+   - For each config id, capture:
+     - exact server command and environment variables,
+     - observed distributed setup from logs, such as SP groups, CFG groups, HSDP shard/replicate sizes, VAE patch size,
+     - exact request command for every scenario,
+     - final server response metrics and client-side elapsed time,
+     - output artifact paths and any failed/empty responses.
+   - Use one warmup request per scenario, then at least three measured repeats for the shortlisted config. For exploratory matrix pruning, one measured request is acceptable only if the margin is large; label it as one-shot.
+   - Compare configurations by stage, not only end-to-end latency. A configuration can improve `diffuse` while hurting `vae.decode`; record both effects.
+   - Select the best config only after checking the target metric, dominant stages, memory headroom, and output correctness.
+
+5. **Run targeted A/B tests.**
+   - Change one variable per test.
+   - Keep model, input, seed, request parameters, GPU placement, and warmup policy fixed.
+   - Record latency, stage timings, memory, output quality, and logs.
+   - Report comparison tables in ms. Include at least end-to-end client time, server `inference_time`, server stage generation time if available, `vae.encode`, `diffuse`, `vae.decode`, and peak memory.
+
+6. **Collect diagnostic trace only after narrowing hypotheses.**
+   - Use torch profiler for a small number of requests.
+   - Run two separate diagnostic traces instead of mixing concerns:
+     - **Operator/shape trace**: enable `torch_profiler_record_shapes=True` and keep stack collection disabled. Use this to rank CUDA kernels, NCCL collectives, attention/MLP/norm/RoPE work, and shape-specific hot operators.
+     - **Host-stack trace**: enable `torch_profiler_with_stack=True` and normally keep shape collection disabled. Use this to map CPU/Python host gaps, synchronization points, scheduler paths, and request handling overhead.
+   - Keep both trace commands and reports separate from baseline/benchmark commands. Torch profiler latency is diagnostic only and must not be used as the final latency claim.
+   - Prefer profiling only the narrowed dominant scenario, for example the highest-resolution/video-length case where `diffuse` dominates.
+   - If profiler endpoints are available, run one warmup request first, then call `/start_profile`, run one profiled request, and call `/stop_profile`. This keeps model initialization and warmup out of the diagnostic trace.
+   - Start by analyzing rank 0 only. Expand to more ranks only if rank 0 suggests imbalance, unclear GPU idle/free bubbles, high NCCL wait, server timing mismatch, CFG branch imbalance, or USP group stragglers.
+   - Diagnostic reports must be written to disk and preserve: server command, profiler config, warmup/request/polling commands, trace artifact paths, rank analyzed, analyzer output or summary, and the decision about whether additional ranks are necessary.
+   - Analyze both rank-level balance and device-level free bubbles when additional ranks are opened.
+
+7. **Analyze host, communication, and operators.**
+   - Find GPU idle/free intervals and map each large gap to the enclosing CPU/Python code.
+   - Separate real GPU idle from profiler overhead such as CUPTI `Command Buffer Full`.
+   - Compare NCCL kernel time to user annotations; annotations can overcount nested intervals.
+   - Rank operator work by total CUDA time and by repeated small-kernel launch count.
+
+8. **Produce an optimization plan.**
+   - Classify candidates as P0/P1/P2.
+   - For each candidate, state necessity, expected benefit, implementation path, validation plan, and quality risk.
+   - Do not implement high-risk operator rewrites before proving the operator is a bottleneck for the target shapes.
+   - End the plan with a user-facing candidate selection table. The assistant
+     should not automatically choose a risky optimization just because it is
+     technically possible. Present the options clearly and let the user decide
+     which item is worth implementing based on latency target, engineering
+     budget, memory headroom, and quality tolerance.
+   - Organize the plan by optimization layer, not by a flat list of ideas:
+     - host/runtime optimization,
+     - measurement/benchmark reliability,
+     - parallelism and communication,
+     - VAE encode/decode and media pre/post processing,
+     - operator fusion and layout cleanup,
+     - attention main-path optimization,
+     - algorithmic, precision, or approximation changes.
+   - For each layer, tie every candidate back to evidence from baseline
+     metrics, diagnostic trace, source code, or output quality requirements.
+     Do not list generic optimizations without a trace or workload reason.
+
+## Priority Rules
+
+- **P0:** low risk, likely useful, or required for trustworthy measurement. Examples: real baseline, warmup, targeted parallel A/B, disabling avoidable `empty_cache`, scheduler coefficient caching.
+- **P1:** meaningful code changes with contained risk. Examples: cross-attention KV caching, VAE gather/broadcast reduction, AdaLayerNorm/RMSNorm/RoPE fusion after trace evidence.
+- **P2:** high implementation or quality risk. Examples: FA to LA replacement, custom Triton/CUDA fused kernels, FP8/quantization, sparsity/Rainfusion-style acceleration.
+
+Every implemented optimization needs A/B validation and quality regression. A/B means same workload and hardware before/after. Quality regression means checking generated image/video stability, artifacts, temporal flicker, and seed behavior when precision or approximate kernels change.
+
+## Optimization Layers
+
+After baseline, parallel-search, and diagnostic traces, summarize optimization
+opportunities by layer. This is the core of the performance analysis: the goal
+is to connect evidence to a scoped implementation and a validation plan.
+
+### Host and Runtime Optimization
+
+Purpose: remove CPU/Python stalls, synchronization points, allocator overhead,
+and request-path overhead that leave GPU lanes empty.
+
+Evidence to look for:
+
+- High `idle_pct` or large `GAP` blocks in `trace_analyzer.py`.
+- Host-stack trace lines such as `torch.cuda.empty_cache`,
+  `cudaStreamSynchronize`, `cudaDeviceSynchronize`, Python locks, scheduler
+  waits, image/video preprocessing, or repeated small allocation paths.
+- Difference between client wall-clock time and server `inference_time_s`.
+
+Typical candidates:
+
+- Make avoidable `torch.cuda.empty_cache()` optional or guard it by memory
+  headroom.
+- Cache scheduler coefficients, timesteps, masks, or other tiny repeated CPU
+  computations when the request shape/steps are fixed.
+- Remove avoidable host-device synchronizations and blocking logging/stat calls.
+- Move expensive preprocessing out of the critical path or cache fixed prompt,
+  image, and transform work for benchmark scenarios.
+- Ensure benchmark scripts record client-side elapsed time, HTTP status, output
+  path, and server response metrics.
+
+Priority guidance:
+
+- Usually P0 when the change is measurement reliability or an obvious removable
+  synchronization.
+- Usually P1 when it changes scheduling, memory lifetime, or request execution
+  order.
+
+Validation:
+
+- Re-run non-profiler baseline with same workload and seed.
+- Confirm peak memory headroom if disabling cache cleanup.
+- Confirm generated output exists and quality/seed behavior is unchanged.
+
+### Parallelism and Communication Optimization
+
+Purpose: choose the right decomposition for CFG branches, sequence tokens, model
+weights, VAE tiles, and rank topology.
+
+Evidence to look for:
+
+- Baseline A/B across `CFG`, `USP/SP`, `Ring`, `HSDP/FSDP`, and VAE patch
+  parallelism.
+- Stage timing shifts: `diffuse`, `vae.encode`, `vae.decode`, and server
+  end-to-end.
+- NCCL kernel time from trace, not only `user_annotation` time.
+- Rank imbalance across SP group ranks or CFG branch ranks.
+- Memory headroom and OOM risk.
+
+Typical candidates:
+
+- `CFG=2 x USP=world/2` versus `CFG=1 x USP=world` for CFG-enabled models.
+- VAE patch parallel on/off or patch size tuning.
+- HSDP/FSDP on/off only if both configurations fit memory.
+- Ulysses versus Ulysses+Ring only after long-sequence `diffuse` is confirmed
+  dominant and all-to-all is suspected.
+- Rank mapping/topology changes if all-rank traces show stragglers or NCCL wait.
+- Buffer reuse or preallocation for FSDP/HSDP all-gather paths.
+
+Priority guidance:
+
+- P0/P1 for configuration-only changes with strong measured wins.
+- P1 for buffer reuse or rank mapping changes.
+- P2 for invasive distributed algorithm changes.
+
+Validation:
+
+- Measure by stage and memory, not only end-to-end.
+- Use one-variable A/B with identical prompt/media/seed/shape/steps.
+- When communication is suspected, compare rank0-3 in one USP group and rank0
+  versus rank4 across CFG branches for `CFG=2 x USP=4`.
+
+### VAE and Media Pipeline Optimization
+
+Purpose: reduce encode/decode, tiling, split/gather, and media conversion time.
+
+Evidence to look for:
+
+- Large `vae.encode` or `vae.decode` in low-overhead stage timings.
+- Host-stack gaps in VAE tile split, gather, merge, broadcast, or image/video
+  transforms.
+- VAE kernels or cuDNN convolution in operator trace.
+- Whether every rank needs the final decoded tensor.
+
+Typical candidates:
+
+- Keep VAE patch parallel enabled when it has clear measured benefit.
+- Reduce VAE gather/broadcast to only ranks that need the final media output.
+- Reuse tile metadata, split buffers, or gather buffers.
+- Evaluate bf16/autocast behavior for VAE only with visual quality checks.
+- Avoid redundant image conversion, resize, or tensor construction in repeated
+  benchmark runs.
+
+Priority guidance:
+
+- P0/P1 if VAE is a large share of the target workload or if a host gap is
+  obvious and low risk.
+- Lower priority when `diffuse` dominates and VAE is already patch-parallelized.
+
+Validation:
+
+- Compare `vae.encode`, `vae.decode`, server end-to-end, and peak memory.
+- Check output video integrity, artifacts, flicker, and seed stability.
+
+### Operator Fusion and Layout Cleanup
+
+Purpose: reduce high-frequency small kernels, memory bandwidth pressure, layout
+conversions, and launch overhead in transformer and VAE blocks.
+
+Evidence to look for:
+
+- Top operator tables showing many `aten::copy_`, `aten::cat`,
+  `split_with_sizes_copy`, `aten::add`, `aten::mul`, `aten::div`, norm,
+  activation, RoPE, or reshape/layout kernels.
+- `ops_rankN.xlsx` `by_shape` sheet showing repeated small shapes inside the
+  same block path.
+- Trace lanes showing many short kernels between larger GEMM/attention kernels.
+- Source code patterns with repeated elementwise chains or layout conversions.
+
+Typical fusion targets:
+
+- AdaLayerNorm / RMSNorm / LayerNorm plus scale/shift fusion.
+- RoPE fusion with Q/K layout preparation when shapes are stable.
+- Residual add, scale, gate, and elementwise chains.
+- MLP gate/up/down cleanup, such as fusing activation and multiply around
+  GEMM outputs when feasible.
+- QKV projection and reshape/split/cat path cleanup.
+- Attention pre/post layout cleanup to avoid unnecessary copies, cats, and
+  splits around sequence parallel all-to-all.
+
+Priority guidance:
+
+- P1 when implemented with existing PyTorch/Triton/local helper patterns and
+  validated against exact outputs or tolerances.
+- P2 when it requires custom CUDA/Triton kernels, changes numerics, or touches
+  attention math directly.
+
+Validation:
+
+- First prove the operator family is material for the target shape.
+- Use non-profiler A/B for latency and stage timing.
+- Use quality regression checks for generated video stability.
+- Check compile behavior and graph breaks if using `torch.compile`.
+
+### Attention Main-Path Optimization
+
+Purpose: address the dominant self-attention cost when FlashAttention or other
+attention kernels dominate CUDA time.
+
+Evidence to look for:
+
+- Operator trace where FlashAttention/SDPA kernels dominate total CUDA time.
+- Attention shape from `ops_rankN.xlsx` `by_shape`, model code, or trace
+  metadata.
+- Whether attention cost scales with latent frames, latent H/W, patch size, or
+  CFG duplication.
+- Layout/copy/all-to-all work around attention.
+
+Typical candidates:
+
+- Verify the attention backend and shape are on the intended fast kernel path.
+- Compare supported attention backends only with identical workload and quality
+  settings.
+- Reduce attention input size by safe model/config choices when allowed:
+  latent resolution, frame count, patching, boundary ratio, or windowing.
+- Remove avoidable layout conversions before/after attention.
+- Reuse condition-side KV or other static inputs if the model structure allows.
+- Consider custom kernels, sparse/window/linear attention, or approximation only
+  after quality risk is accepted.
+
+Priority guidance:
+
+- P1 for backend/config/layout changes with preserved math.
+- P2 for approximate attention, sparsity, custom kernels, or any change that can
+  alter quality/temporal consistency.
+
+Validation:
+
+- Always include output quality and seed behavior checks.
+- Compare `diffuse`, server end-to-end, peak memory, and attention kernel time
+  in diagnostic traces if needed.
+
+### Algorithmic, Precision, and Approximation Optimization
+
+Purpose: reduce mathematical work or precision cost beyond local code cleanup.
+
+Evidence to look for:
+
+- A single operator family dominates even after low-risk runtime, parallel, and
+  fusion work.
+- Memory bandwidth or compute utilization suggests precision or quantization
+  could matter.
+- The user explicitly allows quality-preserving or approximate methods.
+
+Typical candidates:
+
+- FP8/quantization for transformer or selected projections.
+- Sparsity or Rainfusion-style acceleration.
+- Reduced steps, scheduler changes, distillation, or caching across frames.
+- Approximate attention or linear attention.
+
+Priority guidance:
+
+- Usually P2 because quality, numerics, and implementation risk are high.
+
+Validation:
+
+- Requires strict A/B, visual quality review, temporal flicker checks, seed
+  stability, and possibly human evaluation.
+
+### Interpolation, Super-Resolution, and E2E Pipeline Optimization
+
+Purpose: optimize the whole user-visible video product, not only the base
+diffusion invocation. Some deployments trade base-model latency against
+post-processing, interpolation, or super-resolution stages.
+
+Evidence to look for:
+
+- E2E latency breakdown across base generation, interpolation, super-resolution,
+  encoding, storage, and response streaming.
+- Fast/slow GPU or fast/slow stage analysis across multiple cards and pipeline
+  stages.
+- User quality target: resolution, FPS, temporal smoothness, and acceptable
+  post-processing artifacts.
+
+Typical candidates:
+
+- Add or optimize a frame interpolation stage when it reduces required base
+  model frames for the same perceived FPS.
+- Add or optimize a super-resolution model when generating lower base
+  resolution plus SR is faster for the target quality.
+- Analyze E2E2 pipeline behavior: client request, service scheduling, diffusion,
+  VAE/media, post-process, file write, and response.
+- Identify fast/slow cards or stages and rebalance pipeline placement.
+
+Priority guidance:
+
+- P1 when using proven interpolation/SR components without changing diffusion
+  math.
+- P2 when quality risk is high or the pipeline adds significant operational
+  complexity.
+
+Validation:
+
+- Measure E2E wall-clock, per-stage server timings, output FPS/resolution,
+  artifacts, flicker, and user-visible quality.
+
+### Optimization Candidate Library
+
+Use this table as a compact menu, not as automatic recommendations. Pick items
+only when baseline metrics, trace evidence, source inspection, or quality
+tolerance supports them.
+
+| Layer | Candidate | Evidence | Priority | Validation focus |
+|---|---|---|---|---|
+| Measurement | Freeze server/request/benchmark commands | Missing or drifting commands | P0 | Repeatable non-profiler A/B |
+| Measurement | Separate baseline and diagnostic profiler runs | Profiler used for latency claims | P0 | Low-overhead stage timings |
+| Host/runtime | Guard or remove avoidable `empty_cache` | Host-stack gaps or sync stalls | P0/P1 | Latency, peak memory, OOM safety |
+| Host/runtime | Cache scheduler coefficients/timesteps | Repeated tiny CPU/GPU work | P0/P1 | Same seed/output, stage timing |
+| Host/runtime | Reduce framework scheduling overhead | Client time exceeds server time | P1 | E2E latency, throughput |
+| Parallel | `CFG=2 x USP=world/2` vs `USP=world` | CFG doubles forward work | P0/P1 | `diffuse`, NCCL, memory |
+| Parallel | Tune VAE patch parallelism | VAE encode/decode is material | P0/P1 | VAE time, output correctness |
+| Parallel | HSDP on/off or buffer reuse | HSDP affects memory/all-gather | P1 | Memory, latency, OOM risk |
+| Parallel | Ulysses vs Ulysses+Ring | Long sequence all-to-all suspected | P1/P2 | Rank balance, NCCL kernels |
+| Cross-attn | Disable SP for short condition tokens | Cross-attn comm exceeds compute | P1 | `diffuse`, correctness |
+| VAE/media | Reduce VAE gather/broadcast | Rank traces show VAE wait | P1 | Rank balance, output file |
+| VAE/media | Reuse tile metadata/buffers | Tile split/merge host gaps | P1 | `vae.encode/decode`, memory |
+| VAE/media | VAE bf16/autocast | VAE float kernels are slow | P1/P2 | Artifacts, flicker, seed stability |
+| Operator fusion | AdaLayerNorm/LayerNorm fusion | Norm plus scale/shift kernels | P1 | Numeric tolerance, latency |
+| Operator fusion | RMSNorm fusion | Many small RMSNorm kernels | P1 | Numeric tolerance, latency |
+| Operator fusion | RoPE cache/fuse/layout cleanup | RoPE copy/reshape kernels | P1/P2 | Kernel count, correctness |
+| Operator layout | QKV or attention layout cleanup | Copy/cat/split around attention | P1 | Copy kernels, compile behavior |
+| Attention | Verify backend fast path | FA/SDPA dominates trace | P1 | `diffuse`, attention kernels |
+| Attention | FA to LA or selected-head LA | Attention remains dominant | P2 | Quality, temporal stability |
+| Precision | Transformer FP8/quantization | Compute/bandwidth bound and allowed | P2 | Quality, speed, stability |
+| Sparsity | Rainfusion-style acceleration | DiT compute remains dominant | P2 | Prompt diversity, quality |
+| Pipeline | Frame interpolation | Fewer base frames can meet FPS | P1/P2 | E2E latency, motion artifacts |
+| Pipeline | Super-resolution | Lower base res plus SR may win | P1/P2 | Detail quality, artifacts |
+| E2E | Fast/slow-card analysis | Multi-card stragglers | P0/P1 | Per-rank/stage wall-clock |
+
+### Optimization Plan Template
+
+Use this table shape when reporting the next work items:
+
+| Priority | Layer | Candidate | Evidence | Expected benefit | Implementation path | Validation | Quality risk |
+|---|---|---|---|---|---|---|---|
+| P0 | Host/runtime | Guard `empty_cache` | Host-stack gap points to `torch.cuda.empty_cache` | Small latency reduction, less idle | Add config/env guard | Non-profiler A/B, memory check | Low |
+| P1 | Operator fusion | RMSNorm/AdaLayerNorm fusion | High-frequency norm/elementwise kernels | Lower launch/bandwidth overhead | Use existing fusion helper or targeted Triton | A/B + output check | Medium |
+| P1/P2 | Attention | Attention layout/backend investigation | FA kernel dominates CUDA time | Potentially large | Inspect shapes/backend and remove layout copies | A/B + trace + quality | Medium/high |
+
+Then present a short selection prompt using the same rows:
+
+```text
+Which candidate should we implement next?
+
+1. P0 Host/runtime: guard empty_cache
+   - Expected benefit: small but low-risk latency reduction.
+   - Risk: possible memory increase/OOM if memory headroom is insufficient.
+
+2. P1 Operator fusion: inspect by_shape and implement first norm/RoPE/layout fusion
+   - Expected benefit: medium if high-frequency small kernels are confirmed.
+   - Risk: numerical/compile/quality validation needed.
+
+3. P1/P2 Attention: FA/LA/backend/layout investigation
+   - Expected benefit: potentially large.
+   - Risk: high quality and implementation risk.
+```
+
+If the user has not chosen an item, default to explaining tradeoffs and asking
+which candidate to execute. Only proceed autonomously on low-risk P0 measurement
+or instrumentation fixes.
+
+## Analysis Helpers
+
+PyTorch profiler traces are Chrome/Perfetto-compatible JSON files, usually
+`trace_rankN.json` or `trace_rankN.json.gz`. They normally contain a top-level
+`traceEvents` list, though some exporters emit the raw event list directly.
+
+Use the checked-in analyzer from the repository root:
+
+```bash
+python3 .claude/skills/diffusion-perf-opt/scripts/trace_analyzer.py \
+  vllm_profile/.../trace_rank0.json.gz \
+  --min-gap-ms 5 \
+  --topn 20
+```
+
+For rank imbalance or communication questions, pass all relevant rank traces in
+one command. For host gaps, lower `--min-gap-ms` to `1` and use a host-stack
+trace. Read `gpu_span_s`, `busy_union_s`, `idle_union_s`, `idle_pct`, `GAP`
+blocks, `Top GPU/operator events by total duration`, and `Top NCCL-like events
+by category`. Treat `cat=user_annotation` NCCL ranges as enclosing annotations;
+prefer `cat=kernel` or `cat=gpu_user_annotation` for real device work.
+
+The analyzer summarizes timing only. It does not parse tensor shapes, attribute
+overlap to individual streams, prove quality, or provide final latency claims.
+Use `ops_rankN.xlsx` or PyTorch key averages for shape analysis, and re-test any
+optimization with non-profiler baseline commands.
+
+Read `references/optimization-playbook.md` when drafting the optimization table or comparing candidate techniques.
+
+## vLLM Omni Heuristics
+
+- Cross-attention usually should not use USP/SP when text/image condition token count is much smaller than latent video tokens. Confirm via trace; in Wan2.2 I2V, self-attention dominates cross-attention.
+- VAE bf16/autocast is often worthwhile but requires visual quality checks.
+- VAE patch parallel can help decode/encode but may add gather/merge/broadcast overhead. Check whether all ranks need the final decoded tensor.
+- HSDP/FSDP is primarily a memory strategy. If the model fits without it, run an on/off latency comparison.
+- Scheduler work can create small host/device gaps; cache tiny solve coefficients when timesteps/order are known.
+- `torch.cuda.empty_cache()` can prevent OOM but creates synchronization/idle. Make it optional if memory headroom is sufficient.
+- `Command Buffer Full` in profiler output is profiler overhead, not a model optimization target.

.claude/skills/diffusion-perf-opt/agents/openai.yaml

@@ -0,0 +1,4 @@
+interface:
+  display_name: "vLLM Omni Perf Opt"
+  short_description: "Profile and optimize vLLM Omni diffusion performance."
+  default_prompt: "Analyze and optimize a vLLM Omni diffusion workload using baselines, traces, parallel strategy modeling, and prioritized optimization plans."

.claude/skills/diffusion-perf-opt/references/optimization-playbook.md

@@ -0,0 +1,82 @@
+# Optimization Playbook
+
+Use this reference when turning trace evidence into prioritized vLLM Omni optimizations.
+
+## Parallel Strategy
+
+Start from workload shape:
+
+- Long video tokens, high resolution, many frames: self-attention and FFN dominate. USP/SP is usually valuable.
+- CFG enabled with positive and negative prompts: transformer forward count doubles. CFG parallel may beat USP on small/moderate sequence lengths.
+- Cross-attention: skip SP/USP when condition tokens are much smaller than latent tokens; extra all-to-all can exceed compute saved.
+- HSDP/FSDP: use to fit memory first. If memory allows, compare without HSDP because all-gathers and lazy init add overhead.
+- VAE parallel: useful when encode/decode is material. Check gather/merge/broadcast and whether every rank needs final decoded output.
+
+Candidate matrices:
+
+- 2 cards: `USP=2`, `CFG=2`, HSDP on/off, VAE PP on/off.
+- 4 cards: `USP=4`, `CFG=2 x USP=2`, `USP=2 x HSDP`, VAE PP on/off.
+- 8 cards: add pipeline/stage parallel only after measuring orchestration and inter-stage transfer overhead.
+
+## Host And Free Bubble Analysis
+
+Look for device intervals where no kernel, memcpy, or memset is active.
+
+Common causes:
+
+- First request FSDP/HSDP lazy init.
+- Python preprocessing: image load, resize, `to_tensor`, tokenizer, request marshaling.
+- `torch.cuda.empty_cache()` or explicit synchronizations.
+- Scheduler small linear algebra such as tiny `torch.linalg.solve`.
+- VAE tile split/merge Python loops.
+- Profiler overhead such as CUPTI command buffer flush/full.
+
+Also inspect trace-external time:
+
+- API request parsing and multipart upload.
+- Engine/orchestrator dispatch.
+- Worker RPC and queue wait.
+- Output serialization, video write, base64, HTTP response.
+
+## Operator Optimization Candidates
+
+Evaluate necessity before implementation:
+
+- **FA to LA / LA for head:** high potential only if attention is the main CUDA hotspot and target shapes fit LA well.
+- **Cross-attention BSND/no transpose:** do only if transpose/copy around cross-attn is measurable.
+- **AdaLayerNorm fusion:** fuse norm, scale/shift, gate, residual, and dtype conversion if repeated elementwise kernels are significant.
+- **RMSNorm fusion:** useful for Q/K norms and VAE norms when small-kernel launch count is high.
+- **FP32 LayerNorm to bf16:** quality-sensitive; run visual regression.
+- **LA preprocess:** integrate only if preprocess overhead is cached or amortized.
+- **RoPE move/cache/fuse:** useful when RoPE small kernels are visible around every self-attn block.
+- **VAE RMSNorm replacement:** only after VAE internal trace shows norm as a real component of decode/encode.
+- **Scheduler fusion/cache:** cache coefficient solves and fuse CFG combine with scheduler arithmetic when possible.
+
+Before writing a custom kernel, check:
+
+- Local vLLM Omni implementations.
+- Upstream vLLM kernels and attention backends.
+- SGLang, FlashInfer, FlashAttention, and Triton examples.
+- Shape constraints and layout expectations.
+
+## Quantization And Sparsity
+
+Treat these as high-risk optimizations:
+
+- Rainfusion/sparse diffusion can be high reward but needs model-specific quality validation.
+- VAE decode quantization can reduce decode time but usually has smaller end-to-end upside than transformer attention/FFN work.
+- FP8 or approximate kernels require seed-stable visual regression, temporal flicker checks, and prompt diversity.
+
+## Output Format For Recommendations
+
+Return a table with:
+
+- Priority: P0/P1/P2.
+- Optimization.
+- Evidence from trace or code.
+- Expected benefit.
+- Risk/quality impact.
+- Implementation path.
+- Validation command or A/B test.
+
+State clearly when a recommendation is a hypothesis requiring new trace with shapes.

.claude/skills/diffusion-perf-opt/scripts/trace_analyzer.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""Summarize torch profiler Chrome traces for vLLM Omni diffusion optimization."""
+
+from __future__ import annotations
+
+import argparse
+import collections
+import gzip
+import json
+from pathlib import Path
+from typing import Any
+
+GPU_CATS = {"kernel", "gpu_memcpy", "gpu_memset"}
+CPU_CATS = {"python_function", "user_annotation", "cpu_op", "cuda_runtime", "cuda_driver"}
+
+
+def open_trace(path: Path):
+    if path.suffix == ".gz":
+        return gzip.open(path, "rt")
+    return path.open("rt")
+
+
+def event_name(event: dict[str, Any]) -> str:
+    return str(event.get("name", ""))
+
+
+def summarize_trace(path: Path, min_gap_us: float, topn: int) -> None:
+    with open_trace(path) as f:
+        data = json.load(f)
+
+    events = data if isinstance(data, list) else data.get("traceEvents", [])
+    gpu: list[tuple[float, float, float, str, str, Any, Any]] = []
+    cpu: list[tuple[float, float, float, str, str, Any, Any]] = []
+    by_gpu_name: dict[str, list[float]] = collections.defaultdict(lambda: [0, 0.0, 0.0])
+    by_name: dict[str, list[float]] = collections.defaultdict(lambda: [0, 0.0, 0.0])
+    nccl: dict[tuple[str, str], list[float]] = collections.defaultdict(lambda: [0, 0.0, 0.0])
+
+    for event in events:
+        dur = event.get("dur")
+        ts = event.get("ts")
+        if dur is None or ts is None or dur <= 0:
+            continue
+        cat = str(event.get("cat", ""))
+        name = event_name(event)
+        row = (float(ts), float(ts + dur), float(dur), name, cat, event.get("pid"), event.get("tid"))
+        stat = by_name[name]
+        stat[0] += 1
+        stat[1] += dur
+        stat[2] = max(stat[2], dur)
+        if cat in GPU_CATS:
+            gpu.append(row)
+            gstat = by_gpu_name[name]
+            gstat[0] += 1
+            gstat[1] += dur
+            gstat[2] = max(gstat[2], dur)
+        elif cat in CPU_CATS:
+            cpu.append(row)
+        if "nccl" in name.lower():
+            nstat = nccl[(cat, name)]
+            nstat[0] += 1
+            nstat[1] += dur
+            nstat[2] = max(nstat[2], dur)
+
+    print(f"\n== {path}")
+    print(f"events={len(events)} gpu_events={len(gpu)} cpu_events={len(cpu)}")
+    if not gpu:
+        print("No GPU events found.")
+        return
+
+    gpu.sort()
+    merged: list[list[Any]] = []
+    for start, end, dur, name, cat, pid, tid in gpu:
+        if not merged or start > merged[-1][1]:
+            merged.append([start, end, [(start, end, dur, name, cat, pid, tid)]])
+        else:
+            merged[-1][1] = max(merged[-1][1], end)
+            merged[-1][2].append((start, end, dur, name, cat, pid, tid))
+
+    span = max(end for _, end, *_ in gpu) - min(start for start, *_ in gpu)
+    busy = sum(end - start for start, end, _ in merged)
+    idle = span - busy
+    print(
+        f"gpu_span_s={span / 1e6:.3f} "
+        f"busy_union_s={busy / 1e6:.3f} "
+        f"idle_union_s={idle / 1e6:.3f} "
+        f"idle_pct={idle / span * 100:.2f}"
+    )
+
+    interesting_cpu = [
+        row
+        for row in cpu
+        if row[2] >= 1000
+        and (
+            row[4] in {"python_function", "user_annotation"}
+            or "cudaStreamSynchronize" in row[3]
+            or "cudaDeviceSynchronize" in row[3]
+            or "cudaLaunch" in row[3]
+            or "cudaMemcpy" in row[3]
+        )
+    ]
+
+    gaps = []
+    for idx in range(1, len(merged)):
+        gap_start = merged[idx - 1][1]
+        gap_end = merged[idx][0]
+        gap_dur = gap_end - gap_start
+        if gap_dur >= min_gap_us:
+            prev_event = max(merged[idx - 1][2], key=lambda x: x[1])
+            next_event = min(merged[idx][2], key=lambda x: x[0])
+            mid = (gap_start + gap_end) / 2
+            containers = [row for row in interesting_cpu if row[0] <= mid <= row[1]]
+            containers = sorted(containers, key=lambda x: x[2])[:8]
+            gaps.append((gap_dur, gap_start, gap_end, prev_event, next_event, containers))
+
+    print(f"gaps_ge_{min_gap_us / 1000:.3f}ms count={len(gaps)} sum_s={sum(g[0] for g in gaps) / 1e6:.3f}")
+    for gap_dur, gap_start, gap_end, prev_event, next_event, containers in sorted(gaps, reverse=True)[:topn]:
+        print(f"\nGAP {gap_dur / 1000:.3f} ms ts={gap_start:.0f}->{gap_end:.0f}")
+        print(f"  prev {prev_event[4]} {prev_event[2] / 1000:.3f} ms {prev_event[3][:160]}")
+        print(f"  next {next_event[4]} {next_event[2] / 1000:.3f} ms {next_event[3][:160]}")
+        for row in containers:
+            print(f"  in   {row[4]} {row[2] / 1000:.3f} ms {row[3][:180]}")
+
+    print("\nTop GPU/operator events by total duration:")
+    for name, (count, total, max_dur) in sorted(by_gpu_name.items(), key=lambda kv: kv[1][1], reverse=True)[:topn]:
+        print(f"  {int(count):8d} total={total / 1e6:9.3f}s max={max_dur / 1000:9.3f}ms {name[:180]}")
+
+    print("\nTop NCCL-like events by category:")
+    for (cat, name), (count, total, max_dur) in sorted(nccl.items(), key=lambda kv: kv[1][1], reverse=True)[:topn]:
+        print(f"  {int(count):8d} total={total / 1e6:9.3f}s max={max_dur / 1000:9.3f}ms cat={cat} {name[:160]}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("traces", nargs="+", type=Path, help="trace.json or trace.json.gz files")
+    parser.add_argument("--min-gap-ms", type=float, default=5.0, help="minimum GPU idle gap to print")
+    parser.add_argument("--topn", type=int, default=20, help="number of gaps/hotspots to print")
+    args = parser.parse_args()
+
+    for trace in args.traces:
+        summarize_trace(trace, args.min_gap_ms * 1000.0, args.topn)
+
+
+if __name__ == "__main__":
+    main()

.claude/skills/readme.md

@@ -18,6 +18,9 @@ include:
 
 - `add-diffusion-model`: guides integration of a new diffusion model into
   `vllm-omni`
+- `diffusion-perf-opt`: guides diffusion model performance optimization,
+  including profiling traces, parallel strategies, stage timing analysis, and
+  benchmark-driven tuning
 - `add-omni-model`: covers addition of new omni-modality model support
 - `add-tts-model`: covers integration of new TTS models and related serving
   workflows

README.md

@@ -9,7 +9,7 @@ Easy, fast, and cheap omni-modality model serving for everyone
 </h3>
 
 <p align="center">
-| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://discuss.vllm.ai"><b>User Forum</b></a> | <a href="https://slack.vllm.ai"><b>Developer Slack</b></a> | <a href="docs/assets/WeChat.jpg"><b>WeChat</b></a> | <a href="https://arxiv.org/abs/2602.02204"><b>Paper</b></a> | <a href="https://docs.google.com/presentation/d/111-L8zF7A1j_YI_cR8JsblofdScdRr2f/edit?usp=sharing&ouid=110473603432222024453&rtpof=true&sd=true"><b>Slides</b></a> |
+| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://deepwiki.com/vllm-project/vllm-omni"><b>DeepWiki</b></a> | <a href="https://discuss.vllm.ai"><b>User Forum</b></a> | <a href="https://slack.vllm.ai"><b>Developer Slack</b></a> | <a href="docs/assets/WeChat.jpg"><b>WeChat</b></a> | <a href="https://arxiv.org/abs/2602.02204"><b>Paper</b></a> | <a href="https://docs.google.com/presentation/d/111-L8zF7A1j_YI_cR8JsblofdScdRr2f/edit?usp=sharing&ouid=110473603432222024453&rtpof=true&sd=true"><b>Slides</b></a> |
 </p>
 
 

benchmarks/diffusion/README.md

@@ -30,7 +30,7 @@ python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 **Notes**
 
-- The benchmark talks to `http://<host>:<port>/v1/chat/completions`.
+- By default, image tasks talk to `http://<host>:<port>/v1/chat/completions`; video tasks talk to `/v1/videos`.
 - If you run the server on another host or port, pass `--base-url` accordingly.
 
 ## 2. Supported Datasets
@@ -95,8 +95,9 @@ You can point to your own trace using `--dataset-path`.
 
 ### Basic flags
 
-- `--base-url`: Server address (the script calls `.../v1/chat/completions`).
+- `--base-url`: Server address; `--endpoint` selects the path appended to this base URL.
 - `--model`: The OpenAI-compatible `model` field.
+- `--endpoint`: API endpoint path. Leading `/` is optional, e.g. `/v1/videos` or `v1/videos`.
 - `--task`: Task type (e.g., `t2i`, `t2v`, `i2i`, `i2v`).
 - `--dataset`: Dataset mode (`vbench` / `trace` / `random`).
 - `--num-prompts`: Number of requests to send.

benchmarks/diffusion/backends.py

@@ -151,7 +151,7 @@ async def async_request_chat_completions(
     return output
 
 
-async def async_request_openai_images(
+async def async_request_openai_image_generations(
     input: RequestFuncInput,
     session: aiohttp.ClientSession,
     pbar: tqdm | None = None,
@@ -348,12 +348,38 @@ async def async_request_v1_videos(
     return output
 
 
+LEGACY_BACKEND_ENDPOINT_ALIASES = {
+    "vllm-omni": "/v1/chat/completions",
+    "openai": "/v1/images/generations",
+}
+
+
+def normalize_endpoint(value: str) -> str:
+    endpoint = str(value).strip()
+    if not endpoint:
+        raise ValueError("endpoint must not be empty.")
+    endpoint = LEGACY_BACKEND_ENDPOINT_ALIASES.get(
+        endpoint,
+        LEGACY_BACKEND_ENDPOINT_ALIASES.get(endpoint.lstrip("/"), endpoint),
+    )
+    if not endpoint.startswith("/"):
+        endpoint = f"/{endpoint}"
+    return endpoint
+
+
+def endpoint_filename_token(value: str) -> str:
+    token = normalize_endpoint(value).lstrip("/")
+    for bad in ("/", "\\", ":", "*", "?", '"', "<", ">", "|"):
+        token = token.replace(bad, "_")
+    return token or "endpoint"
+
+
 backends_function_mapping = {
     "2i": {
-        "vllm-omni": (async_request_chat_completions, "/v1/chat/completions"),
-        "openai": (async_request_openai_images, "/v1/images/generations"),
+        "/v1/chat/completions": (async_request_chat_completions, "/v1/chat/completions"),
+        "/v1/images/generations": (async_request_openai_image_generations, "/v1/images/generations"),
     },
     "2v": {
-        "v1/videos": (async_request_v1_videos, "/v1/videos"),
+        "/v1/videos": (async_request_v1_videos, "/v1/videos"),
     },
 }

benchmarks/diffusion/diffusion_benchmark_serving.py

@@ -6,31 +6,31 @@
 Benchmark online serving for diffusion models (Image/Video Generation).
 If you want to use i2v, i2i dataset, you should `uv pip install gdown` first
 
-Supports multiple backends:
-    - vllm-omni: Uses /v1/chat/completions endpoint (default)
-    - openai: Uses /v1/images/generations endpoint
-    - v1/videos: Use /v1/videos endpoint
+Supports multiple endpoints:
+    - /v1/chat/completions: OpenAI chat-compatible image requests
+    - /v1/images/generations: OpenAI image generation requests
+    - /v1/videos: Async video jobs
 
 Usage:
-    # Video (v1/videos backend)
+    # Video (/v1/videos endpoint)
     t2v:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend v1/videos --dataset vbench --task t2v --num-prompts 10 \
+        --endpoint /v1/videos --dataset vbench --task t2v --num-prompts 10 \
         --height 480 --width 640 --fps 16 --num-frames 80
 
     i2v:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend v1/videos --dataset vbench --task i2v --num-prompts 10
+        --endpoint /v1/videos --dataset vbench --task i2v --num-prompts 10
 
 
-    # Image (vllm-omni backend)
+    # Image (/v1/chat/completions endpoint)
     t2i:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend vllm-omni --dataset vbench --task t2i --num-prompts 10 \
+        --endpoint /v1/chat/completions --dataset vbench --task t2i --num-prompts 10 \
         --height 1024 --width 1024
 
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend vllm-omni --dataset random --task t2i --num-prompts 1 \
+        --endpoint /v1/chat/completions --dataset random --task t2i --num-prompts 1 \
         --max-concurrency 1 --enable-negative-prompt \
         --random-request-config '[
             {"width":512,"height":512,"num_inference_steps":20,"weight":0.15},
@@ -41,18 +41,18 @@ Usage:
 
     i2i:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend vllm-omni --dataset vbench --task i2i --num-prompts 10
+        --endpoint /v1/chat/completions --dataset vbench --task i2i --num-prompts 10
 
-    # Image (openai backend)
+    # Image (/v1/images/generations endpoint)
     t2i:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend openai --dataset vbench --task t2i --num-prompts 10 \
+        --endpoint /v1/images/generations --dataset vbench --task t2i --num-prompts 10 \
         --height 1024 --width 1024 --port 3000
 
     # Video (v1/videos)
     t2v:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend v1/videos --dataset random --task t2v --num-prompts 1 \
+        --endpoint /v1/videos --dataset random --task t2v --num-prompts 1 \
         --max-concurrency 1 --enable-negative-prompt \
         --random-request-config '[
             {"width":854,"height":480,"num_inference_steps":18,"num_frames":120,"fps":24,"weight":1}
@@ -80,7 +80,12 @@ from typing import Any
 import aiohttp
 import numpy as np
 import requests
-from backends import RequestFuncInput, RequestFuncOutput, backends_function_mapping
+from backends import (
+    RequestFuncInput,
+    RequestFuncOutput,
+    backends_function_mapping,
+    normalize_endpoint,
+)
 from PIL import Image
 from tqdm.asyncio import tqdm
 
@@ -883,6 +888,14 @@ def wait_for_service(base_url: str, timeout: int = 120) -> None:
         time.sleep(1)
 
 
+def _default_endpoint_for_task(task: str) -> str:
+    if task in {"t2v", "i2v", "ti2v"}:
+        return "/v1/videos"
+    if task in {"t2i", "i2i", "ti2i"}:
+        return "/v1/chat/completions"
+    raise ValueError(f"Unsupported task for endpoint resolution: {task}")
+
+
 async def benchmark(args):
     # Construct base_url if not provided
     if args.base_url is None:
@@ -902,18 +915,23 @@ async def benchmark(args):
             f"Valid image tasks: {sorted(IMAGE_TASKS)}"
         )
 
-    valid_backends = sorted(backends_function_mapping[task_type].keys())
+    raw_endpoint = args.endpoint if args.endpoint is not None else args.backend
+    if raw_endpoint is None:
+        raw_endpoint = _default_endpoint_for_task(args.task)
+    args.endpoint = normalize_endpoint(raw_endpoint)
 
-    if args.backend not in valid_backends:
+    valid_endpoints = sorted(backends_function_mapping[task_type].keys())
+
+    if args.endpoint not in valid_endpoints:
         logger.error(
-            f"Invalid backend '{args.backend}' for task '{args.task}' (task type: '{task_type}').\n"
-            f"Valid backends for this task type: {valid_backends}\n"
-            f"Example usage: --task {args.task} --backend {valid_backends[0]}"
+            f"Invalid endpoint '{args.endpoint}' for task '{args.task}' (task type: '{task_type}').\n"
+            f"Valid endpoints for this task type: {valid_endpoints}\n"
+            f"Example usage: --task {args.task} --endpoint {valid_endpoints[0]}"
         )
-        raise ValueError("Backend validation failed. See log above for valid options.")
+        raise ValueError("Endpoint validation failed. See log above for valid options.")
 
-    # Setup API URL and request function based on backend
-    request_func, api_url = backends_function_mapping[task_type][args.backend]
+    # Setup API URL and request function based on endpoint.
+    request_func, api_url = backends_function_mapping[task_type][args.endpoint]
     api_url = f"{args.base_url}{api_url}"
 
     if args.dataset == "vbench":
@@ -976,15 +994,15 @@ async def benchmark(args):
     metrics = calculate_metrics(outputs, total_duration, requests_list, args, args.slo)
 
     # Add configuration info to metrics for JSON output
-    metrics["backend"] = args.backend
+    metrics["endpoint"] = args.endpoint
     metrics["model"] = args.model
     metrics["dataset"] = args.dataset
     metrics["task"] = args.task
 
-    print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=60, c="="))
+    print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
 
     # Section 1: Configuration
-    print("{:<40} {:<15}".format("Backend:", args.backend))
+    print("{:<40} {:<15}".format("Endpoint:", args.endpoint))
     print("{:<40} {:<15}".format("Model:", args.model))
     print("{:<40} {:<15}".format("Dataset:", args.dataset))
     print("{:<40} {:<15}".format("Task:", args.task))
@@ -1047,12 +1065,17 @@ if __name__ == "__main__":
     parser.add_argument("--host", type=str, default="localhost", help="Server host.")
     parser.add_argument("--port", type=int, default=8091, help="Server port.")
     parser.add_argument("--model", type=str, default="default", help="Model name.")
+    parser.add_argument(
+        "--endpoint",
+        type=str,
+        default=None,
+        help=("Endpoint path to target. Leading '/' is optional, e.g. /v1/videos or v1/videos."),
+    )
     parser.add_argument(
         "--backend",
         type=str,
-        default="vllm-omni",
-        choices=["vllm-omni", "openai", "v1/videos"],
-        help="Backend to target the benchmark to.",
+        default=None,
+        help=argparse.SUPPRESS,
     )
     parser.add_argument(
         "--dataset",

benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md

@@ -58,7 +58,7 @@ Record these parameters when reporting performance results.
 
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --backend vllm-omni \
+    --endpoint /v1/chat/completions \
     --dataset <DATASET_NAME> \
     --task t2i \
     --num-prompts <N> \
@@ -71,7 +71,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 | Parameter              | Description                       |
 | ---------------------- | --------------------------------- |
-| `--backend`            | Serving backend (use `vllm-omni`) |
+| `--endpoint`           | API endpoint (use `/v1/chat/completions`; leading `/` is optional) |
 | `--dataset`            | Dataset name (`random` or custom) |
 | `--task`               | Task type (e.g., `t2i`)           |
 | `--num-prompts`        | Total number of requests          |
@@ -129,7 +129,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --backend vllm-omni \
+    --endpoint /v1/chat/completions \
     --dataset random \
     --task t2i \
     --num-prompts 1 \

benchmarks/diffusion/performance_dashboard/wan_2_2_serving_performance.md

@@ -53,7 +53,7 @@ Record these parameters when reporting performance results.
 
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --backend v1/videos \
+    --endpoint /v1/videos \
     --dataset <DATASET_NAME> \
     --task t2v \
     --num-prompts <N> \
@@ -66,7 +66,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 | Parameter              | Description                       |
 | ---------------------- | --------------------------------- |
-| `--backend`            | Serving backend (use `v1/videos`) |
+| `--endpoint`           | API endpoint (use `/v1/videos`; leading `/` is optional) |
 | `--dataset`            | Dataset name (`random` or custom) |
 | `--task`               | Task type (e.g., `t2v`)           |
 | `--num-prompts`        | Total number of requests          |
@@ -120,7 +120,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --backend v1/videos \
+    --endpoint /v1/videos \
     --dataset random \
     --task t2v \
     --num-prompts 1 \

benchmarks/diffusion/quantization_quality.py

@@ -148,6 +148,7 @@ def _build_omni_kwargs(args, quantization=None):
 def _generate_image(omni, args, prompt, seed):
     """Generate a single image and return (PIL.Image, time_seconds, memory_gib)."""
     from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+    from vllm_omni.outputs import OmniRequestOutput
     from vllm_omni.platforms import current_omni_platform
 
     generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(seed)
@@ -165,8 +166,9 @@ def _generate_image(omni, args, prompt, seed):
     elapsed = time.perf_counter() - start
     peak_mem = torch.accelerator.max_memory_allocated() / (1024**3)
 
-    first = outputs[0]
-    req_out = first.request_output[0] if hasattr(first, "request_output") else first
+    req_out = OmniRequestOutput.unwrap_result(outputs)
+    if not req_out.images:
+        raise ValueError("Could not extract image output from result.")
     img = req_out.images[0]
     return img, elapsed, peak_mem
 

docker/Dockerfile.rocm

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.20.0
+ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.21.0
 FROM ${BASE_IMAGE} AS base
 
 # Declare a variable to know if we want to use the nightly build or the stable build.

docker/Dockerfile.xpu

@@ -88,7 +88,7 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
 ENV UV_INDEX_STRATEGY="unsafe-best-match"
 ENV UV_LINK_MODE="copy"
 
-ARG VLLM_VERSION=v0.20.0
+ARG VLLM_VERSION=v0.21.0
 RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
 WORKDIR /workspace/vllm
 

docs/.nav.yml

@@ -6,10 +6,11 @@ nav:
     - getting_started/installation/*
   - Serving:
     - OpenAI-Compatible API:
-      - Diffusion Chat API: serving/diffusion_chat_api.md
+      - Chat Completions API: serving/chat_completions_api.md
       - Image Generation: serving/image_generation_api.md
       - Image Edit: serving/image_edit_api.md
       - Text to Speech: serving/speech_api.md
+      - Videos: serving/videos_api.md
       - Audio Generation: serving/audio_generate_api.md
       - Streaming Video Input: serving/video_stream_api.md
   - Examples:

docs/api/README.md

@@ -99,7 +99,6 @@ Configuration classes.
 - [vllm_omni.diffusion.cache.teacache.config.TeaCacheConfig][]
 - [vllm_omni.distributed.omni_connectors.utils.config.ConnectorSpec][]
 - [vllm_omni.distributed.omni_connectors.utils.config.OmniTransferConfig][]
-- [vllm_omni.model_executor.models.cosyvoice3.config.CosyVoice3Config][]
 - [vllm_omni.model_executor.models.fish_speech.configuration_fish_speech.FishSpeechConfig][]
 - [vllm_omni.model_executor.models.fish_speech.configuration_fish_speech.FishSpeechFastARConfig][]
 - [vllm_omni.model_executor.models.fish_speech.configuration_fish_speech.FishSpeechSlowARConfig][]
@@ -116,10 +115,14 @@ Configuration classes.
 - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderConfig][]
 - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderDiTConfig][]
 - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1EncoderConfig][]
+- [vllm_omni.transformers_utils.configs.cosyvoice3.CosyVoice3Config][]
 - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Config][]
 - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLConfig][]
 - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLTextConfig][]
 - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLVisionConfig][]
+- [vllm_omni.transformers_utils.configs.omnivoice.OmniVoiceConfig][]
+- [vllm_omni.transformers_utils.configs.voxcpm.VoxCPMConfig][]
+- [vllm_omni.transformers_utils.configs.voxcpm2.VoxCPM2Config][]
 
 ## Workers
 

docs/community/governance.md

@@ -39,11 +39,34 @@ Committers have write access and merge rights. They typically have deep expertis
 - [@linyueqian](https://github.com/linyueqian): TTS and Omni Support
 - [@lishunyang12](https://github.com/lishunyang12): Quantization and Configuration
 - [@princepride](https://github.com/princepride): Diffusion and Omni Support
-- [@SamitHuang](https://github.com/SamitHuang): RL and Diffusion
+- [@RuixiangMa](https://github.com/RuixiangMa): Diffusion models, parallel, cache, and docs
+- [@SamitHuang](https://github.com/SamitHuang): RL, Diffusion, and cache
 - [@tzhouam](https://github.com/tzhouam): Engine and New Model Support
-- [@wtomin](https://github.com/wtomin): Diffusion and Parallelism
+- [@wtomin](https://github.com/wtomin): Diffusion models, parallel, and docs
 - [@ZeldaHuang](https://github.com/ZeldaHuang): Omni Support
-- [@ZJY0516](https://github.com/ZJY0516): Diffusion and CustomOp
+- [@ZJY0516](https://github.com/ZJY0516): Diffusion attention backend, kernel fusion, and CustomOp
+- [@yuanheng-zhao](https://github.com/yuanheng-zhao): Diffusion cache, offload, and Omni Support
+
+## Diffusion Workload Division
+
+This section breaks down diffusion-related responsibilities. If you have PRs touching these areas, please ping the listed owners for review.
+
+### Runtime-related
+
+- Diffusion models: @RuixiangMa, @wtomin
+
+### Optimization
+
+- Parallel: @RuixiangMa, @wtomin
+- Attention backend: @ZJY0516
+- Cache: @yuanheng-zhao, @RuixiangMa, @SamitHuang
+- Offload: @yuanheng-zhao
+- Quantization: @lishunyang12, @david6666666
+- Kernel fusion / communication-computation: @ZJY0516
+
+### Docs & Test
+
+- Diffusion docs: @RuixiangMa, @wtomin
 
 ## Meetings
 

docs/design/feature/pipeline_parallel.md

@@ -0,0 +1,356 @@
+# Pipeline Parallel
+
+This section describes how to add Pipeline Parallelism (PP) to a diffusion pipeline. We use the Wan2.2 text-to-video and
+image-to-video pipelines as the reference implementations.
+
+## Implementation Checklist
+
+Adding Pipeline Parallel support requires:
+
+1. ✅ **Inherit the mixin** - add `PipelineParallelMixin` before `CFGParallelMixin` in the pipeline class
+2. ✅ **Make stage forward resumable** - support `intermediate_tensors` in the transformer forward path
+3. ✅ **Return the right object type** - `IntermediateTensors` on non-last PP ranks, final model output on the last rank
+4. ✅ **Use the standard denoising loop** - call `predict_noise_maybe_with_cfg()` and `scheduler_step_maybe_with_cfg()`
+5. ✅ **Test parity** - compare PP results against the single-GPU baseline
+
+---
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Step-by-Step Implementation](#step-by-step-implementation)
+- [Testing](#testing)
+- [Troubleshooting](#troubleshooting)
+- [Reference Implementations](#reference-implementations)
+
+---
+
+## Overview
+
+### What is Pipeline Parallelism?
+
+Pipeline Parallelism splits the denoising transformer into multiple sequential stages and places each stage on a
+different rank. Instead of every rank holding the full DiT, each PP rank owns only a slice of the layers.
+
+For each denoising step:
+
+1. Rank 0 starts the forward pass using the current latents.
+2. Each intermediate rank receives hidden states from the previous rank, runs its local layer slice, and forwards the
+   intermediate tensors downstream.
+3. The last PP rank produces the final noise prediction.
+4. The last PP rank applies the scheduler step and sends the updated latents back to rank 0 for the next timestep.
+
+This reduces per-rank model memory and enables larger diffusion transformers to run across multiple GPUs. It can also be
+combined with CFG-Parallel, where each PP pipeline carries one CFG branch.
+
+### Architecture
+
+vLLM-Omni provides `PipelineParallelMixin` to encapsulate the PP communication pattern for diffusion pipelines.
+
+| Method                            | Purpose                       | Automatic Behavior                                                              |
+|-----------------------------------|-------------------------------|---------------------------------------------------------------------------------|
+| `diffuse()`                       | Denoising loop boundary       | Wrapped by `PipelineParallelMixin` to flush pending async PP sends on exit      |
+| `predict_noise_maybe_with_cfg()`  | Predict noise with PP support | Runs partial forwards on non-last PP ranks, combines with CFG logic when needed |
+| `scheduler_step_maybe_with_cfg()` | Step scheduler with PP sync   | Runs scheduler on the last PP rank, returns updated latents to rank 0           |
+| `_sync_pp_send()`                 | Flush pending async sends     | Waits for outstanding `isend` handles before later collectives or decode        |
+
+`PipelineParallelMixin` is intentionally a pipeline-level abstraction. Your model-specific `predict_noise()` still
+defines how a local stage executes.
+
+### How It Works
+
+`predict_noise_maybe_with_cfg()` automatically switches between these modes:
+
+- **PP disabled** (`pipeline_parallel_size == 1`):
+    - Falls back to `CFGParallelMixin.predict_noise_maybe_with_cfg()`
+- **PP only** (`pipeline_parallel_size > 1`, `cfg_parallel_size == 1`):
+    - Rank 0 starts with the input latents
+    - Middle ranks receive `intermediate_tensors`, run their local layer range, and asynchronously send downstream
+    - The last rank returns the final noise prediction
+    - When CFG is enabled in this mode (sequential CFG), both the positive and negative branches run through the same
+      PP pipeline, doubling the communication volume per denoising step. Prefer PP + CFG-Parallel when
+      `cfg_parallel_size > 1` is available to avoid this overhead.
+- **PP + CFG-Parallel** (`pipeline_parallel_size > 1`, `cfg_parallel_size > 1`):
+    - Each PP pipeline carries one CFG branch
+    - The last PP rank all-gathers across the CFG group
+    - CFG combination happens on every last PP rank in the CFG group, matching the non-PP CFG-parallel behavior
+
+`scheduler_step_maybe_with_cfg()` keeps the denoising loop consistent:
+
+- **PP disabled**:
+    - Falls back to `scheduler_step_maybe_with_cfg()`
+- **PP enabled**:
+    - Only the last PP rank has `noise_pred` and runs the scheduler step
+    - The resulting latents are sent back to rank 0
+    - Rank 0 receives an `AsyncLatents` wrapper that resolves only when the tensor is next consumed
+
+This asynchronous design avoids unnecessary blocking between denoising steps.
+
+When a pipeline class is defined, `PipelineParallelMixin` wraps its `diffuse()` method and calls `_sync_pp_send()` in a
+`finally` block after `diffuse()` returns or raises. This keeps model code free of explicit PP cleanup while still
+ensuring that the final non-blocking PP send completes before decode or later collectives.
+
+---
+
+## Step-by-Step Implementation
+
+### Step 1: Inherit `PipelineParallelMixin` and `CFGParallelMixin`
+
+`PipelineParallelMixin` **requires** `CFGParallelMixin` and must appear before it in the class MRO. This is enforced at
+class definition time via `__init_subclass__`: defining a pipeline that inherits `PipelineParallelMixin` without
+`CFGParallelMixin`, or after `CFGParallelMixin`, raises a `TypeError` immediately on import.
+
+`PipelineParallelMixin` delegates noise prediction, CFG combination, and scheduler stepping to `CFGParallelMixin`, which
+supplies `predict_noise()`, `predict_noise_maybe_with_cfg()`, `scheduler_step_maybe_with_cfg()`, and
+`combine_cfg_noise()`.
+
+**Example:**
+
+```python
+from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
+from vllm_omni.diffusion.distributed.pipeline_parallel import PipelineParallelMixin
+import torch.nn as nn
+
+
+class YourPipeline(nn.Module, PipelineParallelMixin, CFGParallelMixin):
+    ...
+```
+
+The order matters: `PipelineParallelMixin` must be listed before `CFGParallelMixin` so calls to
+`predict_noise_maybe_with_cfg()` and `scheduler_step_maybe_with_cfg()` resolve to the PP-aware wrappers, while their
+`super()` calls delegate to the CFG implementation when PP is disabled or after the last PP stage.
+
+### Step 2: Make model forward and `predict_noise()` PP-aware
+
+The PP mixin injects `intermediate_tensors` into the normal `predict_noise()` call. Your model forward path must support
+two inputs:
+
+- Normal input from rank 0, typically passed as `hidden_states` or `x`
+- `intermediate_tensors` from upstream PP ranks
+
+The standard model forward pattern is:
+
+1. If `intermediate_tensors` is present, read the local hidden state from it.
+2. Run only this rank's layer slice.
+3. Return `IntermediateTensors(...)` on non-last PP ranks.
+4. Return the final model output on the last PP rank. `CFGParallelMixin.predict_noise()` already follows this contract
+   for common pipelines.
+
+**Minimal example:**
+
+```python
+from vllm.sequence import IntermediateTensors
+from vllm_omni.diffusion.distributed.parallel_state import get_pp_group
+
+
+def forward(self, hidden_states=None, intermediate_tensors=None, **kwargs):
+    if intermediate_tensors is not None:
+        hidden_states = intermediate_tensors["hidden_states"]
+
+    for i in range(self.start_layer, self.end_layer):
+        hidden_states = self.layers[i](hidden_states)
+
+    pp_group = get_pp_group()
+    if not pp_group.is_last_rank:
+        return IntermediateTensors({"hidden_states": hidden_states})
+    return (hidden_states,)
+```
+
+### Step 3: Partition the transformer layers
+
+The local module on each PP rank must expose only that rank's layer slice. In the reference tests and vLLM model
+implementations, this is typically done with vLLM utilities such as `make_layers(...)`, which fill missing ranges with
+`PPMissingLayer`. Besides partitioning the layers themselves, model authors should also wire
+`make_empty_intermediate_tensors_factory(...)` for intermediate tensor allocation and `is_pp_missing_parameter(...)` for
+PP-aware weight loading.
+
+To prepare a transformer for PP, implement the following pieces in order.
+
+#### 3.1 Split the transformer layers across PP ranks
+
+Each PP rank should own only its local layer range, typically exposed as `[start_layer, end_layer)`. In practice this is
+usually done with `make_layers(...)`, which constructs local layers and fills missing ranges with `PPMissingLayer`.
+
+The goal is:
+
+- every PP rank knows its `[start_layer, end_layer)` range
+- non-local layers do not execute on this rank
+- the forward can resume from incoming `intermediate_tensors`
+
+By default, layers are distributed evenly across PP ranks using `get_pp_indices()`. When the number of layers is not
+divisible by the PP size, remaining layers are assigned to middle partitions to balance compute and memory. You can
+override this with the `VLLM_PP_LAYER_PARTITION` environment variable to specify an exact per-rank layer count:
+
+```bash
+# Example: 40 layers across 4 PP ranks, assigning 8 / 12 / 12 / 8 layers
+export VLLM_PP_LAYER_PARTITION=8,12,12,8
+```
+
+The value must be a comma-separated list of integers whose length equals `pipeline_parallel_size` and whose sum equals
+the total number of transformer layers.
+
+#### 3.2 Expose `make_empty_intermediate_tensors`
+
+Your transformer module should expose `self.make_empty_intermediate_tensors`, typically created with
+`make_empty_intermediate_tensors_factory(...)`.
+
+This is important because PP ranks need a consistent way to allocate placeholder `IntermediateTensors` with the expected
+keys and hidden dimension.
+
+**Example:**
+
+```python
+from vllm.model_executor.models.utils import make_empty_intermediate_tensors_factory
+
+self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
+    ["hidden_states"],
+    inner_dim,
+)
+```
+
+For Wan2.2, the intermediate payload between PP stages is the token sequence stored under `"hidden_states"`, so the
+factory is created with that key and the transformer hidden size.
+
+#### 3.3 Return `IntermediateTensors` on non-last PP ranks
+
+Your model `forward()` or custom `predict_noise()` implementation should consume `intermediate_tensors` on non-first
+ranks and return `IntermediateTensors(...)` on non-last ranks.
+
+This lets each PP stage resume from the upstream hidden states and pass the local result to the next stage.
+
+#### 3.4 Skip non-local weights during `load_weights()`
+
+When a model is PP-partitioned, many parameters in the checkpoint belong to layers that do not exist on the current
+rank. `load_weights()` must therefore skip parameters for missing PP stages using `is_pp_missing_parameter(...)`.
+
+If this is not done, weight loading will either fail or incorrectly try to load tensors into `PPMissingLayer`
+placeholders.
+
+The Wan2.2 transformer is the best reference here: it uses `is_pp_missing_parameter(...)` before loading both remapped
+and fused parameters.
+
+If your model has multiple transformer variants, PP still works as long as each selected transformer obeys the same
+contract.
+
+### Step 4: Use the standard denoising contract
+
+vLLM-Omni diffusion pipelines already route denoising through `diffuse()`, `predict_noise_maybe_with_cfg()`, and
+`scheduler_step_maybe_with_cfg()`. `PipelineParallelMixin` overrides those standard helpers when
+`pipeline_parallel_size > 1`, so model integrations should not add separate PP-specific helper names or manual post-loop
+synchronization.
+
+In PP mode:
+
+- Non-last PP ranks return `None` from `predict_noise_maybe_with_cfg()` after sending `IntermediateTensors` downstream.
+- The last PP rank returns the final prediction, and with CFG-Parallel every last PP rank in the CFG group receives the
+  combined result.
+- Rank 0 receives `AsyncLatents` from `scheduler_step_maybe_with_cfg()`, which resolves only when the latents are
+  consumed.
+- Pending non-blocking sends are flushed automatically when `diffuse()` exits.
+
+---
+
+## Testing
+
+Use an offline inference script with `pipeline_parallel_size > 1`.
+
+```bash
+python examples/offline_inference/text_to_video/text_to_video.py \
+--model=Wan-AI/Wan2.2-TI2V-5B-Diffusers \
+--width=1280 \
+--height=704 \
+--guidance-scale=5.0 \
+--prompt="Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage" \
+--output=t2v_5B_pp2.mp4 \
+--pipeline-parallel-size=2
+```
+
+For PP + CFG-Parallel together:
+
+```bash
+python examples/offline_inference/text_to_video/text_to_video.py \
+--model=Wan-AI/Wan2.2-TI2V-5B-Diffusers \
+--width=1280 \
+--height=704 \
+--guidance-scale=5.0 \
+--prompt="Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage" \
+--output=t2v_5B_pp2_cfg2.mp4 \
+--pipeline-parallel-size=2 \
+--cfg-parallel-size=2
+```
+
+**Verify:**
+
+1. The run completes without hangs at the PP stage boundary
+2. Output quality matches the non-PP baseline within normal numerical variance
+3. Peak memory per GPU drops compared with the single-rank model
+4. There are no outstanding communication errors before decode
+
+---
+
+## Troubleshooting
+
+### Issue: `TypeError` on import — `CFGParallelMixin` missing
+
+**Symptoms:** Importing a pipeline that inherits `PipelineParallelMixin` raises:
+
+```
+TypeError: YourPipeline inherits PipelineParallelMixin but not CFGParallelMixin.
+```
+
+or:
+
+```
+TypeError: YourPipeline must inherit PipelineParallelMixin before CFGParallelMixin ...
+```
+
+**Cause:** `PipelineParallelMixin` enforces via `__init_subclass__` that the subclass also inherits `CFGParallelMixin`
+and lists `PipelineParallelMixin` first in the MRO.
+
+**Solution:** Add `CFGParallelMixin` to the pipeline's base classes after `PipelineParallelMixin`:
+
+```python
+from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
+from vllm_omni.diffusion.distributed.pipeline_parallel import PipelineParallelMixin
+
+
+class YourPipeline(nn.Module, PipelineParallelMixin, CFGParallelMixin):
+    ...
+```
+
+### Issue: Non-last PP ranks crash when calling `predict_noise`
+
+**Symptoms:** Shape or missing-input errors on ranks other than the first or last PP rank.
+
+**Cause:** The model forward path assumes direct input tensors and ignores `intermediate_tensors`.
+
+**Solution:** Update the transformer `forward()` or custom `predict_noise()` path to load hidden states from
+`intermediate_tensors` when present.
+
+### Issue: PP outputs differ from single-GPU baseline
+
+**Symptoms:** The PP run finishes but produces numerically inconsistent outputs.
+
+**Causes & Solutions:**
+
+- **Local layer partitioning is wrong**
+    - Verify each rank runs only its `[start_layer, end_layer)` slice
+- **Non-last ranks return a plain tensor instead of `IntermediateTensors`**
+    - Return `IntermediateTensors({...})` until the last PP stage
+- **CFG branch wiring is incorrect**
+    - When CFG is enabled, confirm positive and negative kwargs are passed exactly as in the non-PP path
+
+---
+
+## Reference Implementations
+
+Complete examples in the codebase:
+
+| Component               | Path                                                       | Notes                                                                                                                   |
+|-------------------------|------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------|
+| `PipelineParallelMixin` | `vllm_omni/diffusion/distributed/pipeline_parallel.py`     | Core PP communication and scheduler helpers                                                                             |
+| `CFGParallelMixin`      | `vllm_omni/diffusion/distributed/cfg_parallel.py`          | Default `predict_noise()` tuple normalization and CFG helper fallback                                                   |
+| Wan2.2 transformer      | `vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py`  | Reference for layer partitioning, `IntermediateTensors`, `make_empty_intermediate_tensors`, and PP-aware weight loading |
+| Wan2.2 T2V pipeline     | `vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py`     | Reference PP + CFG integration for text-to-video                                                                        |
+| Wan2.2 I2V pipeline     | `vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py` | Reference PP + CFG integration for image-to-video                                                                       |
+| PP tests                | `tests/diffusion/distributed/test_pipeline_parallel.py`    | Baseline parity and async communication tests                                                                           |

docs/getting_started/installation/gpu/rocm.inc.md

@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm==0.21.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.21.0/rocm721
+uv pip install vllm==0.21.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.21.0/rocm722
 ```
 
 #### Installation of vLLM-Omni
@@ -37,10 +37,10 @@ uv pip install onnxruntime-rocm
 If you do not need to modify source code of vLLM, you can directly install the stable 0.21.0 release version of the library
 
 ```bash
-uv pip install vllm==0.21.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.21.0/rocm721
+uv pip install vllm==0.21.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.21.0/rocm722
 ```
 
-The pre-built 0.21.0 vLLM wheel targets ROCm 7.2.1. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The pre-built 0.21.0 vLLM wheel targets ROCm 7.2.2. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
 
 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source

docs/serving/chat_completions_api.md

@@ -1,4 +1,4 @@
-# Diffusion Chat Completions API
+# Chat Completions API
 
 vLLM-Omni supports generating and editing images via the `/v1/chat/completions`
 endpoint using diffusion models. This page explains how to pass generation

docs/serving/videos_api.md

@@ -0,0 +1,152 @@
+# Videos API
+
+vLLM-Omni provides an OpenAI-compatible video generation API for diffusion
+video models. The API supports asynchronous video jobs through `/v1/videos` and
+a synchronous benchmark-oriented endpoint through `/v1/videos/sync`.
+
+Each server instance runs a single model specified at startup with
+`vllm serve <model> --omni`.
+
+## Quick Start
+
+### Start the Server
+
+```bash
+vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --omni --port 8091
+```
+
+### Create a Video Job
+
+```bash
+create_response=$(curl -s http://localhost:8091/v1/videos \
+  -F "prompt=A cinematic tracking shot of a mountain lake at sunrise" \
+  -F "width=1280" \
+  -F "height=720" \
+  -F "num_frames=80" \
+  -F "fps=16" \
+  -F "num_inference_steps=40")
+
+video_id=$(echo "${create_response}" | jq -r '.id')
+```
+
+### Poll and Download
+
+```bash
+curl -s "http://localhost:8091/v1/videos/${video_id}" | jq .
+curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4
+```
+
+## API Reference
+
+### Endpoints
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| `/v1/videos` | `POST` | Create an asynchronous video generation job |
+| `/v1/videos/sync` | `POST` | Generate a video synchronously and return raw video bytes |
+| `/v1/videos/{video_id}` | `GET` | Retrieve job status and metadata |
+| `/v1/videos` | `GET` | List stored video jobs |
+| `/v1/videos/{video_id}/content` | `GET` | Download generated video content |
+| `/v1/videos/{video_id}` | `DELETE` | Delete a video job and stored output |
+
+### Request Parameters
+
+`POST /v1/videos` and `POST /v1/videos/sync` accept `multipart/form-data`.
+
+#### OpenAI-style fields
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `prompt` | string | **required** | Text prompt for video generation |
+| `model` | string | server's model | Optional model name |
+| `seconds` | string | null | Requested clip duration in seconds |
+| `size` | string | null | Requested output size in `WIDTHxHEIGHT` format |
+| `user` | string | null | Optional user identifier |
+
+#### vLLM-Omni extension fields
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `input_reference` | file | null | Uploaded reference image for image-to-video requests |
+| `image_reference` | string | null | JSON-encoded reference image payload; do not combine with `input_reference` |
+| `width` | integer | model default | Output video width |
+| `height` | integer | model default | Output video height |
+| `num_frames` | integer | model default | Number of generated frames |
+| `fps` | integer | model default | Output frames per second |
+| `num_inference_steps` | integer | model default | Number of diffusion steps |
+| `guidance_scale` | number | null | CFG guidance scale for the low-noise stage |
+| `guidance_scale_2` | number | null | CFG guidance scale for the high-noise stage |
+| `boundary_ratio` | number | null | Boundary split ratio for multi-stage denoising |
+| `flow_shift` | number | null | Scheduler flow-shift value |
+| `true_cfg_scale` | number | null | True CFG scale when supported by the model |
+| `seed` | integer | null | Random seed for reproducibility |
+| `negative_prompt` | string | null | Text describing what to avoid in the generated video |
+| `enable_frame_interpolation` | boolean | null | Enable post-generation frame interpolation |
+| `frame_interpolation_exp` | integer | null | Interpolation exponent; `1=2x`, `2=4x`, and so on |
+| `frame_interpolation_scale` | number | null | RIFE inference scale |
+| `frame_interpolation_model_path` | string | null | Local path or Hugging Face repo for the interpolation model |
+| `lora` | string | null | JSON-encoded LoRA configuration object |
+| `extra_params` | string | null | JSON-encoded object for additional model-specific parameters |
+
+### Create Response
+
+`POST /v1/videos` returns a job record:
+
+```json
+{
+  "id": "video-123",
+  "status": "queued",
+  "created_at": 1701234567
+}
+```
+
+The final content is available from `/v1/videos/{video_id}/content` after the
+job status becomes `completed`.
+
+### Synchronous Response
+
+`POST /v1/videos/sync` blocks until generation finishes and returns raw video
+bytes. It is useful for benchmarks and simple scripts that do not need job
+storage or polling.
+
+## Examples
+
+### Image-to-Video
+
+```bash
+curl -s http://localhost:8091/v1/videos \
+  -F "prompt=animate this image with subtle camera movement" \
+  -F "input_reference=@input.png" \
+  -F "width=1280" \
+  -F "height=720" \
+  -F "num_frames=80" \
+  -F "fps=16"
+```
+
+### Synchronous Generation
+
+```bash
+curl -X POST http://localhost:8091/v1/videos/sync \
+  -F "prompt=A small robot walking through a neon city" \
+  -F "width=854" \
+  -F "height=480" \
+  -F "num_frames=80" \
+  -F "fps=16" \
+  -o output.mp4
+```
+
+## Storage
+
+Set `VLLM_OMNI_STORAGE_PATH` to control where asynchronous video outputs are
+stored:
+
+```bash
+export VLLM_OMNI_STORAGE_PATH=/var/tmp/vllm-omni-videos
+```
+
+## Model-Specific Examples
+
+For complete text-to-video and image-to-video walkthroughs, see:
+
+- [Text-to-Video](../user_guide/examples/online_serving/text_to_video.md)
+- [Image-to-Video](../user_guide/examples/online_serving/image_to_video.md)

docs/user_guide/diffusion/frame_interpolation.md

@@ -28,7 +28,6 @@ Frame interpolation is currently supported for:
 
 - `WanPipeline` (Wan2.2 text-to-video)
 - `WanImageToVideoPipeline`
-- `Wan22TI2VPipeline`
 
 ## Request Parameters
 

docs/user_guide/diffusion/parallelism/overview.md

@@ -4,13 +4,14 @@ This guide covers the parallelism methods in vLLM-Omni for speeding up diffusion
 
 ## Supported Methods
 
-| Method | Description |
-|--------|-------------|
-| **[Tensor Parallelism](tensor_parallel.md)** | Shards DiT weights across GPUs to reduce per-GPU memory |
-| **[Sequence Parallelism](sequence_parallel.md)** | Splits sequence dimension across GPUs (Ulysses-SP, Ring-Attention, or hybrid) for high-resolution images and videos |
-| **[CFG-Parallel](cfg_parallel.md)** | Runs CFG positive/negative branches on separate GPUs for ~1.8x speedup on guided generation |
-| **[VAE Patch Parallelism](vae_patch_parallel.md)** | Distributes VAE decode spatially across GPUs to reduce peak VAE memory |
-| **[HSDP](hsdp.md)** | Shards full model weights via PyTorch FSDP2 to enable large-model inference on memory-constrained GPUs |
-| **[Expert Parallelism](expert_parallel.md)** | Shards MoE expert blocks across GPUs for MoE models (e.g. HunyuanImage3.0) |
+| Method                                             | Description                                                                                                         |
+|----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------|
+| **[Tensor Parallelism](tensor_parallel.md)**       | Shards DiT weights across GPUs to reduce per-GPU memory                                                             |
+| **[Sequence Parallelism](sequence_parallel.md)**   | Splits sequence dimension across GPUs (Ulysses-SP, Ring-Attention, or hybrid) for high-resolution images and videos |
+| **[CFG-Parallel](cfg_parallel.md)**                | Runs CFG positive/negative branches on separate GPUs for ~1.8x speedup on guided generation                         |
+| **[Pipeline Parallelism](pipeline_parallel.md)**   | Splits the denoising transformer block-wise across sequential GPU stages to reduce per-GPU model memory             |
+| **[VAE Patch Parallelism](vae_patch_parallel.md)** | Distributes VAE decode spatially across GPUs to reduce peak VAE memory                                              |
+| **[HSDP](hsdp.md)**                                | Shards full model weights via PyTorch FSDP2 to enable large-model inference on memory-constrained GPUs              |
+| **[Expert Parallelism](expert_parallel.md)**       | Shards MoE expert blocks across GPUs for MoE models (e.g. HunyuanImage3.0)                                          |
 
 See [Supported Models](../../diffusion_features.md#supported-models) for per-model compatibility.

docs/user_guide/diffusion/parallelism/pipeline_parallel.md

@@ -0,0 +1,205 @@
+# Pipeline Parallelism Guide
+
+## Table of Content
+
+- [Overview](#overview)
+- [Quick Start](#quick-start)
+- [Example Script](#example-script)
+- [Configuration Parameters](#configuration-parameters)
+- [Best Practices](#best-practices)
+- [Troubleshooting](#troubleshooting)
+- [Summary](#summary)
+
+---
+
+## Overview
+
+Pipeline Parallelism splits the denoising transformer block-wise into sequential stages across GPUs. Each rank owns only
+part of the transformer, which reduces per-GPU model memory and enables larger diffusion models to run across multiple
+devices.
+
+It can also be combined with other distributed methods such as CFG-Parallel, Tensor Parallelism, and Sequence
+Parallelism.
+
+See supported models list in [Supported Models](../../diffusion_features.md#supported-models).
+
+---
+
+## Quick Start
+
+### Basic Usage
+
+Simplest working example:
+
+```python
+from vllm_omni import Omni
+from vllm_omni.diffusion.data import DiffusionParallelConfig
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+
+omni = Omni(
+    model="Wan-AI/Wan2.2-TI2V-5B-Diffusers",
+    parallel_config=DiffusionParallelConfig(
+        pipeline_parallel_size=2,
+    ),
+)
+
+outputs = omni.generate(
+    {"prompt": "A cinematic drone shot over snowy mountains"},
+    OmniDiffusionSamplingParams(
+        num_inference_steps=40,
+        num_frames=81,
+        height=704,
+        width=1280,
+    ),
+)
+```
+
+---
+
+## Example Script
+
+### Offline Inference
+
+Use python scripts under:
+
+- `examples/offline_inference/text_to_video/text_to_video.py`
+- `examples/offline_inference/image_to_video/image_to_video.py`
+
+Text-to-video example:
+
+```bash
+python examples/offline_inference/text_to_video/text_to_video.py \
+--model=Wan-AI/Wan2.2-TI2V-5B-Diffusers \
+--width=1280 \
+--height=704 \
+--guidance-scale=5.0 \
+--prompt="Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage" \
+--output=t2v_5B_pp2.mp4 \
+--pipeline-parallel-size=2
+```
+
+Pipeline Parallelism can also be combined with CFG-Parallel:
+
+```bash
+python examples/offline_inference/text_to_video/text_to_video.py \
+--model=Wan-AI/Wan2.2-TI2V-5B-Diffusers \
+--width=1280 \
+--height=704 \
+--guidance-scale=5.0 \
+--prompt="Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage" \
+--output=t2v_5B_pp2_cfg2.mp4 \
+--pipeline-parallel-size=2 \
+--cfg-parallel-size=2
+```
+
+### Online Serving
+
+Enable Pipeline Parallelism in online serving:
+
+```bash
+# Default PP configuration
+vllm serve Wan-AI/Wan2.2-TI2V-5B-Diffusers --omni --port 8091 --pipeline-parallel-size 2
+
+# PP + CFG-Parallel
+vllm serve Wan-AI/Wan2.2-TI2V-5B-Diffusers --omni --port 8091 \
+  --pipeline-parallel-size 2 \
+  --cfg-parallel-size 2
+```
+
+---
+
+## Configuration Parameters
+
+In `DiffusionParallelConfig`
+
+| Parameter                | Type | Default | Description                                                                                                      |
+|--------------------------|------|---------|------------------------------------------------------------------------------------------------------------------|
+| `pipeline_parallel_size` | int  | 1       | Number of pipeline-parallel stages. Set to a value greater than 1 to split the denoising transformer across GPUs |
+
+> [!NOTE]
+> Total GPU count is the product of all enabled distributed dimensions, for example
+`pipeline_parallel_size * cfg_parallel_size * tensor_parallel_size * ulysses_degree * ring_degree`.
+
+### Manual Layer Partitioning
+
+By default, transformer layers are distributed evenly across PP ranks. You can override this with the
+`VLLM_PP_LAYER_PARTITION` environment variable to assign a specific number of layers to each rank:
+
+```bash
+# Example: 40 layers across 4 PP ranks, assigning 8 / 12 / 12 / 8 layers
+export VLLM_PP_LAYER_PARTITION=8,12,12,8
+```
+
+The value must be a comma-separated list of integers whose length equals `pipeline_parallel_size` and whose sum equals
+the total number of transformer layers. This is useful when you want to balance memory or compute asymmetrically across
+ranks.
+
+
+---
+
+## Best Practices
+
+### When to Use
+
+**Good for:**
+
+- Large diffusion transformers that do not fit comfortably on one GPU
+- Multi-GPU setups where reducing per-GPU model memory is more important than minimizing communication
+- Combining with CFG-Parallel or other distributed methods on supported models
+
+**Not for:**
+
+- Single GPU setups
+- Models that do not support Pipeline Parallelism
+  (check [supported models](../../diffusion_features.md#supported-models))
+- Very small models where inter-stage communication overhead may outweigh the benefit
+
+### Expected Behavior
+
+Pipeline Parallelism primarily reduces per-GPU memory usage by splitting the transformer into stage-local blocks.
+Depending on the model, topology, and resolution, it may also help execution fit into available hardware, but it is not
+primarily a latency optimization.
+
+---
+
+## Troubleshooting
+
+### Common Issue 1: No benefit from Pipeline Parallelism
+
+**Symptoms**: PP is enabled, but latency does not improve or becomes slightly worse.
+
+**Solutions**:
+
+1. **Check your goal:**
+
+```python
+# PP is mainly for memory scaling, not guaranteed latency speedup
+parallel_config = DiffusionParallelConfig(pipeline_parallel_size=2)
+```
+
+2. **Check model support:**
+    - Verify your model in [supported models](../../diffusion_features.md#supported-models)
+    - PP is currently validated only on selected pipelines
+
+3. **Combine with other methods when appropriate:**
+    - PP can be combined with CFG-Parallel, Tensor Parallelism, or Sequence Parallelism on supported models
+
+### Common Issue 2: PP pipeline fails at import
+
+**Symptoms**: Importing a custom pipeline raises a `TypeError` about `CFGParallelMixin` or mixin order.
+
+**Solutions**:
+
+1. Inherit both mixins.
+2. Put `PipelineParallelMixin` before `CFGParallelMixin` in the class base list.
+3. Use `class YourPipeline(nn.Module, PipelineParallelMixin, CFGParallelMixin): ...` as the reference pattern.
+
+---
+
+## Summary
+
+1. ✅ **Enable Pipeline Parallelism** - Set `pipeline_parallel_size > 1` in `DiffusionParallelConfig`
+2. ✅ **Use Supported Models** - Verify your model supports PP in
+   [supported models](../../diffusion_features.md#supported-models)
+3. ✅ **Combine When Needed** - PP can be combined with CFG-Parallel and other distributed methods on supported pipelines
+4. ✅ **Scale for Memory** - Use PP primarily to reduce per-GPU model memory and fit larger transformers

docs/user_guide/diffusion_features.md

@@ -35,14 +35,15 @@ Cache methods trade minimal quality for significant speedup. Quality loss is typ
 
 Parallelism methods distribute computation across GPUs without quality loss (mathematically equivalent to single-GPU).
 
-| Method | Description | Best For |
-|--------|-------------|----------|
-| **[Ulysses-SP](diffusion/parallelism/sequence_parallel.md)** | Sequence parallelism via all-to-all communication | High-resolution images (>1536px) or long videos with 2-8 GPUs |
-| **[Ring-Attention](diffusion/parallelism/sequence_parallel.md)** | Sequence parallelism via ring-based communication | Videos, very long sequences, memory-constrained, with 2-8 GPUs |
-| **[CFG-Parallel](diffusion/parallelism/cfg_parallel.md)** | Splits CFG positive/negative branches across devices | Image editing with CFG guidance (true_cfg_scale > 1) on 2 GPUs |
-| **[Tensor Parallelism](diffusion/parallelism/tensor_parallel.md)** | Shards model weights across devices | Large models that don't fit in single GPU, with 2+ GPUs |
-| **[HSDP](diffusion/parallelism/hsdp.md)** | Weight sharding via FSDP2, redistributed on-demand at runtime | Very large models (14B+) on limited VRAM, combinable with SP |
-| **[Expert Parallelism](diffusion/parallelism/expert_parallel.md)** | Shards MoE expert MLP blocks across devices | MoE diffusion models (e.g., HunyuanImage3.0) |
+| Method                                                                 | Description                                                              | Best For                                                          |
+|------------------------------------------------------------------------|--------------------------------------------------------------------------|-------------------------------------------------------------------|
+| **[Ulysses-SP](diffusion/parallelism/sequence_parallel.md)**           | Sequence parallelism via all-to-all communication                        | High-resolution images (>1536px) or long videos with 2-8 GPUs     |
+| **[Ring-Attention](diffusion/parallelism/sequence_parallel.md)**       | Sequence parallelism via ring-based communication                        | Videos, very long sequences, memory-constrained, with 2-8 GPUs    |
+| **[CFG-Parallel](diffusion/parallelism/cfg_parallel.md)**              | Splits CFG positive/negative branches across devices                     | Image editing with CFG guidance (true_cfg_scale > 1) on 2 GPUs    |
+| **[Tensor Parallelism](diffusion/parallelism/tensor_parallel.md)**     | Shards model weights across devices                                      | Large models that don't fit in single GPU, with 2+ GPUs           |
+| **[Pipeline Parallelism](diffusion/parallelism/pipeline_parallel.md)** | Splits the denoising transformer block-wise across sequential GPU stages | Large diffusion transformers that need lower per-GPU model memory |
+| **[HSDP](diffusion/parallelism/hsdp.md)**                              | Weight sharding via FSDP2, redistributed on-demand at runtime            | Very large models (14B+) on limited VRAM, combinable with SP      |
+| **[Expert Parallelism](diffusion/parallelism/expert_parallel.md)**     | Shards MoE expert MLP blocks across devices                              | MoE diffusion models (e.g., HunyuanImage3.0)                      |
 
 #### Startup Optimization
 
@@ -105,32 +106,32 @@ The following tables show which models support each feature:
 
 ### ImageGen
 
-| Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
-|-------|:----------:|:-----------:|:---------------------:|:--------------:|:-----------------:|:------:|:------------------------:|:--------------------:|:--------------:|:----------------:|
-| **Bagel** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅  | ❌ | ❌ | ❌ |
-| **FLUX.1-dev** | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
-| **FLUX.1-schnell** | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
-| **FLUX.2-klein** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
-| **FLUX.1-Kontext-dev** | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |
-| **FLUX.2-dev** | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |
-| **GLM-Image** | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |
-| **HunyuanImage3** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ |
-| **LongCat-Image** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ |
-| **LongCat-Image-Edit** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ |
-| **MagiHuman** | ❌ | ❌ | ❌ | ❓ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ |
-| **MammothModa2(T2I)** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
-| **Nextstep_1(T2I)** | ❓ | ❓ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ |
-| **OmniGen2** | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
-| **Ovis-Image** | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
-| **Qwen-Image** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ✅ | ✅ |
-| **Qwen-Image-2512** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ✅ | ✅ |
-| **Qwen-Image-Edit** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
-| **Qwen-Image-Edit-2509** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ✅ | ❌ | ❌ |
-| **Qwen-Image-Layered** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
-| **SenseNova-U1** | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ |
-| **Stable-Diffusion3.5** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ (decode) | ❌ | ❌ |
-| **Z-Image** | ✅ | ✅ | ✅ | ❓ | ✅ (TP=2 only) | ✅ | ❌ | ✅ (decode) | ✅ | ❌ |
-| **ERNIE-Image** | ❌ | ✅ | ✅ | ❓ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
+| Model                    | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | 🔀Pipeline-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
+|--------------------------|:---------:|:----------:|:---------------------:|:--------------:|:-----------------:|:-------------------:|:------:|:-------------------------:|:--------------------:|:--------------:|:----------------:|
+| **Bagel**                |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **FLUX.1-dev**           |     ✅     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ✅        |        ❌         |
+| **FLUX.1-schnell**       |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ✅        |        ❌         |
+| **FLUX.2-klein**         |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ✅        |        ❌         |
+| **FLUX.1-Kontext-dev**   |     ❌     |     ❌      |           ❌           |       ❌        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ❌        |        ❌         |
+| **FLUX.2-dev**           |     ✅     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ❌        |        ❌         |
+| **GLM-Image**            |     ❌     |     ❌      |           ❌           |       ✅        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ❌        |        ❌         |
+| **HunyuanImage3**        |     ❌     |     ✅      |           ❌           |       ❌        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ✅        |        ❌         |
+| **LongCat-Image**        |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **LongCat-Image-Edit**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **MagiHuman**            |     ❌     |     ❌      |           ❌           |       ❓        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **MammothModa2(T2I)**    |     ❌     |     ❌      |           ❌           |       ❌        |         ❌         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
+| **Nextstep_1(T2I)**      |     ❓     |     ❓      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **OmniGen2**             |     ❌     |     ✅      |           ❌           |       ❌        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
+| **Ovis-Image**           |     ❌     |     ✅      |           ❌           |       ✅        |         ❌         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **Qwen-Image**           |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ✅        |        ✅         |
+| **Qwen-Image-2512**      |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ✅        |        ✅         |
+| **Qwen-Image-Edit**      |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
+| **Qwen-Image-Edit-2509** |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |        ✅ (decode)         |          ✅           |       ❌        |        ❌         |
+| **Qwen-Image-Layered**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
+| **SenseNova-U1**         |     ❌     |     ❌      |           ❌           |       ❌        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **Stable-Diffusion3.5**  |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
+| **Z-Image**              |     ✅     |     ✅      |           ✅           |       ❓        |   ✅ (TP=2 only)   |          ❌          |   ✅    |             ❌             |      ✅ (decode)      |       ✅        |        ❌         |
+| **ERNIE-Image**          |     ❌     |     ✅      |           ✅           |       ❓        |         ✅         |          ❌          |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
 
 > Notes:
 > 1. Nextstep_1(T2I) does not support cache acceleration methods such as TeaCache or Cache-DiT.
@@ -138,15 +139,15 @@ The following tables show which models support each feature:
 
 ### VideoGen
 
-| Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
-|-------|:----------:|:-----------:|:---------------------:|:--------------:|:-----------------:|:------:|:------------------------:|:--------------------:|:--------------:|:----------------:|
-| **Wan2.2** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
-| **Wan2.1-VACE** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
-| **LTX-2** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
-| **LTX-2.3** | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
-| **Helios** | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
-| **HunyuanVideo-1.5 T2V I2V** | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ✅ | ❌ |
-| **DreamID-Omni** | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+| Model                        | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | Pipeline-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
+|------------------------------|:---------:|:----------:|:---------------------:|:--------------:|:-----------------:|:-----------------:|:------:|:-------------------------:|:--------------------:|:--------------:|:----------------:|
+| **Wan2.2**                   |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ✅         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ❌        |        ❌         |
+| **Wan2.1-VACE**              |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
+| **LTX-2**                    |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **LTX-2.3**                  |     ❌     |     ✅      |           ✅           |       ❌        |         ✅         |         ❌         |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
+| **Helios**                   |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **HunyuanVideo-1.5 T2V I2V** |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |       ✅        |        ❌         |
+| **DreamID-Omni**             |     ❌     |     ❌      |           ❌           |       ✅        |         ❌         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
 
 **Frame Interpolation Support**
 
@@ -155,9 +156,9 @@ The following tables show which models support each feature:
 
 ### AudioGen
 
-| Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
-|-------|:----------:|:-----------:|:---------------------:|:--------------:|:-----------------:|:------:|:------------------------:|:--------------------:|:--------------:|:----------------:|
-| **Stable-Audio-Open** | ✅ | ❌ | ❓ | ❓ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ |
+| Model                 | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | 🔀Pipeline-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
+|-----------------------|:---------:|:----------:|:---------------------:|:--------------:|:-----------------:|:-------------------:|:------:|:-------------------------:|:--------------------:|:--------------:|:----------------:|
+| **Stable-Audio-Open** |     ✅     |     ❌      |           ❓           |       ❓        |         ❌         |          ❌          |   ✅    |             ✅             |          ❌           |       ✅        |        ❌         |
 
 
 ## Feature Compatibility
@@ -257,7 +258,7 @@ Measured on NVIDIA H800:
 
 **Parallelism Methods:**
 
-- **[Parallelism Overview](diffusion/parallelism/overview.md)** - Tensor Parallelism, Sequence Parallelism, CFG Parallelism, HSDP, and Expert Parallelism
+- **[Parallelism Overview](diffusion/parallelism/overview.md)** - Tensor Parallelism, Sequence Parallelism, CFG Parallelism, Pipeline Parallelism, HSDP, and Expert Parallelism
 
 **Memory Optimization:**
 

docs/user_guide/examples/online_serving/glm_image.md

@@ -101,7 +101,7 @@ guides.
 
 When using `/v1/chat/completions`, pass these inside `extra_body` in the curl
 JSON, or via the `extra_body` keyword argument in the OpenAI Python SDK (see the
-[Diffusion Chat API guide](../../../serving/diffusion_chat_api.md)).
+[Chat Completions API guide](../../../serving/chat_completions_api.md)).
 When using the dedicated [`/v1/images/generations`](../../../serving/image_generation_api.md)
 or [`/v1/images/edits`](../../../serving/image_edit_api.md) endpoints, pass
 the supported generation controls as top-level fields directly. For image

docs/user_guide/examples/online_serving/image_to_image.md

@@ -208,7 +208,7 @@ Wrap generation parameters inside `extra_body` in the request JSON:
     ```
 
     For details on how generation parameters are handled across different clients, see the
-    [Diffusion Chat API guide](../../../../serving/diffusion_chat_api.md).
+    [Chat Completions API guide](../../../../serving/chat_completions_api.md).
 
 ### Layered Image Generation (Qwen-Image-Layered)
 
@@ -361,7 +361,7 @@ Provide multiple images in `content` (order matters):
 
 When using `/v1/chat/completions`, pass these inside `extra_body` in the curl
 JSON, or via the `extra_body` keyword argument in the OpenAI Python SDK (see the
-[Diffusion Chat API guide](../../../../serving/diffusion_chat_api.md)).
+[Chat Completions API guide](../../../../serving/chat_completions_api.md)).
 When using the dedicated [`/v1/images/edits`](../../../../serving/image_edit_api.md)
 endpoint, pass the supported generation controls as top-level form fields
 directly. For image dimensions and count, use `size` and `n` rather than

docs/user_guide/examples/online_serving/text_to_image.md

@@ -216,7 +216,7 @@ Wrap generation parameters inside `extra_body` in the request JSON:
     ```
 
     For details on how generation parameters are handled across different clients, see the
-    [Diffusion Chat API guide](../../../../serving/diffusion_chat_api.md).
+    [Chat Completions API guide](../../../../serving/chat_completions_api.md).
 
 ### Multimodal Input (Text + Structured Content)
 
@@ -237,7 +237,7 @@ Wrap generation parameters inside `extra_body` in the request JSON:
 
 When using `/v1/chat/completions`, pass these inside `extra_body` in the curl
 JSON, or via the `extra_body` keyword argument in the OpenAI Python SDK (see the
-[Diffusion Chat API guide](../../../../serving/diffusion_chat_api.md)).
+[Chat Completions API guide](../../../../serving/chat_completions_api.md)).
 When using the dedicated [`/v1/images/generations`](../../../../serving/image_generation_api.md)
 endpoint, pass the supported generation controls as top-level JSON fields
 directly. For image dimensions and count, use `size` and `n` rather than

docs/user_guide/quantization/overview.md

@@ -138,3 +138,98 @@ build_quant_config({"method": "gguf", "gguf_model": "/path/to/model.gguf"})
 build_quant_config({"transformer": {"method": "fp8"}, "vae": None})
 build_quant_config(None)
 ```
+
+## Output Similarity Comparison Tool
+
+Use `vllm_omni.quantization.tools.compare_diffusion_trajectory_similarity`
+to compare a reference diffusion run with a quantized candidate run using the
+same prompt, seed, resolution, scheduler settings, and inference steps. The
+tool compares final decoded images or video frames, and also reports generation
+latency and worker-reported peak memory when available.
+
+This is useful when validating whether online quantization, an offline
+pre-quantized checkpoint, or a new `ignored_layers` choice keeps generation
+quality close to the BF16 reference.
+
+### Online Quantization Example
+
+```bash
+python -m vllm_omni.quantization.tools.compare_diffusion_trajectory_similarity \
+  --task t2i \
+  --model Qwen/Qwen-Image \
+  --candidate-quantization fp8 \
+  --ignored-layers img_mlp \
+  --prompt "a cup of coffee on the table" \
+  --height 512 --width 512 \
+  --num-inference-steps 20 \
+  --seed 142 \
+  --output-json /tmp/qwen_image_fp8_similarity/result.json \
+  --save-output-dir /tmp/qwen_image_fp8_similarity/images \
+  --enforce-eager
+```
+
+### Offline Checkpoint Example
+
+Use `--candidate-model` when the candidate is already quantized or lives at a
+different model path:
+
+```bash
+python -m vllm_omni.quantization.tools.compare_diffusion_trajectory_similarity \
+  --task t2i \
+  --reference-model Qwen/Qwen-Image \
+  --candidate-model /path/to/qwen-image-fp8-checkpoint \
+  --prompt "a cup of coffee on the table" \
+  --height 512 --width 512 \
+  --num-inference-steps 20 \
+  --seed 142 \
+  --output-json /tmp/qwen_image_fp8_checkpoint_similarity/result.json
+```
+
+If the checkpoint does not include a loadable quantization config, pass one
+explicitly:
+
+```bash
+--candidate-quantization-config-json '{"method":"fp8"}'
+```
+
+### Output Metrics
+
+The output JSON includes `output_metrics`, `reference_generation`, and
+`candidate_generation`.
+
+| Metric | Direction | Meaning |
+|--------|-----------|---------|
+| `cosine_similarity` | Higher is better | Vector direction similarity between output pixels or frames. Useful as a broad sanity check. |
+| `mae` | Lower is better | Mean absolute pixel or frame error. For decoded outputs, values are in uint8 pixel units. |
+| `mse` / `rmse` | Lower is better | Squared error and its square root. These penalize localized large differences more than `mae`. |
+| `max_abs` | Lower is better | Worst single-element absolute error. Treat it as an outlier/debug signal, not as a release gate. |
+| `l2` / `relative_l2` | Lower is better | Absolute and reference-normalized L2 distance. `relative_l2` is easier to compare across resolutions. |
+| `psnr_db` | Higher is better | Pixel-space signal-to-noise ratio in dB for uint8 images or frames. |
+| `avg_generation_time_s` | Lower is better | Average wall-clock generation time across measured runs. |
+| `max_peak_memory_mb` | Lower is better | Maximum worker-reported peak device memory across measured runs, when the worker reports it. |
+
+Recommended starting thresholds for same-seed diffusion comparisons:
+
+| Metric | Smoke threshold | Stricter target | Notes |
+|--------|-----------------|-----------------|-------|
+| `psnr_db` | `>= 20.0` | `>= 25.0` | Good for quick image or frame regression checks. |
+| `mae` | `<= 12.0` | `<= 6.0` | Interpreted in decoded uint8 pixel units. |
+| `cosine_similarity` | `>= 0.98` | `>= 0.995` | Less sensitive to global scale than L2-style metrics. |
+| `relative_l2` | `<= 0.20` | `<= 0.08` | Useful when comparing across prompts or resolutions. |
+
+These thresholds are heuristics. Tune them by model family, task, resolution,
+quantization method, and deployment tolerance. For release gating, pair the
+numeric report with visual inspection of saved reference and candidate outputs.
+
+The tool intentionally reports separate quality, latency, and memory metrics
+instead of a single consolidated similarity score. A single score can hide
+important tradeoffs, for example a candidate with good PSNR but a meaningful
+memory regression, or a candidate with low average error but localized visual
+artifacts. If you need a project-specific pass/fail gate, define it as an
+explicit policy over the individual metrics.
+
+Pixel-level metrics do not measure semantic consistency. For higher-cost
+evaluation, you can complement this report with a vision-language judge that
+describes the reference and candidate outputs and compares those descriptions.
+Keep that semantic check separate from this lightweight tool so users can
+choose whether the additional model cost and latency are appropriate.

examples/offline_inference/hunyuan_image3/end2end.py

@@ -77,6 +77,24 @@ def parse_args():
     parser.add_argument("--log-stats", action="store_true", default=False)
     parser.add_argument("--init-timeout", type=int, default=300, help="Initialization timeout in seconds.")
     parser.add_argument("--enforce-eager", action="store_true", help="Disable torch.compile.")
+    parser.add_argument(
+        "--diffusion-kv-cache-dtype",
+        type=str,
+        default=None,
+        help="Diffusion attention KV cache dtype, for example 'fp8'. Separate from vLLM --kv-cache-dtype.",
+    )
+    parser.add_argument(
+        "--diffusion-kv-cache-skip-steps",
+        type=str,
+        default=None,
+        help="Denoising step selector to keep diffusion KV cache in native dtype, for example '0,1,4-6'.",
+    )
+    parser.add_argument(
+        "--diffusion-kv-cache-skip-layers",
+        type=str,
+        default=None,
+        help="Transformer layer selector to keep diffusion KV cache in native dtype, for example '0-2,10'.",
+    )
     parser.add_argument(
         "--additional-config",
         type=str,
@@ -140,6 +158,9 @@ def main():
         "init_timeout": args.init_timeout,
         "enforce_eager": args.enforce_eager,
         "mode": _MODALITY_MODE[args.modality],
+        "diffusion_kv_cache_dtype": args.diffusion_kv_cache_dtype,
+        "diffusion_kv_cache_skip_steps": args.diffusion_kv_cache_skip_steps,
+        "diffusion_kv_cache_skip_layers": args.diffusion_kv_cache_skip_layers,
     }
 
     if additional_config is not None:
@@ -237,6 +258,9 @@ def main():
         print(f"  Inference steps: {args.steps}")
         print(f"  Guidance scale: {args.guidance_scale}")
         print(f"  Seed: {args.seed}")
+        print(f"  diffusion_kv_cache_dtype: {args.diffusion_kv_cache_dtype}")
+        print(f"  diffusion_kv_cache_skip_steps: {args.diffusion_kv_cache_skip_steps}")
+        print(f"  diffusion_kv_cache_skip_layers: {args.diffusion_kv_cache_skip_layers}")
     if args.modality == "text2img":
         print(f"  Output size: {args.width}x{args.height}")
     if args.image_path:

examples/offline_inference/image_to_video/image_to_video.py

@@ -142,6 +142,42 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="Enable layerwise (blockwise) offloading on DiT modules.",
     )
+    parser.add_argument(
+        "--enforce-eager",
+        action="store_true",
+        help="Disable torch.compile and force eager execution.",
+    )
+    parser.add_argument(
+        "--audio-sample-rate",
+        type=int,
+        default=24000,
+        help="Sample rate for audio output when saved (default: 24000).",
+    )
+    parser.add_argument(
+        "--cache-backend",
+        type=str,
+        default=None,
+        choices=["cache_dit", "tea_cache"],
+        help=(
+            "Cache backend to use for acceleration. "
+            "Options: 'cache_dit' (DBCache + SCM + TaylorSeer), 'tea_cache' (Timestep Embedding Aware Cache). "
+            "Default: None (no cache acceleration)."
+        ),
+    )
+    parser.add_argument(
+        "--enable-diffusion-pipeline-profiler",
+        action="store_true",
+        help="Enable diffusion pipeline profiler to display stage durations.",
+    )
+    parser.add_argument(
+        "--quantization",
+        type=str,
+        default=None,
+        choices=["fp8", "mxfp8", "int8", "gguf"],
+        help="Quantization method for the transformer. mxfp8: W8A8 MXFP8 online quant (NPU). fp8: online FP8 (GPU).",
+    )
+
+    # Distributed and parallel execution
     parser.add_argument(
         "--ulysses-degree",
         type=int,
@@ -173,28 +209,6 @@ def parse_args() -> argparse.Namespace:
         default=1,
         help="Number of GPUs used for VAE patch/tile parallelism (decode).",
     )
-    parser.add_argument(
-        "--enforce-eager",
-        action="store_true",
-        help="Disable torch.compile and force eager execution.",
-    )
-    parser.add_argument(
-        "--audio-sample-rate",
-        type=int,
-        default=24000,
-        help="Sample rate for audio output when saved (default: 24000).",
-    )
-    parser.add_argument(
-        "--cache-backend",
-        type=str,
-        default=None,
-        choices=["cache_dit", "tea_cache"],
-        help=(
-            "Cache backend to use for acceleration. "
-            "Options: 'cache_dit' (DBCache + SCM + TaylorSeer), 'tea_cache' (Timestep Embedding Aware Cache). "
-            "Default: None (no cache acceleration)."
-        ),
-    )
     parser.add_argument(
         "--use-hsdp",
         action="store_true",
@@ -219,16 +233,10 @@ def parse_args() -> argparse.Namespace:
         ),
     )
     parser.add_argument(
-        "--quantization",
-        type=str,
-        default=None,
-        choices=["fp8", "mxfp8", "int8", "gguf"],
-        help="Quantization method for the transformer. mxfp8: W8A8 MXFP8 online quant (NPU). fp8: online FP8 (GPU).",
-    )
-    parser.add_argument(
-        "--enable-diffusion-pipeline-profiler",
-        action="store_true",
-        help="Enable diffusion pipeline profiler to display stage durations.",
+        "--pipeline-parallel-size",
+        type=int,
+        default=1,
+        help="Number of pipeline parallel stages.",
     )
     parser.add_argument(
         "--profiler-config",
@@ -326,6 +334,7 @@ def main():
         use_hsdp=args.use_hsdp,
         hsdp_shard_size=args.hsdp_shard_size,
         hsdp_replicate_size=args.hsdp_replicate_size,
+        pipeline_parallel_size=args.pipeline_parallel_size,
     )
     omni_kwargs = dict(
         model=args.model,
@@ -366,7 +375,8 @@ def main():
     print(f"  diffusion_kv_cache_skip_layers(config): {args.diffusion_kv_cache_skip_layers}")
     print(
         f"  Parallel configuration: cfg_parallel_size={args.cfg_parallel_size},"
-        f" tensor_parallel_size={args.tensor_parallel_size}, vae_patch_parallel_size={args.vae_patch_parallel_size}"
+        f" tensor_parallel_size={args.tensor_parallel_size}, vae_patch_parallel_size={args.vae_patch_parallel_size},"
+        f" pipeline_parallel_size={args.pipeline_parallel_size}"
     )
     print(f"  Video size: {args.width}x{args.height}")
     print(f"{'=' * 60}\n")

examples/offline_inference/text_to_speech/README.md

@@ -73,13 +73,15 @@ python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
 ```
 
 ### Voice cloning
-Pass a reference audio. Note that CosyVoice3's `--prompt-text` is a system-style prompt for the GPT stage, not a reference transcript:
+If `--ref-audio` is omitted, the script downloads the upstream
+[`zero_shot_prompt.wav`](https://github.com/FunAudioLLM/CosyVoice/blob/main/asset/zero_shot_prompt.wav) from the CosyVoice repo into the current directory.
+To use your own clip, pass `--ref-audio /path/to/reference.wav`, and modify `--prompt-text` correspondingly.
 ```bash
 python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
     --model pretrained_models/Fun-CosyVoice3-0.5B \
     --tokenizer pretrained_models/Fun-CosyVoice3-0.5B/CosyVoice-BlankEN \
-    --ref-audio prompt.wav \
-    --prompt-text "You are a helpful assistant.<|endofprompt|>Testing my voices. Why should I not?"
+    --ref-audio /path/to/reference.wav \
+    --prompt-text "You are a helpful assistant.<|endofprompt|>Trascript in your ref audio clip"
 ```
 
 ### Streaming
@@ -317,44 +319,39 @@ pip install voxcpm soundfile
 export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
 ```
 
-If the native VoxCPM `config.json` does not contain HF metadata such as `model_type`, prepare a persistent HF-compatible config directory and point the stage configs to it via `VLLM_OMNI_VOXCPM_HF_CONFIG_PATH`:
-
-```bash
-export VOXCPM_MODEL=/path/to/voxcpm-model
-export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config
-mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"
-cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json"
-cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true
-python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)'
-```
-
 ### Quick start
 ```bash
 python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
-    --model "$VOXCPM_MODEL" \
+    --model openbmb/VoxCPM-0.5B \
     --text "This is a split-stage VoxCPM synthesis example running on vLLM Omni."
 ```
 
 ### Voice cloning
 ```bash
 python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
-    --model "$VOXCPM_MODEL" \
+    --model openbmb/VoxCPM-0.5B \
     --text "This sentence is synthesized with a cloned voice." \
     --ref-audio /path/to/reference.wav \
     --ref-text  "The exact transcript spoken in reference.wav."
 ```
 
 ### Streaming
-Pass the async-chunk stage config:
+Pass `--streaming` together with the legacy async-chunk stage config:
 ```bash
 python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
-    --model "$VOXCPM_MODEL" \
+    --model openbmb/VoxCPM-0.5B \
+    --streaming \
     --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
     --text "This is a split-stage VoxCPM streaming example running on vLLM Omni."
 ```
 
+### Persistent HF config (optional)
+The engine auto-patches a missing `model_type` in the checkpoint's `config.json` in-memory at startup (see `vllm_omni/engine/arg_utils.py`), so no manual setup is required.
+
+If you'd rather maintain a persistent patched config dir (e.g. to share across runs or to add fields the auto-patcher doesn't set), export `VLLM_OMNI_VOXCPM_HF_CONFIG_PATH` to its location — both the bundled deploy and async-chunk yamls read it via `${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,}`.
+
 ### Notes
-- `voxcpm.yaml` is the default non-streaming stage config; `voxcpm_async_chunk.yaml` enables streaming.
+- Non-streaming auto-loads `vllm_omni/deploy/voxcpm.yaml` from the model directory name / HF `model_type`. Streaming uses the legacy `voxcpm_async_chunk.yaml` passed via `--stage-configs-path`.
 - Streaming is currently single-request oriented.
 - `--ref-text` must be the real transcript of `--ref-audio`; mismatched text degrades quality.
 - For online serving, see the [VoxCPM section in the online hub](../../online_serving/text_to_speech/README.md#voxcpm). For benchmark reporting, see [`benchmarks/voxcpm`](../../../benchmarks/voxcpm/README.md).

examples/offline_inference/text_to_speech/cosyvoice3/end2end.py

@@ -2,18 +2,32 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 import argparse
 import os
+import urllib.request
+from pathlib import Path
 
 import numpy as np
 import soundfile as sf
 from vllm import SamplingParams
-from vllm.assets.audio import AudioAsset
 from vllm.multimodal.media.audio import load_audio
 
 from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.model_executor.models.cosyvoice3.config import CosyVoice3Config
 from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
 from vllm_omni.model_executor.models.cosyvoice3.utils import extract_text_token
+from vllm_omni.transformers_utils.configs.cosyvoice3 import CosyVoice3Config
+
+# Upstream zero-shot reference clip
+ZERO_SHOT_PROMPT_URL = "https://raw.githubusercontent.com/FunAudioLLM/CosyVoice/main/asset/zero_shot_prompt.wav"
+
+
+def _default_ref_audio() -> str:
+    # Download the upstream zero_shot_prompt.wav into the current dir
+    dest = Path("zero_shot_prompt.wav")
+    if not dest.exists() or dest.stat().st_size == 0:
+        print(f"Downloading default reference audio to {dest}")
+        urllib.request.urlretrieve(ZERO_SHOT_PROMPT_URL, dest)
+
+    return str(dest)
 
 
 def run_e2e():
@@ -36,9 +50,15 @@ def run_e2e():
     parser.add_argument(
         "--prompt-text",
         type=str,
-        default="You are a helpful assistant.<|endofprompt|>Testing my voices. Why should I not?",
+        default="You are a helpful assistant.<|endofprompt|>希望你以后，能够做的比我还好呦!",
+    )
+    parser.add_argument(
+        "--ref-audio",
+        type=str,
+        default=None,
+        help="Path to reference audio for voice cloning. "
+        "If unset, downloads the upstream CosyVoice3 zero-shot prompt audio clip",
     )
-    parser.add_argument("--ref-audio", type=str, default="prompt.wav")
     parser.add_argument(
         "--tokenizer",
         type=str,
@@ -66,24 +86,22 @@ def run_e2e():
     sampling_cfg = {"top_p": 0.8, "top_k": 25, "eos_token_id": 6561 + 1}
 
     print("Model initialized. Preparing inputs...")
-    if args.ref_audio:
-        if not os.path.exists(args.ref_audio):
-            raise FileNotFoundError(f"Audio file not found: {args.ref_audio}")
-        # Load at native sample rate
-        audio_signal, sr = load_audio(args.ref_audio, sr=None)
+    ref_audio_path = args.ref_audio or _default_ref_audio()
+    if not os.path.exists(ref_audio_path):
+        raise FileNotFoundError(f"Audio file not found: {ref_audio_path}")
+    # Load at native sample rate
+    audio_signal, sr = load_audio(ref_audio_path, sr=None)
 
-        # Validate sample rate before processing (similar to original CosyVoice)
-        min_sr = 16000
-        if sr < min_sr:
-            raise ValueError(
-                f"Audio sample rate {sr} Hz is too low. "
-                f"Minimum required: {min_sr} Hz. "
-                f"Please provide audio with sample rate >= {min_sr} Hz."
-            )
+    # Validate sample rate before processing (similar to original CosyVoice)
+    min_sr = 16000
+    if sr < min_sr:
+        raise ValueError(
+            f"Audio sample rate {sr} Hz is too low. "
+            f"Minimum required: {min_sr} Hz. "
+            f"Please provide audio with sample rate >= {min_sr} Hz."
+        )
 
-        audio_data = (audio_signal.astype(np.float32), sr)
-    else:
-        audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
+    audio_data = (audio_signal.astype(np.float32), sr)
 
     prompts = {
         "prompt": args.text,

examples/offline_inference/text_to_speech/voxcpm/end2end.py

@@ -11,12 +11,9 @@ import soundfile as sf
 import torch
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 
-from tests.helpers.stage_config import get_deploy_config_path
 from vllm_omni import AsyncOmni, Omni
 from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 
-DEFAULT_SYNC_STAGE_CONFIG = get_deploy_config_path("voxcpm.yaml")
-
 
 def _build_prompt(args) -> dict[str, Any]:
     additional_information: dict[str, list[Any]] = {
@@ -59,10 +56,6 @@ def _extract_sample_rate(mm: dict[str, Any]) -> int:
     return int(sr_raw)
 
 
-def _is_streaming_stage_config(stage_config_path: str) -> bool:
-    return "async_chunk" in Path(stage_config_path).stem
-
-
 def _save_audio(audio: torch.Tensor, sample_rate: int, output_dir: Path, request_id: str) -> Path:
     output_dir.mkdir(parents=True, exist_ok=True)
     output_path = output_dir / f"output_{request_id}.wav"
@@ -87,6 +80,7 @@ async def _run_streaming(args) -> Path:
     omni = AsyncOmni(
         model=args.model,
         stage_configs_path=args.stage_configs_path,
+        deploy_config=args.deploy_config,
         log_stats=args.log_stats,
         stage_init_timeout=args.stage_init_timeout,
     )
@@ -133,6 +127,7 @@ def _run_sync(args) -> Path:
     omni = Omni(
         model=args.model,
         stage_configs_path=args.stage_configs_path,
+        deploy_config=args.deploy_config,
         log_stats=args.log_stats,
         stage_init_timeout=args.stage_init_timeout,
     )
@@ -164,11 +159,27 @@ def _run_sync(args) -> Path:
 def parse_args():
     parser = FlexibleArgumentParser(description="Minimal offline VoxCPM example for vLLM Omni.")
     parser.add_argument("--model", type=str, required=True, help="Local VoxCPM model directory.")
+    parser.add_argument(
+        "--deploy-config",
+        type=str,
+        default=None,
+        help=("Override the deploy config path."),
+    )
     parser.add_argument(
         "--stage-configs-path",
         type=str,
-        default=DEFAULT_SYNC_STAGE_CONFIG,
-        help=("Stage config path. Use voxcpm.yaml for non-streaming or voxcpm_async_chunk.yaml for streaming."),
+        default=None,
+        help=(
+            "Legacy stage_args yaml path. Required for streaming "
+            "(vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml); "
+            "leave unset for non-streaming to use the auto-resolved deploy config."
+        ),
+    )
+    parser.add_argument(
+        "--streaming",
+        action="store_true",
+        default=False,
+        help="Stream audio chunks as they arrive via AsyncOmni.",
     )
     parser.add_argument("--text", type=str, required=True, help="Input text for synthesis.")
     parser.add_argument("--ref-audio", type=str, default=None, help="Reference audio path for voice cloning.")
@@ -194,9 +205,10 @@ def parse_args():
 
 
 def main(args) -> None:
-    route = "streaming" if _is_streaming_stage_config(args.stage_configs_path) else "sync"
+    route = "streaming" if args.streaming else "sync"
     print(f"Model: {args.model}")
-    print(f"Stage config: {args.stage_configs_path}")
+    print(f"Deploy config: {args.deploy_config or '<auto from HF model_type>'}")
+    print(f"Stage configs path: {args.stage_configs_path or '<unused>'}")
     print(f"Route: {route}")
     if route == "streaming":
         asyncio.run(_run_streaming(args))

examples/offline_inference/text_to_video/text_to_video.py

@@ -142,6 +142,32 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="Enable layerwise (blockwise) offloading on DiT modules.",
     )
+    parser.add_argument(
+        "--audio-sample-rate",
+        type=int,
+        default=24000,
+        help="Sample rate for audio output when saved (default: 24000).",
+    )
+    parser.add_argument(
+        "--enable-diffusion-pipeline-profiler",
+        action="store_true",
+        help="Enable diffusion pipeline profiler to display stage durations.",
+    )
+    parser.add_argument(
+        "--profiler-config",
+        type=parse_profiler_config,
+        default=None,
+        help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
+    )
+    parser.add_argument(
+        "--quantization",
+        type=str,
+        default=None,
+        choices=["fp8", "mxfp8", "int8", "gguf"],
+        help="Quantization method for the transformer. mxfp8: W8A8 MXFP8 online quant (NPU). fp8: online FP8 (GPU).",
+    )
+
+    # Distributed and parallel execution
     parser.add_argument(
         "--ulysses-mode",
         type=str,
@@ -174,41 +200,23 @@ def parse_args() -> argparse.Namespace:
         default=1,
         help="Number of GPUs used for tensor parallelism (TP) inside the DiT.",
     )
-    parser.add_argument(
-        "--audio-sample-rate",
-        type=int,
-        default=24000,
-        help="Sample rate for audio output when saved (default: 24000).",
-    )
     parser.add_argument(
         "--vae-patch-parallel-size",
         type=int,
         default=1,
         help="Number of GPUs used for VAE patch/tile parallelism (decode).",
     )
+    parser.add_argument(
+        "--pipeline-parallel-size",
+        type=int,
+        default=1,
+        help="Number of pipeline parallel stages.",
+    )
     parser.add_argument(
         "--enable-expert-parallel",
         action="store_true",
         help="Enable expert parallelism for MoE layers.",
     )
-    parser.add_argument(
-        "--enable-diffusion-pipeline-profiler",
-        action="store_true",
-        help="Enable diffusion pipeline profiler to display stage durations.",
-    )
-    parser.add_argument(
-        "--profiler-config",
-        type=parse_profiler_config,
-        default=None,
-        help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
-    )
-    parser.add_argument(
-        "--quantization",
-        type=str,
-        default=None,
-        choices=["fp8", "mxfp8", "int8", "gguf"],
-        help="Quantization method for the transformer. mxfp8: W8A8 MXFP8 online quant (NPU). fp8: online FP8 (GPU).",
-    )
     parser.add_argument(
         "--use-hsdp",
         action="store_true",
@@ -262,6 +270,7 @@ def main():
         cfg_parallel_size=args.cfg_parallel_size,
         tensor_parallel_size=args.tensor_parallel_size,
         vae_patch_parallel_size=args.vae_patch_parallel_size,
+        pipeline_parallel_size=args.pipeline_parallel_size,
         enable_expert_parallel=args.enable_expert_parallel,
     )
 
@@ -307,7 +316,8 @@ def main():
     print(
         f"  Parallel configuration: ulysses_degree={args.ulysses_degree}, ring_degree={args.ring_degree},"
         f" cfg_parallel_size={args.cfg_parallel_size}, tensor_parallel_size={args.tensor_parallel_size},"
-        f" vae_patch_parallel_size={args.vae_patch_parallel_size}, enable_expert_parallel={args.enable_expert_parallel}"
+        f" vae_patch_parallel_size={args.vae_patch_parallel_size}, pipeline_parallel_size={args.pipeline_parallel_size},"
+        f" enable_expert_parallel={args.enable_expert_parallel}"
     )
     print(f"  Video size: {args.width}x{args.height}")
     print(f"{'=' * 60}\n")

pyproject.toml

@@ -252,3 +252,4 @@ fullset = "fullset"
 Vai = "Vai"
 tockens = "tockens"
 CANN = "CANN"
+ND = "ND"

recipes/Baidu/ERNIE-Image.md

@@ -1,4 +1,6 @@
-# ERNIE-Image for text-to-image generation
+# ERNIE-Image
+
+> Text-to-image online serving (ERNIE-Image 8B)
 
 ## Summary
 

recipes/LTX/LTX-2.3.md

@@ -1,4 +1,4 @@
-# LTX-2.3 Text-to-Video with Audio on 1x GPU (96GB VRAM)
+# LTX-2.3
 
 > 22B parameter text-to-video + audio generation model served via vLLM-Omni
 

recipes/LTX/LTX-2.md

@@ -1,6 +1,6 @@
-# LTX-2 T2V and I2V
+# LTX-2
 
-> LTX-2 for text-to-video & image-to-video on 1x H200 141GB
+> Text-to-video and image-to-video serving
 
 ## Summary
 

recipes/Qwen/Qwen-Image-Edit.md

@@ -0,0 +1,171 @@
+# Qwen-Image-Edit
+
+> Image editing serving
+
+## Summary
+
+- Vendor: Qwen
+- Model: `Qwen/Qwen-Image-Edit`
+- Task: Text-guided single-image editing
+- Mode: Online serving with the OpenAI-compatible multimodal endpoint
+- Maintainer: Community
+
+## When to use this recipe
+
+Use this recipe as a baseline for serving `Qwen/Qwen-Image-Edit` for single-image editing on H200 141GB. Two configurations are validated below, sharing one software environment:
+
+| Configuration | Parallelism | Notes |
+| --- | --- | --- |
+| `1x H200` | none | Single-card baseline; ~60 GiB peak VRAM on a 141 GB card |
+| `2x H200` | TP=2 | Lower single-request latency (≈ 1.60× speedup) and lower per-GPU VRAM |
+
+## References
+
+- Model: <https://huggingface.co/Qwen/Qwen-Image-Edit>
+- Example guide: [`examples/online_serving/image_to_image/README.md`](../../examples/online_serving/image_to_image/README.md)
+- User guide: [`docs/user_guide/examples/online_serving/image_to_image.md`](../../docs/user_guide/examples/online_serving/image_to_image.md)
+
+## Hardware Support
+
+### GPU
+
+The two hardware configurations below share the same software environment.
+
+#### Environment
+
+- OS: Linux
+- Python: 3.12+
+- Driver / runtime: CUDA-capable runtime matching the repository build
+- vLLM: use versions from your current checkout, >=0.20.0
+- vLLM-Omni: use versions from your current checkout
+
+#### 1x H200 141GB
+
+##### Command
+
+```bash
+CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen-Image-Edit --omni \
+  --port 8092
+```
+
+The bundled launcher is equivalent:
+
+```bash
+bash examples/online_serving/image_to_image/run_server.sh
+```
+
+The server is ready when the log shows `INFO:     Application startup complete.` (preceded by `Started server process [PID]` and `Waiting for application startup.`).
+
+##### Verification
+
+Run a single-image edit against the public example asset:
+
+```bash
+wget https://vllm-public-assets.s3.us-west-2.amazonaws.com/omni-assets/qwen-bear.png
+
+PROMPT="Let this mascot dance under the moon, surrounded by floating stars and poetic bubbles such as 'Be Kind'"
+IMG_B64=$(base64 -w0 qwen-bear.png)
+
+curl -s http://localhost:8092/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d "$(jq -n --arg prompt "$PROMPT" --arg img "$IMG_B64" '{
+    messages: [{
+      role: "user",
+      content: [
+        {type: "text", text: $prompt},
+        {type: "image_url", image_url: {url: ("data:image/png;base64," + $img)}}
+      ]
+    }],
+    extra_body: {
+      height: 1024,
+      width: 1024,
+      num_inference_steps: 50,
+      guidance_scale: 1.0,
+      seed: 42
+    }
+  }')" \
+  | jq -r '.choices[0].message.content[0].image_url.url' \
+  | cut -d',' -f2- | base64 -d > qwen_image_edit_1card.png
+```
+
+The decoded PNG is a 1024×1024 RGB image of the bear mascot dancing under stylized moonlight, surrounded by floating stars and a "Be Kind" text bubble.
+
+You can also run the offline CLI without standing up a server:
+
+```bash
+python examples/offline_inference/image_to_image/image_edit.py \
+  --image qwen-bear.png \
+  --prompt "Let this mascot dance under the moon, ..." \
+  --output qwen_image_edit_offline.png \
+  --num-inference-steps 50 \
+  --cfg-scale 4.0
+```
+
+Validated reference numbers for 1024×1024, 50 denoising steps, batch size 1, `guidance_scale=1.0`, seed 42 (`Cold` = first request after startup, `Warm` = identical subsequent request):
+
+| Cold latency | Warm latency | Peak GPU memory |
+| ---:         | ---:         | ---:            |
+| 17.80 s      | 17.78 s      | 61,091 MiB (~59.66 GiB) |
+
+##### Notes
+
+- Generation parameters live under `extra_body` for `/v1/chat/completions`, or as top-level form fields when using `/v1/images/edits`. See [`examples/online_serving/image_to_image/README.md`](../../examples/online_serving/image_to_image/README.md) for the full request / response schema.
+- Memory: a 1024×1024 / 50-step edit uses ~60 GiB on a single H200, well within the 141 GB headroom.
+- Multi-image variants of Qwen-Image-Edit (e.g. `Qwen-Image-Edit-2509`) are intentionally out of scope for this recipe and are not validated here.
+
+#### 2x H200 141GB (TP=2)
+
+##### Command
+
+```bash
+CUDA_VISIBLE_DEVICES=0,1 vllm serve Qwen/Qwen-Image-Edit --omni \
+  --port 8092 \
+  --tensor-parallel-size 2
+```
+
+In addition to the `Application startup complete.` line, the TP=2 server's `non-default args` log line echoes `'tensor_parallel_size': 2`, and a separate `Worker N: Initialized device and distributed environment` line is emitted for each rank — one per visible GPU.
+
+##### Verification
+
+Run the same `curl` command as in the **1x H200** Verification section above against the TP=2 server (same prompt, same seed, same image). Save the result to `qwen_image_edit_2card_tp2.png` so the two outputs sit side by side for comparison. During TP=2 inference, `nvidia-smi` shows sustained `~99 %` utilization on both GPUs, which is the expected signal that TP wiring is active.
+
+Same workload as the 1x table above (1024×1024, 50 steps, `guidance_scale=1.0`, seed 42):
+
+| Configuration | Cold latency | Warm latency | Peak per-GPU memory |
+| ---           | ---:         | ---:         | ---:                |
+| 1x H200       | 17.80 s      | 17.78 s      | 61,091 MiB (~59.66 GiB) |
+| 2x H200 TP=2  | 11.16 s      | 11.09 s      | 49,377 MiB (~48.22 GiB) |
+
+TP=2 reduces single-request warm latency from `17.78 s` to `11.09 s` (≈ **1.60×**) and drops per-GPU peak VRAM by ~11.4 GiB.
+
+##### Notes
+
+- TP=2 shards model weights across both cards, which is what produces both the latency reduction and the per-GPU VRAM drop above.
+- CFG-Parallel (`--cfg-parallel-size 2`) is supported on the Qwen-Image family but only when `cfg_scale > 1`. The verification command above uses `guidance_scale=1.0` to match the public example image, so it does not exercise CFG-parallel; a separate run with `cfg_scale > 1` is needed if you want to benchmark that path.
+- For deeper acceleration knobs (Cache-DiT, sequence parallel, HSDP) see [`docs/user_guide/diffusion/parallelism/overview.md`](../../docs/user_guide/diffusion/parallelism/overview.md); this recipe intentionally documents only the validated TP=2 baseline.
+
+#### Profiling
+
+The numbers below are from a warm request at 1024×1024, 50 steps, and `guidance_scale=1.0`, measured with `--enable-diffusion-pipeline-profiler` and a 250 ms `nvidia-smi --query-gpu=memory.used` sampler. For TP=2, latency uses rank max, and VRAM is per GPU.
+
+Per-stage latency:
+
+| Stage | TP=1 | TP=2 | Notes |
+| --- | ---: | ---: | --- |
+| `text_encoder.forward` | 0.132 s | 0.135 s | ~1.0× |
+| `vae.encode` | 0.040 s | 0.041 s | ~1.0× |
+| `diffuse` | 17.105 s | 10.382 s | **1.65×** speedup |
+| `vae.decode` | 0.064 s | 0.066 s | ~1.0× |
+| `forward` | 17.372 s | 10.652 s | 1.63× speedup |
+
+Per-GPU VRAM:
+
+| Snapshot | TP=1 | TP=2 | Notes |
+| --- | ---: | ---: | --- |
+| Static VRAM | 58,777 MiB | 47,065 MiB | after model load, before request |
+| Peak VRAM | 61,091 MiB | 49,377 MiB | during inference |
+| Request-time increment | 2,314 MiB | 2,312 MiB | peak minus static |
+
+The latency gain mainly comes from the DiT denoising stage (`diffuse` in the table above). The other measured stages, such as text encoder and VAE encode/decode, stay essentially unchanged between TP=1 and TP=2, which is consistent with them not benefiting meaningfully from TP in this pipeline. The `forward` row's 1.63× speedup tracks `diffuse`, since `diffuse` accounts for about 98% of total pipeline time.
+
+For memory, TP=2 saves about 11.4 GiB per GPU, mostly from static model-state memory, likely dominated by tensor-parallel sharding of the DiT weights. The request-time increment is almost unchanged, so TP=2 does not materially reduce transient inference-time memory for this single-image, batch-1 workload.

recipes/Qwen/Qwen-Image.md

@@ -1,4 +1,6 @@
-# Qwen-Image for text-to-image serving with optional continuous batching on 1x A100 80GB
+# Qwen-Image
+
+> Text-to-image serving with optional step-wise continuous batching
 
 ## Summary
 
@@ -102,7 +104,7 @@ To replay the batching benefit with matched warmup:
 
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
-  --backend vllm-omni \
+  --endpoint /v1/chat/completions \
   --dataset vbench \
   --task t2i \
   --model Qwen/Qwen-Image \

recipes/Qwen/Qwen3-Omni.md

@@ -1,5 +1,7 @@
 # Qwen3-Omni
 
+> Online serving for multimodal chat
+
 ## Summary
 
 - Vendor: Qwen

recipes/Qwen/Qwen3-TTS.md

@@ -1,4 +1,6 @@
-# Qwen3-TTS for text-to-speech
+# Qwen3-TTS
+
+> Text-to-speech serving (CustomVoice / VoiceDesign / Base)
 
 ## Summary
 

recipes/README.md

@@ -19,34 +19,29 @@ recipes/
   Qwen/
     Qwen3-Omni.md
     Qwen3-TTS.md
-  Tencent-Hunyuan/
-    HunyuanVideo.md
+  Tencent/
+    Covo-Audio-Chat.md
 ```
 
 ## Available Recipes
 
-- [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md): text-to-image serving recipe for
-  Qwen-Image on `1x A100 80GB`, including optional step-wise continuous batching replay
-- [`Qwen/Qwen3-Omni.md`](./Qwen/Qwen3-Omni.md): online serving recipe for
-  multimodal chat on `1x A100 80GB`
-- [`Tencent/Covo-Audio-Chat.md`](./Tencent/Covo-Audio-Chat.md): online
-  serving recipe for audio chat on `1x A100 80GB`
-- [`Qwen/Qwen3-TTS.md`](./Qwen/Qwen3-TTS.md): text-to-speech serving recipe
-  for Qwen3-TTS (CustomVoice / VoiceDesign / Base) on `1x H100/A100 80GB`
-- [`LTX/LTX-2.md`](./LTX/LTX-2.md): text-to-video and image-to-video serving
-  recipe for LTX-2 on `1x H200 141GB`
-- [`Wan-AI/Wan2.2-I2V.md`](./Wan-AI/Wan2.2-I2V.md): image-to-video serving
-  recipe for Wan2.2 14B on `8x Ascend NPU (A2/A3)`
-- [`Tencent-Hunyuan/HunyuanImage-3.0-Instruct.md`](./Tencent-Hunyuan/HunyuanImage-3.0-Instruct.md):
-  DiT-only text-to-image serving and benchmark recipe for HunyuanImage-3.0-Instruct
-  on `4x H100/H800 80GB`
-- [`inclusionAI/Ming-flash-omni-2.0.md`](./inclusionAI/Ming-flash-omni-2.0.md):
-  online serving recipe for multimodal chat (`4x H100 80GB`) and standalone TTS (`1x H100 80GB`)
-- [`Baidu/ERNIE-Image.md`](./Baidu/ERNIE-Image.md): text-to-image serving
-  online serving recipe for ERNIE-Image 8B on `1x RTX 4090 24GB` or `2x RTX 4090 24GB`
-- [`fishaudio/Fish-Speech-S2-Pro.md`](./fishaudio/Fish-Speech-S2-Pro.md): online serving recipe for TTS on `1x A800 80GB`
-- [`audiox/AudioX.md`](./audiox/AudioX.md): offline + online recipe for AudioX
-  unified text/video→audio diffusion on `1x L4 24GB`
+| Recipe | Task | Hardware |
+|--------|------|----------|
+| [`audiox/AudioX.md`](./audiox/AudioX.md) | Offline + online unified text/video→audio diffusion | 1x L4 24GB |
+| [`Baidu/ERNIE-Image.md`](./Baidu/ERNIE-Image.md) | Text-to-image online serving (ERNIE-Image 8B) | 1x or 2x RTX 4090 24GB |
+| [`fishaudio/Fish-Speech-S2-Pro.md`](./fishaudio/Fish-Speech-S2-Pro.md) | Online serving for TTS | 1x A800 80GB |
+| [`inclusionAI/Ming-flash-omni-2.0.md`](./inclusionAI/Ming-flash-omni-2.0.md) | Online serving for multimodal chat + standalone TTS | 4x H100 / 1x H100 80GB |
+| [`LTX/LTX-2.md`](./LTX/LTX-2.md) | Text-to-video and image-to-video serving | 1x H200 141GB |
+| [`LTX/LTX-2.3.md`](./LTX/LTX-2.3.md) | Text-to-video with audio generation (22B) | 1x GPU (96GB VRAM) |
+| [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md) | Text-to-image serving with step-wise continuous batching replay | 1x A100 80GB |
+| [`Qwen/Qwen-Image-Edit.md`](./Qwen/Qwen-Image-Edit.md) | Text-guided single-image editing | 1x or 2x H200 141GB |
+| [`Qwen/Qwen3-Omni.md`](./Qwen/Qwen3-Omni.md) | Online serving for multimodal chat | 1x A100 80GB |
+| [`Qwen/Qwen3-TTS.md`](./Qwen/Qwen3-TTS.md) | Text-to-speech serving (CustomVoice / VoiceDesign / Base) | 1x H100/A100 80GB |
+| [`SenseNova/SenseNova-U1.md`](./SenseNova/SenseNova-U1.md) | Unified image generation and understanding | 1x H200 (144GB) |
+| [`Tencent/Covo-Audio-Chat.md`](./Tencent/Covo-Audio-Chat.md) | Online serving for audio chat | 1x A100 80GB |
+| [`Tencent/HunyuanImage-3.0-Instruct.md`](./Tencent/HunyuanImage-3.0-Instruct.md) | DiT-only text-to-image serving and benchmark | 4x H100/H800 80GB |
+| [`Wan-AI/Wan2.2-I2V.md`](./Wan-AI/Wan2.2-I2V.md) | Image-to-video serving (Wan2.2 14B) | 8x Ascend NPU (A2/A3) |
+| [`Wan-AI/Wan2.2-S2V.md`](./Wan-AI/Wan2.2-S2V.md) | Speech-to-video serving (Wan2.2 14B) | 2x A100/H100 80GB |
 
 Within a single recipe file, include different hardware support sections such
 as `GPU`, `ROCm`, and `NPU`, and add concrete tested configurations like

recipes/SenseNova/SenseNova-U1.md

@@ -1,4 +1,6 @@
-# SenseNova-U1 for unified image generation and understanding
+# SenseNova-U1
+
+> Unified image generation and understanding
 
 ## Summary
 

recipes/TEMPLATE.md

@@ -1,6 +1,6 @@
 # Recipe Title
 
-> Example: Qwen3-Omni for speech chat on 1x A100 80GB
+> Example: Qwen3-Omni for speech chat
 
 ## Summary
 

recipes/Tencent/Covo-Audio-Chat.md

@@ -1,4 +1,6 @@
-# Covo-Audio-Chat for audio chat on 1x A100 80GB
+# Covo-Audio-Chat
+
+> Online serving for audio chat
 
 ## Summary
 

recipes/Tencent/HunyuanImage-3.0-Instruct.md

@@ -1,7 +1,6 @@
-# HunyuanImage-3.0-Instruct DiT Image Generation on 4x GPU
+# HunyuanImage-3.0-Instruct
 
-> DiT-only text-to-image recipe for HunyuanImage-3.0-Instruct with FP8,
-> tensor parallelism, sequence parallelism, and CFG parallelism.
+> DiT-only text-to-image serving and benchmark
 
 ## Summary
 

recipes/Wan-AI/Wan2.2-I2V.md

@@ -1,4 +1,6 @@
-# Wan2.2 Image To Video
+# Wan2.2 Image-to-Video
+
+> Image-to-video serving (Wan2.2 14B)
 
 ## Summary
 

recipes/Wan-AI/Wan2.2-S2V.md

@@ -1,4 +1,6 @@
-# Wan2.2 Speech To Video
+# Wan2.2 Speech-to-Video
+
+> Speech-to-video serving (Wan2.2 14B)
 
 ## Summary
 

recipes/fishaudio/Fish-Speech-S2-Pro.md

@@ -1,5 +1,7 @@
 # Fish Speech S2 Pro
 
+> Online serving for TTS
+
 ## Summary
 
 - Vendor: FishAudio

recipes/inclusionAI/Ming-flash-omni-2.0.md

@@ -1,4 +1,6 @@
-# Ming-flash-omni 2.0 for omni-speech chat and standalone TTS
+# Ming-flash-omni 2.0
+
+> Online serving for multimodal chat + standalone TTS
 
 ## Summary
 

tests/benchmarks/metrics/test_metrics.py

@@ -63,5 +63,96 @@ def test_total_input_aggregated_from_output_prompt_len():
     )
 
 
+# ============================================================================
+# TTFT suppression for pure-audio (TTS) benchmarks
+# ============================================================================
+
+
+class _EmptyAwareTokenizer:
+    """Minimal tokenizer stub: token count == len(text), so '' -> 0 tokens.
+
+    Mirrors production where a TTS speech endpoint returns empty generated_text,
+    making total_output == 0 (the real CI path uses a real tokenizer, not None).
+    """
+
+    def __call__(self, text, add_special_tokens=False):
+        class _R:
+            pass
+
+        r = _R()
+        r.input_ids = [0] * len(text)
+        return r
+
+
+def _make_tts_output(prompt_len: int) -> MixRequestFuncOutput:
+    """Pure-TTS output: no text tokens, only audio. ttft is left unset (0.0)."""
+    output = MixRequestFuncOutput()
+    output.success = True
+    output.prompt_len = prompt_len
+    output.output_tokens = 0
+    output.generated_text = ""
+    output.ttft = 0.0
+    output.text_latency = 1.0
+    output.latency = 1.0
+    output.start_time = 0.0
+    output.itl = []
+    output.audio_ttfp = 0.05
+    output.audio_rtf = 0.2
+    output.audio_duration = 5.0
+    output.audio_frames = 120000
+    output.input_audio_duration = 0.0
+    output.error = ""
+    return output
+
+
+_TTS_PERCENTILE_METRICS = ["ttft", "e2el", "audio_rtf", "audio_ttfp", "audio_duration"]
+
+
+def test_tts_benchmark_omits_ttft(capsys):
+    """Pure-TTS run (total_output == 0) must not print a Time to First Token section."""
+    outputs = [_make_tts_output(100), _make_tts_output(120)]
+
+    calculate_metrics(
+        input_requests=[],
+        outputs=outputs,
+        dur_s=10.0,
+        tokenizer=_EmptyAwareTokenizer(),
+        selected_percentiles=[99.0],
+        goodput_config_dict={},
+        task_type=TaskType.GENERATION,
+        selected_percentile_metrics=_TTS_PERCENTILE_METRICS,
+        max_concurrency=None,
+        request_rate=float("inf"),
+        benchmark_duration=10.0,
+    )
+
+    out = capsys.readouterr().out
+    assert "Time to First Token" not in out, "TTS bench must not surface a meaningless TTFT"
+    assert "Time to First Packet" in out, "audio TTFP must still be reported"
+    assert "End-to-end Latency" in out, "e2el must still be reported"
+
+
+def test_text_benchmark_still_reports_ttft(capsys):
+    """Regression guard: real text generation (total_output > 0) keeps TTFT."""
+    outputs = [_make_output(100, output_tokens=10), _make_output(120, output_tokens=10)]
+
+    calculate_metrics(
+        input_requests=[],
+        outputs=outputs,
+        dur_s=10.0,
+        tokenizer=_EmptyAwareTokenizer(),
+        selected_percentiles=[99.0],
+        goodput_config_dict={},
+        task_type=TaskType.GENERATION,
+        selected_percentile_metrics=_TTS_PERCENTILE_METRICS,
+        max_concurrency=None,
+        request_rate=float("inf"),
+        benchmark_duration=10.0,
+    )
+
+    out = capsys.readouterr().out
+    assert "Time to First Token" in out, "text benchmarks must keep TTFT"
+
+
 if __name__ == "__main__":
     pytest.main([__file__, "-v", "-s"])

tests/benchmarks/test_diffusion_backends_metrics.py

@@ -1,6 +1,11 @@
 import pytest
 
-from benchmarks.diffusion.backends import RequestFuncInput, async_request_chat_completions
+from benchmarks.diffusion.backends import (
+    RequestFuncInput,
+    async_request_chat_completions,
+    endpoint_filename_token,
+    normalize_endpoint,
+)
 
 
 class _MockResponse:
@@ -29,6 +34,32 @@ class _MockSession:
         return _MockResponse(self._payload)
 
 
+@pytest.mark.core_model
+@pytest.mark.benchmark
+@pytest.mark.cpu
+def test_endpoint_normalization_accepts_optional_leading_slash():
+    assert normalize_endpoint("v1/videos") == "/v1/videos"
+    assert normalize_endpoint("/v1/videos") == "/v1/videos"
+    assert normalize_endpoint("v1/chat/completions") == "/v1/chat/completions"
+    assert normalize_endpoint("v1/images/generations") == "/v1/images/generations"
+
+
+@pytest.mark.core_model
+@pytest.mark.benchmark
+@pytest.mark.cpu
+def test_endpoint_normalization_accepts_legacy_backend_aliases():
+    assert normalize_endpoint("vllm-omni") == "/v1/chat/completions"
+    assert normalize_endpoint("openai") == "/v1/images/generations"
+
+
+@pytest.mark.core_model
+@pytest.mark.benchmark
+@pytest.mark.cpu
+def test_endpoint_filename_token_drops_leading_slash():
+    assert endpoint_filename_token("/v1/videos") == "v1_videos"
+    assert endpoint_filename_token("v1/chat/completions") == "v1_chat_completions"
+
+
 @pytest.mark.core_model
 @pytest.mark.benchmark
 @pytest.mark.cpu

tests/conftest.py

@@ -36,6 +36,7 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config):
 _ASSERTION_EXPORT_NAMES = (
     "assert_audio_speech_response",
     "assert_diffusion_response",
+    "assert_http_error",
     "assert_image_diffusion_response",
     "assert_image_valid",
     "assert_omni_response",
@@ -54,6 +55,7 @@ _MEDIA_EXPORT_NAMES = (
 _STAGE_CONFIG_EXPORT_NAMES = ("modify_stage_config",)
 _RUNTIME_EXPORT_NAMES = (
     "DiffusionResponse",
+    "HttpResponse",
     "OmniResponse",
     "OmniRunner",
     "OmniRunnerHandler",

tests/dfx/conftest.py

@@ -279,6 +279,10 @@ def create_benchmark_indices(
     return indices
 
 
+# Omni perf/stability: ``vllm bench serve --omni`` result template (used if bench exits without writing JSON).
+OMNI_RESULT_TEMPLATE_PATH = Path(__file__).resolve().parent / "perf" / "scripts" / "result_omni_template.json"
+
+
 def _safe_filename_token(value: Any | None, *, default: str = "na") -> str:
     """Make a single path segment safe for result filenames on common filesystems."""
     if value is None:
@@ -289,7 +293,7 @@ def _safe_filename_token(value: Any | None, *, default: str = "na") -> str:
     return s if s else default
 
 
-def _resolve_baseline_value(
+def resolve_baseline_value(
     baseline_raw: Any,
     *,
     sweep_index: int | None,
@@ -330,7 +334,7 @@ def _baseline_thresholds_for_step(
 ) -> dict[str, Any]:
     """Resolve baseline config to one threshold per metric for this iteration."""
     return {
-        metric_name: _resolve_baseline_value(
+        metric_name: resolve_baseline_value(
             baseline_raw,
             sweep_index=sweep_index,
             max_concurrency=max_concurrency,
@@ -354,7 +358,12 @@ def run_benchmark(
     random_input_len: Any | None = None,
     random_output_len: Any | None = None,
 ) -> dict[str, Any]:
-    """Run one ``vllm bench serve --omni`` iteration and return parsed metrics."""
+    """Run one ``vllm bench serve --omni`` iteration and return parsed metrics.
+
+    After ``vllm bench`` writes the JSON, ``result["baseline"]`` holds the resolved per-metric thresholds
+    (when ``baseline_config`` is provided). If the benchmark exits without writing a result file,
+    ``result_omni_template.json`` is used as a fallback.
+    """
     current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
     ri = _safe_filename_token(random_input_len)
     ro = _safe_filename_token(random_output_len)
@@ -403,8 +412,17 @@ def run_benchmark(
         result_dir = "./"
 
     result_path = os.path.join(result_dir, result_filename)
-    with open(result_path, encoding="utf-8") as f:
-        result = json.load(f)
+    if not os.path.exists(result_path):
+        with open(OMNI_RESULT_TEMPLATE_PATH, encoding="utf-8") as f:
+            template_result: dict[str, Any] = json.load(f)
+        Path(result_path).parent.mkdir(parents=True, exist_ok=True)
+        with open(result_path, "w", encoding="utf-8") as f:
+            json.dump(template_result, f, ensure_ascii=False, indent=2)
+        print(f"Benchmark result file not generated, fallback to template: {result_path}")
+        result = template_result
+    else:
+        with open(result_path, encoding="utf-8") as f:
+            result = json.load(f)
 
     if baseline_config:
         result["baseline"] = _baseline_thresholds_for_step(
@@ -432,3 +450,12 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         default=None,
         help=("Path to benchmark config JSON. Example: --test-config-file tests/dfx/perf/tests/test_tts.json"),
     )
+    parser.addoption(
+        "--assert-baseline",
+        action="store_true",
+        default=False,
+        help=(
+            "When set, omni/diffusion perf runners compare metrics against the baseline block in the JSON config "
+            "(default: off)."
+        ),
+    )

tests/dfx/perf/scripts/diffusion_result_template.json

@@ -1,7 +1,7 @@
 [
   {
     "test_name": null,
-    "backend": null,
+    "endpoint": null,
     "timestamp": null,
     "server_params": {
       "model": null,
@@ -43,7 +43,7 @@
       "stage_durations_mean": {},
       "stage_durations_p50": {},
       "stage_durations_p99": {},
-      "backend": null,
+      "endpoint": null,
       "model": null,
       "dataset": null,
       "task": null
@@ -51,6 +51,7 @@
     "log_file": null,
     "Model": null,
     "Framework": null,
+    "API Endpoint": null,
     "Hardware": null,
     "Deployment": null,
     "Task": null,

tests/dfx/perf/scripts/run_benchmark.py

@@ -1,8 +1,6 @@
 import json
 import os
-import subprocess
 import threading
-from datetime import datetime
 from pathlib import Path
 from typing import Any
 
@@ -14,11 +12,17 @@ from tests.dfx.conftest import (
     create_unique_server_params,
     get_benchmark_params_for_server,
     load_configs,
+    resolve_baseline_value,
+    run_benchmark,
 )
 from tests.helpers.runtime import OmniServer
 
 pytestmark = [pytest.mark.full_model]
 
+# Compare metrics to each test JSON ``baseline`` block only when pytest is run with ``--assert-baseline``
+# (registered in ``tests/dfx/conftest.py``; default: off). ``run_benchmark`` and ``_resolve_baseline_value`` are
+# defined in the same module.
+
 
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
 
@@ -47,7 +51,6 @@ if CONFIG_FILE_PATH is None:
     CONFIG_FILE_PATH = _DEFAULT_CONFIG_FILE
 
 BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH)
-OMNI_RESULT_TEMPLATE_PATH = Path(__file__).parent / "result_omni_template.json"
 
 
 DEPLOY_CONFIGS_DIR = Path(__file__).parent.parent / "deploy"
@@ -88,101 +91,6 @@ def omni_server(request):
         print("OmniServer stopped")
 
 
-def _safe_filename_token(value: Any | None, *, default: str = "na") -> str:
-    """Make a single path segment safe for result filenames on common filesystems."""
-    if value is None:
-        return default
-    s = str(value).strip()
-    for bad in ("/", "\\", ":", "*", "?", '"', "<", ">", "|"):
-        s = s.replace(bad, "_")
-    return s if s else default
-
-
-def run_benchmark(
-    args: list,
-    test_name: str,
-    flow,
-    dataset_name: str,
-    num_prompt,
-    *,
-    baseline_config: dict[str, Any] | None = None,
-    sweep_index: int | None = None,
-    request_rate: Any | None = None,
-    max_concurrency: Any | None = None,
-    random_input_len: Any | None = None,
-    random_output_len: Any | None = None,
-) -> Any:
-    """Run a single benchmark iteration and return the parsed result JSON.
-
-    After ``vllm bench`` writes the JSON, ``result["baseline"]`` holds the same
-    per-metric resolved thresholds as ``assert_result`` (via ``_baseline_thresholds_for_step``).
-    When ``random_input_len`` / ``random_output_len`` are set, they are also written into the result JSON;
-    omitted keys when not configured.
-    """
-    current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
-    ri = _safe_filename_token(random_input_len)
-    ro = _safe_filename_token(random_output_len)
-    result_filename = f"result_{test_name}_{dataset_name}_{flow}_{num_prompt}_in{ri}_out{ro}_{current_dt}.json"
-    if "--result-filename" in args:
-        print(f"The result file will be overwritten by {result_filename}")
-    command = (
-        ["vllm", "bench", "serve", "--omni"]
-        + args
-        + [
-            "--save-result",
-            "--result-dir",
-            os.environ.get("BENCHMARK_DIR", "tests"),
-            "--result-filename",
-            result_filename,
-        ]
-    )
-    process = subprocess.Popen(
-        command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1, universal_newlines=True
-    )
-
-    for line in iter(process.stdout.readline, ""):
-        print(line, end=" ")
-
-    for line in iter(process.stderr.readline, ""):
-        print(line, end=" ")
-
-    if "--result-dir" in command:
-        index = command.index("--result-dir")
-        result_dir = command[index + 1]
-    else:
-        result_dir = "./"
-
-    result_path = os.path.join(result_dir, result_filename)
-    if not os.path.exists(result_path):
-        with open(OMNI_RESULT_TEMPLATE_PATH, encoding="utf-8") as f:
-            template_result: dict[str, Any] = json.load(f)
-        Path(result_path).parent.mkdir(parents=True, exist_ok=True)
-        with open(result_path, "w", encoding="utf-8") as f:
-            json.dump(template_result, f, ensure_ascii=False, indent=2)
-        print(f"Benchmark result file not generated, fallback to template: {result_path}")
-        result = template_result
-    else:
-        with open(result_path, encoding="utf-8") as f:
-            result = json.load(f)
-
-    if baseline_config:
-        result["baseline"] = _baseline_thresholds_for_step(
-            baseline_config,
-            sweep_index=sweep_index,
-            request_rate=request_rate,
-            max_concurrency=max_concurrency,
-        )
-    else:
-        result["baseline"] = {}
-    if random_input_len is not None:
-        result["random_input_len"] = random_input_len
-    if random_output_len is not None:
-        result["random_output_len"] = random_output_len
-    with open(result_path, "w", encoding="utf-8") as f:
-        json.dump(result, f, ensure_ascii=False, indent=2)
-    return result
-
-
 benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
 
 
@@ -212,76 +120,23 @@ def benchmark_params(request, omni_server):
     }
 
 
-def _resolve_baseline_value(
-    baseline_raw: Any,
-    *,
-    sweep_index: int | None,
-    max_concurrency: Any = None,
-    request_rate: Any = None,
-) -> Any:
-    """Pick the baseline threshold for this sweep step.
-
-    Supported shapes per metric:
-    - **Scalar** — same threshold for every concurrency / QPS.
-    - **List** — aligned with ``max_concurrency`` / ``request_rate`` sweep order; use ``sweep_index``.
-    - **Dict** — keyed by concurrency or rate, e.g. ``{"1": 500, "4": 800}`` (keys are strings in JSON).
-
-    For dict lookup, ``max_concurrency`` is preferred when both are set (concurrency sweep).
-    """
-    if baseline_raw is None:
-        # If no baseline is set, the maximum value will be used.
-        return 100000
-    if isinstance(baseline_raw, dict):
-        if max_concurrency is not None:
-            for key in (max_concurrency, str(max_concurrency)):
-                if key in baseline_raw:
-                    return baseline_raw[key]
-        if request_rate is not None:
-            for key in (request_rate, str(request_rate)):
-                if key in baseline_raw:
-                    return baseline_raw[key]
-        raise KeyError(
-            f"baseline dict has no key for max_concurrency={max_concurrency!r} "
-            f"or request_rate={request_rate!r}; keys={list(baseline_raw.keys())!r}"
-        )
-    if isinstance(baseline_raw, (list, tuple)):
-        return baseline_raw[sweep_index]
-    return baseline_raw
-
-
-def _baseline_thresholds_for_step(
-    baseline_data: dict[str, Any],
-    *,
-    sweep_index: int | None = None,
-    max_concurrency: Any = None,
-    request_rate: Any = None,
-) -> dict[str, Any]:
-    """Resolve ``test.json`` ``baseline`` block to one threshold per metric (same as ``assert_result``)."""
-    return {
-        metric_name: _resolve_baseline_value(
-            baseline_raw,
-            sweep_index=sweep_index,
-            max_concurrency=max_concurrency,
-            request_rate=request_rate,
-        )
-        for metric_name, baseline_raw in baseline_data.items()
-    }
-
-
 def assert_result(
     result,
     params,
     num_prompt,
     *,
+    assert_baseline: bool,
     sweep_index: int | None = None,
-    max_concurrency: Any = None,
-    request_rate: Any = None,
+    max_concurrency: Any | None = None,
+    request_rate: Any | None = None,
 ) -> None:
     assert result["completed"] == num_prompt, "Request failures exist"
+    if not assert_baseline:
+        return
     baseline_data = params.get("baseline", {})
     for metric_name, baseline_raw in baseline_data.items():
         current_value = result[metric_name]
-        baseline_value = _resolve_baseline_value(
+        baseline_value = resolve_baseline_value(
             baseline_raw,
             sweep_index=sweep_index,
             max_concurrency=max_concurrency,
@@ -300,7 +155,7 @@ def assert_result(
 @pytest.mark.benchmark
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 @pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True)
-def test_performance_benchmark(omni_server, benchmark_params):
+def test_performance_benchmark(omni_server, benchmark_params, request):
     test_name = benchmark_params["test_name"]
     params = benchmark_params["params"]
     dataset_name = params.get("dataset_name", "")
@@ -312,6 +167,8 @@ def test_performance_benchmark(omni_server, benchmark_params):
     print(f"Running benchmark for model: {model}")
     print(f"Benchmark parameters: {benchmark_params}")
 
+    assert_baseline = request.config.getoption("--assert-baseline", default=False)
+
     def to_list(value, default=None):
         if value is None:
             return [] if default is None else [default]
@@ -350,7 +207,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
         elif not isinstance(value, bool):
             args.extend([arg_name, str(value)])
 
-    # QPS test (sweep_index aligns with qps_list / num_prompt_list for this loop)
+    # QPS / request-rate sweep
     for i, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)):
         args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)]
         result = run_benchmark(
@@ -369,12 +226,13 @@ def test_performance_benchmark(omni_server, benchmark_params):
         assert_result(
             result,
             params,
-            num_prompt=num_prompt,
+            num_prompt,
+            assert_baseline=assert_baseline,
             sweep_index=i,
             request_rate=qps,
         )
 
-    # concurrency test (sweep_index aligns with max_concurrency_list for separate thresholds per concurrency)
+    # concurrency test
     for i, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
         args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"]
         result = run_benchmark(
@@ -393,7 +251,8 @@ def test_performance_benchmark(omni_server, benchmark_params):
         assert_result(
             result,
             params,
-            num_prompt=num_prompt,
+            num_prompt,
+            assert_baseline=assert_baseline,
             sweep_index=i,
             max_concurrency=concurrency,
         )

tests/dfx/perf/scripts/run_diffusion_benchmark.py

@@ -5,16 +5,17 @@ This runner separates two concepts:
 
 1. ``server_type``: how the serving process is started.
    Currently only ``vllm-omni`` is supported here.
-2. ``benchmark_backend``: which serving API the benchmark client calls.
-   Examples: ``vllm-omni`` for ``/v1/chat/completions`` and ``v1/videos``
-   for async video jobs.
+2. ``benchmark_endpoint``: which serving API the benchmark client calls.
+   Examples: ``/v1/chat/completions`` and ``/v1/videos``.
 
 A config JSON file is REQUIRED via --test-config-file:
   pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
 
+Optional: ``--assert-baseline`` compares metrics to the ``baseline`` block in each benchmark entry (default: off).
+
 All benchmark results for a session are consolidated into a single JSON file under
 BENCHMARK_RESULT_DIR (override via the DIFFUSION_BENCHMARK_DIR environment variable).
-Each entry in the file contains the test metadata (test_name, backend, benchmark_params,
+Each entry in the file contains the test metadata (test_name, endpoint, benchmark_params,
 timestamp) together with the raw metrics returned by the benchmark script.
 """
 
@@ -33,6 +34,8 @@ from typing import Any, cast
 import psutil
 import pytest
 
+from benchmarks.diffusion.backends import endpoint_filename_token, normalize_endpoint
+
 pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
 
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
@@ -400,7 +403,7 @@ def _unique_server_params(configs: list[dict[str, Any]]) -> list[dict[str, Any]]
                 "model": cfg["server_params"]["model"],
                 "serve_args_dict": serve_args_dict,
                 "serve_args": _build_serve_args(serve_args_dict),
-                "benchmark_backend": cfg.get("benchmark_backend"),
+                "benchmark_endpoint": cfg.get("benchmark_endpoint", cfg.get("benchmark_backend")),
                 "server_params": cfg["server_params"],
             }
         )
@@ -482,7 +485,7 @@ def run_benchmark(
     model: str,
     params: dict[str, Any],
     test_name: str,
-    backend: str = "vllm-omni",
+    endpoint: str = "/v1/chat/completions",
     server_cfg: dict[str, Any] | None = None,
     source_file: str = "",
 ) -> dict[str, Any]:
@@ -490,18 +493,19 @@ def run_benchmark(
 
     The raw metrics are written to a temporary file by the subprocess.  After
     the run completes the metrics are merged with full metadata (test_name,
-    backend, benchmark_params, timestamp, flat reporting fields) and appended
+    endpoint, benchmark_params, timestamp, flat reporting fields) and appended
     to the session-wide aggregated JSON file (AGGREGATED_RESULT_FILE).  The
     temporary file is removed afterwards.  Subprocess stdout/stderr are tee'd
     to a .log file under BENCHMARK_RESULT_DIR/logs/; its path is stored in
     the record.
     """
     timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+    endpoint = normalize_endpoint(endpoint)
 
     log_dir = BENCHMARK_RESULT_DIR / "logs"
     log_dir.mkdir(parents=True, exist_ok=True)
-    backend_label = backend.replace("/", "_")
-    log_file = log_dir / f"{test_name}_{backend_label}_{timestamp}.log"
+    endpoint_label = endpoint_filename_token(endpoint)
+    log_file = log_dir / f"{test_name}_{endpoint_label}_{timestamp}.log"
 
     with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="diffusion_bench_tmp_", delete=False) as tmp:
         tmp_result_file = Path(tmp.name)
@@ -517,8 +521,8 @@ def run_benchmark(
         str(port),
         "--model",
         model,
-        "--backend",
-        backend,
+        "--endpoint",
+        endpoint,
         "--dataset",
         params.get("dataset", "random"),
         "--task",
@@ -545,7 +549,7 @@ def run_benchmark(
     # cause truncated or out-of-order log output when stdout is piped).
     cmd = [cmd[0], "-u"] + cmd[1:]
 
-    print(f"\nRunning benchmark (backend={backend}): {' '.join(cmd)}")
+    print(f"\nRunning benchmark (endpoint={endpoint}): {' '.join(cmd)}")
     print(f"  Log file: {log_file}")
 
     # Redirect stdout + stderr directly to the log file at the OS level
@@ -597,7 +601,7 @@ def run_benchmark(
 
     record: dict[str, Any] = {
         "test_name": test_name,
-        "backend": backend,
+        "endpoint": endpoint,
         "timestamp": timestamp,
         "server_params": server_cfg.get("server_params"),
         "benchmark_params": params,
@@ -605,7 +609,7 @@ def run_benchmark(
         "log_file": str(log_file),
         "Model": model,
         "Framework": server_type,
-        "API Backend": backend,
+        "API Endpoint": endpoint,
         "Hardware": "",
         "Deployment": "",
         "Task": params.get("task", "t2i"),
@@ -650,12 +654,14 @@ def run_benchmark(
 # ---------------------------------------------------------------------------
 
 
-def assert_result(result: dict[str, Any], params: dict[str, Any]) -> None:
-    """Assert that benchmark metrics satisfy the configured baselines."""
+def assert_result(result: dict[str, Any], params: dict[str, Any], *, assert_baseline: bool) -> None:
+    """Assert request completion; optionally assert metrics against ``baseline`` in *params*."""
     num_prompts = params.get("num-prompts", 10)
     completed = result.get("completed_requests", result.get("completed", 0))
     assert completed == num_prompts, f"Expected {num_prompts} completed requests, got {completed}"
 
+    if not assert_baseline:
+        return
     if params.get("skip-performance-assertion", False):
         print("Skipping performance assertions.")
         return
@@ -669,21 +675,21 @@ def assert_result(result: dict[str, Any], params: dict[str, Any]) -> None:
             assert current <= threshold, f"{metric}: {current:.4f} > baseline {threshold}"
 
 
-def _default_benchmark_backend_for_task(task: str) -> str:
-    """Return the default client-side benchmark backend for a diffusion task."""
+def _default_benchmark_endpoint_for_task(task: str) -> str:
+    """Return the default client-side benchmark endpoint for a diffusion task."""
     if task in {"t2v", "i2v", "ti2v"}:
-        return "v1/videos"
+        return "/v1/videos"
     if task in {"t2i", "i2i", "ti2i"}:
-        return "vllm-omni"
-    raise ValueError(f"Unsupported task for benchmark backend resolution: {task}")
+        return "/v1/chat/completions"
+    raise ValueError(f"Unsupported task for benchmark endpoint resolution: {task}")
 
 
-def _resolve_benchmark_backend(server_cfg: dict[str, Any], params: dict[str, Any]) -> str:
+def _resolve_benchmark_endpoint(server_cfg: dict[str, Any], params: dict[str, Any]) -> str:
     """Resolve which serving API the benchmark client should call."""
-    configured = server_cfg.get("benchmark_backend")
+    configured = server_cfg.get("benchmark_endpoint")
     if configured:
-        return cast(str, configured)
-    return _default_benchmark_backend_for_task(cast(str, params.get("task", "t2i")))
+        return normalize_endpoint(cast(str, configured))
+    return _default_benchmark_endpoint_for_task(cast(str, params.get("task", "t2i")))
 
 
 # ---------------------------------------------------------------------------
@@ -697,21 +703,19 @@ def _resolve_benchmark_backend(server_cfg: dict[str, Any], params: dict[str, Any
     indirect=True,
 )
 @pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True)
-def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
-    """Run the diffusion performance benchmark and assert against baselines.
+def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, request):
+    """Run the diffusion performance benchmark and verify request completion.
 
     One server is started per unique parallel configuration (module scope).
     For each server, all benchmark parameter sets defined in the config JSON
-    are executed sequentially; results are asserted against the baselines.
+    are executed sequentially; metrics are recorded to the aggregated result file.
 
-    Tracked metrics:
-        - throughput_qps          (higher is better)
-        - latency_p50, latency_p99 (lower is better)
+    Pass ``--assert-baseline`` to compare ``throughput_qps``, ``latency_mean``, etc. to the JSON ``baseline`` block.
     """
     test_name = benchmark_params["test_name"]
     params = benchmark_params["params"]
     server_cfg = getattr(diffusion_server, "server_cfg", {})
-    backend = _resolve_benchmark_backend(server_cfg, params)
+    endpoint = _resolve_benchmark_endpoint(server_cfg, params)
 
     result = run_benchmark(
         host=diffusion_server.host,
@@ -719,13 +723,13 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
         model=diffusion_server.model,
         params=params,
         test_name=test_name,
-        backend=backend,
+        endpoint=endpoint,
         server_cfg=server_cfg,
         source_file=cast(str, CONFIG_FILE_PATH),
     )
 
     print(f"\n{'=' * 60}")
-    print(f"Results for {test_name} (server={diffusion_server.server_type}, backend={backend}):")
+    print(f"Results for {test_name} (server={diffusion_server.server_type}, endpoint={endpoint}):")
     for key in (
         "throughput_qps",
         "latency_mean",
@@ -742,4 +746,5 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
     print(f"\n  Aggregated results: {AGGREGATED_RESULT_FILE}")
     print("=" * 60)
 
-    assert_result(result, params)
+    assert_baseline = request.config.getoption("--assert-baseline", default=False)
+    assert_result(result, params, assert_baseline=assert_baseline)

tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json

@@ -3,31 +3,67 @@
         "test_name": "test_qwen3_omni_chunk",
         "server_params": {
             "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-            "extra_cli_args": ["--async-chunk"]
+            "extra_cli_args": [
+                "--async-chunk"
+            ]
         },
         "benchmark_params": [
             {
                 "dataset_name": "random",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [4, 16, 32, 64, 128],
-                "max_concurrency": [1, 4, 8, 16, 32],
+                "num_prompts": [
+                    4,
+                    16,
+                    32,
+                    64,
+                    128
+                ],
+                "max_concurrency": [
+                    1,
+                    4,
+                    8,
+                    16,
+                    32
+                ],
                 "random_input_len": 2500,
                 "random_output_len": 900,
                 "ignore_eos": true,
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [1000, 3000, 5000, 7000, 9000],
-                    "mean_audio_ttfp_ms": [1000, 3000, 5000, 7000, 9000],
-                    "mean_audio_rtf": [0.2, 0.35, 0.6, 0.85, 0.9]
+                    "mean_ttft_ms": [
+                        400,
+                        300,
+                        400,
+                        600,
+                        900
+                    ],
+                    "mean_e2el_ms": [
+                        26000,
+                        43000,
+                        52000,
+                        72000,
+                        109000
+                    ],
+                    "mean_audio_rtf": [
+                        0.15,
+                        0.30,
+                        0.35,
+                        0.50,
+                        0.75
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [10],
-                "request_rate": [0.1],
+                "num_prompts": [
+                    10
+                ],
+                "request_rate": [
+                    0.1
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -42,17 +78,27 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [2000],
-                    "mean_audio_ttfp_ms": [2000],
-                    "mean_audio_rtf": [0.25]
+                    "mean_ttft_ms": [
+                        600
+                    ],
+                    "mean_e2el_ms": [
+                        6000
+                    ],
+                    "mean_audio_rtf": [
+                        0.20
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [40],
-                "request_rate": [0.5],
+                "num_prompts": [
+                    40
+                ],
+                "request_rate": [
+                    0.5
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -69,17 +115,27 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [6000],
-                    "mean_audio_ttfp_ms": [6000],
-                    "mean_audio_rtf": [0.7]
+                    "mean_ttft_ms": [
+                        300
+                    ],
+                    "mean_e2el_ms": [
+                        7000
+                    ],
+                    "mean_audio_rtf": [
+                        0.25
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [100],
-                "request_rate": [1.0],
+                "num_prompts": [
+                    100
+                ],
+                "request_rate": [
+                    1.0
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -98,40 +154,79 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [12000],
-                    "mean_audio_ttfp_ms": [12000],
-                    "mean_audio_rtf": [1.0]
+                    "mean_ttft_ms": [
+                        700
+                    ],
+                    "mean_e2el_ms": [
+                        11000
+                    ],
+                    "mean_audio_rtf": [
+                        0.35
+                    ]
                 }
             },
             {
                 "dataset_name": "random",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [4, 16, 32, 64, 128],
-                "max_concurrency": [1, 4, 8, 16, 32],
+                "num_prompts": [
+                    4,
+                    16,
+                    32,
+                    64,
+                    128
+                ],
+                "max_concurrency": [
+                    1,
+                    4,
+                    8,
+                    16,
+                    32
+                ],
                 "random_input_len": 2500,
                 "random_output_len": 900,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [1000, 3000, 5000, 7000, 9000]
+                    "mean_ttft_ms": [
+                        100,
+                        300,
+                        500,
+                        800,
+                        1500
+                    ],
+                    "mean_e2el_ms": [
+                        5000,
+                        8000,
+                        10000,
+                        14000,
+                        20000
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [10],
-                "request_rate": [0.1],
+                "num_prompts": [
+                    10
+                ],
+                "request_rate": [
+                    0.1
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "random_mm_base_items_per_request": 1,
                 "random_mm_num_mm_items_range_ratio": 0.5,
@@ -143,21 +238,32 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [2000]
+                    "mean_ttft_ms": [
+                        500
+                    ],
+                    "mean_e2el_ms": [
+                        1000
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [40],
-                "request_rate": [0.5],
+                "num_prompts": [
+                    40
+                ],
+                "request_rate": [
+                    0.5
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "random_mm_base_items_per_request": 2,
                 "random_mm_num_mm_items_range_ratio": 0.5,
@@ -171,21 +277,32 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [6000]
+                    "mean_ttft_ms": [
+                        200
+                    ],
+                    "mean_e2el_ms": [
+                        800
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [100],
-                "request_rate": [1.0],
+                "num_prompts": [
+                    100
+                ],
+                "request_rate": [
+                    1.0
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "random_mm_base_items_per_request": 3,
                 "random_mm_num_mm_items_range_ratio": 0.5,
@@ -201,7 +318,12 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [6000]
+                    "mean_ttft_ms": [
+                        600
+                    ],
+                    "mean_e2el_ms": [
+                        1300
+                    ]
                 }
             }
         ]

tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json

@@ -3,31 +3,67 @@
         "test_name": "test_qwen3_omni",
         "server_params": {
             "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-            "extra_cli_args": ["--no-async-chunk"]
+            "extra_cli_args": [
+                "--no-async-chunk"
+            ]
         },
         "benchmark_params": [
             {
                 "dataset_name": "random",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [4, 16, 32, 64, 128],
-                "max_concurrency": [1, 4, 8, 16, 32],
+                "num_prompts": [
+                    4,
+                    16,
+                    32,
+                    64,
+                    128
+                ],
+                "max_concurrency": [
+                    1,
+                    4,
+                    8,
+                    16,
+                    32
+                ],
                 "random_input_len": 2500,
                 "random_output_len": 900,
                 "ignore_eos": true,
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [1000, 3000, 5000, 7000, 9000],
-                    "mean_audio_ttfp_ms": [30000, 60000, 90000, 120000, 150000],
-                    "mean_audio_rtf": [0.35, 0.45, 0.55, 0.65, 0.75]
+                    "mean_ttft_ms": [
+                        400,
+                        300,
+                        400,
+                        600,
+                        900
+                    ],
+                    "mean_e2el_ms": [
+                        32000,
+                        34000,
+                        45000,
+                        61000,
+                        90000
+                    ],
+                    "mean_audio_rtf": [
+                        0.20,
+                        0.25,
+                        0.30,
+                        0.45,
+                        0.60
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [10],
-                "request_rate": [0.1],
+                "num_prompts": [
+                    10
+                ],
+                "request_rate": [
+                    0.1
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -42,17 +78,27 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [2000],
-                    "mean_audio_ttfp_ms": [10000],
-                    "mean_audio_rtf": [0.25]
+                    "mean_ttft_ms": [
+                        500
+                    ],
+                    "mean_e2el_ms": [
+                        6000
+                    ],
+                    "mean_audio_rtf": [
+                        0.20
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [40],
-                "request_rate": [0.5],
+                "num_prompts": [
+                    40
+                ],
+                "request_rate": [
+                    0.5
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -69,17 +115,27 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [6000],
-                    "mean_audio_ttfp_ms": [15000],
-                    "mean_audio_rtf": [0.45]
+                    "mean_ttft_ms": [
+                        300
+                    ],
+                    "mean_e2el_ms": [
+                        7000
+                    ],
+                    "mean_audio_rtf": [
+                        0.20
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [100],
-                "request_rate": [1.0],
+                "num_prompts": [
+                    100
+                ],
+                "request_rate": [
+                    1.0
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -98,9 +154,15 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [12000],
-                    "mean_audio_ttfp_ms": [18000],
-                    "mean_audio_rtf": [0.9]
+                    "mean_ttft_ms": [
+                        700
+                    ],
+                    "mean_e2el_ms": [
+                        9000
+                    ],
+                    "mean_audio_rtf": [
+                        0.30
+                    ]
                 }
             }
         ]

tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json

@@ -4,7 +4,9 @@
         "server_params": {
             "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
             "use_omni": false,
-            "extra_cli_args": ["--no-enable-prefix-caching"]
+            "extra_cli_args": [
+                "--no-enable-prefix-caching"
+            ]
         },
         "benchmark_params": [
             {
@@ -12,8 +14,20 @@
                 "backend": "vllm",
                 "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
                 "endpoint": "/v1/completions",
-                "num_prompts": [4, 16, 32, 64, 128],
-                "max_concurrency": [1, 4, 8, 16, 32],
+                "num_prompts": [
+                    4,
+                    16,
+                    32,
+                    64,
+                    128
+                ],
+                "max_concurrency": [
+                    1,
+                    4,
+                    8,
+                    16,
+                    32
+                ],
                 "random_input_len": 2500,
                 "random_output_len": 900,
                 "temperature": 0.4,
@@ -23,11 +37,26 @@
                 "repetition_penalty": 1.05,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [1000, 3000, 5000, 7000, 9000]
+                    "mean_ttft_ms": [
+                        300,
+                        300,
+                        1000,
+                        600,
+                        800
+                    ],
+                    "mean_e2el_ms": [
+                        6000,
+                        8000,
+                        11000,
+                        13000,
+                        17000
+                    ]
                 }
             },
             {
@@ -35,8 +64,12 @@
                 "backend": "vllm",
                 "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
                 "endpoint": "/v1/completions",
-                "num_prompts": [10],
-                "request_rate": [0.1],
+                "num_prompts": [
+                    10
+                ],
+                "request_rate": [
+                    0.1
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -47,7 +80,9 @@
                 "repetition_penalty": 1.05,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "random_mm_base_items_per_request": 1,
                 "random_mm_num_mm_items_range_ratio": 0.5,
@@ -59,7 +94,12 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [2000]
+                    "mean_ttft_ms": [
+                        30
+                    ],
+                    "mean_e2el_ms": [
+                        540
+                    ]
                 }
             },
             {
@@ -67,8 +107,12 @@
                 "backend": "vllm",
                 "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
                 "endpoint": "/v1/completions",
-                "num_prompts": [40],
-                "request_rate": [0.5],
+                "num_prompts": [
+                    40
+                ],
+                "request_rate": [
+                    0.5
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -79,7 +123,9 @@
                 "repetition_penalty": 1.05,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "random_mm_base_items_per_request": 2,
                 "random_mm_num_mm_items_range_ratio": 0.5,
@@ -93,7 +139,12 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [6000]
+                    "mean_ttft_ms": [
+                        30
+                    ],
+                    "mean_e2el_ms": [
+                        600
+                    ]
                 }
             },
             {
@@ -101,8 +152,12 @@
                 "backend": "vllm",
                 "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
                 "endpoint": "/v1/completions",
-                "num_prompts": [100],
-                "request_rate": [1.0],
+                "num_prompts": [
+                    100
+                ],
+                "request_rate": [
+                    1.0
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -113,7 +168,9 @@
                 "repetition_penalty": 1.05,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "random_mm_base_items_per_request": 3,
                 "random_mm_num_mm_items_range_ratio": 0.5,
@@ -129,7 +186,12 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [6000]
+                    "mean_ttft_ms": [
+                        40
+                    ],
+                    "mean_e2el_ms": [
+                        700
+                    ]
                 }
             }
         ]

tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json

@@ -22,10 +22,10 @@
                 "num-input-images": 2,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.05,
-                    "latency_mean": 18,
-                    "peak_memory_mb_max": 78500,
-                    "peak_memory_mb_mean": 78500
+                    "throughput_qps": 0.06,
+                    "latency_mean": 20,
+                    "peak_memory_mb_max": 57000,
+                    "peak_memory_mb_mean": 57000
                 }
             },
             {
@@ -41,9 +41,9 @@
                 "enable-negative-prompt": true,
                 "baseline": {
                     "throughput_qps": 0.01,
-                    "latency_mean": 70,
-                    "peak_memory_mb_max": 81000,
-                    "peak_memory_mb_mean": 81000
+                    "latency_mean": 60,
+                    "peak_memory_mb_max": 67000,
+                    "peak_memory_mb_mean": 67000
                 }
             }
         ]
@@ -75,10 +75,10 @@
                 "num-input-images": 2,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.1,
-                    "latency_mean": 12,
-                    "peak_memory_mb_max": 69000,
-                    "peak_memory_mb_mean": 69000
+                    "throughput_qps": 0.15,
+                    "latency_mean": 10,
+                    "peak_memory_mb_max": 55000,
+                    "peak_memory_mb_mean": 55000
                 }
             },
             {
@@ -93,10 +93,10 @@
                 "num-input-images": 2,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.03,
-                    "latency_mean": 28,
-                    "peak_memory_mb_max": 69000,
-                    "peak_memory_mb_mean": 69000
+                    "throughput_qps": 0.05,
+                    "latency_mean": 20,
+                    "peak_memory_mb_max": 55000,
+                    "peak_memory_mb_mean": 55000
                 }
             }
         ]
@@ -138,10 +138,10 @@
                 "num-input-images": 2,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.10,
-                    "latency_mean": 12,
-                    "peak_memory_mb_max": 73000,
-                    "peak_memory_mb_mean": 73000
+                    "throughput_qps": 0.21,
+                    "latency_mean": 10,
+                    "peak_memory_mb_max": 57000,
+                    "peak_memory_mb_mean": 57000
                 }
             },
             {
@@ -156,10 +156,10 @@
                 "num-input-images": 2,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.05,
+                    "throughput_qps": 0.08,
                     "latency_mean": 20,
-                    "peak_memory_mb_max": 81000,
-                    "peak_memory_mb_mean": 81000
+                    "peak_memory_mb_max": 66000,
+                    "peak_memory_mb_mean": 66000
                 }
             }
         ]

tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json

@@ -21,10 +21,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.05,
-                    "latency_mean": 15.0,
-                    "peak_memory_mb_max": 72500,
-                    "peak_memory_mb_mean": 72500
+                    "throughput_qps": 0.1,
+                    "latency_mean": 10,
+                    "peak_memory_mb_max": 57000,
+                    "peak_memory_mb_mean": 57000
                 }
             },
             {
@@ -38,10 +38,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.01,
-                    "latency_mean": 65.6,
-                    "peak_memory_mb_max": 80777,
-                    "peak_memory_mb_mean": 80777
+                    "throughput_qps": 0.02,
+                    "latency_mean": 50,
+                    "peak_memory_mb_max": 65000,
+                    "peak_memory_mb_mean": 65000
                 }
             }
         ]
@@ -72,10 +72,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.10,
-                    "latency_mean": 7.2,
-                    "peak_memory_mb_max": 68100,
-                    "peak_memory_mb_mean": 68100
+                    "throughput_qps": 0.21,
+                    "latency_mean": 10,
+                    "peak_memory_mb_max": 56000,
+                    "peak_memory_mb_mean": 56000
                 }
             },
             {
@@ -89,10 +89,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.03,
-                    "latency_mean": 24.0,
-                    "peak_memory_mb_max": 68100,
-                    "peak_memory_mb_mean": 68100
+                    "throughput_qps": 0.06,
+                    "latency_mean": 20,
+                    "peak_memory_mb_max": 56000,
+                    "peak_memory_mb_mean": 56000
                 }
             }
         ]
@@ -133,10 +133,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.1,
-                    "latency_mean": 6.5,
-                    "peak_memory_mb_max": 72600,
-                    "peak_memory_mb_mean": 72600
+                    "throughput_qps": 0.26,
+                    "latency_mean": 10,
+                    "peak_memory_mb_max": 58000,
+                    "peak_memory_mb_mean": 58000
                 }
             },
             {
@@ -150,10 +150,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.05,
-                    "latency_mean": 16.0,
-                    "peak_memory_mb_max": 81000,
-                    "peak_memory_mb_mean": 81000
+                    "throughput_qps": 0.1,
+                    "latency_mean": 10,
+                    "peak_memory_mb_max": 67000,
+                    "peak_memory_mb_mean": 67000
                 }
             }
         ]

tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json

@@ -21,10 +21,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.02,
-                    "latency_mean": 40.0,
-                    "peak_memory_mb_max": 70000,
-                    "peak_memory_mb_mean": 70000
+                    "throughput_qps": 0.05,
+                    "latency_mean": 20,
+                    "peak_memory_mb_max": 63000,
+                    "peak_memory_mb_mean": 63000
                 }
             },
             {
@@ -38,10 +38,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.005,
-                    "latency_mean": 80.0,
-                    "peak_memory_mb_max": 70000,
-                    "peak_memory_mb_mean": 70000
+                    "throughput_qps": 0.03,
+                    "latency_mean": 30,
+                    "peak_memory_mb_max": 63000,
+                    "peak_memory_mb_mean": 63000
                 }
             }
         ]

tests/dfx/perf/tests/test_qwen_image_vllm_omni.json

@@ -21,9 +21,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.30,
-                    "latency_mean": 3.50,
-                    "peak_memory_mb_mean": 67000
+                    "throughput_qps": 0.35,
+                    "latency_mean": 3,
+                    "peak_memory_mb_mean": 56000
                 }
             },
             {
@@ -37,9 +37,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.037,
-                    "latency_mean": 27.0,
-                    "peak_memory_mb_mean": 74000
+                    "throughput_qps": 0.03,
+                    "latency_mean": 26,
+                    "peak_memory_mb_mean": 65000
                 }
             }
         ]
@@ -67,9 +67,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.30,
-                    "latency_mean": 3.50,
-                    "peak_memory_mb_mean": 67000
+                    "throughput_qps": 0.35,
+                    "latency_mean": 3,
+                    "peak_memory_mb_mean": 56000
                 }
             },
             {
@@ -83,9 +83,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.037,
-                    "latency_mean": 27.0,
-                    "peak_memory_mb_mean": 74000
+                    "throughput_qps": 0.03,
+                    "latency_mean": 26,
+                    "peak_memory_mb_mean": 65000
                 }
             }
         ]
@@ -116,9 +116,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.1,
-                    "latency_mean": 9.1,
-                    "peak_memory_mb_mean": 61000
+                    "throughput_qps": 0.11,
+                    "latency_mean": 9,
+                    "peak_memory_mb_mean": 55000
                 }
             }
         ]
@@ -159,9 +159,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.1,
-                    "latency_mean": 3.50,
-                    "peak_memory_mb_mean": 67000
+                    "throughput_qps": 0.46,
+                    "latency_mean": 3,
+                    "peak_memory_mb_mean": 56000
                 }
             },
             {
@@ -175,9 +175,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.1,
-                    "latency_mean": 6.15,
-                    "peak_memory_mb_mean": 74000
+                    "throughput_qps": 0.18,
+                    "latency_mean": 6,
+                    "peak_memory_mb_mean": 67000
                 }
             }
         ]

tests/dfx/perf/tests/test_tts.json

@@ -4,7 +4,9 @@
     "server_params": {
       "model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base",
       "stage_overrides": {
-        "1": {"devices": "1"}
+        "1": {
+          "devices": "1"
+        }
       }
     },
     "benchmark_params": [
@@ -15,13 +17,21 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "linyueqian/seed-tts-eval-subset",
-        "num_prompts": [20],
-        "max_concurrency": [1],
+        "num_prompts": [
+          20
+        ],
+        "max_concurrency": [
+          1
+        ],
         "seed_tts_locale": "en",
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [350],
-          "median_audio_rtf": [0.25]
+          "median_audio_ttfp_ms": [
+            3000
+          ],
+          "median_audio_rtf": [
+            0.6
+          ]
         }
       },
       {
@@ -31,14 +41,34 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "linyueqian/seed-tts-eval-subset",
-        "num_prompts": [80, 128, 128],
-        "max_concurrency": [8, 16, 64],
+        "num_prompts": [
+          80,
+          128,
+          128
+        ],
+        "max_concurrency": [
+          8,
+          16,
+          64
+        ],
         "seed_tts_locale": "en",
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [3500, 30000, 7000],
-          "median_audio_rtf": [0.75, 10.0, 1.5],
-          "audio_throughput": [10.0, 1.0, 40.0]
+          "median_audio_ttfp_ms": [
+            1000,
+            3000,
+            14000
+          ],
+          "median_audio_rtf": [
+            0.70,
+            1.15,
+            3.90
+          ],
+          "audio_throughput": [
+            12,
+            14,
+            14
+          ]
         }
       },
       {
@@ -49,13 +79,19 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "linyueqian/seed-tts-eval-subset",
-        "num_prompts": [200],
-        "max_concurrency": [4],
+        "num_prompts": [
+          200
+        ],
+        "max_concurrency": [
+          4
+        ],
         "seed_tts_locale": "en",
         "seed_tts_wer_eval": true,
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "mean_audio_rtf": [0.45]
+          "mean_audio_rtf": [
+            0.45
+          ]
         }
       }
     ]
@@ -65,7 +101,9 @@
     "server_params": {
       "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
       "stage_overrides": {
-        "1": {"devices": "1"}
+        "1": {
+          "devices": "1"
+        }
       }
     },
     "benchmark_params": [
@@ -76,14 +114,26 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
-        "num_prompts": [20],
-        "max_concurrency": [1],
+        "num_prompts": [
+          20
+        ],
+        "max_concurrency": [
+          1
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
+        "extra_body": {
+          "voice": "Vivian",
+          "language": "English",
+          "task_type": "CustomVoice"
+        },
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [150],
-          "median_audio_rtf": [0.15]
+          "median_audio_ttfp_ms": [
+            2000
+          ],
+          "median_audio_rtf": [
+            0.40
+          ]
         }
       },
       {
@@ -93,15 +143,39 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
-        "num_prompts": [80, 128, 128],
-        "max_concurrency": [8, 16, 64],
+        "num_prompts": [
+          80,
+          128,
+          128
+        ],
+        "max_concurrency": [
+          8,
+          16,
+          64
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
+        "extra_body": {
+          "voice": "Vivian",
+          "language": "English",
+          "task_type": "CustomVoice"
+        },
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [1500, 30000, 3000],
-          "median_audio_rtf": [0.30, 10.0, 0.60],
-          "audio_throughput": [30.0, 1.0, 100.0]
+          "median_audio_ttfp_ms": [
+            300,
+            1000,
+            5000
+          ],
+          "median_audio_rtf": [
+            0.25,
+            0.35,
+            1.10
+          ],
+          "audio_throughput": [
+            30,
+            40,
+            40
+          ]
         }
       },
       {
@@ -112,14 +186,24 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
-        "num_prompts": [200],
-        "max_concurrency": [4],
+        "num_prompts": [
+          200
+        ],
+        "max_concurrency": [
+          4
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
+        "extra_body": {
+          "voice": "Vivian",
+          "language": "English",
+          "task_type": "CustomVoice"
+        },
         "seed_tts_wer_eval": true,
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "mean_audio_rtf": [0.35]
+          "mean_audio_rtf": [
+            0.35
+          ]
         }
       },
       {
@@ -129,15 +213,29 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
-        "num_prompts": [100],
-        "request_rate": [2.0],
+        "num_prompts": [
+          100
+        ],
+        "request_rate": [
+          2.0
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
+        "extra_body": {
+          "voice": "Vivian",
+          "language": "English",
+          "task_type": "CustomVoice"
+        },
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [3000],
-          "median_audio_rtf": [0.25],
-          "audio_throughput": [4.0]
+          "median_audio_ttfp_ms": [
+            100
+          ],
+          "median_audio_rtf": [
+            0.20
+          ],
+          "audio_throughput": [
+            10
+          ]
         }
       },
       {
@@ -147,14 +245,25 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_design",
-        "num_prompts": [20],
-        "max_concurrency": [1],
+        "num_prompts": [
+          20
+        ],
+        "max_concurrency": [
+          1
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"task_type": "VoiceDesign", "language": "English"},
+        "extra_body": {
+          "task_type": "VoiceDesign",
+          "language": "English"
+        },
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [150],
-          "median_audio_rtf": [0.15]
+          "median_audio_ttfp_ms": [
+            100
+          ],
+          "median_audio_rtf": [
+            0.15
+          ]
         }
       },
       {
@@ -164,15 +273,38 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_design",
-        "num_prompts": [80, 128, 128],
-        "max_concurrency": [8, 16, 64],
+        "num_prompts": [
+          80,
+          128,
+          128
+        ],
+        "max_concurrency": [
+          8,
+          16,
+          64
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"task_type": "VoiceDesign", "language": "English"},
+        "extra_body": {
+          "task_type": "VoiceDesign",
+          "language": "English"
+        },
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [1500, 30000, 3500],
-          "median_audio_rtf": [0.35, 10.0, 0.70],
-          "audio_throughput": [25.0, 1.0, 80.0]
+          "median_audio_ttfp_ms": [
+            100,
+            1000,
+            5000
+          ],
+          "median_audio_rtf": [
+            0.25,
+            0.40,
+            1.20
+          ],
+          "audio_throughput": [
+            30,
+            40,
+            40
+          ]
         }
       },
       {
@@ -182,15 +314,28 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_design",
-        "num_prompts": [100],
-        "request_rate": [2.0],
+        "num_prompts": [
+          100
+        ],
+        "request_rate": [
+          2.0
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"task_type": "VoiceDesign", "language": "English"},
+        "extra_body": {
+          "task_type": "VoiceDesign",
+          "language": "English"
+        },
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [3500],
-          "median_audio_rtf": [0.30],
-          "audio_throughput": [4.0]
+          "median_audio_ttfp_ms": [
+            100
+          ],
+          "median_audio_rtf": [
+            0.20
+          ],
+          "audio_throughput": [
+            9
+          ]
         }
       }
     ]

tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json

@@ -30,9 +30,9 @@
                     }
                 ],
                 "baseline": {
-                    "throughput_qps": 0.034,
-                    "latency_mean": 26.0,
-                    "peak_memory_mb_mean": 80000
+                    "throughput_qps": 0.04,
+                    "latency_mean": 30,
+                    "peak_memory_mb_mean": 75000
                 }
             }
         ]
@@ -72,9 +72,9 @@
                     }
                 ],
                 "baseline": {
-                    "throughput_qps": 0.042,
-                    "latency_mean": 21.6,
-                    "peak_memory_mb_mean": 55300
+                    "throughput_qps": 0.06,
+                    "latency_mean": 20,
+                    "peak_memory_mb_mean": 48000
                 }
             },
             {
@@ -97,9 +97,9 @@
                     }
                 ],
                 "baseline": {
-                    "throughput_qps": 0.0085,
-                    "latency_mean": 101.6,
-                    "peak_memory_mb_mean": 65200
+                    "throughput_qps": 0.01,
+                    "latency_mean": 90,
+                    "peak_memory_mb_mean": 56000
                 }
             }
         ]

tests/dfx/reliability/invalid_param_test/conftest.py

@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Shared env for ``http_invalid`` (real :func:`omni_server`)."""
+
+from __future__ import annotations
+
+import io
+import os
+
+import pytest
+from PIL import Image
+
+# Match ``tests/e2e/online_serving/*`` module-level env for subprocess serve.
+os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
+os.environ.setdefault("VLLM_TEST_CLEAN_GPU_MEMORY", "0")
+
+
+@pytest.fixture
+def tiny_png_bytes() -> bytes:
+    img = Image.new("RGB", (32, 32), color="gray")
+    buf = io.BytesIO()
+    img.save(buf, format="PNG")
+    return buf.getvalue()

tests/dfx/reliability/invalid_param_test/test_invalid_audio_diffusion.py

@@ -0,0 +1,118 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""``POST /v1/audio/generate`` validation (live Stable Audio Open server)."""
+
+from __future__ import annotations
+
+import pytest
+
+from tests.helpers.mark import hardware_marks
+from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
+
+pytestmark = [pytest.mark.slow, pytest.mark.diffusion]
+
+# https://github.com/vllm-project/vllm-omni/issues/3649 — invalid inputs still HTTP 200 / timeout / missing validation.
+_SKIP_ISSUE_3649 = pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/3649")
+
+_PARAMS = [
+    pytest.param(
+        OmniServerParams(
+            model="stabilityai/stable-audio-open-1.0",
+            server_args=[
+                "--trust-remote-code",
+                "--enforce-eager",
+                "--model-class-name",
+                "StableAudioPipeline",
+            ],
+        ),
+        id="stable_audio_open",
+        marks=hardware_marks(res={"cuda": "L4"}),
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    "overrides, err_message",
+    [
+        pytest.param({"input": ""}, ("cannot be empty", "input"), id="input_empty", marks=_SKIP_ISSUE_3649),
+        pytest.param(
+            {"input": "   "}, ("cannot be empty", "input"), id="input_whitespace_only", marks=_SKIP_ISSUE_3649
+        ),
+        pytest.param({"input": 123}, ("input", "a valid string"), id="input_wrong_type"),
+        pytest.param(
+            {"response_format": "mpeg"},
+            ("response_format", "literal_error", "Input should be"),
+            id="response_format_invalid",
+        ),
+        pytest.param({"speed": 0.1}, ("speed", "greater_than_equal", "0.25"), id="speed_too_low"),
+        pytest.param({"speed": 99.0}, ("speed", "less_than_equal", "4"), id="speed_too_high"),
+        pytest.param({"stream_format": 1}, ("stream_format", "literal_error", "audio"), id="stream_format_wrong_type"),
+        pytest.param(
+            {"stream_format": "sse"},
+            ("sse", "stream_format", "value_error", "not a supported stream_format"),
+            id="stream_format_sse_blocked",
+        ),
+        pytest.param(
+            {"audio_length": -1.0}, ("audio_end_in_s", "higher than", "audio_start_in_s"), id="audio_length_negative"
+        ),
+        pytest.param({"audio_length": 0}, "audio_length", id="audio_length_zero", marks=_SKIP_ISSUE_3649),
+        # ``OmniOpenAIServingAudioGenerate`` only forwards ``audio_start`` when ``audio_length`` is set.
+        pytest.param(
+            {"audio_start": -0.5, "audio_length": 5.0},
+            ("audio_start_in_s", "between", "0", "512"),
+            id="audio_start_negative",
+        ),
+        pytest.param(
+            {"negative_prompt": ["noise"]}, ("negative_prompt", "a valid string"), id="negative_prompt_wrong_type"
+        ),
+        pytest.param({"guidance_scale": -1.0}, "guidance_scale", id="guidance_scale_negative", marks=_SKIP_ISSUE_3649),
+        pytest.param({"guidance_scale": 0}, "guidance_scale", id="guidance_scale_zero", marks=_SKIP_ISSUE_3649),
+        pytest.param(
+            {"num_inference_steps": 0}, "num_inference_steps", id="num_inference_steps_zero", marks=_SKIP_ISSUE_3649
+        ),
+        pytest.param(
+            {"num_inference_steps": -1}, ("number of steps", "non-negative"), id="num_inference_steps_negative"
+        ),
+        pytest.param(
+            {"num_inference_steps": 6000},
+            "num_inference_steps",
+            id="num_inference_steps_above_max",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            {"guidance_scale": 1001.0}, "guidance_scale", id="guidance_scale_above_max", marks=_SKIP_ISSUE_3649
+        ),
+        pytest.param(
+            {"audio_length": 86401.0}, ("Requested audio length", "exceeds maximum"), id="audio_length_above_max"
+        ),
+        pytest.param(
+            {"audio_start": 86401.0, "audio_length": 1.0},
+            ("audio_start_in_s", "between", "0", "512"),
+            id="audio_start_above_max",
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
+def test_audio_generate_invalid_field_values(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    overrides: dict[str, object],
+    err_message: str | tuple[str, ...],
+) -> None:
+    body = {"model": omni_server.model, "input": "ambient electronic pad"}
+    body.update(overrides)
+    openai_client.send_audio_generate_http_request(
+        {"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
+    )
+
+
+@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
+def test_audio_generate_missing_input(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
+    openai_client.send_audio_generate_http_request(
+        {
+            "json": {"model": omni_server.model},
+            "timeout": 120,
+            "err_code": 400,
+            "err_message": ("missing", "input", "field required"),
+        }
+    )

tests/dfx/reliability/invalid_param_test/test_invalid_audio_speech.py

@@ -0,0 +1,862 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""HTTP + WebSocket validation for Qwen3-TTS Base: ``/v1/audio/speech``, ``/v1/audio/speech/stream``, batch, voices."""
+
+from __future__ import annotations
+
+import io
+import json
+import uuid
+import wave
+from typing import Any
+
+import pytest
+
+from tests.helpers.mark import hardware_marks
+from tests.helpers.media import load_test_audio_data_url
+from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
+from tests.helpers.stage_config import get_deploy_config_path
+
+# Speech / voice-upload numeric caps for these tests only (not exported from vllm_omni).
+# Align where possible with serving_speech defaults: _TTS_MAX_INSTRUCTIONS_LENGTH (500),
+# _TTS_MAX_NEW_TOKENS_MAX (4096). Voice form max lengths are not public constants in prod;
+# values below keep tests self-contained and deterministic.
+_SPEECH_API_MAX_INSTRUCTIONS_CHARS = 500
+_SPEECH_API_MAX_NEW_TOKENS = 4096
+_VOICE_UPLOAD_MAX_CONSENT_LEN = 1024
+_VOICE_UPLOAD_MAX_REF_TEXT_CHARS = 8192
+_VOICE_UPLOAD_MAX_SPEAKER_DESCRIPTION_CHARS = 2048
+REF_AUDIO_URL = load_test_audio_data_url("qwen3_tts/clone_2.wav")
+REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
+# Qwen3-TTS 0.6B expects 1024-dim speaker embeddings (see serving_speech / talker hidden_size).
+_QWEN3_TTS_06B_SPEAKER_EMBEDDING_DIM = 1024
+# Pop-only marker for ``test_speech_invalid_field_values``: omit ``ref_audio`` / ``ref_text`` so Base
+# ``speaker_embedding`` cases mirror standalone embedding-only JSON (no ref_audio clash).
+_SPEECH_INVALID_EMBEDDING_ONLY_BODY = "__speech_invalid_embedding_only_body__"
+# Batch-only: merged items inherit batch ``ref_audio`` / ``ref_text`` because ``_pick`` treats item ``None``
+# as "unset". Strip clone fields from the batch envelope before asserting CustomVoice preset errors.
+_SPEECH_BATCH_NO_CLONE_FIELDS = "__speech_batch_no_clone_fields__"
+
+_SKIP_ISSUE_3649 = pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/3649")
+
+
+pytestmark = [pytest.mark.slow, pytest.mark.tts]
+
+_L4_SPEECH_HW = hardware_marks(res={"cuda": "L4"})
+_SPEECH_SERVER_ARGS = ["--trust-remote-code", "--disable-log-stats"]
+
+_QWEN3_TTS_SPEECH = [
+    pytest.param(
+        OmniServerParams(
+            model="Qwen/Qwen3-TTS-12Hz-0.6B-Base",
+            stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
+            server_args=_SPEECH_SERVER_ARGS,
+        ),
+        id="qwen3_tts",
+        marks=_L4_SPEECH_HW,
+    ),
+]
+
+
+def _apply_batch_overrides(body: dict[str, object], *, loc: str, overrides: dict[str, object]) -> None:
+    if loc == "batch":
+        body.update(overrides)
+        return
+    items = body["items"]
+    assert isinstance(items, list) and len(items) >= 1
+    first = items[0]
+    assert isinstance(first, dict)
+    body["items"] = [{**first, **overrides}]
+
+
+def _pcm_wav_mono_bytes(*, duration_s: float = 1.1, sample_rate: int = 44100) -> bytes:
+    buf = io.BytesIO()
+    with wave.open(buf, "wb") as wf:
+        wf.setnchannels(1)
+        wf.setsampwidth(2)
+        wf.setframerate(sample_rate)
+        nframes = max(1, int(sample_rate * duration_s))
+        wf.writeframes(b"\x00\x00" * nframes)
+    return buf.getvalue()
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# POST /v1/audio/speech (JSON)
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
+def test_speech_malformed_json(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
+    openai_client.send_audio_speech_http_request(
+        {
+            "raw_body": "{",
+            "timeout": 120,
+            "err_code": 400,
+            "err_message": ("json", "decode", "invalid"),
+        }
+    )
+
+
+@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
+def test_speech_missing_required_fields(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
+    openai_client.send_audio_speech_http_request(
+        {
+            "json": {"model": omni_server.model},
+            "timeout": 120,
+            "err_code": 400,
+            "err_message": ("input", "field required", "missing"),
+        }
+    )
+
+
+@pytest.mark.parametrize(
+    "overrides, err_message",
+    [
+        pytest.param({"input": ""}, ("input", "empty"), id="input_empty"),
+        pytest.param({"input": "   "}, ("input", "empty"), id="input_whitespace_only"),
+        pytest.param({"voice": ""}, "Invalid voice", id="voice_empty", marks=_SKIP_ISSUE_3649),
+        pytest.param(
+            {"instructions": 123}, ("instructions", "string_type", "valid string"), id="instructions_wrong_type"
+        ),
+        pytest.param(
+            {"response_format": "mpeg"}, ("response_format", "literal_error", "wav"), id="response_format_invalid"
+        ),
+        pytest.param({"stream_format": 1}, ("stream_format", "literal_error", "audio"), id="stream_format_wrong_type"),
+        pytest.param(
+            {"stream_format": "sse"}, ("stream_format", "value_error", "audio"), id="stream_format_sse_blocked"
+        ),
+        pytest.param({"stream": "wrong_type"}, ("stream", "bool_parsing", "validation error"), id="stream_wrong_type"),
+        pytest.param(
+            {
+                "voice": "vivian",
+                "task_type": "CustomVoice",
+                "ref_audio": None,
+                "ref_text": None,
+            },
+            ("CustomVoice", "does not support"),
+            id="customvoice_task_not_supported",
+        ),
+        pytest.param(
+            {"task_type": "InvalidEnum"}, ("task_type", "literal_error", "CustomVoice"), id="task_type_invalid"
+        ),
+        pytest.param({"language": ""}, ("language", "invalid language", "chinese"), id="language_empty"),
+        pytest.param({"ref_audio": "ftp://example.com/a.wav"}, ("ref_audio", "URL"), id="ref_audio_bad_scheme"),
+        pytest.param(
+            {"ref_audio": "not_a_valid_uri"},
+            ("ref_audio", "url"),
+            id="ref_audio_invalid_uri",
+        ),
+        pytest.param({"ref_text": ["x"]}, ("ref_text", "string_type", "valid string"), id="ref_text_wrong_type"),
+        pytest.param(
+            {"x_vector_only_mode": "wrong"},
+            ("x_vector_only_mode", "valid boolean"),
+            id="x_vector_only_mode_string_not_boolean",
+        ),
+        pytest.param(
+            {"speaker_embedding": [0.01] * _QWEN3_TTS_06B_SPEAKER_EMBEDDING_DIM},
+            ("mutually exclusive", "speaker_embedding", "ref_audio"),
+            id="speaker_embedding_with_ref_audio_mutually_exclusive",
+        ),
+        pytest.param({"max_new_tokens": 0}, ("max_new_tokens", "least 1"), id="max_new_tokens_below_min"),
+        pytest.param(
+            {"max_new_tokens": _SPEECH_API_MAX_NEW_TOKENS + 1},
+            ("max_new_tokens", "exceed 4096"),
+            id="max_new_tokens_above_max",
+        ),
+        pytest.param({"seed": -1}, ("seed", "greater_than_equal", "0"), id="seed_negative"),
+        pytest.param({"seed": 2**63}, ("seed", "less_than_equal", "9223372036854775807"), id="seed_above_max"),
+        pytest.param(
+            {"initial_codec_chunk_frames": -1},
+            ("initial_codec_chunk_frames", "greater_than_equal", "0"),
+            id="initial_codec_chunk_frames_negative",
+        ),
+        pytest.param(
+            {"instructions": "x" * (_SPEECH_API_MAX_INSTRUCTIONS_CHARS + 1)},
+            ("instructions", "too long"),
+            id="instructions_exceed_api_limit",
+        ),
+        # Base clone: whitespace-only ref_text fails serving validation (not pydantic).
+        pytest.param(
+            {
+                "task_type": "Base",
+                "ref_audio": "data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEARKwAAIhYAQACABAAZGF0YQAAAAA=",
+                "ref_text": "   ",
+            },
+            ("ref_text", "base task", "non-empty"),
+            id="ref_text_whitespace_only_base_clone",
+        ),
+        pytest.param(
+            {
+                _SPEECH_INVALID_EMBEDDING_ONLY_BODY: True,
+                "task_type": "Base",
+                "speaker_embedding": [],
+            },
+            ("speaker_embedding", "non-empty"),
+            id="base_speaker_embedding_empty",
+        ),
+        pytest.param(
+            {
+                _SPEECH_INVALID_EMBEDDING_ONLY_BODY: True,
+                "task_type": "Base",
+                "x_vector_only_mode": True,
+                "speaker_embedding": [0.01] * 512,
+            },
+            ("speaker_embedding", "dimensions", str(_QWEN3_TTS_06B_SPEAKER_EMBEDDING_DIM)),
+            id="base_speaker_embedding_wrong_dim_short",
+        ),
+        pytest.param(
+            {
+                _SPEECH_INVALID_EMBEDDING_ONLY_BODY: True,
+                "task_type": "Base",
+                "x_vector_only_mode": True,
+                "speaker_embedding": [0.01] * 2048,
+            },
+            ("speaker_embedding", "2048", "dimensions", str(_QWEN3_TTS_06B_SPEAKER_EMBEDDING_DIM)),
+            id="base_speaker_embedding_wrong_dim_long",
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
+def test_speech_invalid_field_values(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    overrides: dict[str, object],
+    err_message: str | tuple[str, ...],
+) -> None:
+    ov = dict(overrides)
+    embedding_only = bool(ov.pop(_SPEECH_INVALID_EMBEDDING_ONLY_BODY, False))
+    body = {
+        "model": omni_server.model,
+        "input": "Hello.",
+        "voice": "clone",
+        "language": "English",
+        "response_format": "wav",
+        "ref_audio": REF_AUDIO_URL,
+        "ref_text": REF_TEXT,
+    }
+    if embedding_only:
+        body.pop("ref_audio", None)
+        body.pop("ref_text", None)
+    body.update(ov)
+    openai_client.send_audio_speech_http_request(
+        {"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
+    )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# POST /v1/audio/speech/batch (JSON)
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
+def test_speech_batch_empty_items(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
+    openai_client.send_audio_speech_batch_http_request(
+        {"json": {"items": []}, "timeout": 120, "err_code": (400, 422), "err_message": ("items", "least 1 item")}
+    )
+
+
+@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
+def test_speech_batch_missing_items(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
+    openai_client.send_audio_speech_batch_http_request(
+        {
+            "json": {"model": omni_server.model},
+            "timeout": 120,
+            "err_code": (400, 422),
+            "err_message": ("items", "field required", "missing"),
+        }
+    )
+
+
+# Align field-level checks with ``POST /v1/audio/speech`` where ``BatchSpeechRequest`` /
+# ``SpeechBatchItem`` expose the same knobs. Not on batch schema (single-speech only):
+# ``stream``, ``stream_format``, ``seed``, ``speaker_embedding`` — see dedicated speech tests.
+@pytest.mark.parametrize(
+    "loc, overrides, err_message",
+    [
+        pytest.param("item", {"input": ""}, ("input", "empty"), id="item_input_empty", marks=_SKIP_ISSUE_3649),
+        pytest.param(
+            "item",
+            {"input": "   "},
+            ("input", "empty"),
+            id="item_input_whitespace_only",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param("batch", {"voice": ""}, "Invalid voice", id="batch_voice_empty", marks=_SKIP_ISSUE_3649),
+        pytest.param("item", {"voice": ""}, "Invalid voice", id="item_voice_empty", marks=_SKIP_ISSUE_3649),
+        pytest.param(
+            "batch",
+            {
+                "voice": "nonexistent_voice_xyz",
+                "task_type": "CustomVoice",
+                "ref_audio": None,
+                "ref_text": None,
+            },
+            "Invalid voice",
+            id="batch_voice_unknown_preset",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "item",
+            {
+                _SPEECH_BATCH_NO_CLONE_FIELDS: True,
+                "voice": "nonexistent_voice_xyz",
+                "task_type": "CustomVoice",
+            },
+            "Invalid voice",
+            id="item_voice_unknown_preset",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "batch",
+            {"instructions": 123},
+            ("instructions", "string_type", "valid string"),
+            id="batch_instructions_wrong_type",
+        ),
+        pytest.param(
+            "item",
+            {"instructions": 123},
+            ("instructions", "string_type", "valid string"),
+            id="item_instructions_wrong_type",
+        ),
+        pytest.param(
+            "batch",
+            {"instructions": "x" * (_SPEECH_API_MAX_INSTRUCTIONS_CHARS + 1)},
+            ("instructions", "too long"),
+            id="batch_instructions_exceed_api_limit",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "item",
+            {"instructions": "x" * (_SPEECH_API_MAX_INSTRUCTIONS_CHARS + 1)},
+            ("instructions", "too long"),
+            id="item_instructions_exceed_api_limit",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "batch",
+            {"response_format": "mpeg"},
+            ("response_format", "literal_error", "wav"),
+            id="batch_response_format_invalid",
+        ),
+        pytest.param(
+            "item",
+            {"response_format": "mpeg"},
+            ("response_format", "literal_error", "wav"),
+            id="item_response_format_invalid",
+        ),
+        pytest.param("batch", {"speed": 0.1}, ("speed", "greater_than_equal", "0.25"), id="batch_speed_too_low"),
+        pytest.param("item", {"speed": 10.0}, ("speed", "less_than_equal", "4"), id="item_speed_too_high"),
+        pytest.param(
+            "batch",
+            {"task_type": "InvalidEnum"},
+            ("task_type", "literal_error", "CustomVoice"),
+            id="batch_task_type_invalid",
+        ),
+        pytest.param(
+            "item",
+            {"task_type": "InvalidEnum"},
+            ("task_type", "literal_error", "CustomVoice"),
+            id="item_task_type_invalid",
+        ),
+        pytest.param(
+            "batch",
+            {"language": ""},
+            ("language", "invalid language", "chinese"),
+            id="batch_language_empty",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "item",
+            {"language": ""},
+            ("language", "invalid language", "chinese"),
+            id="item_language_empty",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "batch",
+            {"ref_audio": "ftp://example.com/a.wav"},
+            ("ref_audio", "URL"),
+            id="batch_ref_audio_bad_scheme",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "item",
+            {"ref_audio": "ftp://example.com/a.wav"},
+            ("ref_audio", "URL"),
+            id="item_ref_audio_bad_scheme",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "batch",
+            {"ref_audio": "not_a_valid_uri"},
+            ("ref_audio", "url"),
+            id="batch_ref_audio_invalid_uri",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "item",
+            {"ref_audio": "not_a_valid_uri"},
+            ("ref_audio", "url"),
+            id="item_ref_audio_invalid_uri",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "batch",
+            {"ref_text": ["x"]},
+            ("ref_text", "string_type", "valid string"),
+            id="batch_ref_text_wrong_type",
+        ),
+        pytest.param(
+            "item",
+            {"ref_text": ["x"]},
+            ("ref_text", "string_type", "valid string"),
+            id="item_ref_text_wrong_type",
+        ),
+        pytest.param(
+            "batch",
+            {"x_vector_only_mode": "wrong_type"},
+            ("x_vector_only_mode", "bool_parsing", "validation error"),
+            id="batch_x_vector_only_mode_wrong_type",
+        ),
+        pytest.param(
+            "item",
+            {"x_vector_only_mode": "wrong_type"},
+            ("x_vector_only_mode", "bool_parsing", "validation error"),
+            id="item_x_vector_only_mode_wrong_type",
+        ),
+        pytest.param(
+            "batch",
+            {"max_new_tokens": 0},
+            ("max_new_tokens", "least 1"),
+            id="batch_max_new_tokens_below_min",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "item",
+            {"max_new_tokens": 0},
+            ("max_new_tokens", "least 1"),
+            id="item_max_new_tokens_below_min",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "batch",
+            {"max_new_tokens": _SPEECH_API_MAX_NEW_TOKENS + 1},
+            ("max_new_tokens", "exceed 4096"),
+            id="batch_max_new_tokens_above_max",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "item",
+            {"max_new_tokens": _SPEECH_API_MAX_NEW_TOKENS + 1},
+            ("max_new_tokens", "exceed 4096"),
+            id="item_max_new_tokens_above_max",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "batch",
+            {"initial_codec_chunk_frames": -1},
+            ("initial_codec_chunk_frames", "greater_than_equal", "0"),
+            id="batch_initial_codec_chunk_frames_negative",
+        ),
+        pytest.param(
+            "item",
+            {"initial_codec_chunk_frames": -1},
+            ("initial_codec_chunk_frames", "greater_than_equal", "0"),
+            id="item_initial_codec_chunk_frames_negative",
+        ),
+        # Base clone: whitespace-only ref_text (same serving check as single speech).
+        pytest.param(
+            "batch",
+            {
+                "task_type": "Base",
+                "ref_audio": "data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEARKwAAIhYAQACABAAZGF0YQAAAAA=",
+                "ref_text": "   ",
+            },
+            ("ref_text", "base task", "non-empty"),
+            id="batch_ref_text_whitespace_only_base_clone",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "item",
+            {
+                "task_type": "Base",
+                "ref_audio": "data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEARKwAAIhYAQACABAAZGF0YQAAAAA=",
+                "ref_text": "   ",
+            },
+            ("ref_text", "base task", "non-empty"),
+            id="item_ref_text_whitespace_only_base_clone",
+            marks=_SKIP_ISSUE_3649,
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
+def test_speech_batch_invalid_field_values(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    loc: str,
+    overrides: dict[str, object],
+    err_message: str | tuple[str, ...],
+) -> None:
+    ov = dict(overrides)
+    strip_clone = bool(ov.pop(_SPEECH_BATCH_NO_CLONE_FIELDS, False))
+    body = {
+        "model": omni_server.model,
+        "voice": "clone",
+        "ref_audio": REF_AUDIO_URL,
+        "ref_text": REF_TEXT,
+        "language": "English",
+        "response_format": "wav",
+        "items": [{"input": "Hello."}],
+    }
+    if strip_clone:
+        body.pop("ref_audio", None)
+        body.pop("ref_text", None)
+    _apply_batch_overrides(body, loc=loc, overrides=ov)
+    openai_client.send_audio_speech_batch_http_request(
+        {"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
+    )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# WebSocket /v1/audio/speech/stream
+# ─────────────────────────────────────────────────────────────────────────────
+
+# ``test_speech_stream_invalid_requests``: build ``session.config`` JSON with ``omni_server.model``.
+_SPEECH_STREAM_WS_SESSION_STREAM_AUDIO_WAV_FRAMES = object()
+
+
+@pytest.mark.parametrize(
+    "send_frames_spec, err_message",
+    [
+        pytest.param("{not-json", "Invalid JSON", id="invalid_json"),
+        pytest.param(
+            json.dumps({"type": "input.text", "text": "hello"}),
+            "Expected session.config",
+            id="wrong_first_message_type",
+        ),
+        pytest.param(
+            _SPEECH_STREAM_WS_SESSION_STREAM_AUDIO_WAV_FRAMES,
+            ("invalid session config", "response_format", "pcm"),
+            id="session_config_stream_audio_requires_pcm",
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
+def test_speech_stream_invalid_requests(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    send_frames_spec: Any,
+    err_message: str | tuple[str, ...],
+) -> None:
+    """Bad first WebSocket text frames: non-JSON, wrong ``type``, or invalid ``session.config``."""
+    if send_frames_spec is _SPEECH_STREAM_WS_SESSION_STREAM_AUDIO_WAV_FRAMES:
+        send_frames = json.dumps(
+            {
+                "type": "session.config",
+                "model": omni_server.model,
+                "response_format": "wav",
+                "stream_audio": True,
+            }
+        )
+    else:
+        send_frames = send_frames_spec
+    assert isinstance(send_frames, str)
+    openai_client.send_audio_speech_stream_ws_request(
+        {
+            "send_frames": send_frames,
+            "timeout": 120,
+            "ws_max_size": None,
+            "ws_json_type": "error",
+            "err_message": err_message,
+        }
+    )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# POST /v1/audio/voices (multipart + embedding-only) and DELETE /v1/audio/voices/{name}
+# ─────────────────────────────────────────────────────────────────────────────
+
+# Templates for ``test_voices_create_invalid_requests``: resolved in ``_finalize_voices_form_data``.
+_VF_LONG_CONSENT = object()
+_VF_LONG_REF_TEXT = object()
+_VF_LONG_SPEAKER_DESC = object()
+_VF_EMBEDDING_NAN_JSON = object()
+
+
+def _finalize_voices_form_data(template: dict[str, Any], uuid_hex: str) -> dict[str, Any]:
+    out: dict[str, Any] = {}
+    for key, val in template.items():
+        if val is _VF_LONG_CONSENT:
+            out[key] = "x" * (_VOICE_UPLOAD_MAX_CONSENT_LEN + 1)
+        elif val is _VF_LONG_REF_TEXT:
+            out[key] = "y" * (_VOICE_UPLOAD_MAX_REF_TEXT_CHARS + 1)
+        elif val is _VF_LONG_SPEAKER_DESC:
+            out[key] = "z" * (_VOICE_UPLOAD_MAX_SPEAKER_DESCRIPTION_CHARS + 1)
+        elif val is _VF_EMBEDDING_NAN_JSON:
+            out[key] = json.dumps([0.1] * 1023 + [float("nan")])
+        elif isinstance(val, str):
+            out[key] = val.replace("{uuid}", uuid_hex)
+        else:
+            out[key] = val
+    return out
+
+
+def _voices_upload_multipart_files(kind: str | None) -> dict[str, Any] | None:
+    """``None`` => no multipart file (embedding-only or neither)."""
+    if kind is None:
+        return None
+    if kind == "wav_ok":
+        return {"audio_sample": ("clip.wav", _pcm_wav_mono_bytes(), "audio/wav")}
+    if kind == "wav_short":
+        return {"audio_sample": ("clip.wav", _pcm_wav_mono_bytes(duration_s=0.3), "audio/wav")}
+    if kind == "wav_bad_body":
+        return {"audio_sample": ("clip.wav", b"not valid wav content", "audio/wav")}
+    if kind == "wav_pdf_type":
+        return {"audio_sample": ("clip.wav", _pcm_wav_mono_bytes(), "application/pdf")}
+    raise AssertionError(f"unknown multipart kind {kind!r}")
+
+
+@pytest.mark.parametrize(
+    "multipart_kind, form_template, err_message",
+    [
+        pytest.param(
+            None,
+            {"consent": "c1", "name": "v1"},
+            ("audio_sample", "speaker_embedding", "provided"),
+            id="neither_audio_nor_embedding",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_ok",
+            {"name": "v_miss_consent_{uuid}"},
+            ("missing", "consent", "field required"),
+            id="missing_consent",
+        ),
+        pytest.param(
+            "wav_ok",
+            {"consent": "consent_ok"},
+            ("missing", "name", "field required"),
+            id="missing_name",
+        ),
+        pytest.param(
+            "wav_ok",
+            {"consent": "", "name": "v_empty_consent_{uuid}"},
+            ("missing", "consent", "field required"),
+            id="empty_consent",
+        ),
+        pytest.param(
+            "wav_ok",
+            {"consent": "   ", "name": "v_ws_consent_{uuid}"},
+            "consent",
+            id="whitespace_consent",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_ok",
+            {"consent": "bad/consent", "name": "v_bad_cons_{uuid}"},
+            "consent",
+            id="consent_path_sep",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_ok",
+            {"consent": _VF_LONG_CONSENT, "name": "v_long_cons_{uuid}"},
+            ("consent", "too long", "Failed to save"),
+            id="consent_too_long",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_ok",
+            {"consent": "consent_ok", "name": ""},
+            ("missing", "name", "field required"),
+            id="empty_name",
+        ),
+        pytest.param(
+            "wav_ok",
+            {"consent": "consent_ok", "name": "evil/name"},
+            ("voice name", "invalid voice name"),
+            id="invalid_name_path_sep",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_ok",
+            {"consent": "consent_ok", "name": "v_long_ref_{uuid}", "ref_text": _VF_LONG_REF_TEXT},
+            "ref_text",
+            id="ref_text_too_long",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_ok",
+            {
+                "consent": "consent_ok",
+                "name": "v_long_desc_{uuid}",
+                "speaker_description": _VF_LONG_SPEAKER_DESC,
+            },
+            "speaker_description",
+            id="speaker_description_too_long",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_pdf_type",
+            {"consent": "consent_ok", "name": "v_bad_mime_{uuid}"},
+            ("mime", "unsupported", "audio/mp4"),
+            id="audio_unsupported_mime",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_short",
+            {"consent": "consent_ok", "name": "v_short_{uuid}"},
+            ("too short", "reference audio"),
+            id="audio_too_short",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_bad_body",
+            {"consent": "consent_ok", "name": "v_bad_wav_{uuid}"},
+            ("decode", "audio", "Format not recognised"),
+            id="audio_decode_error",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            None,
+            {
+                "speaker_embedding": "not valid json [[[",
+                "consent": "consent_emb",
+                "name": "v_bad_json_{uuid}",
+            },
+            ("speaker_embedding", "valid JSON"),
+            id="speaker_embedding_invalid_json",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            None,
+            {
+                "speaker_embedding": json.dumps([]),
+                "consent": "consent_emb",
+                "name": "v_empty_emb_{uuid}",
+            },
+            ("speaker_embedding", "non-empty"),
+            id="speaker_embedding_empty",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            None,
+            {
+                "speaker_embedding": _VF_EMBEDDING_NAN_JSON,
+                "consent": "consent_emb",
+                "name": "v_nan_emb_{uuid}",
+            },
+            ("speaker_embedding", "finite"),
+            id="speaker_embedding_nan",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            None,
+            {
+                "speaker_embedding": json.dumps([0.01] * 512),
+                "consent": "consent_emb",
+                "name": "v_wrong_dim_{uuid}",
+            },
+            ("expected", "dimensions", "1024"),
+            id="speaker_embedding_wrong_dim",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_ok",
+            {
+                "consent": "consent_emb",
+                "name": "v_both_{uuid}",
+                "speaker_embedding": json.dumps([0.01] * 1024),
+            },
+            ("mutually exclusive", "audio_sample", "speaker_embedding"),
+            id="audio_and_embedding_mutually_exclusive",
+            marks=_SKIP_ISSUE_3649,
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
+def test_voices_create_invalid_requests(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    multipart_kind: str | None,
+    form_template: dict[str, Any],
+    err_message: str | tuple[str, ...],
+) -> None:
+    """Invalid ``POST /v1/audio/voices`` bodies (multipart or embedding-only)."""
+    uid = uuid.uuid4().hex[:8]
+    cfg: dict[str, Any] = {
+        "data": _finalize_voices_form_data(form_template, uid),
+        "timeout": 120,
+        "err_code": 400,
+        "err_message": err_message,
+    }
+    files = _voices_upload_multipart_files(multipart_kind)
+    if files is not None:
+        cfg["files"] = files
+    openai_client.send_audio_voices_create_http_request(cfg)
+
+
+@pytest.mark.parametrize(
+    "voice_name, err_code, err_message",
+    [
+        pytest.param(
+            "missing-voice-404",
+            404,
+            "not found",
+            id="not_found",
+        ),
+        pytest.param(
+            "   ",
+            404,
+            "not found",
+            id="whitespace_only",
+        ),
+        pytest.param(
+            "evil/name",
+            404,
+            "not found",
+            id="path_separator",
+        ),
+        pytest.param(
+            ".",
+            405,
+            ("Method Not Allowed", "detail"),
+            id="dot_single",
+        ),
+        pytest.param(
+            "..",
+            404,
+            "not found",
+            id="dot_dot",
+        ),
+        pytest.param(
+            "bad\x00voice",
+            404,
+            "not found",
+            id="nul_byte",
+        ),
+        pytest.param(
+            "",
+            405,
+            "Method Not Allowed",
+            id="empty_name_path",
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
+def test_voices_delete_invalid_requests(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    voice_name: str,
+    err_code: int | tuple[int, ...],
+    err_message: str | tuple[str, ...],
+) -> None:
+    """DELETE ``/v1/audio/voices/{name}``: missing voice (404), odd segments (often 404), ``.``paths (405)."""
+    openai_client.send_audio_voices_delete_http_request(
+        {
+            "name": voice_name,
+            "timeout": 120,
+            "err_code": err_code,
+            "err_message": err_message,
+        }
+    )

tests/dfx/reliability/invalid_param_test/test_invalid_image_editing.py

@@ -0,0 +1,90 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""``POST /v1/images/edits`` multipart validation (live ``Qwen/Qwen-Image-Edit``)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+
+from tests.helpers.mark import hardware_marks
+from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
+
+pytestmark = [pytest.mark.slow, pytest.mark.diffusion]
+
+_PARAMS = [
+    pytest.param(
+        OmniServerParams(model="Qwen/Qwen-Image-Edit"),
+        id="qwen_image_edit",
+        marks=hardware_marks(res={"cuda": "H100"}),
+    ),
+]
+
+# Populated in ``test_images_edits_invalid_requests`` from ``omni_server.model``.
+_IMAGE_EDITS_SERVER_MODEL = object()
+
+
+def _finalize_edits_form_data(template: dict[str, Any], server_model: str) -> dict[str, Any]:
+    return {k: (server_model if v is _IMAGE_EDITS_SERVER_MODEL else v) for k, v in template.items()}
+
+
+@pytest.mark.parametrize(
+    "include_image, data_template, err_message, err_code",
+    [
+        pytest.param(
+            False,
+            {"prompt": "make it brighter", "model": _IMAGE_EDITS_SERVER_MODEL},
+            ("image", "required", "Unprocessable Entity"),
+            422,
+            id="missing_image",
+        ),
+        pytest.param(
+            True,
+            {"prompt": "edit", "model": "wrong-model-id"},
+            ("model", "mismatch"),
+            400,
+            id="model_mismatch",
+        ),
+        pytest.param(
+            True,
+            {"prompt": "edit", "response_format": "url"},
+            ("response_format", "b64_json", "supported"),
+            400,
+            id="unsupported_response_format",
+        ),
+        pytest.param(
+            True,
+            {"prompt": "edit", "model": _IMAGE_EDITS_SERVER_MODEL, "output_compression": "101"},
+            ("less_than_equal", "output_compression", "100"),
+            400,
+            id="output_compression_above_max",
+        ),
+        pytest.param(
+            True,
+            {"prompt": "edit", "model": _IMAGE_EDITS_SERVER_MODEL, "layers": "2"},
+            ("Invalid layers", "3", "1"),
+            400,
+            id="invalid_layers",
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
+def test_images_edits_invalid_requests(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    tiny_png_bytes: bytes,
+    include_image: bool,
+    data_template: dict[str, Any],
+    err_message: str | tuple[str, ...],
+    err_code: int | tuple[int, ...],
+) -> None:
+    cfg: dict[str, Any] = {
+        "data": _finalize_edits_form_data(data_template, omni_server.model),
+        "timeout": 300,
+        "err_code": err_code,
+        "err_message": err_message,
+    }
+    if include_image:
+        cfg["files"] = {"image": ("x.png", tiny_png_bytes, "image/png")}
+    openai_client.send_images_edits_http_request(cfg)

tests/dfx/reliability/invalid_param_test/test_invalid_image_generation.py

@@ -0,0 +1,142 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""``POST /v1/images/generations`` validation (live ``Qwen/Qwen-Image``)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+
+from tests.helpers.mark import hardware_marks
+from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
+
+pytestmark = [pytest.mark.slow, pytest.mark.diffusion]
+
+_SKIP_ISSUE_3649 = pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/3649")
+
+_PARAMS = [
+    pytest.param(
+        OmniServerParams(model="Qwen/Qwen-Image"),
+        id="qwen_image",
+        marks=hardware_marks(res={"cuda": "H100"}),
+    ),
+]
+
+# Full replacement body for ``test_images_generations_invalid_requests`` (no minimal prompt/model merge).
+_IMAGES_GEN_MISSING_PROMPT_JSON = object()
+
+
+def _minimal_images_gen_json(omni_server: OmniServer) -> dict[str, object]:
+    return {"prompt": "a simple red apple icon", "model": omni_server.model}
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# POST /v1/images/generations (JSON)
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize(
+    "body_spec, err_message",
+    [
+        pytest.param(_IMAGES_GEN_MISSING_PROMPT_JSON, ("prompt", "Field required", "Missing"), id="missing_prompt"),
+        pytest.param(
+            {"prompt": "cat", "model": "not-the-loaded-checkpoint"},
+            ("model", "mismatch"),
+            id="model_mismatch",
+        ),
+        pytest.param({"prompt": ""}, ("prompt", "empty"), id="prompt_empty", marks=_SKIP_ISSUE_3649),
+        pytest.param({"prompt": "   "}, ("prompt", "empty"), id="prompt_whitespace_only", marks=_SKIP_ISSUE_3649),
+        pytest.param({"prompt": 123}, ("prompt", "string_type", "valid string"), id="prompt_wrong_type"),
+        pytest.param({"n": 0}, ("n", "greater_than_equal", "1"), id="n_below_min"),
+        pytest.param({"n": 11}, ("n", "less_than_equal", "10"), id="n_above_max"),
+        pytest.param({"size": "1024"}, ("size", "value_error", "WIDTHxHEIGHT"), id="size_missing_height_token"),
+        pytest.param({"size": "not-a-size"}, ("size", "value_error", "WIDTHxHEIGHT"), id="size_missing_separator"),
+        pytest.param({"size": "1024xabc"}, ("Invalid size format", "integers"), id="size_non_integer_height"),
+        pytest.param({"size": "1024x"}, ("Invalid size format", "integers"), id="size_incomplete_height"),
+        pytest.param({"size": "0x1024"}, ("Invalid size", "positive integers"), id="size_non_positive_width"),
+        pytest.param(
+            {"response_format": "url"},
+            ("response_format", "value_error", "b64_json"),
+            id="response_format_not_b64_json",
+        ),
+        pytest.param(
+            {"use_system_prompt": "bogus"},
+            ("use_system_prompt", "value_error", "dynamic"),
+            id="use_system_prompt_invalid",
+        ),
+        pytest.param(
+            {"num_inference_steps": 0},
+            ("num_inference_steps", "'greater_than_equal", "1"),
+            id="num_inference_steps_below_min",
+        ),
+        pytest.param(
+            {"num_inference_steps": 201},
+            ("num_inference_steps", "less_than_equal", "200"),
+            id="num_inference_steps_above_max",
+        ),
+        pytest.param(
+            {"guidance_scale": -1.0}, ("guidance_scale", "greater_than_equal", "0"), id="guidance_scale_negative"
+        ),
+        pytest.param(
+            {"guidance_scale": 21.0}, ("guidance_scale", "less_than_equal", "20"), id="guidance_scale_above_max"
+        ),
+        pytest.param(
+            {"true_cfg_scale": -1.0}, ("true_cfg_scale", "greater_than_equal", "0"), id="true_cfg_scale_negative"
+        ),
+        pytest.param(
+            {"true_cfg_scale": 21.0}, ("true_cfg_scale", "less_than_equal", "20"), id="true_cfg_scale_above_max"
+        ),
+        pytest.param({"layers": 2}, ("layers", "value_error", "3", "10"), id="layers_below_min"),
+        pytest.param({"layers": 11}, ("layers", "value_error", "3", "10"), id="layers_above_max"),
+        pytest.param({"seed": -1}, ("seed", "greater_than_equal", "0"), id="seed_negative", marks=_SKIP_ISSUE_3649),
+        pytest.param(
+            {"seed": 2**32},
+            ("seed", "less_than_equal", "4294967295"),
+            id="seed_above_uint32",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            {"output_format": "gif"},
+            ("output_format", "value_error", "b64_json"),
+            id="output_format_invalid",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            {"vae_use_slicing": "wrong_type"},
+            ("vae_use_slicing", "bool_parsing", "validation error"),
+            id="vae_use_slicing_wrong_type",
+        ),
+        pytest.param({"user": 123}, ("user", "string_type", "valid string"), id="user_wrong_type"),
+        pytest.param(
+            {"negative_prompt": ["noise"]},
+            ("negative_prompt", "string_type", "valid string"),
+            id="negative_prompt_wrong_type",
+        ),
+        pytest.param(
+            {"generator_device": 1},
+            ("generator_device", "string_type", "valid string"),
+            id="generator_device_wrong_type",
+        ),
+        pytest.param(
+            {"lora": {"foo": "bar"}}, ("lora", "both name and path", "required"), id="lora_missing_required_fields"
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
+def test_images_generations_invalid_requests(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    body_spec: Any,
+    err_message: str | tuple[str, ...],
+) -> None:
+    """Invalid JSON bodies for ``POST /v1/images/generations`` (missing prompt, model mismatch, bad fields)."""
+    if body_spec is _IMAGES_GEN_MISSING_PROMPT_JSON:
+        body: dict[str, object] = {"size": "1024x1024"}
+    else:
+        assert isinstance(body_spec, dict)
+        body = _minimal_images_gen_json(omni_server)
+        body.update(body_spec)
+    openai_client.send_images_generations_http_request(
+        {"json": body, "timeout": 300, "err_code": 400, "err_message": err_message}
+    )

tests/dfx/reliability/invalid_param_test/test_invalid_omni_chat.py

@@ -0,0 +1,216 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Invalid inputs on Qwen3-Omni: ``POST /v1/chat/completions``, ``WS /v1/video/chat/stream``, ``WS /v1/realtime``."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+import pytest
+
+from tests.helpers.mark import hardware_test
+from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
+from tests.helpers.stage_config import get_deploy_config_path
+
+pytestmark = [pytest.mark.slow, pytest.mark.omni]
+
+_SKIP_ISSUE_3649 = pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/3649")
+
+
+def _minimal_chat_json(omni_server: OmniServer) -> dict[str, object]:
+    """Minimal valid chat body; individual tests override one offending field."""
+    return {
+        "model": omni_server.model,
+        "messages": [{"role": "user", "content": "hello"}],
+        "stream": False,
+    }
+
+
+_QWEN3_OMNI_SERVER = [
+    pytest.param(
+        OmniServerParams(
+            model="Qwen/Qwen3-Omni-30B-A3B-Instruct",
+            stage_config_path=get_deploy_config_path("qwen3_omni_moe.yaml"),
+            server_args=["--async-chunk"],
+        ),
+        id="qwen3_omni",
+    ),
+]
+
+
+def _chat_completions_request_without_expectations(omni_server: OmniServer, case_id: str) -> dict[str, Any]:
+    """Build ``send_chat_completions_http_request`` kwargs excluding ``err_code`` / ``err_message``."""
+    if case_id == "malformed_json":
+        return {"raw_body": "{not-json", "timeout": 120}
+    if case_id == "missing_messages":
+        return {"json": {"model": omni_server.model, "stream": False}, "timeout": 120}
+    if case_id == "model_mismatch":
+        return {
+            "json": {
+                "model": "this-model-is-not-served-on-this-instance",
+                "messages": [{"role": "user", "content": "hello"}],
+                "stream": False,
+            },
+            "timeout": 120,
+        }
+    body = _minimal_chat_json(omni_server)
+    if case_id == "stream_wrong_type":
+        body["stream"] = "not-a-boolean"  # type: ignore[assignment]
+    elif case_id == "temperature_wrong_type":
+        body["temperature"] = "hot"  # type: ignore[assignment]
+    elif case_id == "max_tokens_wrong_type":
+        body["max_tokens"] = "not-an-int"  # type: ignore[assignment] # type: ignore[assignment]
+    elif case_id == "modalities_list_bad":
+        body["modalities"] = [123]  # type: ignore[assignment]
+    elif case_id == "response_format_json_schema_incomplete":
+        body["response_format"] = {"type": "json_schema"}
+    elif case_id == "logprobs_wrong_type":
+        body["logprobs"] = "yes"  # type: ignore[assignment]
+    elif case_id == "logprobs_top_without_enabled":
+        body["logprobs"] = False
+        body["top_logprobs"] = 5
+    elif case_id == "speaker_unknown":
+        body["speaker"] = "zz_invalid_qwen3_omni_chat_speaker_xyz"
+    else:
+        raise AssertionError(f"unknown chat completions invalid case_id {case_id!r}")
+    return {"json": body, "timeout": 120}
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# POST /v1/chat/completions
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
+@pytest.mark.parametrize(
+    "case_id, err_code, err_message",
+    [
+        pytest.param(
+            "malformed_json",
+            400,
+            ("json_invalid", "decode error"),
+            id="malformed_json",
+        ),
+        pytest.param("missing_messages", 400, ("messages", "missing", "Field required"), id="missing_messages"),
+        pytest.param(
+            "model_mismatch",
+            404,
+            ("model", "does not exist", "NotFoundError"),
+            id="model_mismatch",
+        ),
+        pytest.param("stream_wrong_type", 400, ("stream", "bool_parsing", "not-a-boolean"), id="invalid_stream_type"),
+        pytest.param(
+            "temperature_wrong_type",
+            400,
+            ("temperature", "float_parsing", "a valid number"),
+            id="invalid_temperature_type",
+        ),
+        pytest.param(
+            "max_tokens_wrong_type", 400, ("max_tokens", "int_parsing", "integer"), id="invalid_max_tokens_type"
+        ),
+        pytest.param(
+            "modalities_list_bad",
+            400,
+            ("modalities", "value_error", ""),
+            id="modalities_list_bad_element",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "response_format_json_schema_incomplete",
+            400,
+            ("response_format", "value_error", "json_schema"),
+            id="invalid_response_format_json_schema",
+        ),
+        pytest.param("logprobs_wrong_type", 400, "logprobs", id="logprobs_wrong_type", marks=_SKIP_ISSUE_3649),
+        pytest.param(
+            "logprobs_top_without_enabled",
+            400,
+            ("top_logprobs", "logprobs", "True"),
+            id="logprobs_top_without_logprobs_true",
+        ),
+        pytest.param(
+            "speaker_unknown",
+            400,
+            ("Invalid speaker", "Supported"),
+            id="speaker_unknown_preset",
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _QWEN3_OMNI_SERVER, indirect=True)
+def test_chat_completions_invalid_requests(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    case_id: str,
+    err_code: int | tuple[int, ...],
+    err_message: str | tuple[str, ...],
+) -> None:
+    cfg = _chat_completions_request_without_expectations(omni_server, case_id)
+    cfg["err_code"] = err_code
+    cfg["err_message"] = err_message
+    openai_client.send_chat_completions_http_request(cfg)[0]
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# WS /v1/video/chat/stream
+# ─────────────────────────────────────────────────────────────────────────────
+
+# First-frame JSON with ``omni_server.model`` for ``test_video_chat_stream_invalid_requests``.
+_VIDEO_CHAT_WS_SESSION_MODALITIES_SCALAR = object()
+
+
+@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
+@pytest.mark.parametrize(
+    "send_frames_spec, err_message",
+    [
+        pytest.param("{not-json", "Invalid JSON", id="invalid_json"),
+        pytest.param(
+            _VIDEO_CHAT_WS_SESSION_MODALITIES_SCALAR,
+            "Invalid session config",
+            id="session_modalities_scalar",
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _QWEN3_OMNI_SERVER, indirect=True)
+def test_video_chat_stream_invalid_requests(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    send_frames_spec: Any,
+    err_message: str,
+) -> None:
+    if send_frames_spec is _VIDEO_CHAT_WS_SESSION_MODALITIES_SCALAR:
+        send_frames = json.dumps({"type": "session.config", "model": omni_server.model, "modalities": "text"})
+    else:
+        send_frames = send_frames_spec
+    assert isinstance(send_frames, str)
+    openai_client.send_video_chat_stream_ws_request(
+        {
+            "send_frames": send_frames,
+            "timeout": 120,
+            "ws_max_size": None,
+            "ws_json_type": "error",
+            "err_message": err_message,
+        }
+    )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# WS /v1/realtime
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
+@pytest.mark.parametrize("omni_server", _QWEN3_OMNI_SERVER, indirect=True)
+def test_realtime_rejected_when_async_chunk_enabled(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+) -> None:
+    openai_client.send_realtime_ws_request(
+        {
+            "timeout": 120,
+            "ws_max_size": None,
+            "ws_json_type": "error",
+            "ws_error_code": "unsupported",
+            "err_message": "async_chunk",
+        }
+    )

tests/dfx/reliability/invalid_param_test/test_invalid_server_control.py

@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Omni sleep/wakeup JSON validation (live ``Qwen/Qwen-Image`` process)."""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+import pytest
+
+from tests.helpers.mark import hardware_marks
+from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
+
+pytestmark = [pytest.mark.slow, pytest.mark.diffusion, pytest.mark.omni]
+
+_SKIP_ISSUE_3649 = pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/3649")
+
+_QWEN_IMAGE = [
+    pytest.param(
+        OmniServerParams(model="Qwen/Qwen-Image"),
+        id="qwen_image",
+        marks=hardware_marks(res={"cuda": "H100"}),
+    ),
+]
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# POST omni sleep / omni wakeup (invalid JSON)
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize(
+    "endpoint, json_body, err_message",
+    [
+        pytest.param("sleep", {"level": 2}, ("stage_ids", "Field required", "Missing"), id="sleep_missing_stage_ids"),
+        pytest.param(
+            "sleep",
+            {"stage_ids": [], "level": 2},
+            ("stage_ids", "Field required", "Missing"),
+            id="sleep_empty_stage_ids",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "sleep",
+            {"stage_ids": "not-a-list", "level": 2},
+            ("stage_ids", "list_type", "a valid list"),
+            id="sleep_invalid_stage_ids_type",
+        ),
+        pytest.param(
+            "sleep",
+            {"stage_ids": [0], "level": "high"},
+            ("level", "int_parsing", "a valid integer"),
+            id="sleep_invalid_level_type",
+        ),
+        pytest.param(
+            "sleep",
+            {"stage_ids": [0], "level": -1},
+            ("level", "greater"),
+            id="sleep_negative_level",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param("wakeup", {}, ("stage_ids", "missing", "Field required"), id="wakeup_missing_stage_ids"),
+        pytest.param(
+            "wakeup",
+            {"stage_ids": None},
+            ("stage_ids", "list_type", "a valid list"),
+            id="wakeup_invalid_stage_ids_type",
+        ),
+        pytest.param(
+            "wakeup",
+            {"stage_ids": []},
+            ("stage_ids", "length"),
+            id="wakeup_empty_stage_ids",
+            marks=_SKIP_ISSUE_3649,
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _QWEN_IMAGE, indirect=True)
+def test_omni_sleep_wakeup_invalid_requests(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    endpoint: Literal["sleep", "wakeup"],
+    json_body: dict[str, Any],
+    err_message: str | tuple[str, ...],
+) -> None:
+    cfg: dict[str, Any] = {
+        "json": json_body,
+        "timeout": 120,
+        "err_code": 400,
+        "err_message": err_message,
+    }
+    if endpoint == "sleep":
+        openai_client.send_omni_sleep_http_request(cfg)
+    else:
+        openai_client.send_omni_wakeup_http_request(cfg)

tests/dfx/reliability/invalid_param_test/test_invalid_video_generation.py

@@ -0,0 +1,171 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""HTTP video job routes (live ``Wan-AI/Wan2.2-T2V-A14B-Diffusers``).
+
+WebSocket ``/v1/video/chat/stream`` and ``/v1/realtime`` invalid-input checks live in
+``test_invalid_omni_chat.py`` with the same Qwen3-Omni server module fixture (Wan cannot serve those paths).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+
+from tests.helpers.mark import hardware_marks
+from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
+
+pytestmark = [pytest.mark.slow, pytest.mark.diffusion]
+
+_WAN_T2V = [
+    pytest.param(
+        OmniServerParams(model="Wan-AI/Wan2.2-T2V-A14B-Diffusers"),
+        id="wan22_t2v",
+        marks=hardware_marks(res={"cuda": "H100"}),
+    ),
+]
+
+
+def _minimal_video_form_data() -> dict[str, str]:
+    return {"prompt": "a cat walking on grass"}
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# POST /v1/videos (async)
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize(
+    "form_extra, err_message",
+    [
+        pytest.param(None, ("prompt", "Field required", "Missing"), id="missing_prompt"),
+        pytest.param({"seconds": "0"}, ("seconds", "string_pattern_mismatch", "1"), id="seconds_zero"),
+        pytest.param({"size": "bad"}, ("size", "string_pattern_mismatch"), id="size_bad"),
+        pytest.param(
+            {"num_inference_steps": "0"},
+            ("num_inference_steps", "greater_than_equal", "1"),
+            id="num_inference_steps_zero",
+        ),
+        pytest.param(
+            {"guidance_scale": "25"}, ("guidance_scale", "less_than_equal", "20"), id="guidance_scale_above_max"
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _WAN_T2V, indirect=True)
+def test_post_videos_invalid_requests(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    form_extra: dict[str, str] | None,
+    err_message: str | tuple[str, ...],
+) -> None:
+    data: dict[str, str] = {} if form_extra is None else {**_minimal_video_form_data(), **form_extra}
+    openai_client.send_videos_create_http_request(
+        {"data": data, "timeout": 120, "err_code": 400, "err_message": err_message}
+    )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# POST /v1/videos/sync
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize(
+    "form_extra, err_message",
+    [
+        pytest.param(None, ("prompt", "Field required", "Missing"), id="missing_prompt"),
+        pytest.param({"seconds": "0"}, ("seconds", "string_pattern_mismatch", "1"), id="seconds_zero"),
+        pytest.param({"size": "bad"}, ("size", "string_pattern_mismatch"), id="size_bad"),
+        pytest.param(
+            {"num_inference_steps": "201"},
+            ("num_inference_steps", "less_than_equal", "200"),
+            id="num_inference_steps_above_max",
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _WAN_T2V, indirect=True)
+def test_post_videos_sync_invalid_requests(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    form_extra: dict[str, str] | None,
+    err_message: str | tuple[str, ...],
+) -> None:
+    data: dict[str, str] = {} if form_extra is None else {**_minimal_video_form_data(), **form_extra}
+    openai_client.send_videos_sync_http_request(
+        {"data": data, "timeout": 120, "err_code": 400, "err_message": err_message}
+    )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# GET /v1/videos (list)
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize(
+    "params, err_message",
+    [
+        pytest.param({"limit": -1}, ("limit", "greater_than_equal", "0"), id="invalid_limit_negative"),
+        pytest.param({"limit": 101}, ("limit", "less_than_equal", "100"), id="limit_above_max"),
+        pytest.param({"order": "newest"}, ("order", "literal_error", "asc", "desc"), id="invalid_order"),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _WAN_T2V, indirect=True)
+def test_get_videos_list_invalid_requests(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    params: dict[str, Any],
+    err_message: str | tuple[str, ...],
+) -> None:
+    openai_client.send_videos_list_http_request(
+        {"params": params, "timeout": 120, "err_code": 400, "err_message": err_message}
+    )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# GET /v1/videos/{video_id}
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize("omni_server", _WAN_T2V, indirect=True)
+def test_get_video_invalid_requests(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
+    openai_client.send_video_retrieve_http_request(
+        {
+            "video_id": "does-not-exist",
+            "timeout": 120,
+            "err_code": 404,
+            "err_message": ("not found", "video"),
+        }
+    )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# DELETE /v1/videos/{video_id}
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize("omni_server", _WAN_T2V, indirect=True)
+def test_delete_video_invalid_requests(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
+    openai_client.send_video_delete_http_request(
+        {
+            "video_id": "does-not-exist",
+            "timeout": 120,
+            "err_code": 404,
+            "err_message": ("not found", "video"),
+        }
+    )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# GET /v1/videos/{video_id}/content
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize("omni_server", _WAN_T2V, indirect=True)
+def test_get_video_content_invalid_requests(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
+    openai_client.send_video_content_http_request(
+        {
+            "video_id": "does-not-exist",
+            "timeout": 120,
+            "err_code": 404,
+            "err_message": ("not found", "video"),
+        }
+    )

tests/diffusion/distributed/test_autoencoder_kl_hunyuan.py

@@ -0,0 +1,79 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+import torch
+
+from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_hunyuan import (
+    DistributedAutoencoderKLHunyuan,
+)
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+class _DummyDistributedAutoencoderKLHunyuan(DistributedAutoencoderKLHunyuan):
+    def __init__(self):
+        torch.nn.Module.__init__(self)
+        self.tile_latent_min_size = 8
+        self.tile_sample_min_size = 8
+        self.tile_overlap_factor = 0.25
+        self.use_spatial_tiling = False
+
+    @property
+    def dtype(self) -> torch.dtype:
+        return torch.float32
+
+    def decoder(self, x: torch.Tensor) -> torch.Tensor:
+        return x + 10
+
+    def encoder(self, x: torch.Tensor) -> torch.Tensor:
+        return x + 20
+
+    def blend_v(self, _a: torch.Tensor, b: torch.Tensor, _blend_extent: int) -> torch.Tensor:
+        return b
+
+    def blend_h(self, _a: torch.Tensor, b: torch.Tensor, _blend_extent: int) -> torch.Tensor:
+        return b
+
+
+def test_hunyuan_vae_use_tiling_aliases_spatial_tiling():
+    # Verify use_tiling property maps to use_spatial_tiling.
+    vae = _DummyDistributedAutoencoderKLHunyuan()
+
+    assert not vae.use_tiling
+
+    vae.use_tiling = True
+
+    assert vae.use_spatial_tiling
+
+
+def test_hunyuan_vae_decode_tiles_round_trip():
+    # Validate decode tile split/exec/merge returns expected reconstructed tensor.
+    vae = _DummyDistributedAutoencoderKLHunyuan()
+    z = torch.arange(144, dtype=torch.float32).reshape(1, 1, 1, 12, 12)
+
+    tile_tasks, grid_spec = vae.tile_split(z)
+    decoded_tiles = {task.grid_coord: vae.tile_exec(task) for task in tile_tasks}
+    output = vae.tile_merge(decoded_tiles, grid_spec)
+
+    assert grid_spec.split_dims == (3, 4)
+    assert grid_spec.grid_shape == (2, 2)
+    assert grid_spec.tile_spec == {"blend_extent": 2, "row_limit": 6}
+    assert len(tile_tasks) == 4
+    assert torch.equal(output, z + 10)
+
+
+def test_hunyuan_vae_encode_tiles_round_trip():
+    # Validate encode tile split/exec/merge returns expected latent tensor.
+    vae = _DummyDistributedAutoencoderKLHunyuan()
+    x = torch.arange(144, dtype=torch.float32).reshape(1, 1, 1, 12, 12)
+
+    tile_tasks, grid_spec = vae.encode_tile_split(x)
+    encoded_tiles = {task.grid_coord: vae.encode_tile_exec(task) for task in tile_tasks}
+    output = vae.encode_tile_merge(encoded_tiles, grid_spec)
+
+    assert grid_spec.split_dims == (3, 4)
+    assert grid_spec.grid_shape == (2, 2)
+    assert grid_spec.tile_spec == {"blend_extent": 2, "row_limit": 6}
+    assert len(tile_tasks) == 4
+    assert torch.equal(output, x + 20)

tests/diffusion/distributed/test_pipeline_parallel.py

@@ -0,0 +1,802 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import os
+import socket
+
+import pytest
+import torch
+from vllm.model_executor.models.utils import PPMissingLayer, make_empty_intermediate_tensors_factory, make_layers
+from vllm.sequence import IntermediateTensors
+from vllm.v1.worker.gpu_worker import AsyncIntermediateTensors
+
+import vllm_omni.diffusion.distributed.pipeline_parallel as pp_module
+from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
+from vllm_omni.diffusion.distributed.parallel_state import (
+    destroy_distributed_env,
+    get_classifier_free_guidance_rank,
+    get_pp_group,
+    init_distributed_environment,
+    initialize_model_parallel,
+)
+from vllm_omni.diffusion.distributed.pipeline_parallel import AsyncLatents, PipelineParallelMixin
+from vllm_omni.platforms import current_omni_platform
+
+pytestmark = [pytest.mark.parallel]
+
+
+def _find_free_port() -> str:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("127.0.0.1", 0))
+        return str(s.getsockname()[1])
+
+
+def update_environment_variables(envs_dict: dict[str, str]) -> None:
+    for k, v in envs_dict.items():
+        os.environ[k] = v
+
+
+# ---------------------------------------------------------------------------
+# Shared stubs used by both unit and distributed tests
+# ---------------------------------------------------------------------------
+
+
+class FakeWork:
+    """Drop-in for torch.distributed.Work that records whether wait() was called."""
+
+    def __init__(self):
+        self.waited = False
+
+    def wait(self):
+        self.waited = True
+
+
+class SimpleScheduler:
+    """Minimal diffusion-step scheduler: latents -= 0.1 * noise_pred."""
+
+    def step(self, noise_pred: torch.Tensor, t, latents: torch.Tensor, return_dict: bool = False):
+        return (latents - 0.1 * noise_pred,)
+
+
+class FakeVAE:
+    def __init__(self, distributed_enabled: bool = False):
+        self.calls = 0
+        self.distributed_enabled = distributed_enabled
+
+    def decode(self, z: torch.Tensor):
+        """Original decode docstring."""
+        self.calls += 1
+        return (z + 1,)
+
+    def is_distributed_enabled(self) -> bool:
+        return self.distributed_enabled
+
+
+class MockPipelineParallel(PipelineParallelMixin, CFGParallelMixin):
+    """Minimal pipeline used to exercise PipelineParallelMixin.
+
+    Uses vLLM's ``make_layers`` for layer partitioning — the same utility used
+    by real DiT models — so the PP layer-split logic is exercised faithfully.
+
+    Each layer's weights are seeded by ``seed + layer_index`` so that layer ``i``
+    is initialized identically on every rank regardless of which ranks are active,
+    allowing the distributed output to be compared against the single-GPU baseline.
+
+    Args:
+        num_layers: Total number of Linear layers.
+        dim:        Input / hidden dimension.
+        seed:       Base RNG seed; layer ``i`` uses ``seed + i``.
+        device:     Target device for layer weights (default: CPU).
+        dtype:      Target dtype for layer weights (default: float32).
+    """
+
+    def __init__(
+        self,
+        num_layers: int = 4,
+        dim: int = 64,
+        seed: int = 42,
+        device: torch.device | None = None,
+        dtype: torch.dtype = torch.float32,
+    ):
+        self.start_layer, self.end_layer, self.layers = make_layers(
+            num_layers,
+            lambda prefix: torch.nn.Linear(dim, dim, bias=False),
+            prefix="layers",
+        )
+
+        for i, layer in enumerate(self.layers):
+            if not isinstance(layer, PPMissingLayer):
+                torch.manual_seed(seed + i)
+                torch.nn.init.normal_(layer.weight, mean=0.0, std=0.02)
+
+        self.layers.to(device=device, dtype=dtype)
+
+        self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(["hidden_states"], dim)
+        self.scheduler = SimpleScheduler()
+
+    def predict_noise(self, x=None, intermediate_tensors=None, **_kwargs) -> torch.Tensor | IntermediateTensors:
+        """Layer-split forward pass.
+
+        * First PP rank: uses ``x`` from caller kwargs.
+        * Later PP ranks: overrides ``x`` with ``intermediate_tensors["hidden_states"]``
+          (which transparently waits for the async receive).
+        * Non-last PP ranks return ``IntermediateTensors``; the last rank
+          returns the plain noise-prediction tensor.
+        """
+        if intermediate_tensors is not None:
+            x = intermediate_tensors["hidden_states"]
+
+        for i in range(self.start_layer, self.end_layer):
+            x = self.layers[i](x)
+
+        pp_group = get_pp_group()
+        if not pp_group.is_last_rank:
+            return IntermediateTensors({"hidden_states": x})
+        return x
+
+
+# ---------------------------------------------------------------------------
+# 1.  AsyncLatents – unit tests (no distributed env required)
+# ---------------------------------------------------------------------------
+
+
+class TestAsyncLatents:
+    """Verifies the lazy-resolution behaviour of AsyncLatents without a real process group."""
+
+    pytestmark = [pytest.mark.cpu]
+
+    def _make(self, tensor: torch.Tensor, handles: list | None = None, postproc: list | None = None) -> AsyncLatents:
+        return AsyncLatents({"latents": tensor}, handles or [], postproc or [])
+
+    def test_resolve_returns_wrapped_tensor(self):
+        t = torch.randn(2, 4)
+        al = self._make(t)
+        assert al._resolve() is t
+
+    def test_attribute_access_resolves(self):
+        t = torch.randn(2, 4)
+        al = self._make(t)
+        assert al.shape == t.shape
+        assert al.dtype == t.dtype
+
+    def test_torch_function_protocol(self):
+        """torch ops that receive an AsyncLatents should see the underlying tensor."""
+        t = torch.randn(2, 4)
+        al = self._make(t)
+        mask = torch.ones_like(t)
+        result = mask * al  # triggers __torch_function__
+        torch.testing.assert_close(result, mask * t)
+
+    def test_torch_function_with_list_arg(self):
+        """__torch_function__ must unwrap AsyncLatents inside list/tuple args."""
+        t = torch.randn(2, 4)
+        al = self._make(t)
+        result = torch.cat([al, al], dim=0)
+        torch.testing.assert_close(result, torch.cat([t, t], dim=0))
+
+    def test_torch_tensor_conversion(self):
+        """torch.as_tensor on an AsyncLatents must share storage with the underlying tensor (no copy)."""
+        t = torch.randn(2, 4)
+        al = self._make(t)
+        result = torch.as_tensor(al)
+        assert result.data_ptr() == t.data_ptr(), "torch.as_tensor copied the data instead of sharing storage"
+
+    def test_handles_are_waited_before_resolve(self):
+        t = torch.randn(2, 4)
+        h1, h2 = FakeWork(), FakeWork()
+        al = self._make(t, handles=[h1, h2])
+        _ = al.shape  # trigger resolution
+        assert h1.waited and h2.waited, "Not all handles were waited on"
+
+    def test_postproc_callbacks_invoked_on_resolve(self):
+        t = torch.randn(2, 4)
+        log: list[int] = []
+        al = self._make(t, postproc=[lambda: log.append(1), lambda: log.append(2)])
+        _ = al.shape
+        assert log == [1, 2], f"postproc not called in order: {log}"
+
+    def test_idempotent_resolve(self):
+        """handle.wait() must not be called twice if _resolve() is called twice."""
+        t = torch.randn(2, 4)
+        h = FakeWork()
+        al = self._make(t, handles=[h])
+        _ = al.shape  # first resolve
+        h.waited = False  # reset sentinel
+        _ = al.dtype  # second resolve
+        assert not h.waited, "handle.wait() was called a second time"
+
+
+# ---------------------------------------------------------------------------
+# 2.  _sync_pp_send / diffuse wrapper – unit tests (no distributed env required)
+# ---------------------------------------------------------------------------
+
+
+class TestSyncPPSend:
+    """Verifies PipelineParallelMixin's internal PP-send flush."""
+
+    pytestmark = [pytest.mark.cpu]
+
+    @staticmethod
+    def _make_pipeline() -> PipelineParallelMixin:
+        # Instantiate a bare mixin — no layers, no distributed env needed.
+        # _sync_pp_send only touches _pp_send_work, so this is sufficient.
+        class _BarePP(PipelineParallelMixin, CFGParallelMixin):
+            pass
+
+        return _BarePP()
+
+    def test_noop_when_work_list_empty(self):
+        pipeline = self._make_pipeline()
+        pipeline._sync_pp_send()
+        assert pipeline._pp_send_work == []
+
+    def test_waits_all_pending_handles(self):
+        pipeline = self._make_pipeline()
+        works = [FakeWork(), FakeWork(), FakeWork()]
+        pipeline._pp_send_work = works
+        pipeline._sync_pp_send()
+        assert all(w.waited for w in works), "Some handles were not waited on"
+
+    def test_clears_work_list_after_sync(self):
+        pipeline = self._make_pipeline()
+        pipeline._pp_send_work = [FakeWork()]
+        pipeline._sync_pp_send()
+        assert pipeline._pp_send_work == []
+
+
+class TestDiffuseWrapper:
+    """Verifies that PipelineParallelMixin flushes pending sends when diffuse() exits."""
+
+    pytestmark = [pytest.mark.cpu]
+
+    def test_diffuse_flushes_pending_sends_on_success(self):
+        work = FakeWork()
+
+        class _DiffusePP(PipelineParallelMixin, CFGParallelMixin):
+            def diffuse(self):
+                self._pp_send_work = [work]
+                return "done"
+
+        pipeline = _DiffusePP()
+
+        assert pipeline.diffuse() == "done"
+        assert work.waited
+        assert pipeline._pp_send_work == []
+
+    def test_diffuse_flushes_pending_sends_on_exception(self):
+        work = FakeWork()
+
+        class _DiffusePP(PipelineParallelMixin, CFGParallelMixin):
+            def diffuse(self):
+                self._pp_send_work = [work]
+                raise RuntimeError("boom")
+
+        pipeline = _DiffusePP()
+
+        with pytest.raises(RuntimeError, match="boom"):
+            pipeline.diffuse()
+        assert work.waited
+        assert pipeline._pp_send_work == []
+
+    def test_diffuse_wrapper_preserves_metadata(self):
+        class _DiffusePP(PipelineParallelMixin, CFGParallelMixin):
+            def diffuse(self):
+                """Original diffuse docstring."""
+                return "done"
+
+        assert _DiffusePP.diffuse.__name__ == "diffuse"
+        assert _DiffusePP.diffuse.__doc__ == "Original diffuse docstring."
+
+
+class TestVaeDecodeGuard:
+    pytestmark = [pytest.mark.cpu]
+
+    @staticmethod
+    def _make_pipeline(distributed_enabled: bool = False) -> PipelineParallelMixin:
+        class _DecodePP(PipelineParallelMixin, CFGParallelMixin):
+            def __init__(self):
+                self.vae = FakeVAE(distributed_enabled=distributed_enabled)
+
+        return _DecodePP()
+
+    @staticmethod
+    def _set_rank(monkeypatch, world_size: int, first_stage: bool) -> None:
+        monkeypatch.setattr(pp_module, "get_pipeline_parallel_world_size", lambda: world_size)
+        monkeypatch.setattr(pp_module, "is_pipeline_first_stage", lambda: first_stage)
+
+    def test_calls_original_decode_when_pp_disabled(self, monkeypatch):
+        self._set_rank(monkeypatch, world_size=1, first_stage=True)
+        pipeline = self._make_pipeline()
+        z = torch.ones(2, 3)
+
+        output = pipeline.vae.decode(z)[0]
+
+        assert pipeline.vae.calls == 1
+        torch.testing.assert_close(output, z + 1)
+
+    def test_calls_original_decode_on_first_stage(self, monkeypatch):
+        self._set_rank(monkeypatch, world_size=2, first_stage=True)
+        pipeline = self._make_pipeline()
+        z = torch.ones(2, 3)
+
+        output = pipeline.vae.decode(z)[0]
+
+        assert pipeline.vae.calls == 1
+        torch.testing.assert_close(output, z + 1)
+
+    def test_skips_decode_on_non_first_stage(self, monkeypatch):
+        self._set_rank(monkeypatch, world_size=2, first_stage=False)
+        pipeline = self._make_pipeline()
+        z = torch.ones(2, 3)
+
+        output = pipeline.vae.decode(z)
+
+        assert pipeline.vae.calls == 0
+        assert output == (None,)
+
+    def test_calls_original_decode_when_distributed_vae_enabled(self, monkeypatch):
+        self._set_rank(monkeypatch, world_size=2, first_stage=False)
+        pipeline = self._make_pipeline(distributed_enabled=True)
+        z = torch.ones(2, 3)
+
+        output = pipeline.vae.decode(z)[0]
+
+        assert pipeline.vae.calls == 1
+        torch.testing.assert_close(output, z + 1)
+
+    def test_decode_wrapper_preserves_metadata(self):
+        pipeline = self._make_pipeline()
+
+        assert pipeline.vae.decode.__name__ == "decode"
+        assert pipeline.vae.decode.__doc__ == "Original decode docstring."
+
+
+@pytest.mark.cpu
+def test_pipeline_parallel_requires_cfg_mixin():
+    with pytest.raises(TypeError, match="inherits PipelineParallelMixin but not CFGParallelMixin"):
+
+        class _MissingCFG(PipelineParallelMixin):
+            pass
+
+
+@pytest.mark.cpu
+def test_pipeline_parallel_requires_mro_before_cfg_mixin():
+    with pytest.raises(TypeError, match="must inherit PipelineParallelMixin before CFGParallelMixin"):
+
+        class _WrongOrder(CFGParallelMixin, PipelineParallelMixin):
+            pass
+
+
+# ---------------------------------------------------------------------------
+# Distributed test helpers
+# ---------------------------------------------------------------------------
+
+
+def init_dist(local_rank: int, world_size: int, master_port: str) -> torch.device:
+    """Initialise the distributed environment for a spawned worker."""
+    device = torch.device(f"{current_omni_platform.device_type}:{local_rank}")
+    current_omni_platform.set_device(device)
+    update_environment_variables(
+        {
+            "RANK": str(local_rank),
+            "LOCAL_RANK": str(local_rank),
+            "WORLD_SIZE": str(world_size),
+            "MASTER_ADDR": "localhost",
+            "MASTER_PORT": master_port,
+        }
+    )
+    init_distributed_environment()
+    return device
+
+
+def make_pipeline_and_inputs(
+    test_config: dict, dtype: torch.dtype, device: torch.device, do_true_cfg: bool = False
+) -> tuple["MockPipelineParallel", dict, dict | None]:
+    """Create a MockPipelineParallel and seeded inputs from a test_config dict.
+
+    Must be called after ``initialize_model_parallel`` so that ``make_layers``
+    can read the PP group to determine this rank's layer slice.
+
+    Returns ``(pipeline, positive_kwargs, negative_kwargs)``.
+    ``negative_kwargs`` is ``None`` when ``do_true_cfg=False``.
+    """
+    pipeline = MockPipelineParallel(
+        num_layers=test_config["num_layers"],
+        dim=test_config["dim"],
+        seed=test_config["model_seed"],
+        device=device,
+        dtype=dtype,
+    )
+
+    torch.manual_seed(test_config["input_seed"])
+    if torch.cuda.is_available():
+        torch.cuda.manual_seed_all(test_config["input_seed"])
+    pos_x = {"x": torch.randn(test_config["batch_size"], test_config["dim"], dtype=dtype, device=device)}
+
+    negative_kwargs = None
+    if do_true_cfg:
+        torch.manual_seed(test_config["input_seed"] + 1)
+        if torch.cuda.is_available():
+            torch.cuda.manual_seed_all(test_config["input_seed"] + 1)
+        neg_x = torch.randn(test_config["batch_size"], test_config["dim"], dtype=dtype, device=device)
+        negative_kwargs = {"x": neg_x}
+
+    return pipeline, pos_x, negative_kwargs
+
+
+# ---------------------------------------------------------------------------
+# 3.  isend_tensor_dict / irecv_tensor_dict  (2 GPUs)
+# ---------------------------------------------------------------------------
+
+
+def isend_irecv_worker(local_rank: int, world_size: int, master_port: str, result_queue):
+    device = init_dist(local_rank, world_size, master_port)
+    initialize_model_parallel(pipeline_parallel_size=world_size)
+    pp_group = get_pp_group()
+
+    if pp_group.is_first_rank:
+        torch.manual_seed(77)
+        if torch.cuda.is_available():
+            torch.cuda.manual_seed_all(77)
+        tensor = torch.randn(3, 5, dtype=torch.float32, device=device)
+        handles = pp_group.isend_tensor_dict({"t": tensor})
+        for h in handles:
+            h.wait()
+        result_queue.put(("sent", tensor.cpu()))
+    else:
+        received = AsyncIntermediateTensors(*pp_group.irecv_tensor_dict())
+        result_queue.put(("received", received["t"].cpu()))
+
+    if torch.distributed.is_initialized():
+        torch.distributed.barrier()
+    destroy_distributed_env()
+
+
+@pytest.mark.gpu
+@pytest.mark.skipif(current_omni_platform.get_device_count() < 2, reason="Need at least 2 GPUs")
+@pytest.mark.parametrize("pp_size", [2])
+def test_isend_irecv_tensor_dict(pp_size: int):
+    """isend_tensor_dict / irecv_tensor_dict transfer a tensor dict without loss."""
+    mp_context = torch.multiprocessing.get_context("spawn")
+    manager = mp_context.Manager()
+    q = manager.Queue()
+
+    port = _find_free_port()
+    torch.multiprocessing.spawn(isend_irecv_worker, args=(pp_size, port, q), nprocs=pp_size)
+
+    results = {label: tensor for label, tensor in [q.get(), q.get()]}
+    torch.testing.assert_close(
+        results["received"], results["sent"], rtol=0, atol=0, msg="isend/irecv transferred tensor incorrectly"
+    )
+
+
+# ---------------------------------------------------------------------------
+# 4.  predict_noise_maybe_with_cfg
+# ---------------------------------------------------------------------------
+
+_baseline_cache: dict[tuple, torch.Tensor] = {}
+
+
+def compute_single_gpu_baseline(test_config: dict, dtype: torch.dtype, do_true_cfg: bool) -> torch.Tensor:
+    """Compute expected single-GPU output using the same MockPipelineParallel.
+
+    Initializes a trivial distributed env (world_size=1) so that ``make_layers`` and the PP/CFG mixins work normally.
+    Results are cached so identical configs are only computed once.
+    """
+    key = (
+        test_config["num_layers"],
+        test_config["dim"],
+        test_config["batch_size"],
+        test_config["model_seed"],
+        test_config["input_seed"],
+        test_config["cfg_scale"],
+        dtype,
+        do_true_cfg,
+    )
+    if key in _baseline_cache:
+        return _baseline_cache[key]
+
+    device = init_dist(0, 1, _find_free_port())
+    initialize_model_parallel(pipeline_parallel_size=1)
+
+    pipeline, positive_kwargs, negative_kwargs = make_pipeline_and_inputs(
+        test_config, dtype, device, do_true_cfg=do_true_cfg
+    )
+
+    with torch.inference_mode():
+        noise_pred = pipeline.predict_noise_maybe_with_cfg(
+            do_true_cfg=do_true_cfg,
+            true_cfg_scale=test_config["cfg_scale"],
+            positive_kwargs=positive_kwargs,
+            negative_kwargs=negative_kwargs,
+            cfg_normalize=False,
+        )
+
+    destroy_distributed_env()
+
+    _baseline_cache[key] = noise_pred.cpu()
+    return _baseline_cache[key]
+
+
+def predict_noise_worker(
+    local_rank: int,
+    world_size: int,
+    master_port: str,
+    pp_size: int,
+    cfg_size: int,
+    do_true_cfg: bool,
+    dtype: torch.dtype,
+    test_config: dict,
+    result_queue,
+):
+    """Generic predict-noise worker parameterized by PP and CFG topology."""
+    device = init_dist(local_rank, world_size, master_port)
+    initialize_model_parallel(pipeline_parallel_size=pp_size, cfg_parallel_size=cfg_size)
+
+    pp_group = get_pp_group()
+    cfg_rank = get_classifier_free_guidance_rank()
+
+    pipeline, positive_kwargs, negative_kwargs = make_pipeline_and_inputs(
+        test_config, dtype, device, do_true_cfg=do_true_cfg
+    )
+
+    with torch.inference_mode():
+        noise_pred = pipeline.predict_noise_maybe_with_cfg(
+            do_true_cfg=do_true_cfg,
+            true_cfg_scale=test_config["cfg_scale"],
+            positive_kwargs=positive_kwargs,
+            negative_kwargs=negative_kwargs,
+            cfg_normalize=False,
+        )
+    # This worker exercises predict_noise_maybe_with_cfg directly, bypassing diffuse().
+    # Flush the non-last PP rank's async send before barrier / process teardown.
+    pipeline._sync_pp_send()
+
+    if pp_group.is_last_rank:
+        assert noise_pred is not None
+        if cfg_rank == 0:
+            result_queue.put(noise_pred.cpu())
+    else:
+        assert noise_pred is None
+
+    if torch.distributed.is_initialized():
+        torch.distributed.barrier()
+    destroy_distributed_env()
+
+
+@pytest.mark.gpu
+@pytest.mark.parametrize(
+    "pp_size, cfg_size, do_true_cfg, dtype, num_layers, input_seed, rtol, atol",
+    [
+        pytest.param(
+            2,
+            1,
+            False,
+            torch.float32,
+            4,
+            100,
+            1e-5,
+            1e-5,
+            marks=pytest.mark.skipif(current_omni_platform.get_device_count() < 2, reason="Need at least 2 GPUs"),
+            id="pp2-no_cfg-float32",
+        ),
+        pytest.param(
+            2,
+            1,
+            False,
+            torch.bfloat16,
+            4,
+            100,
+            1e-2,
+            1e-2,
+            marks=pytest.mark.skipif(current_omni_platform.get_device_count() < 2, reason="Need at least 2 GPUs"),
+            id="pp2-no_cfg-bfloat16",
+        ),
+        pytest.param(
+            2,
+            1,
+            True,
+            torch.bfloat16,
+            4,
+            100,
+            1e-2,
+            1e-2,
+            marks=pytest.mark.skipif(current_omni_platform.get_device_count() < 2, reason="Need at least 2 GPUs"),
+            id="pp2-seq_cfg-bfloat16",
+        ),
+        pytest.param(
+            2,
+            2,
+            True,
+            torch.bfloat16,
+            4,
+            100,
+            1e-2,
+            1e-2,
+            marks=pytest.mark.skipif(current_omni_platform.get_device_count() < 4, reason="Need at least 4 GPUs"),
+            id="pp2-cfg2-bfloat16",
+        ),
+        pytest.param(
+            3,
+            1,
+            False,
+            torch.bfloat16,
+            6,
+            100,
+            1e-2,
+            1e-2,
+            marks=pytest.mark.skipif(current_omni_platform.get_device_count() < 3, reason="Need at least 3 GPUs"),
+            id="pp3-no_cfg-bfloat16",
+        ),
+    ],
+)
+def test_predict_noise(pp_size, cfg_size, do_true_cfg, dtype, num_layers, input_seed, rtol, atol):
+    """predict_noise_maybe_with_cfg output matches the single-GPU baseline across PP / CFG topologies."""
+    test_config = {
+        "num_layers": num_layers,
+        "dim": 64,
+        "batch_size": 2,
+        "cfg_scale": 7.5,
+        "model_seed": 42,
+        "input_seed": input_seed,
+    }
+
+    baseline_out = compute_single_gpu_baseline(test_config, dtype, do_true_cfg)
+
+    mp_context = torch.multiprocessing.get_context("spawn")
+    manager = mp_context.Manager()
+    pp_q = manager.Queue()
+
+    world_size = pp_size * cfg_size
+    port = _find_free_port()
+    torch.multiprocessing.spawn(
+        predict_noise_worker,
+        args=(world_size, port, pp_size, cfg_size, do_true_cfg, dtype, test_config, pp_q),
+        nprocs=world_size,
+    )
+
+    pp_out = pp_q.get()
+
+    assert baseline_out.shape == pp_out.shape
+    torch.testing.assert_close(
+        pp_out,
+        baseline_out,
+        rtol=rtol,
+        atol=atol,
+        msg=f"PP={pp_size} cfg={cfg_size} {'with' if do_true_cfg else 'no'} CFG output differs from baseline ({dtype=})",
+    )
+
+
+# ---------------------------------------------------------------------------
+# 5.  scheduler_step_maybe_with_cfg
+# ---------------------------------------------------------------------------
+
+
+def compute_scheduler_step_baseline(test_config: dict, do_true_cfg: bool) -> torch.Tensor:
+    """Single-GPU reference: predict_noise + scheduler_step."""
+    device = init_dist(0, 1, _find_free_port())
+    initialize_model_parallel(pipeline_parallel_size=1)
+
+    pipeline, positive_kwargs, negative_kwargs = make_pipeline_and_inputs(
+        test_config, torch.float32, device, do_true_cfg=do_true_cfg
+    )
+    latents = positive_kwargs["x"]
+    t = torch.tensor(500, device=device)
+
+    with torch.inference_mode():
+        noise_pred = pipeline.predict_noise_maybe_with_cfg(
+            do_true_cfg=do_true_cfg,
+            true_cfg_scale=test_config["cfg_scale"],
+            positive_kwargs=positive_kwargs,
+            negative_kwargs=negative_kwargs,
+            cfg_normalize=False,
+        )
+        result = pipeline.scheduler_step_maybe_with_cfg(
+            noise_pred=noise_pred, t=t, latents=latents, do_true_cfg=do_true_cfg
+        )
+
+    destroy_distributed_env()
+    return result.cpu()
+
+
+def scheduler_step_worker(
+    local_rank: int,
+    world_size: int,
+    master_port: str,
+    pp_size: int,
+    cfg_size: int,
+    do_true_cfg: bool,
+    test_config: dict,
+    result_queue,
+):
+    device = init_dist(local_rank, world_size, master_port)
+    initialize_model_parallel(pipeline_parallel_size=pp_size, cfg_parallel_size=cfg_size)
+
+    pp_group = get_pp_group()
+    cfg_rank = get_classifier_free_guidance_rank()
+
+    pipeline, positive_kwargs, negative_kwargs = make_pipeline_and_inputs(
+        test_config, torch.float32, device, do_true_cfg=do_true_cfg
+    )
+    latents = positive_kwargs["x"]
+    t = torch.tensor(500, device=device)
+
+    with torch.inference_mode():
+        noise_pred = pipeline.predict_noise_maybe_with_cfg(
+            do_true_cfg=do_true_cfg,
+            true_cfg_scale=test_config["cfg_scale"],
+            positive_kwargs=positive_kwargs,
+            negative_kwargs=negative_kwargs,
+            cfg_normalize=False,
+        )
+        latents = pipeline.scheduler_step_maybe_with_cfg(
+            noise_pred=noise_pred, t=t, latents=latents, do_true_cfg=do_true_cfg
+        )
+    # This worker exercises scheduler_step_maybe_with_cfg directly, bypassing diffuse().
+    # Flush the last PP rank's async latent send before barrier / process teardown.
+    pipeline._sync_pp_send()
+
+    if pp_group.is_first_rank and cfg_rank == 0:
+        resolved = latents.contiguous()
+        result_queue.put(resolved.cpu())
+
+    if torch.distributed.is_initialized():
+        torch.distributed.barrier()
+    destroy_distributed_env()
+
+
+@pytest.mark.gpu
+@pytest.mark.parametrize(
+    "pp_size, cfg_size, do_true_cfg, input_seed",
+    [
+        pytest.param(
+            2,
+            1,
+            False,
+            300,
+            marks=pytest.mark.skipif(current_omni_platform.get_device_count() < 2, reason="Need at least 2 GPUs"),
+            id="pp2-no_cfg",
+        ),
+        pytest.param(
+            2,
+            2,
+            True,
+            600,
+            marks=pytest.mark.skipif(current_omni_platform.get_device_count() < 4, reason="Need at least 4 GPUs"),
+            id="pp2-cfg2-true_cfg",
+        ),
+    ],
+)
+def test_scheduler_step(pp_size, cfg_size, do_true_cfg, input_seed):
+    """Rank 0 latents after scheduler_step match the single-GPU baseline across PP / CFG topologies."""
+    test_config = {
+        "num_layers": 4,
+        "dim": 64,
+        "batch_size": 2,
+        "cfg_scale": 7.5,
+        "model_seed": 42,
+        "input_seed": input_seed,
+    }
+
+    baseline = compute_scheduler_step_baseline(test_config, do_true_cfg)
+
+    mp_context = torch.multiprocessing.get_context("spawn")
+    manager = mp_context.Manager()
+    q = manager.Queue()
+
+    port = _find_free_port()
+    world_size = pp_size * cfg_size
+    torch.multiprocessing.spawn(
+        scheduler_step_worker,
+        args=(world_size, port, pp_size, cfg_size, do_true_cfg, test_config, q),
+        nprocs=world_size,
+    )
+
+    result = q.get()
+    torch.testing.assert_close(
+        result,
+        baseline,
+        rtol=0,
+        atol=0,
+        msg=f"PP={pp_size} CFG={cfg_size} scheduler step latents on rank 0 do not match single-GPU baseline",
+    )

tests/diffusion/models/flux2/test_flux2_klein_num_inference_steps.py

@@ -0,0 +1,119 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from vllm_omni.diffusion.models.flux2_klein.pipeline_flux2_klein import (
+    Flux2KleinPipeline,
+)
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
+
+
+class _StopAfterCheckInputsError(Exception):
+    pass
+
+
+def _make_pipeline():
+    pipeline = object.__new__(Flux2KleinPipeline)
+    pipeline.vae_scale_factor = 8
+    pipeline.is_distilled = True
+    pipeline._guidance_scale = 0.0
+    return pipeline
+
+
+def _make_minimal_request(
+    prompt="valid prompt",
+    *,
+    num_inference_steps=None,
+):
+    """Build a minimal OmniDiffusionRequest-like object that forward() reads from."""
+    params = OmniDiffusionSamplingParams(
+        height=512,
+        width=512,
+        num_inference_steps=num_inference_steps,
+        seed=42,
+    )
+    req = MagicMock()
+    req.sampling_params = params
+    req.prompts = [prompt]
+    req.multi_modal_data = {}
+    return req
+
+
+def _capture_num_inference_steps(pipe, steps):
+    """Run pipe.forward() and capture the num_inference_steps passed to check_inputs."""
+    captured = {}
+
+    def fake_check_inputs(**kwargs):
+        captured["num_inference_steps"] = kwargs.get("num_inference_steps")
+        raise _StopAfterCheckInputsError
+
+    original = pipe.check_inputs
+    pipe.check_inputs = fake_check_inputs
+    try:
+        req = _make_minimal_request(num_inference_steps=steps)
+        pipe.forward(req)
+    except _StopAfterCheckInputsError:
+        pass
+    finally:
+        pipe.check_inputs = original
+    return captured.get("num_inference_steps")
+
+
+# --- Forward resolution tests (the actual regression) ---
+
+
+def test_forward_preserves_zero_not_default():
+    """#3703: num_inference_steps=0 must reach validation as 0, not as default 50."""
+    pipe = _make_pipeline()
+    captured = _capture_num_inference_steps(pipe, 0)
+    assert captured == 0, f"Expected 0, got {captured}"
+
+
+def test_forward_preserves_negative():
+    pipe = _make_pipeline()
+    captured = _capture_num_inference_steps(pipe, -1)
+    assert captured == -1
+
+
+def test_forward_none_uses_default():
+    """None means unset — forward() should substitute the pipeline default."""
+    pipe = _make_pipeline()
+    captured = _capture_num_inference_steps(pipe, None)
+    assert captured == 50
+
+
+def test_forward_preserves_positive():
+    pipe = _make_pipeline()
+    captured = _capture_num_inference_steps(pipe, 9)
+    assert captured == 9
+
+
+# --- check_inputs validation tests ---
+
+
+@pytest.mark.parametrize("steps", [0, -1])
+def test_check_inputs_rejects_non_positive(steps):
+    pipe = _make_pipeline()
+    with pytest.raises(ValueError):
+        pipe.check_inputs(
+            prompt="valid prompt",
+            height=512,
+            width=512,
+            num_inference_steps=steps,
+        )
+
+
+def test_check_inputs_accepts_none():
+    pipe = _make_pipeline()
+    pipe.check_inputs(prompt="valid prompt", height=512, width=512, num_inference_steps=None)
+
+
+@pytest.mark.parametrize("steps", [1, 2])
+def test_check_inputs_accepts_positive(steps):
+    pipe = _make_pipeline()
+    pipe.check_inputs(prompt="valid prompt", height=512, width=512, num_inference_steps=steps)

tests/diffusion/models/flux2/test_flux2_klein_validation.py

@@ -0,0 +1,80 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+import torch
+
+from vllm_omni.diffusion.models.flux2_klein.pipeline_flux2_klein import (
+    Flux2KleinPipeline,
+)
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
+
+
+def _make_pipeline():
+    pipeline = object.__new__(Flux2KleinPipeline)
+    pipeline.vae_scale_factor = 8
+    pipeline.is_distilled = True
+    pipeline._guidance_scale = 0.0
+    return pipeline
+
+
+def _check(pipeline, prompt, prompt_embeds=None):
+    pipeline.check_inputs(
+        prompt=prompt,
+        height=512,
+        width=512,
+        prompt_embeds=prompt_embeds,
+        num_inference_steps=4,
+        guidance_scale=0.0,
+    )
+
+
+@pytest.mark.parametrize(
+    "prompt",
+    [
+        "",
+        "   ",
+    ],
+)
+def test_rejects_empty_or_whitespace_prompt(prompt):
+    pipe = _make_pipeline()
+    with pytest.raises(ValueError, match="empty or whitespace-only"):
+        _check(pipe, prompt)
+
+
+@pytest.mark.parametrize(
+    "prompt",
+    [
+        ["valid prompt", ""],
+        ["valid prompt", "   "],
+        ["   "],
+    ],
+)
+def test_rejects_list_with_empty_or_whitespace_element(prompt):
+    pipe = _make_pipeline()
+    with pytest.raises(ValueError, match="empty or whitespace-only"):
+        _check(pipe, prompt)
+
+
+@pytest.mark.parametrize(
+    "prompt",
+    [
+        "valid prompt",
+        ["valid prompt", "another valid prompt"],
+    ],
+)
+def test_accepts_valid_prompts(prompt):
+    pipe = _make_pipeline()
+    _check(pipe, prompt)
+
+
+def test_rejects_none_without_embeds():
+    pipe = _make_pipeline()
+    with pytest.raises(ValueError):
+        _check(pipe, None)
+
+
+def test_accepts_none_with_prompt_embeds():
+    pipe = _make_pipeline()
+    _check(pipe, None, prompt_embeds=torch.randn(1, 256, 4096))

tests/diffusion/models/wan2_2/test_wan22_ti2v_pipeline.py

@@ -1,97 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-from types import SimpleNamespace
-
-import pytest
-import torch
-from PIL import Image
-from torch import nn
-
-from tests.diffusion.models.wan2_2.conftest import StubTransformer, StubVAE, noop_progress_bar
-from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_ti2v import (
-    Wan22TI2VPipeline,
-    get_wan22_ti2v_pre_process_func,
-)
-
-pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
-
-
-def _make_ti2v_pipeline() -> Wan22TI2VPipeline:
-    pipeline = object.__new__(Wan22TI2VPipeline)
-    nn.Module.__init__(pipeline)
-    pipeline.device = torch.device("cpu")
-    pipeline.transformer = StubTransformer(in_channels=4, out_channels=4)
-    pipeline.vae = StubVAE(z_dim=4)
-    pipeline.vae_scale_factor_temporal = 4
-    pipeline.vae_scale_factor_spatial = 8
-    pipeline.progress_bar = noop_progress_bar
-    return pipeline
-
-
-def test_ti2v_preprocess_uses_720p_area_for_image_condition() -> None:
-    preprocess = get_wan22_ti2v_pre_process_func(SimpleNamespace())
-    request = SimpleNamespace(
-        prompts=[{"prompt": "p", "multi_modal_data": {"image": Image.new("RGB", (320, 160), "blue")}}],
-        sampling_params=SimpleNamespace(height=None, width=None),
-    )
-
-    result = preprocess(request)
-
-    assert result.sampling_params.height == 672
-    assert result.sampling_params.width == 1344
-    assert result.prompts[0]["multi_modal_data"]["image"].size == (1344, 672)
-
-
-def test_ti2v_diffuse_without_image_condition_expands_patch_timesteps() -> None:
-    pipeline = _make_ti2v_pipeline()
-    latents = torch.zeros(1, 4, 2, 4, 4)
-    calls = []
-
-    def fake_predict_noise_maybe_with_cfg(**kwargs):
-        calls.append(kwargs)
-        return torch.ones_like(latents)
-
-    pipeline.predict_noise_maybe_with_cfg = fake_predict_noise_maybe_with_cfg  # type: ignore[method-assign]
-    pipeline.scheduler_step_maybe_with_cfg = lambda noise, t, current, cfg: current + noise  # type: ignore[method-assign]
-
-    result = pipeline.diffuse(
-        latents=latents,
-        timesteps=torch.tensor([7]),
-        prompt_embeds=torch.zeros(1, 2, 3),
-        negative_prompt_embeds=torch.zeros(1, 2, 3),
-        guidance_scale=3.0,
-        dtype=torch.float32,
-        attention_kwargs={"a": "b"},
-        num_latent_frames=2,
-        latent_height=4,
-        latent_width=4,
-    )
-
-    positive = calls[0]["positive_kwargs"]
-    assert calls[0]["do_true_cfg"] is True
-    assert positive["timestep"].shape == (1, 8)
-    torch.testing.assert_close(positive["timestep"], torch.full((1, 8), 7, dtype=positive["timestep"].dtype))
-    torch.testing.assert_close(positive["hidden_states"], latents)
-    torch.testing.assert_close(result, torch.ones_like(latents))
-
-
-def test_ti2v_prepare_i2v_latents_encodes_condition_and_masks_first_frame() -> None:
-    pipeline = _make_ti2v_pipeline()
-    latents, latent_condition, first_frame_mask = pipeline.prepare_i2v_latents(
-        image=torch.zeros(1, 3, 16, 16),
-        batch_size=1,
-        num_channels_latents=4,
-        height=16,
-        width=16,
-        num_frames=5,
-        dtype=torch.float32,
-        device=torch.device("cpu"),
-        generator=None,
-        latents=torch.zeros(1, 4, 2, 2, 2),
-    )
-
-    torch.testing.assert_close(latents, torch.zeros(1, 4, 2, 2, 2))
-    assert latent_condition.shape == (1, 4, 1, 2, 2)
-    assert first_frame_mask[:, :, 0].sum() == 0
-    assert first_frame_mask[:, :, 1].sum() == 4

tests/diffusion/quantization/test_quantization_quality.py

@@ -32,17 +32,31 @@ Optional artifact dump:
 from __future__ import annotations
 
 import gc
+import importlib.util
 import json
 import os
+import sys
 from dataclasses import dataclass
 from pathlib import Path
+from types import SimpleNamespace
 
 import numpy as np
 import pytest
 import torch
+from PIL import Image
 
 from tests.helpers.mark import hardware_marks
 
+_REPO_ROOT = Path(__file__).resolve().parents[3]
+_BENCH_MODULE_PATH = _REPO_ROOT / "benchmarks" / "diffusion" / "quantization_quality.py"
+_BENCH_MODULE_NAME = "benchmarks.diffusion.quantization_quality"
+
+if _BENCH_MODULE_NAME not in sys.modules:
+    _spec = importlib.util.spec_from_file_location(_BENCH_MODULE_NAME, _BENCH_MODULE_PATH)
+    _mod = importlib.util.module_from_spec(_spec)
+    sys.modules[_BENCH_MODULE_NAME] = _mod
+    _spec.loader.exec_module(_mod)
+
 # ---------------------------------------------------------------------------
 # Configuration — add new quantization methods / models here
 # ---------------------------------------------------------------------------
@@ -316,6 +330,37 @@ def _unload(omni):
 # Tests
 # ---------------------------------------------------------------------------
 
+
+def test_benchmark_generate_image_unwraps_nested_omni_request_output(monkeypatch):
+    from benchmarks.diffusion.quantization_quality import _generate_image as benchmark_generate_image
+    from vllm_omni.outputs import OmniRequestOutput
+    from vllm_omni.platforms import current_omni_platform
+
+    monkeypatch.setattr(current_omni_platform, "device_type", "cpu", raising=False)
+    monkeypatch.setattr(torch.accelerator, "reset_peak_memory_stats", lambda: None, raising=False)
+    monkeypatch.setattr(torch.accelerator, "max_memory_allocated", lambda: 0, raising=False)
+
+    image = Image.new("RGB", (2, 2))
+    inner = OmniRequestOutput.from_diffusion(request_id="req", images=[image])
+    outer = OmniRequestOutput(
+        request_id="req",
+        stage_id=0,
+        final_output_type="image",
+        request_output=inner,
+        finished=True,
+    )
+
+    class DummyOmni:
+        def generate(self, *_args, **_kwargs):
+            return [outer]
+
+    args = SimpleNamespace(height=2, width=2, num_inference_steps=1)
+    output, _elapsed, peak_mem = benchmark_generate_image(DummyOmni(), args, "prompt", 42)
+
+    assert output is image
+    assert peak_mem == 0.0
+
+
 _marks = hardware_marks(res={"cuda": "H100"})
 _OUTPUT_DIR = Path(os.environ["VLLM_OMNI_QUALITY_OUTPUT_DIR"]) if "VLLM_OMNI_QUALITY_OUTPUT_DIR" in os.environ else None
 

tests/diffusion/quantization/test_trajectory_similarity_tool.py

@@ -0,0 +1,126 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import math
+import sys
+from types import SimpleNamespace
+
+import numpy as np
+import pytest
+import torch
+from PIL import Image
+
+from vllm_omni.quantization.tools.compare_diffusion_trajectory_similarity import (
+    VariantRun,
+    _build_variant_config,
+    _request_peak_memory_mb,
+    _run_summary,
+    compute_tensor_metrics,
+    compute_uint8_image_metrics,
+    metric_guidance,
+    parse_args,
+    summarize_output_image_metrics,
+)
+
+
+def test_compute_tensor_metrics_identical_tensors():
+    metrics = compute_tensor_metrics(torch.ones(2, 3), torch.ones(2, 3))
+
+    assert metrics["cosine_similarity"] == pytest.approx(1.0)
+    assert metrics["mae"] == 0.0
+    assert metrics["mse"] == 0.0
+    assert metrics["rmse"] == 0.0
+    assert metrics["max_abs"] == 0.0
+    assert metrics["l2"] == 0.0
+    assert metrics["relative_l2"] == 0.0
+
+
+def test_compute_uint8_image_metrics_adds_psnr():
+    lhs = np.zeros((2, 2, 3), dtype=np.uint8)
+    rhs = np.zeros((2, 2, 3), dtype=np.uint8)
+
+    metrics = compute_uint8_image_metrics(lhs, rhs)
+
+    assert math.isinf(metrics["psnr_db"])
+
+
+def test_summarize_output_image_metrics_stacks_pil_images():
+    reference = [Image.fromarray(np.zeros((2, 2, 3), dtype=np.uint8))]
+    candidate = [Image.fromarray(np.ones((2, 2, 3), dtype=np.uint8))]
+
+    summary = summarize_output_image_metrics(reference, candidate)
+
+    assert summary["num_images"] == 1
+    assert summary["image0_metrics"]["mae"] == 1.0
+    assert summary["all_images_metrics"]["mse"] == 1.0
+
+
+def test_run_summary_reports_worker_peak_memory():
+    summary = _run_summary(
+        VariantRun(
+            label="candidate",
+            result=object(),
+            generation_times_s=[1.0, 3.0],
+            peak_memory_mb=[100.0, 150.0],
+        )
+    )
+
+    assert summary["peak_memory_mb"] == 150.0
+    assert summary["avg_peak_memory_mb"] == 125.0
+    assert summary["max_peak_memory_mb"] == 150.0
+    assert summary["per_run_peak_memory_mb"] == [100.0, 150.0]
+
+
+def test_request_peak_memory_prefers_inner_diffusion_output():
+    inner = type("InnerOutput", (), {"peak_memory_mb": 321.0})()
+    outer = type("OuterOutput", (), {"request_output": inner, "peak_memory_mb": 123.0})()
+
+    assert _request_peak_memory_mb(outer) == 321.0
+
+
+def test_metric_guidance_describes_thresholds():
+    guidance = metric_guidance()
+
+    assert "cosine_similarity" in guidance["descriptions"]
+    assert guidance["recommended_thresholds"]["output_images_or_frames_uint8"]["psnr_db"]["recommended_min"] == 20.0
+    assert (
+        guidance["recommended_thresholds"]["performance"]["max_peak_memory_ratio_candidate_over_reference"][
+            "recommended_max"
+        ]
+        == 1.00
+    )
+
+
+def test_candidate_model_can_point_to_offline_checkpoint_without_online_quantization():
+    args = SimpleNamespace(
+        model="Qwen/Qwen-Image",
+        candidate_model="Qwen/Qwen-Image-FP8",
+        candidate_quantization=None,
+        candidate_quantization_config_json=None,
+        candidate_ignored_layers=None,
+        ignored_layers=None,
+        candidate_gguf_model=None,
+        candidate_load_format="default",
+    )
+
+    config = _build_variant_config(args, "candidate")
+
+    assert config.model == "Qwen/Qwen-Image-FP8"
+    assert config.quantization is None
+    assert config.quantization_config is None
+
+
+def test_step_execution_defaults_to_false(monkeypatch):
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        [
+            "compare_diffusion_trajectory_similarity.py",
+            "--output-json",
+            "result.json",
+        ],
+    )
+
+    args = parse_args()
+
+    assert args.step_execution is False

tests/diffusion/test_diffusion_engine_cleanup.py

@@ -0,0 +1,141 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import asyncio
+import queue
+import threading
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+
+from vllm_omni.diffusion.diffusion_engine import DiffusionEngine
+from vllm_omni.diffusion.request import OmniDiffusionRequest
+from vllm_omni.diffusion.sched import DiffusionRequestStatus, RequestScheduler
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
+
+
+def _make_request(
+    request_id: str,
+    *,
+    request_ids: list[str] | None = None,
+) -> OmniDiffusionRequest:
+    return OmniDiffusionRequest(
+        prompts=[f"prompt_{request_id}"],
+        sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
+        request_ids=request_ids or [request_id],
+        request_id=request_id,
+    )
+
+
+def _make_engine() -> DiffusionEngine:
+    engine = DiffusionEngine.__new__(DiffusionEngine)
+    engine.scheduler = RequestScheduler()
+    engine.scheduler.initialize(SimpleNamespace())
+    engine.executor = SimpleNamespace(shutdown=Mock())
+    engine._rpc_lock = threading.RLock()
+    engine._cv = threading.Condition(engine._rpc_lock)
+    engine._out_queue = {}
+    engine._closed = False
+    engine._shutdown_complete = False
+    engine.abort_queue = queue.Queue()
+    engine._loop_started = False
+    engine.stop_event = None
+    engine.worker_thread = None
+    return engine
+
+
+def test_close_completes_pending_async_waiters() -> None:
+    engine = _make_engine()
+    event_loop = asyncio.new_event_loop()
+    try:
+        future = event_loop.create_future()
+        engine._out_queue["pending-req"] = future
+
+        engine.close()
+
+        assert future.done()
+        assert future.result().error == "DiffusionEngine is closed."
+    finally:
+        event_loop.close()
+
+
+def test_handle_finished_requests_ignores_already_drained_waiter() -> None:
+    class RacingOutQueue(dict):
+        def __contains__(self, key):
+            return True
+
+        def pop(self, key, default=None):
+            return default
+
+    engine = _make_engine()
+    engine._out_queue = RacingOutQueue()
+    engine._finalize_finished_request = Mock(side_effect=AssertionError("should not finalize drained waiters"))
+
+    engine._handle_finished_requests({"pending-req"})
+
+    engine._finalize_finished_request.assert_not_called()
+
+
+def test_abort_parent_request_id_aborts_batched_scheduler_request() -> None:
+    engine = _make_engine()
+    request = _make_request("batch-parent", request_ids=["batch-parent-0", "batch-parent-1"])
+    sched_req_id = engine.scheduler.add_request(request)
+
+    engine.abort("batch-parent")
+    engine._process_aborts_queue()
+
+    state = engine.scheduler.get_request_state(sched_req_id)
+    assert state is not None
+    assert state.status == DiffusionRequestStatus.FINISHED_ABORTED
+
+
+def test_close_rejects_late_async_requests() -> None:
+    engine = _make_engine()
+    event_loop = asyncio.new_event_loop()
+    try:
+        engine.main_loop = event_loop
+        engine.close()
+
+        with pytest.raises(RuntimeError, match="closed"):
+            engine.add_request(_make_request("late-req"))
+    finally:
+        event_loop.close()
+
+
+def test_close_resets_loop_started_for_dead_worker_thread() -> None:
+    engine = _make_engine()
+    engine._loop_started = True
+    engine.worker_thread = SimpleNamespace(is_alive=Mock(return_value=False))
+
+    engine.close()
+
+    assert engine._loop_started is False
+
+
+def test_close_defers_resource_shutdown_until_worker_thread_stops() -> None:
+    engine = _make_engine()
+    engine.scheduler.close = Mock()
+    engine._loop_started = True
+    worker_thread = SimpleNamespace(
+        is_alive=Mock(side_effect=[True, True, False, False]),
+        join=Mock(),
+    )
+    engine.worker_thread = worker_thread
+
+    engine.close()
+
+    worker_thread.join.assert_called_once_with(timeout=10)
+    engine.scheduler.close.assert_not_called()
+    engine.executor.shutdown.assert_not_called()
+    assert engine._shutdown_complete is False
+    assert engine._loop_started is True
+
+    engine.close()
+
+    engine.scheduler.close.assert_called_once()
+    engine.executor.shutdown.assert_called_once()
+    assert engine._shutdown_complete is True
+    assert engine._loop_started is False

tests/diffusion/test_diffusion_engine_rpc_routing.py

@@ -136,6 +136,7 @@ def _make_engine_with_loop(
     calls and a real ``RequestScheduler``.
     """
     engine = DiffusionEngine.__new__(DiffusionEngine)
+    engine._closed = False
     engine.executor = _ConcurrencyTrackingExecutor(rpc_delay=rpc_delay)
 
     sched = RequestScheduler()
@@ -371,6 +372,7 @@ def test_collective_rpc_before_loop_starts_calls_executor_directly():
     on the caller's thread without enqueueing.
     """
     engine = DiffusionEngine.__new__(DiffusionEngine)
+    engine._closed = False
     engine._loop_started = False
     engine._rpc_lock = threading.RLock()
     engine._cv = threading.Condition(engine._rpc_lock)
@@ -391,6 +393,7 @@ def test_collective_rpc_before_loop_starts_serializes_concurrent_callers():
     serialize them so they cannot race on the shared executor MQ pair.
     """
     engine = DiffusionEngine.__new__(DiffusionEngine)
+    engine._closed = False
     engine._loop_started = False
     engine._rpc_lock = threading.RLock()
     engine._cv = threading.Condition(engine._rpc_lock)

tests/diffusion/test_diffusion_scheduler.py

@@ -305,17 +305,55 @@ class TestRequestScheduler:
             prompts=["prompt_map_a", "prompt_map_b"],
             sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
             request_ids=["map-a", "map-b"],
+            request_id="map-parent",
         )
 
         sched_req_id = self.scheduler.add_request(request)
 
         assert self.scheduler.get_sched_req_id("map-a") == sched_req_id
         assert self.scheduler.get_sched_req_id("map-b") == sched_req_id
+        assert self.scheduler.get_sched_req_id("map-parent") == sched_req_id
 
         self.scheduler.pop_request_state(sched_req_id)
 
         assert self.scheduler.get_sched_req_id("map-a") is None
         assert self.scheduler.get_sched_req_id("map-b") is None
+        assert self.scheduler.get_sched_req_id("map-parent") is None
+
+    def test_parent_request_id_registration_failure_rolls_back_child_ids(self) -> None:
+        self.scheduler.add_request(
+            OmniDiffusionRequest(
+                prompts=["prompt_existing"],
+                sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
+                request_ids=["existing-child"],
+                request_id="duplicate-parent",
+            )
+        )
+
+        colliding_request = OmniDiffusionRequest(
+            prompts=["prompt_unique"],
+            sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
+            request_ids=["unique-child"],
+            request_id="duplicate-parent",
+        )
+
+        with pytest.raises(ValueError, match="duplicate-parent"):
+            self.scheduler.add_request(colliding_request)
+
+        assert self.scheduler.get_request_state("unique-child") is None
+        assert self.scheduler.get_sched_req_id("unique-child") is None
+
+        valid_request = OmniDiffusionRequest(
+            prompts=["prompt_unique"],
+            sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
+            request_ids=["unique-child"],
+            request_id="unique-parent",
+        )
+        sched_req_id = self.scheduler.add_request(valid_request)
+
+        assert sched_req_id == "unique-child"
+        assert self.scheduler.get_sched_req_id("unique-child") == sched_req_id
+        assert self.scheduler.get_sched_req_id("unique-parent") == sched_req_id
 
 
 class TestDiffusionEngine:
@@ -324,6 +362,8 @@ class TestDiffusionEngine:
         engine.scheduler = RequestScheduler()
         engine.scheduler.initialize(SimpleNamespace())
         engine._rpc_lock = threading.RLock()
+        engine._cv = threading.Condition(engine._rpc_lock)
+        engine._closed = False
         engine.abort_queue = queue.Queue()
 
         request = _make_request("engine")
@@ -343,6 +383,8 @@ class TestDiffusionEngine:
         engine = DiffusionEngine.__new__(DiffusionEngine)
         engine.scheduler = scheduler
         engine._rpc_lock = threading.RLock()
+        engine._cv = threading.Condition(engine._rpc_lock)
+        engine._closed = False
         engine.abort_queue = queue.Queue()
         engine.execute_fn = mocker.Mock(return_value=runner_output)
 
@@ -394,7 +436,8 @@ class TestDiffusionEngine:
     @pytest.mark.asyncio
     async def test_step_raises_aborted_error(self, mocker: MockerFixture) -> None:
         engine = DiffusionEngine.__new__(DiffusionEngine)
-        engine._loop_started = False
+        engine._closed = False
+        engine._loop_started = True
         engine._init_lock = asyncio.Lock()
         engine.main_loop = asyncio.get_running_loop()
         engine.stop_event = threading.Event()
@@ -410,6 +453,7 @@ class TestDiffusionEngine:
         engine = DiffusionEngine.__new__(DiffusionEngine)
         engine._rpc_lock = threading.RLock()
         engine._cv = threading.Condition(engine._rpc_lock)
+        engine._closed = False
         engine.scheduler = RequestScheduler()
         engine.scheduler.initialize(SimpleNamespace())
         engine.abort_queue = queue.Queue()

tests/diffusion/test_diffusion_step_pipeline.py

@@ -305,6 +305,7 @@ def _make_engine(scheduler, execute_fn=None) -> DiffusionEngine:
     engine.execute_fn = execute_fn
     engine._rpc_lock = threading.RLock()
     engine._cv = threading.Condition(engine._rpc_lock)
+    engine._closed = False
     engine.abort_queue = queue.Queue()
     return engine
 

tests/diffusion/test_inline_stage_diffusion_client.py

@@ -5,6 +5,7 @@ from types import SimpleNamespace
 from unittest.mock import MagicMock, patch
 
 import pytest
+from vllm.v1.engine.exceptions import EngineDeadError
 
 from vllm_omni.diffusion.data import OmniDiffusionConfig
 from vllm_omni.diffusion.inline_stage_diffusion_client import InlineStageDiffusionClient
@@ -19,6 +20,10 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
 def mock_engine():
     with patch("vllm_omni.diffusion.inline_stage_diffusion_client.DiffusionEngine") as mock:
         engine_instance = MagicMock()
+        engine_instance.executor = SimpleNamespace(
+            register_failure_callback=MagicMock(),
+            check_health=MagicMock(),
+        )
         mock.make_engine.return_value = engine_instance
         yield engine_instance
 
@@ -97,6 +102,41 @@ def test_inline_shutdown(client, mock_engine):
     mock_engine.close.assert_called_once()
 
 
+def test_inline_registers_executor_failure_callback(client, mock_engine):
+    mock_engine.executor.register_failure_callback.assert_called_once()
+
+
+def test_inline_executor_failure_marks_engine_dead(client, mock_engine):
+    callback = mock_engine.executor.register_failure_callback.call_args.args[0]
+
+    callback()
+
+    assert client._engine_dead is True
+    with pytest.raises(EngineDeadError, match="inline diffusion engine is dead"):
+        client.get_diffusion_output_nowait()
+
+
+def test_inline_returns_queued_output_before_engine_dead(client, mock_engine):
+    callback = mock_engine.executor.register_failure_callback.call_args.args[0]
+    output = OmniRequestOutput.from_diffusion(request_id="req-queued", images=[])
+    client._output_queue.put_nowait(output)
+
+    callback()
+
+    assert client.get_diffusion_output_nowait() is output
+    with pytest.raises(EngineDeadError, match="inline diffusion engine is dead"):
+        client.get_diffusion_output_nowait()
+
+
+def test_inline_check_health_marks_engine_dead(client, mock_engine):
+    mock_engine.executor.check_health.side_effect = EngineDeadError()
+
+    with pytest.raises(EngineDeadError):
+        client.check_health()
+
+    assert client._engine_dead is True
+
+
 def test_inline_client_requires_replica_id(mock_engine):
     metadata = SimpleNamespace(
         stage_id=0,

tests/diffusion/test_multiproc_engine_concurrency.py

@@ -79,6 +79,7 @@ def _make_engine(num_gpus: int = 1):
     engine.executor = executor
     engine._rpc_lock = threading.RLock()
     engine._cv = threading.Condition(engine._rpc_lock)
+    engine._closed = False
     engine._loop_started = False
     engine._rpc_queue = queue.Queue()
     engine.abort_queue = queue.Queue()

tests/distributed/omni_connectors/test_chunk_transfer_adapter.py

@@ -700,3 +700,232 @@ def test_wire_round_trip_struct_to_dict_contract():
     assert set(decoded.keys()) == set(expected.keys())
     assert set(decoded["meta"].keys()) == set(expected["meta"].keys())
     assert set(decoded["codes"].keys()) == set(expected["codes"].keys())
+
+
+# ---------------------------------------------------------------
+# Deferred finish for upstream-completed requests
+# ---------------------------------------------------------------
+
+
+def _build_deferred_finish_scheduler(mocker, *, running, pending_finish_reqs):
+    """Build a mock scheduler with requests queued for deferred finish."""
+    adapter_mock = mocker.MagicMock()
+    adapter_mock.finished_requests = {r.request_id for r in pending_finish_reqs}
+    cleanup_calls = []
+    adapter_mock.cleanup = lambda *a, **kw: cleanup_calls.append((a, kw))
+
+    scheduler = mocker.MagicMock()
+    scheduler.chunk_transfer_adapter = adapter_mock
+    scheduler.connector = None
+    scheduler.ec_connector = None
+    scheduler.perf_metrics = None
+    scheduler.log_stats = False
+    scheduler.recompute_kv_load_failures = False
+    scheduler.structured_output_manager = mocker.MagicMock()
+    scheduler.structured_output_manager.should_advance.return_value = False
+    scheduler.finished_req_ids_dict = {}
+    scheduler.kv_cache_manager.take_events.return_value = None
+    scheduler.kv_event_publisher = mocker.MagicMock()
+    scheduler._pending_finish_reqs = list(pending_finish_reqs)
+
+    scheduler._handle_stopped_request = mocker.MagicMock(return_value=True)
+    scheduler._free_request = mocker.MagicMock(return_value=None)
+    scheduler._get_routed_experts = mocker.MagicMock(return_value=None)
+    scheduler.running = list(running)
+    scheduler.waiting = mocker.MagicMock()
+    scheduler.waiting.remove_requests = mocker.MagicMock()
+    scheduler.make_stats = mocker.MagicMock(return_value=None)
+    scheduler.requests = {r.request_id: r for r in running}
+
+    scheduler_output = SimpleNamespace(
+        num_scheduled_tokens={},
+        scheduled_spec_decode_tokens={},
+        num_invalid_spec_tokens=0,
+    )
+    model_runner_output = SimpleNamespace(
+        sampled_token_ids=None,
+        logprobs=None,
+        prompt_logprobs_dict={},
+        pooler_output=None,
+        num_nans_in_logits=None,
+        kv_connector_output=None,
+        cudagraph_stats=None,
+        req_id_to_index={},
+    )
+    return scheduler, scheduler_output, model_runner_output, cleanup_calls
+
+
+def test_deferred_finish_emits_finished_output(mocker: MockerFixture):
+    """A request whose upstream completed with no remaining tokens should
+    emit a FINISHED EngineCoreOutput, free resources, and clean up adapter state."""
+    from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
+
+    request = _HashableRequest(
+        request_id="req-df1",
+        external_req_id="ext-df1",
+        status=RequestStatus.RUNNING,
+        is_finished=lambda: False,
+        num_computed_tokens=16,
+        num_prompt_tokens=16,
+        prompt_token_ids=list(range(1, 17)),
+        num_output_placeholders=0,
+        sampling_params=None,
+        pooling_params=None,
+        stop_reason=None,
+        client_index=0,
+        take_events=lambda: [],
+        trace_headers=None,
+        has_encoder_inputs=False,
+        take_prefill_stats=lambda: None,
+        num_nans_in_logits=0,
+        get_finished_reason=lambda: "stop",
+    )
+    scheduler, sched_out, model_out, cleanup_calls = _build_deferred_finish_scheduler(
+        mocker,
+        running=[request],
+        pending_finish_reqs=[request],
+    )
+    scheduler._free_request.return_value = {"mock": "kv_params"}
+
+    result = OmniGenerationScheduler.update_from_output(scheduler, sched_out, model_out)
+
+    assert request.status == RequestStatus.FINISHED_STOPPED
+    scheduler._handle_stopped_request.assert_called_once_with(request)
+    scheduler._free_request.assert_called_once_with(request)
+    assert len(cleanup_calls) == 1
+    assert cleanup_calls[0][0] == ("req-df1", "ext-df1")
+
+    eco = result[0]
+    assert len(eco.outputs) == 1
+    assert eco.outputs[0].request_id == "req-df1"
+    assert eco.outputs[0].finish_reason == "stop"
+    assert eco.outputs[0].kv_transfer_params == {"mock": "kv_params"}
+    assert scheduler._pending_finish_reqs == []
+
+
+def test_deferred_finish_empty_prompt(mocker: MockerFixture):
+    """A request that never received any tokens (finished immediately upstream)
+    should still emit a FINISHED output and clean up."""
+    from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
+
+    request = _HashableRequest(
+        request_id="req-df2",
+        external_req_id="ext-df2",
+        status=RequestStatus.WAITING,
+        is_finished=lambda: False,
+        num_computed_tokens=0,
+        num_prompt_tokens=0,
+        prompt_token_ids=[],
+        num_output_placeholders=0,
+        sampling_params=None,
+        pooling_params=None,
+        stop_reason=None,
+        client_index=0,
+        take_events=lambda: [],
+        trace_headers=None,
+        has_encoder_inputs=False,
+        take_prefill_stats=lambda: None,
+        num_nans_in_logits=0,
+        get_finished_reason=lambda: "stop",
+    )
+    scheduler, sched_out, model_out, cleanup_calls = _build_deferred_finish_scheduler(
+        mocker,
+        running=[],
+        pending_finish_reqs=[request],
+    )
+
+    result = OmniGenerationScheduler.update_from_output(scheduler, sched_out, model_out)
+
+    assert request.status == RequestStatus.FINISHED_STOPPED
+    scheduler._free_request.assert_called_once_with(request)
+    assert len(cleanup_calls) == 1
+    eco = result[0]
+    assert len(eco.outputs) == 1
+    assert eco.outputs[0].finish_reason == "stop"
+    assert scheduler._pending_finish_reqs == []
+
+
+def test_deferred_finish_skips_already_finished(mocker: MockerFixture):
+    """A request that was aborted between schedule() and update_from_output()
+    should be skipped without emitting output or freeing resources twice."""
+    from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
+
+    request = _HashableRequest(
+        request_id="req-df3",
+        external_req_id="ext-df3",
+        status=RequestStatus.FINISHED_ABORTED,
+        is_finished=lambda: True,
+        num_computed_tokens=0,
+        num_prompt_tokens=0,
+        prompt_token_ids=[],
+        num_output_placeholders=0,
+        sampling_params=None,
+        pooling_params=None,
+        stop_reason=None,
+        client_index=0,
+        take_events=lambda: [],
+        trace_headers=None,
+        has_encoder_inputs=False,
+        take_prefill_stats=lambda: None,
+        num_nans_in_logits=0,
+        get_finished_reason=lambda: "stop",
+    )
+    scheduler, sched_out, model_out, cleanup_calls = _build_deferred_finish_scheduler(
+        mocker,
+        running=[],
+        pending_finish_reqs=[request],
+    )
+
+    result = OmniGenerationScheduler.update_from_output(scheduler, sched_out, model_out)
+
+    scheduler._handle_stopped_request.assert_not_called()
+    scheduler._free_request.assert_not_called()
+    assert len(cleanup_calls) == 0
+    assert 0 not in result
+    assert scheduler._pending_finish_reqs == []
+
+
+def test_deferred_finish_not_finished_still_emits_output(mocker: MockerFixture):
+    """When _handle_stopped_request returns False (resumable request), the
+    output must still be emitted so the client stream doesn't hang."""
+    from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
+
+    request = _HashableRequest(
+        request_id="req-df4",
+        external_req_id="ext-df4",
+        status=RequestStatus.RUNNING,
+        is_finished=lambda: False,
+        num_computed_tokens=16,
+        num_prompt_tokens=16,
+        prompt_token_ids=list(range(1, 17)),
+        num_output_placeholders=0,
+        sampling_params=None,
+        pooling_params=None,
+        stop_reason=None,
+        client_index=0,
+        take_events=lambda: [],
+        trace_headers=None,
+        has_encoder_inputs=False,
+        take_prefill_stats=lambda: None,
+        num_nans_in_logits=0,
+        get_finished_reason=lambda: "stop",
+    )
+    scheduler, sched_out, model_out, cleanup_calls = _build_deferred_finish_scheduler(
+        mocker,
+        running=[request],
+        pending_finish_reqs=[request],
+    )
+    scheduler._handle_stopped_request.return_value = False
+
+    result = OmniGenerationScheduler.update_from_output(scheduler, sched_out, model_out)
+
+    scheduler._handle_stopped_request.assert_called_once_with(request)
+    scheduler._free_request.assert_not_called()
+    assert len(cleanup_calls) == 0
+
+    eco = result[0]
+    assert len(eco.outputs) == 1
+    assert eco.outputs[0].request_id == "req-df4"
+    assert eco.outputs[0].finish_reason == "stop"
+    assert eco.outputs[0].kv_transfer_params is None
+    assert scheduler._pending_finish_reqs == []

tests/distributed/omni_coordinator/test_load_balancer.py

@@ -6,18 +6,18 @@ from time import time
 import pytest
 
 from vllm_omni.distributed.omni_coordinator import (
-    InstanceInfo,
     LeastQueueLengthBalancer,
     RandomBalancer,
+    ReplicaInfo,
+    ReplicaStatus,
     RoundRobinBalancer,
-    StageStatus,
 )
 
 pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
 
 
 def test_load_balancer_select_returns_valid_index():
-    """Verify RandomBalancer.select() returns a valid index for instances."""
+    """Verify RandomBalancer.select() returns a valid index for replicas."""
     # Task structure mirrors async_omni; RandomBalancer ignores task contents.
     task: dict = {
         "request_id": "test",
@@ -26,30 +26,30 @@ def test_load_balancer_select_returns_valid_index():
     }
 
     now = time()
-    instances = [
-        InstanceInfo(
+    replicas = [
+        ReplicaInfo(
             input_addr="tcp://host:10001",
             output_addr="tcp://host:10001-out",
             stage_id=0,
-            status=StageStatus.UP,
+            status=ReplicaStatus.UP,
             queue_length=0,
             last_heartbeat=now,
             registered_at=now,
         ),
-        InstanceInfo(
+        ReplicaInfo(
             input_addr="tcp://host:10002",
             output_addr="tcp://host:10002-out",
             stage_id=0,
-            status=StageStatus.UP,
+            status=ReplicaStatus.UP,
             queue_length=1,
             last_heartbeat=now,
             registered_at=now,
         ),
-        InstanceInfo(
+        ReplicaInfo(
             input_addr="tcp://host:10003",
             output_addr="tcp://host:10003-out",
             stage_id=1,
-            status=StageStatus.UP,
+            status=ReplicaStatus.UP,
             queue_length=2,
             last_heartbeat=now,
             registered_at=now,
@@ -58,38 +58,38 @@ def test_load_balancer_select_returns_valid_index():
 
     balancer = RandomBalancer()
 
-    index = balancer.select(task, instances)
+    index = balancer.select(task, replicas)
 
     assert isinstance(index, int)
-    assert 0 <= index < len(instances)
+    assert 0 <= index < len(replicas)
 
 
-def test_round_robin_balancer_cycles_instances():
+def test_round_robin_balancer_cycles_replicas():
     now = time()
-    instances = [
-        InstanceInfo(
+    replicas = [
+        ReplicaInfo(
             input_addr="tcp://host:10001",
             output_addr="tcp://host:10001-out",
             stage_id=0,
-            status=StageStatus.UP,
+            status=ReplicaStatus.UP,
             queue_length=2,
             last_heartbeat=now,
             registered_at=now,
         ),
-        InstanceInfo(
+        ReplicaInfo(
             input_addr="tcp://host:10002",
             output_addr="tcp://host:10002-out",
             stage_id=0,
-            status=StageStatus.UP,
+            status=ReplicaStatus.UP,
             queue_length=1,
             last_heartbeat=now,
             registered_at=now,
         ),
-        InstanceInfo(
+        ReplicaInfo(
             input_addr="tcp://host:10003",
             output_addr="tcp://host:10003-out",
             stage_id=1,
-            status=StageStatus.UP,
+            status=ReplicaStatus.UP,
             queue_length=0,
             last_heartbeat=now,
             registered_at=now,
@@ -97,35 +97,35 @@ def test_round_robin_balancer_cycles_instances():
     ]
 
     balancer = RoundRobinBalancer()
-    results = [balancer.select({}, instances) for _ in range(5)]
+    results = [balancer.select({}, replicas) for _ in range(5)]
 
     # Default start_index=0 => 0,1,2,0,1
     assert results == [0, 1, 2, 0, 1]
 
 
-def test_round_robin_balancer_empty_instances_raises():
-    with pytest.raises(ValueError, match="instances must not be empty"):
+def test_round_robin_balancer_empty_replicas_raises():
+    with pytest.raises(ValueError, match="replicas must not be empty"):
         RoundRobinBalancer().select({}, [])
 
 
 def test_round_robin_balancer_after_large_index_and_shorter_list():
-    """Large start_index % len(instances) then counter wraps with shorter list."""
+    """Large start_index % len(replicas) then counter wraps with shorter list."""
     now = time()
     two = [
-        InstanceInfo(
+        ReplicaInfo(
             input_addr="tcp://host:10001",
             output_addr="tcp://host:10001-out",
             stage_id=0,
-            status=StageStatus.UP,
+            status=ReplicaStatus.UP,
             queue_length=0,
             last_heartbeat=now,
             registered_at=now,
         ),
-        InstanceInfo(
+        ReplicaInfo(
             input_addr="tcp://host:10002",
             output_addr="tcp://host:10002-out",
             stage_id=0,
-            status=StageStatus.UP,
+            status=ReplicaStatus.UP,
             queue_length=0,
             last_heartbeat=now,
             registered_at=now,
@@ -138,30 +138,30 @@ def test_round_robin_balancer_after_large_index_and_shorter_list():
 
 def test_least_queue_length_balancer_picks_min_queue():
     now = time()
-    instances = [
-        InstanceInfo(
+    replicas = [
+        ReplicaInfo(
             input_addr="tcp://host:10001",
             output_addr="tcp://host:10001-out",
             stage_id=0,
-            status=StageStatus.UP,
+            status=ReplicaStatus.UP,
             queue_length=2,
             last_heartbeat=now,
             registered_at=now,
         ),
-        InstanceInfo(
+        ReplicaInfo(
             input_addr="tcp://host:10002",
             output_addr="tcp://host:10002-out",
             stage_id=0,
-            status=StageStatus.UP,
+            status=ReplicaStatus.UP,
             queue_length=0,
             last_heartbeat=now,
             registered_at=now,
         ),
-        InstanceInfo(
+        ReplicaInfo(
             input_addr="tcp://host:10003",
             output_addr="tcp://host:10003-out",
             stage_id=1,
-            status=StageStatus.UP,
+            status=ReplicaStatus.UP,
             queue_length=5,
             last_heartbeat=now,
             registered_at=now,
@@ -169,41 +169,41 @@ def test_least_queue_length_balancer_picks_min_queue():
     ]
 
     balancer = LeastQueueLengthBalancer()
-    index = balancer.select({}, instances)
+    index = balancer.select({}, replicas)
     assert index == 1
 
 
-def test_least_queue_length_balancer_empty_instances_raises():
-    with pytest.raises(ValueError, match="instances must not be empty"):
+def test_least_queue_length_balancer_empty_replicas_raises():
+    with pytest.raises(ValueError, match="replicas must not be empty"):
         LeastQueueLengthBalancer().select({}, [])
 
 
 def test_least_queue_length_balancer_equal_queues_uses_choice(mocker):
     now = time()
-    instances = [
-        InstanceInfo(
+    replicas = [
+        ReplicaInfo(
             input_addr="tcp://host:10001",
             output_addr="tcp://host:10001-out",
             stage_id=0,
-            status=StageStatus.UP,
+            status=ReplicaStatus.UP,
             queue_length=3,
             last_heartbeat=now,
             registered_at=now,
         ),
-        InstanceInfo(
+        ReplicaInfo(
             input_addr="tcp://host:10002",
             output_addr="tcp://host:10002-out",
             stage_id=0,
-            status=StageStatus.UP,
+            status=ReplicaStatus.UP,
             queue_length=3,
             last_heartbeat=now,
             registered_at=now,
         ),
-        InstanceInfo(
+        ReplicaInfo(
             input_addr="tcp://host:10003",
             output_addr="tcp://host:10003-out",
             stage_id=1,
-            status=StageStatus.UP,
+            status=ReplicaStatus.UP,
             queue_length=3,
             last_heartbeat=now,
             registered_at=now,
@@ -214,21 +214,21 @@ def test_least_queue_length_balancer_equal_queues_uses_choice(mocker):
         "vllm_omni.distributed.omni_coordinator.load_balancer.random.choice",
         return_value=2,
     )
-    assert balancer.select({}, instances) == 2
+    assert balancer.select({}, replicas) == 2
 
 
 def test_least_queue_length_balancer_negative_queue_raises():
     now = time()
-    instances = [
-        InstanceInfo(
+    replicas = [
+        ReplicaInfo(
             input_addr="tcp://host:10001",
             output_addr="tcp://host:10001-out",
             stage_id=0,
-            status=StageStatus.UP,
+            status=ReplicaStatus.UP,
             queue_length=-1,
             last_heartbeat=now,
             registered_at=now,
         ),
     ]
     with pytest.raises(ValueError, match="queue_length must be non-negative"):
-        LeastQueueLengthBalancer().select({}, instances)
+        LeastQueueLengthBalancer().select({}, replicas)