docs: align quantization overview

Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
2026-08-12 08:56:19 +08:00
226 changed files with 1720 additions and 17206 deletions
@@ -345,7 +345,7 @@ steps:
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
-          - export CACHE_DIT_VERSION=1.5.0
+          - export CACHE_DIT_VERSION=1.3.0
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
@@ -361,7 +361,7 @@ steps:
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
-          - export CACHE_DIT_VERSION=1.5.0
+          - export CACHE_DIT_VERSION=1.3.0
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json
@@ -377,7 +377,7 @@ steps:
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
-          - export CACHE_DIT_VERSION=1.5.0
+          - export CACHE_DIT_VERSION=1.3.0
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json
@@ -393,7 +393,7 @@ steps:
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
-          - export CACHE_DIT_VERSION=1.5.0
+          - export CACHE_DIT_VERSION=1.3.0
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json
@@ -409,7 +409,7 @@ steps:
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
-          - export CACHE_DIT_VERSION=1.5.0
+          - export CACHE_DIT_VERSION=1.3.0
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json
@@ -227,18 +227,6 @@ steps:
            "
        mirror_hardwares: h100_1
      - label: "Omni · MiniCPM-o 4.5 Perf Test"
        timeout_in_minutes: 60
        source_file_dependencies:
          - tests/dfx/perf/tests/test_minicpmo_4_5_ready.json
          - vllm_omni/deploy/minicpmo_4_5.yaml
          - vllm_omni/model_executor/models/minicpmo_4_5/
          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda'
        mirror_hardwares: h100_1
      - label: "TTS · VoxCPM2 Test"
        source_file_dependencies:
          - tests/e2e/online_serving/test_voxcpm2_tts.py
@@ -115,7 +115,7 @@ steps:
          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export CACHE_DIT_VERSION=1.5.0
+          - export CACHE_DIT_VERSION=1.3.0
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json
@@ -17,15 +17,6 @@ steps:
          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
        commands:
          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
      - label: "Omni · MiniCPM-o 4.5 Perf Test"
        timeout_in_minutes: 60
        mirror_hardwares: a3_npu_2
        env:
          VLLM_WORKER_MULTIPROC_METHOD: spawn
          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and npu and A3 and omni'
  - group: ":card_index_dividers: TTS Model Test"
    key: ready-tts-group
@@ -40,11 +40,9 @@ steps:
    commands:
      - "bash .buildkite/release/scripts/publish-release-wheels-pypi.sh"
  # Auto-skip when NIGHTLY=1 (scheduled); otherwise require manual unblock.
  - block: "Unblock to build release Docker images"
    depends_on: ~
    key: block-build-release-images
    if: build.env("NIGHTLY") != "1"
  - group: "Build release Docker images"
    key: "build-release-images"
@@ -65,7 +63,7 @@ steps:
              --progress plain \
              -f docker/Dockerfile.cuda .
          - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
-          # Fallback default tag (no arch suffix) if arm64 build fails.
+          # re-tag to default image tag and push, just in case arm64 build fails
          - "docker tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
          - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
        env:
@@ -88,26 +86,6 @@ steps:
        env:
          DOCKER_BUILDKIT: "1"
  # NIGHTLY=1 only: promote ECR images to DockerHub nightly tags, then prune.
  - label: "Publish nightly multi-arch image to DockerHub"
    depends_on:
      - build-release-image-x86
      - build-release-image-arm64
    if: build.env("NIGHTLY") == "1"
    key: publish-nightly-images-dockerhub
    agents:
      queue: small_cpu_queue_release
    commands:
      - "bash .buildkite/release/scripts/publish-release-images.sh nightly"
      - "bash .buildkite/release/scripts/cleanup-nightly-builds.sh"
    plugins:
      - docker-login#v3.0.0:
          username: vllmbot
          password-env: DOCKERHUB_TOKEN
    env:
      DOCKER_BUILDKIT: "1"
      DOCKERHUB_USERNAME: "vllmbot"
  - block: "Publish release images to DockerHub"
    key: block-publish-release-images
    depends_on:
@@ -1,147 +0,0 @@
 #!/bin/bash
 # SPDX-License-Identifier: Apache-2.0
 #
 # Clean up old nightly builds from DockerHub (vllm/vllm-omni), keeping the
 # newest 14 tags with the nightly- prefix.
 set -ex
 TAG_PREFIX="nightly-"
 REPO="vllm/vllm-omni"
 REPO_API_URL="https://hub.docker.com/v2/repositories/${REPO}/tags"
 echo "Cleaning up tags with prefix: $TAG_PREFIX in repository: $REPO"
 if [ -z "$DOCKERHUB_TOKEN" ]; then
  echo "Error: DOCKERHUB_TOKEN environment variable is not set"
  exit 1
 fi
 if [ -z "$DOCKERHUB_USERNAME" ]; then
  echo "Error: DOCKERHUB_USERNAME environment variable is not set"
  exit 1
 fi
 echo "Getting DockerHub bearer token..."
 set +x
 BEARER_TOKEN=$(curl -s -X POST \
  -H "Content-Type: application/json" \
  -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_TOKEN\"}" \
  "https://hub.docker.com/v2/users/login" | jq -r '.token')
 set -x
 if [ -z "$BEARER_TOKEN" ] || [ "$BEARER_TOKEN" = "null" ]; then
  echo "Error: Failed to get DockerHub bearer token"
  exit 1
 fi
 get_all_tags() {
  local page=1
  local all_tags=""
  # Hub `name` is a partial match (not a strict prefix). Use it to narrow the
  # list, then enforce startswith() client-side. Exhaust API pages; do not
  # stop merely because a page has no matching tags after filtering.
  local name_query
  name_query=$(printf '%s' "$TAG_PREFIX" | jq -sRr @uri)
  while true; do
    set +x
    local response
    response=$(curl -s -H "Authorization: Bearer $BEARER_TOKEN" \
      "$REPO_API_URL?page=$page&page_size=100&name=${name_query}")
    set -x
    local page_count
    page_count=$(echo "$response" | jq -r '.results | length')
    if [ -z "$page_count" ] || [ "$page_count" = "0" ] || [ "$page_count" = "null" ]; then
      break
    fi
    local tags
    tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" \
      '.results[] | select(.name | startswith($prefix)) | "\(.last_updated)|\(.name)"')
    if [ -n "$tags" ]; then
      all_tags="$all_tags$tags"$'\n'
    fi
    page=$((page + 1))
  done
  echo "$all_tags" | sort -r | cut -d'|' -f2
 }
 delete_tag() {
  local tag_name="$1"
  echo "Deleting tag: $tag_name"
  local delete_url="https://hub.docker.com/v2/repositories/${REPO}/tags/$tag_name"
  set +x
  local response http_code body
  response=$(curl -s -w "\n%{http_code}" -X DELETE \
    -H "Authorization: Bearer $BEARER_TOKEN" "$delete_url")
  set -x
  http_code=$(printf '%s\n' "$response" | tail -n1)
  body=$(printf '%s\n' "$response" | sed '$d')
  if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then
    echo "Successfully deleted tag: $tag_name (HTTP ${http_code})"
    return 0
  fi
  local detail=""
  if echo "$body" | jq -e '.detail' > /dev/null 2>&1; then
    detail=$(echo "$body" | jq -r '.detail')
  elif [ -n "$body" ]; then
    detail="$body"
  else
    detail="(empty response body)"
  fi
  echo "Error: Failed to delete tag $tag_name (HTTP ${http_code}): ${detail}"
  return 1
 }
 echo "Fetching all tags from DockerHub..."
 all_tags=$(get_all_tags)
 if [ -z "$all_tags" ]; then
  echo "No tags found to clean up"
  exit 0
 fi
 total_tags=$(echo "$all_tags" | wc -l)
 echo "Found $total_tags tags"
 tags_to_keep=14
 tags_to_delete=$((total_tags - tags_to_keep))
 if [ $tags_to_delete -le 0 ]; then
  echo "No tags need to be deleted (only $total_tags tags found, keeping $tags_to_keep)"
  exit 0
 fi
 echo "Will delete $tags_to_delete old tags, keeping the newest $tags_to_keep"
 tags_to_delete_list=$(echo "$all_tags" | tail -n +$((tags_to_keep + 1)))
 if [ -z "$tags_to_delete_list" ]; then
  echo "No tags to delete"
  exit 0
 fi
 echo "Deleting old tags..."
 delete_failures=0
 while IFS= read -r tag; do
  if [ -n "$tag" ]; then
    if ! delete_tag "$tag"; then
      delete_failures=$((delete_failures + 1))
    fi
    sleep 1
  fi
 done <<< "$tags_to_delete_list"
 if [ "$delete_failures" -gt 0 ]; then
  echo "Cleanup finished with ${delete_failures} deletion failure(s)"
  exit 1
 fi
 echo "Cleanup completed successfully"
@@ -1,87 +1,50 @@
 #!/bin/bash
 # SPDX-License-Identifier: Apache-2.0
 #
-# Publish Docker images from ECR staging to DockerHub.
+# Publish release Docker images from ECR to DockerHub.
-#
+# Pulls per-arch images, tags with `latest` and versioned tags, pushes them,
-# Usage:
+# then creates and pushes a multi-arch manifest.
 #   publish-release-images.sh            # release mode (default)
 #   publish-release-images.sh release
 #   publish-release-images.sh nightly
 #
 # Release tags: latest, v<version>
 # Nightly tags: nightly, nightly-<commit>
 set -euo pipefail
-MODE="${1:-release}"
+RELEASE_VERSION=$(buildkite-agent meta-data get release-version --default "" | sed 's/^v//')
 if [ -z "${RELEASE_VERSION}" ]; then
  echo "ERROR: release-version metadata not set"
  exit 1
 fi
 COMMIT="$BUILDKITE_COMMIT"
 DOCKERHUB_REPO="vllm/vllm-omni"
 ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-omni-release-repo"
 COMMIT="$BUILDKITE_COMMIT"
 PRIMARY_TAGS=()
 EXTRA_MANIFEST_TAGS=()
 case "${MODE}" in
  release)
    RELEASE_VERSION=$(buildkite-agent meta-data get release-version --default "" | sed 's/^v//')
    if [ -z "${RELEASE_VERSION}" ]; then
      echo "ERROR: release-version metadata not set"
      exit 1
    fi
    PRIMARY_TAGS=("latest" "v${RELEASE_VERSION}")
    ;;
  nightly)
    # Commit-pinned manifest reuses nightly-{x86_64,aarch64} arch images.
    PRIMARY_TAGS=("nightly")
    EXTRA_MANIFEST_TAGS=("nightly-${COMMIT}")
    ;;
  *)
    echo "Usage: $0 {release|nightly}"
    exit 2
    ;;
 esac
 echo "========================================"
-echo "Publishing ${MODE} images to ${DOCKERHUB_REPO}"
+echo "Publishing release images v${RELEASE_VERSION}"
 echo "  Commit: ${COMMIT}"
 echo "  Primary tags: ${PRIMARY_TAGS[*]}"
 if [ ${#EXTRA_MANIFEST_TAGS[@]} -gt 0 ]; then
  echo "  Extra manifests: ${EXTRA_MANIFEST_TAGS[*]}"
 fi
 echo "========================================"
 # Login to ECR to pull staging images
 aws ecr-public get-login-password --region us-east-1 | \
  docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
-SRC_X86="${ECR_REPO}:${COMMIT}-x86_64"
+docker pull "${ECR_REPO}:${COMMIT}-x86_64"
-SRC_ARM="${ECR_REPO}:${COMMIT}-aarch64"
+docker pull "${ECR_REPO}:${COMMIT}-aarch64"
-docker pull "${SRC_X86}"
+docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:latest-x86_64"
-docker pull "${SRC_ARM}"
+docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64"
 docker push "${DOCKERHUB_REPO}:latest-x86_64"
 docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64"
-publish_manifest() {
+docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:latest-aarch64"
-  local tag="$1"
+docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
-  local arch_tag_base="$2"
+docker push "${DOCKERHUB_REPO}:latest-aarch64"
-  docker manifest rm "${DOCKERHUB_REPO}:${tag}" || true
+docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
  docker manifest create \
    "${DOCKERHUB_REPO}:${tag}" \
    "${DOCKERHUB_REPO}:${arch_tag_base}-x86_64" \
    "${DOCKERHUB_REPO}:${arch_tag_base}-aarch64"
  docker manifest push "${DOCKERHUB_REPO}:${tag}"
 }
-for tag in "${PRIMARY_TAGS[@]}"; do
+docker manifest rm "${DOCKERHUB_REPO}:latest" || true
-  docker tag "${SRC_X86}" "${DOCKERHUB_REPO}:${tag}-x86_64"
+docker manifest rm "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" || true
-  docker tag "${SRC_ARM}" "${DOCKERHUB_REPO}:${tag}-aarch64"
+docker manifest create "${DOCKERHUB_REPO}:latest" "${DOCKERHUB_REPO}:latest-x86_64" "${DOCKERHUB_REPO}:latest-aarch64"
-  docker push "${DOCKERHUB_REPO}:${tag}-x86_64"
+docker manifest create "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
-  docker push "${DOCKERHUB_REPO}:${tag}-aarch64"
+docker manifest push "${DOCKERHUB_REPO}:latest"
-  publish_manifest "${tag}" "${tag}"
+docker manifest push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}"
 done
 ARCH_BASE="${PRIMARY_TAGS[0]}"
 for tag in "${EXTRA_MANIFEST_TAGS[@]}"; do
  publish_manifest "${tag}" "${ARCH_BASE}"
 done
 echo ""
-echo "Successfully published ${MODE} images"
+echo "Successfully published release images for v${RELEASE_VERSION}"
@@ -1,6 +1,6 @@
 ---
 name: precheck-pr
-description: Self-check your branch before creating a PR — catch dead code, prevent new model-specific Python examples, verify accuracy/perf claims, validate PR title format, and confirm merge readiness. Use when the user says "precheck", "self review", "pre-submit check", or "check my PR before I open it." Never posts to GitHub.
+description: Self-check your branch before creating a PR — catch dead code, verify accuracy/perf claims, validate PR title format, and confirm merge readiness. Use when the user says "precheck", "self review", "pre-submit check", or "check my PR before I open it." Never posts to GitHub.
 ---
 # PR Pre-Check
@@ -65,8 +65,6 @@ Ask: "Quick mode or full mode?" Then walk the checklist for the detected PR type
 **Also run the Code-Quality sweep on every PR**, regardless of type or mode: the five diff-scoped checks in [references/code-quality.md](references/code-quality.md) — kwargs fragility, broad-except swallow, `Any`/wrong type hints, hot-path `.clone()`/`deepcopy`, and event-loop blocking — plus the advisory conventions (log level, structured logging, synchronization, cleanup, dependencies, naming) in [checklists.md](references/checklists.md). These count only lines the PR *adds* — the pre-existing backlog across the repo is out of scope.
 **Also run the Examples-Policy check on every PR** using [references/examples-policy.md](references/examples-policy.md). Inspect only Python paths introduced by the diff. Treat a new model-, checkpoint-, vendor-, or family-specific Python example as blocking; do not report pre-existing example debt.
 ### Step 5: Print Report
 ```
@@ -79,7 +77,6 @@ Pre-check report for <branch>
  ─────────────────  ──────
  PR title format    ✓
  Code quality       ⚠ 1 broad except, 2 Any hints
  Examples policy    ✗ new model-specific Python example
  PR desc integrity  ✓
  Registry/config    ✓
  Dead code          ⚠ 2 warnings
@@ -47,19 +47,6 @@ Diff-scoped sweep for five fragility patterns that are pervasive in the existing
 Roll the ⚠/✗ counts into the report as a single **Code quality** dimension row.
 ---
 ## All PRs: Examples Policy (run on every PR, quick & full)
 Inspect Python files introduced under `examples/` using
 [examples-policy.md](examples-policy.md). This is a diff-scoped ratchet: existing
 model-specific examples may be edited or removed without triggering the check.
 - [ ] **No new model-specific Python examples:** the diff adds, copies, or renames no Python file whose path or behavior is specific to one model, checkpoint, vendor, or model family. ✗ for any violation. Route inference through a shared task/protocol entrypoint, model request behavior through `vllm_omni/model_extras`, and commands through task docs or `recipes/`.
 - [ ] **New generic example code is justified:** any new task-neutral Python file under `examples/` is an actual user-facing entrypoint rather than reusable production logic, a benchmark, an app, a setup tool, or a reproducer. ⚠ when the placement is questionable.
 Roll the result into the report as a separate **Examples policy** dimension.
 ### Conventions (eyeball — no grep)
 Judgment calls that don't grep cleanly — apply them while reading the diff, not as a mechanical check:
@@ -1,68 +0,0 @@
 # Examples Python Policy
 Prevent new model-specific Python examples without forcing cleanup of the
 existing backlog. Run this check on every PR in quick and full modes.
 ## Scope
 Set the merge base as in the main workflow, then list Python paths introduced
 by the PR, including additions, copies, and renames:
 ```bash
 BASE="$(git merge-base HEAD origin/main)"
 git diff --name-status --diff-filter=ACR --find-renames --find-copies \
  "${BASE}"...HEAD -- 'examples/**/*.py'
 ```
 Inspect destination paths for copies and renames. Existing model-specific files
 that are only modified or deleted are grandfathered and must not be reported.
 ## Blocking classification
 Mark ✗ when a newly introduced Python file does any of the following:
 - Names or scopes a path to one model, checkpoint, vendor, or model family.
 - Hard-codes a model identifier, model-specific prompt, request shape, output
  conversion, or launch configuration behind an otherwise generic filename.
 - Duplicates an existing task/protocol runner to demonstrate one model.
 - Adds a per-model helper below a generic task hub such as `text_to_speech/`.
 Do not infer safety from a generic filename alone. For example, a file named
 `text_to_audio.py` is still model-specific if it only implements one model's
 contract.
 ## Acceptable additions
 A new Python example may pass when it is a genuinely model-neutral,
 user-facing task or protocol entrypoint. It must accept model selection through
 configuration and keep model-specific behavior out of the script.
 Use ⚠ when the file is model-neutral but reusable logic appears misplaced under
 `examples/`. Route code by purpose:
 | Code purpose | Preferred location |
 |---|---|
 | Model prompt, defaults, request/output adaptation | `vllm_omni/model_extras/` or production model modules |
 | Runnable model command and validation evidence | Task documentation or `recipes/` |
 | UI or interactive application | `apps/` |
 | Benchmark or evaluator | `benchmarks/` |
 | Regression reproducer | `tests/` |
 | Download, conversion, or setup utility | `tools/` |
 If a model cannot use an existing shared runner, report the missing generic
 capability. Do not solve the gap by adding a model-specific Python example.
 ## Report format
 Add one row to the pre-check report:
 ```text
 Examples policy    ✓ no new Python example paths
 Examples policy    ⚠ generic helper belongs in tools/
 Examples policy    ✗ examples/offline_inference/new_model/end2end.py is model-specific
 Examples policy    ✗ examples/offline_inference/x_to_y_model_name.py is model-specific
 ```
 For a blocker, name the path, the model-specific behavior, and the appropriate
 shared runner or destination. Do not propose deleting unrelated existing
 examples as part of the PR.
@@ -97,7 +97,6 @@ If docs and live code disagree, verify the code/tests and report the drift.
 | [test-quality-evaluation.md](references/checks/test-quality-evaluation.md) | Tests change, are absent for risky code, or may not exercise production behavior. |
 | [tests-docs-checklist.md](references/checks/tests-docs-checklist.md) | Coverage, CI markers, examples, user docs, or PR evidence need review. |
 | [verification.md](references/checks/verification.md) | Hardware, a server, or a runnable affected path is available for active verification. |
 | [examples-policy.md](../precheck-pr/references/examples-policy.md) | The PR adds, copies, or renames Python under `examples/`; apply the canonical policy shared with `precheck-pr`. |
 ### Delivery and reviewer coordination
@@ -153,13 +152,6 @@ contract they protect or use only the applicable evidence checks.
 Apply every category in [general-checks.md](references/process/general-checks.md) before
 lower-priority comments.
 If the diff census contains an added, copied, or renamed Python path under
 `examples/`, read and apply the canonical
 [examples policy](../precheck-pr/references/examples-policy.md). Treat a new
 model-specific Python example as blocking. Do not flag model-specific example
 debt that the PR only modifies or removes, and do not run the rest of the
 author-oriented `precheck-pr` workflow.
 For each changed value or behavior, trace:
 ```text
@@ -11,15 +11,10 @@ build:
    post_checkout:
      - git fetch --unshallow || true
    pre_install:
      # `pre_install` runs before RTD upgrades pip, and the seeded pip 23.1
      # rejects the `typing_extensions` wheel torch needs (fixed in pip 23.2).
      - python -m pip install --upgrade pip
      # Pre-install CPU torch so the subsequent `pip install .[docs]` does not
      # pull ~2 GB of CUDA 13 libs (cuBLAS/cuDNN/NCCL/...) that exceed RTD's
-      # 1 GB memory cap and time out the mkdocs build. Wheel-only: `--index-url`
+      # 1 GB memory cap and time out the mkdocs build.
-      # replaces PyPI, so an sdist fallback cannot reach its build deps. The
+      - python -m pip install --index-url https://download.pytorch.org/whl/cpu torch
      # quotes keep YAML from reading `:all:` as a mapping key.
      - "python -m pip install --only-binary=:all: --index-url https://download.pytorch.org/whl/cpu torch"
 mkdocs:
  configuration: mkdocs.yml
@@ -158,6 +158,7 @@ def _build_omni_kwargs(args, quantization=None):
 def _generate_image(omni, args, prompt, seed):
    """Generate a single image and return (PIL.Image, time_seconds, memory_gib)."""
    from vllm_omni.inputs.data import OmniDiffusionSamplingParams
    from vllm_omni.outputs import OmniRequestOutput
    from vllm_omni.platforms import current_omni_platform
    generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(seed)
@@ -175,7 +176,7 @@ def _generate_image(omni, args, prompt, seed):
    elapsed = time.perf_counter() - start
    peak_mem = torch.accelerator.max_memory_allocated() / (1024**3)
-    req_out = outputs[0] if isinstance(outputs, list) else outputs
+    req_out = OmniRequestOutput.unwrap_result(outputs)
    if not req_out.images:
        raise ValueError("Could not extract image output from result.")
    img = req_out.images[0]
@@ -205,15 +206,15 @@ def _generate_video(omni, args, prompt, seed):
    elapsed = time.perf_counter() - start
    peak_mem = torch.accelerator.max_memory_allocated() / (1024**3)
-    if isinstance(outputs, list) and isinstance(outputs[0], OmniRequestOutput):
+    first = outputs[0]
-        first = outputs[0]
+    if hasattr(first, "request_output") and isinstance(first.request_output, list):
-    elif isinstance(outputs, OmniRequestOutput):
+        inner = first.request_output[0]
-        first = outputs
+        if isinstance(inner, OmniRequestOutput) and hasattr(inner, "images"):
-    else:
+            frames = inner.images[0] if inner.images else None
-        raise ValueError("Could not extract video frames from output.")
+        else:
-
+            frames = inner
-    if hasattr(first, "images"):
+    elif hasattr(first, "images") and first.images:
-        frames = first.images[0] if first.images else None
+        frames = first.images[0]
    else:
        raise ValueError("Could not extract video frames from output.")
@@ -300,7 +300,7 @@ def benchmark(args: argparse.Namespace) -> None:
        output_idx = 0
        for stage_outputs in omni.generate(all_prompts, sampling_params_list, py_generator=True):
            if stage_outputs.final_output_type == "image":
-                request_output = stage_outputs
+                request_output = stage_outputs.request_output
                request_id = getattr(request_output, "request_id", "")
                images = getattr(request_output, "images", [])
@@ -167,11 +167,9 @@ def main() -> None:
    ]
    native_cmd = [
        sys.executable,
-        "examples/offline_inference/text_to_video/text_to_video.py",
+        "examples/offline_inference/text_to_video/text_to_video_lingbot.py",
        "--model",
        args.model,
        "--model-class-name",
        "LingBotVideoPipeline",
        "--prompt",
        args.prompt,
        "--output",
@@ -1,6 +1,6 @@
 # Prometheus Metrics Design
-This document describes how vLLM-Omni exposes Prometheus metrics for multi-stage pipelines, the constraints that shaped the design, and how the pipeline-level metrics coexist with upstream vLLM per-engine metrics.
+This document describes how vLLM-Omni exposes Prometheus metrics for multi-stage pipelines, the constraints that shaped the design, and how the pipeline-level metrics coexist with upstream vLLM per-engine metrics. This iteration covers the text and audio surface; diffusion / image / video families land in a follow-up.
 ## Objectives
@@ -14,10 +14,7 @@ A: At first, you can check current [issues](https://github.com/vllm-project/vllm
 > Q: Does vLLM-Omni support AWQ or any other quantization?
-A: AWQ is not currently listed as a validated vLLM-Omni path. Quantization
+A: We plan to introduce GGUF FP8 prequantized models and online FP8 quantization in version 0.16.0. Support for other quantization types will follow in future releases. For details, please see our [Q1 quantization roadmap](https://github.com/vllm-project/vllm-omni/issues/1057).
 support depends on the model, pipeline stage, checkpoint format, and hardware
 backend. See the current [quantization support matrix](../user_guide/quantization/overview.md)
 for online methods, pre-quantized checkpoints, and model-level validation.
 > Q: Does vLLM-Omni support multimodal streaming input and output?
@@ -1,23 +1,23 @@
 # Production Metrics
-vLLM-Omni exposes Prometheus metrics via the `/metrics` endpoint on the OpenAI-compatible API server.
+vLLM-Omni exposes Prometheus metrics via the `/metrics` endpoint on the OpenAI-compatible API server. This page covers the text and audio surface; diffusion / image / video metrics are tracked in a follow-up PR.
 ```bash
 vllm-omni serve Qwen/Qwen3-Omni-30B-A3B-Instruct --port 8000 --log-stats
 curl http://localhost:8000/metrics
 ```
-**`--log-stats` is required to populate metric data.** Without the flag, the endpoint still returns `200 OK` but the upstream `vllm:*` wrap is not registered at all, and the 15 `vllm_omni:*` families are registered as placeholders with no data written to them. This keeps the runtime cost essentially zero for deployments that don't need monitoring. With the flag, all ~80 families populate.
+**`--log-stats` is required to populate metric data.** Without the flag, the endpoint still returns `200 OK` but the upstream `vllm:*` wrap is not registered at all, and the 15 `vllm:omni_*` families are registered as placeholders with no data written to them. This keeps the runtime cost essentially zero for deployments that don't need monitoring. With the flag, all ~80 families populate.
 ## Metric Namespaces
 | Prefix | Source | Present when |
 |--------|--------|--------------|
-| `vllm_omni:` | vLLM-Omni orchestrator / audio modality / cross-stage transfer | Pipeline-dependent |
+| `vllm:omni_` | vLLM-Omni orchestrator / audio modality / cross-stage transfer | Pipeline-dependent |
 | `vllm:` | Upstream vLLM engine, wrapped by `OmniPrometheusStatLogger` to expose `{stage, replica}` | Pipeline includes an LLM (AR) stage |
 | `http_` / `process_` | Uvicorn / Python runtime | Always |
-## Pipeline-Level Metrics (`vllm_omni:`)
+## Pipeline-Level Metrics (`vllm:omni_`)
 Defined in `vllm_omni/metrics/prometheus.py`. Track request lifecycle across the full multi-stage pipeline.
@@ -25,66 +25,42 @@ Defined in `vllm_omni/metrics/prometheus.py`. Track request lifecycle across the
 | Metric | Type | Labels | Description |
 |--------|------|--------|-------------|
-| `vllm_omni:num_requests_running` | Gauge | `model_name` | Pipeline-global in-flight requests (dispatched to engine, not yet finalized) |
+| `vllm:omni_num_requests_running` | Gauge | `model_name` | Pipeline-global in-flight requests (dispatched to engine, not yet finalized) |
-| `vllm_omni:num_requests_waiting` | Gauge | `model_name` | Requests waiting in the Orchestrator queue |
+| `vllm:omni_num_requests_waiting` | Gauge | `model_name` | Requests waiting in the Orchestrator queue |
-| `vllm_omni:requests_success_total` | Counter | `model_name`, `finished_reason` | Total requests by completion reason. `finished_reason` ∈ {`stop`, `length`, `abort`, ...} mirroring upstream `vllm:request_success_total`; aborts cover client disconnect / cancellation paths in addition to upstream `FinishReason.ABORT` |
+| `vllm:omni_requests_success_total` | Counter | `model_name`, `finished_reason` | Total requests by completion reason. `finished_reason` ∈ {`stop`, `length`, `abort`, ...} mirroring upstream `vllm:request_success_total`; aborts cover client disconnect / cancellation paths in addition to upstream `FinishReason.ABORT` |
 ### Latency
 | Metric | Type | Labels | Description |
 |--------|------|--------|-------------|
-| `vllm_omni:e2e_request_latency_s` | Histogram | `model_name` | Pipeline-global end-to-end request latency in seconds |
+| `vllm:omni_e2e_request_latency_s` | Histogram | `model_name` | Pipeline-global end-to-end request latency in seconds |
-## Audio Modality Metrics (`vllm_omni:`)
+## Audio Modality Metrics (`vllm:omni_`)
 Emitted at request finalize, except for `audio_ttfp_s` (streaming-hook at the first audio packet) and `audio_underrun_s` / `audio_continuity_ok_total` (streaming finalize, after the chunk stream is exhausted). All carry `{model_name, stage, replica}` plus the listed extra label.
 | Metric | Type | Extra label | Description |
 |--------|------|-------------|-------------|
-| `vllm_omni:audio_ttfp_s` | Histogram | — | Time from request arrival to first audio packet/frame |
+| `vllm:omni_audio_ttfp_s` | Histogram | — | Time from request arrival to first audio packet/frame |
-| `vllm_omni:audio_duration_s` | Histogram | — | Audio content duration (`audio_frames / sample_rate`) |
+| `vllm:omni_audio_duration_s` | Histogram | — | Audio content duration (`audio_frames / sample_rate`) |
-| `vllm_omni:audio_rtf` | Histogram | — | Real-time factor (`stage_gen_time_s / audio_duration_s`); streaming TTS SLO red line `< 1`; uses `RTF_BUCKETS` |
+| `vllm:omni_audio_rtf` | Histogram | — | Real-time factor (`stage_gen_time_s / audio_duration_s`); streaming TTS SLO red line `< 1`; uses `RTF_BUCKETS` |
-| `vllm_omni:audio_frames_total` | Counter | — | Cumulative audio frame count; throughput via `rate()` |
+| `vllm:omni_audio_frames_total` | Counter | — | Cumulative audio frame count; throughput via `rate()` |
-| `vllm_omni:audio_underrun_s` | Histogram | — | Per-request worst-case player deficit; `> 0` indicates listener heard silent gaps |
+| `vllm:omni_audio_underrun_s` | Histogram | — | Per-request worst-case player deficit; `> 0` indicates listener heard silent gaps |
-| `vllm_omni:audio_continuity_ok_total` | Counter | `threshold_ms` | Incremented when the request's worst underrun stayed below `threshold_ms` |
+| `vllm:omni_audio_continuity_ok_total` | Counter | `threshold_ms` | Incremented when the request's worst underrun stayed below `threshold_ms` |
-| `vllm_omni:audio_skipped_requests_total` | Counter | `reason` | Silent-loss counter — code2wav rejected malformed codec input and returned `200 OK` with empty audio |
+| `vllm:omni_audio_skipped_requests_total` | Counter | `reason` | Silent-loss counter — code2wav rejected malformed codec input and returned `200 OK` with empty audio |
 The continuity math comes from `vllm_omni/benchmarks/audio_continuity.py::compute_continuity_stats` so the server-side observation aligns with the bench-side definition.
-## Diffusion Metrics (`vllm_omni:`)
+## Cross-Stage Transfer Metrics (`vllm:omni_`)
 Per-request timing breakdowns for diffusion (image/video) stages. Emitted at request finalize when `stage_metrics.diffusion_metrics` is present. All carry `{model_name, stage, replica}`.
 | Metric | Type | Description |
 |--------|------|-------------|
 | `vllm_omni:diffusion_exec_s` | Histogram | DiT forward pass execution time per request in seconds |
 | `vllm_omni:diffusion_exec_per_step_s` | Histogram | DiT forward pass execution time per denoising step in seconds (`exec_s / num_inference_steps`) |
 | `vllm_omni:diffusion_preprocess_s` | Histogram | Diffusion input preprocessing time per request in seconds |
 | `vllm_omni:diffusion_postprocess_s` | Histogram | Diffusion output postprocessing (VAE decode) time per request in seconds |
 `diffusion_exec_s` uses `SECONDS_BUCKETS` (50 ms → 300 s); the other three use `SECONDS_FAST_BUCKETS` (1 ms → 60 s). `diffusion_exec_per_step_s` is skipped when `num_inference_steps` is not set.
 ### Example PromQL
 ```promql
 # P99 DiT exec time by stage
 histogram_quantile(0.99, rate(vllm_omni:diffusion_exec_s_bucket[5m]))
 # Ratio of preprocess to total exec wall time
 rate(vllm_omni:diffusion_preprocess_s_sum[5m])
  / rate(vllm_omni:diffusion_exec_s_sum[5m])
 ```
 ## Cross-Stage Transfer Metrics (`vllm_omni:`)
 Per-physical-transfer histograms tracking the data hop between adjacent stages. Labels `{model_name, from_stage, from_replica, to_stage, to_replica}` let dashboards attribute latency to specific replica edges. `from_replica` / `to_replica` are resolved from the orchestrator's sticky-routing binding (`stage_pool.get_bound_replica_id(request_id)`), so no extra plumbing through `TransferEdgeStats` is needed.
 | Metric | Type | Description |
 |--------|------|-------------|
-| `vllm_omni:transfer_size_bytes` | Histogram | Per-transfer payload size in bytes |
+| `vllm:omni_transfer_size_bytes` | Histogram | Per-transfer payload size in bytes |
-| `vllm_omni:transfer_tx_s` | Histogram | Sender-side time (serialize + submit to connector) |
+| `vllm:omni_transfer_tx_s` | Histogram | Sender-side time (serialize + submit to connector) |
-| `vllm_omni:transfer_rx_s` | Histogram | Receiver-side time (recv + deserialize) |
+| `vllm:omni_transfer_rx_s` | Histogram | Receiver-side time (recv + deserialize) |
-| `vllm_omni:transfer_in_flight_s` | Histogram | Network in-flight time (TX done → RX recv start) |
+| `vllm:omni_transfer_in_flight_s` | Histogram | Network in-flight time (TX done → RX recv start) |
 ## vLLM Engine Metrics (`vllm:`)
@@ -107,9 +83,9 @@ For the full list of upstream metrics, see [the vLLM docs](https://github.com/vl
 | Metric group | Multi-stage LLM (Qwen3-Omni) |
 |---|---|
-| `vllm_omni:` request tracking + latency | With `--log-stats` |
+| `vllm:omni_` request tracking + latency | With `--log-stats` |
-| `vllm_omni:` audio modality | With `--log-stats`, if pipeline has a talker stage |
+| `vllm:omni_` audio modality | With `--log-stats`, if pipeline has a talker stage |
-| `vllm_omni:` transfer | With `--log-stats`, if pipeline has ≥ 2 stages |
+| `vllm:omni_` transfer | With `--log-stats`, if pipeline has ≥ 2 stages |
 | `vllm:` engine metrics (per `(stage, replica)`) | With `--log-stats` |
 | `vllm:` MFU metrics | With `--log-stats --enable-mfu-metrics` |
@@ -118,5 +94,5 @@ For the full list of upstream metrics, see [the vLLM docs](https://github.com/vl
 - All time-bearing metrics use the `_s` suffix (values in seconds). Buckets are `SECONDS_BUCKETS` for e2e / generation-style values and `SECONDS_FAST_BUCKETS` (1 ms → 60 s) for the fine-grained transfer and audio-underrun values.
 - Counters use the `_total` suffix (auto-appended by `prometheus_client`).
 - Sizes use the `_bytes` suffix.
- All omni-specific families are prefixed `vllm_omni:`. The upstream `unregister_vllm_metrics()` function is monkey-patched (see `vllm_omni/patch.py`) to a scoped version that still strips upstream `vllm:*` collectors so multi-engine init within one process does not crash on duplicate registration, but preserves anything prefixed `vllm_omni:`.
+- All omni-specific families are prefixed `vllm:omni_`. The upstream `unregister_vllm_metrics()` function is monkey-patched (see `vllm_omni/patch.py`) to a scoped version that still strips upstream `vllm:*` collectors so multi-engine init within one process does not crash on duplicate registration, but preserves anything prefixed `vllm:omni_` / `vllm_omni`.
- Text and audio first-output use distinct families (`vllm:time_to_first_token_seconds` reused from upstream for text; `vllm_omni:audio_ttfp_s` for audio) rather than a single metric with a `modality` label.
+- Text and audio first-output use distinct families (`vllm:time_to_first_token_seconds` reused from upstream for text; `vllm:omni_audio_ttfp_s` for audio) rather than a single metric with a `modality` label.
@@ -92,7 +92,7 @@ current support.
 |--------|--------------|-------------|----------|
 | **[FP8](quantization/fp8.md)** | `quantization="fp8"` | FP8 W8A8 on validated transformer stages | Memory reduction, inference speedup |
 | **[INT8](quantization/int8.md)** | `quantization="int8"` | INT8 W8A8 on validated transformer stages | Memory reduction, broad GPU compatibility |
-| **[BitsAndBytes W4](quantization/bitsandbytes.md)** | `quantization="bitsandbytes"` | Online NF4/FP4 weight-only quantization on validated CUDA transformer stages | Reducing transformer weight memory without preparing a separate checkpoint |
+| **[GGUF](quantization/gguf.md)** | `quantization="gguf"` | Native GGUF transformer-only weights (Q4, Q8, etc.) | Memory reduction on consumer GPUs |
 ## Supported Models
@@ -12,7 +12,6 @@ For backend selection and SageAttention usage, see the [Diffusion Attention Back
 | Model | Default Resolution | Default Frames | Default Steps | Guidance | VRAM (BF16) |
 |---|---|---|---|---|---|
 | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB |
 | `robbyant/lingbot-video-dense-1.3b` / `robbyant/lingbot-video-moe-30b-a3b` | 192x320 | 9 | 2 | 3.0 | ~68 GiB (MoE smoke) |
 | `Lightricks/LTX-2` | 512x768 | 121 | 40 | video 3.0 / audio 7.0 | Model-dependent |
 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB |
 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required |
@@ -36,25 +35,6 @@ python text_to_video.py \
  --output t2v_out.mp4
 ```
 ### LingBot-Video
 The shared runner recognizes both official LingBot checkpoint IDs and selects
 the native `LingBotVideoPipeline` automatically:
 ```bash
 python text_to_video.py \
  --model robbyant/lingbot-video-dense-1.3b \
  --prompt "a robotic arm picks up a red block" \
  --height 192 --width 320 --num-frames 9 --num-inference-steps 2 \
  --guidance-scale 3.0 --flow-shift 3.0 --fps 24 \
  --output lingbot_t2v.mp4
 ```
 Use `robbyant/lingbot-video-moe-30b-a3b` for the MoE checkpoint. For local
 paths without `lingbot` in the directory name, add
 `--model-class-name LingBotVideoPipeline`. Model-only options use
 `--extra-body`, for example `'{"batch_cfg": true, "output_type": "np"}'`.
 ### LTX-2
 ```bash
@@ -143,8 +123,7 @@ python text_to_video.py \
 - `--enable-layerwise-offload`: enable layerwise offloading on DiT modules.
 - `--frame-rate`: generation FPS for pipelines that require it (e.g., LTX2).
 - `--audio-sample-rate`: fallback audio sample rate when the pipeline returns audio.
- `--quantization`: online quantization method, for example `fp8`; supported
+- `--quantization`: quantization method (`fp8` for FP8, `gguf` for GGUF).
  pre-quantized checkpoints are normally auto-detected from checkpoint metadata.
 - `--flow-shift`: scheduler flow_shift parameter.
 - `--lora-path`: path to PEFT LoRA adapter folder or checkpoint file.
 - `--lora-scale`: scale factor for LoRA weights.
@@ -90,8 +90,8 @@ vllm serve "Qwen/Qwen-Image" \
    --quantization-config '{"method": "fp8"}'
 ```
-Other vLLM-Omni quantization methods, such as `modelopt`, `mxfp4`, `mxfp8`,
+Other vLLM-Omni quantization methods, such as `gguf`, `modelopt`, `mxfp4`,
-serialized checkpoints, static FP8 configs, and layer-name skip lists
+`mxfp8`, serialized checkpoints, static FP8 configs, and layer-name skip lists
 such as `ignored_layers`, are intentionally not translated by this compatibility
 shortcut. Use Diffusers-native configuration through `diffusers_load_kwargs` or
 a native vLLM-Omni pipeline for those cases.
@@ -10,10 +10,6 @@ checkpoint's `config.json` and auto-detects
 AutoRound is static quantization: no `--quantization` flag is needed at
 inference time when the checkpoint already contains the quantization config.
 The validation terms on this page follow the
 [support levels](overview.md#support-levels) defined in the overview. A listed
 checkpoint is not enough by itself to establish end-to-end model support.
 ## Hardware Support
 | Device | Support |
@@ -25,57 +21,46 @@ checkpoint is not enough by itself to establish end-to-end model support.
 | Intel XPU | ✅ |
 | Ascend NPU | ❌ |
-Legend: `✅` backend available, `❌` unsupported, `⭕` not verified in this
+Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
-guide. This table is method-level backend availability. AutoRound is an Intel
+guide. AutoRound is Intel-supported.
 project; vLLM selects a compatible CUDA or XPU compute backend at load time.
 ## Model Type Support
 ### Diffusion Model (Qwen-Image, Wan2.2)
-| Model | Checkpoint | Scope | Scheme | Validation |
+| Model | Checkpoint | Scope | Scheme | Backend |
-|-------|------------|-------|--------|------------|
+|-------|------------|-------|--------|---------|
-| FLUX.1-dev | `vllm-project-org/FLUX.1-dev-AutoRound-w4a16` | Diffusion transformer | W4A16 | Validated |
+| FLUX.1-dev | `vllm-project-org/FLUX.1-dev-AutoRound-w4a16` | Diffusion transformer | W4A16 | GPTQ-Marlin or Intel-supported AutoRound backend |
-| Qwen-Image | `INC4AI/Qwen-Image-AutoRound-W4A16` | Diffusion transformer | W4A16 | Validated with end-to-end generation and quality comparison |
+| Qwen-Image | Not listed | Diffusion transformer | W4A16 | Not validated |
-| Wan2.2-I2V | `Intel/Wan2.2-I2V-A14B-Diffusers-int4-AutoRound` | Both diffusion transformers | W4A16 | CI-backed |
+| Wan2.2-I2V | `Intel/Wan2.2-I2V-A14B-Diffusers-int4-AutoRound` | Diffusion transformer | W4A16 | GPTQ-Marlin or Intel-supported AutoRound backend |
-| Wan2.2-T2V | `Intel/Wan2.2-T2V-A14B-Diffusers-int4-AutoRound` | Both diffusion transformers | W4A16 | CI-backed |
+| Wan2.2-T2V | `Intel/Wan2.2-T2V-A14B-Diffusers-int4-AutoRound` | Diffusion transformer | W4A16 | GPTQ-Marlin or Intel-supported AutoRound backend |
-| Wan2.2-TI2V | `Intel/Wan2.2-TI2V-5B-Diffusers-int4-AutoRound` | Diffusion transformer | W4A16 | Validated; not in the scheduled Wan AutoRound job |
+| Wan2.2-TI2V | `Intel/Wan2.2-TI2V-5B-Diffusers-int4-AutoRound` | Diffusion transformer | W4A16 | GPTQ-Marlin or Intel-supported AutoRound backend |
 CUDA execution uses GPTQ-Marlin or another compatible vLLM AutoRound backend;
 Intel XPU uses the Intel-supported backend. The text encoder and VAE stay in
 BF16 for all entries above.
 ### World Model (Cosmos3)
-| Model | Checkpoint | Scope | Scheme | Validation |
+| Model | Checkpoint | Scope | Scheme | Backend |
-|-------|------------|-------|--------|------------|
+|-------|------------|-------|--------|---------|
-| Cosmos3-Nano | `Intel/Cosmos3-Nano-int4-AutoRound` | World-model transformer | W4A16 | Integrated; no model-specific end-to-end test is maintained in-tree |
+| Cosmos3-Nano | `Intel/Cosmos3-Nano-int4-AutoRound` | World-model transformer | W4A16 | GPTQ-Marlin or Intel-supported AutoRound backend |
-| Cosmos3-Super | `Intel/Cosmos3-Super-int4-AutoRound` | World-model transformer | W4A16 | Validated with a manual serving run |
+| Cosmos3-Super | `Intel/Cosmos3-Super-int4-AutoRound` | World-model transformer | W4A16 | GPTQ-Marlin or Intel-supported AutoRound backend |
 The Cosmos VAE and guardrail components stay in BF16.
 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
-| Model | Checkpoint | Scope | Scheme | Validation |
+| Model | Checkpoint | Scope | Scheme | Backend |
-|-------|------------|-------|--------|------------|
+|-------|------------|-------|--------|---------|
-| Qwen2.5-Omni-7B | `Intel/Qwen2.5-Omni-7B-int4-AutoRound` | Thinker language-model stage | W4A16 | Validated outside scheduled CI |
+| Qwen2.5-Omni-7B | `Intel/Qwen2.5-Omni-7B-int4-AutoRound` | Language-model stage | W4A16 | AutoRound |
-| Qwen3-Omni-30B-A3B-Instruct | `Intel/Qwen3-Omni-30B-A3B-Instruct-int4-AutoRound` | Thinker language-model stage | W4A16 | Validated outside scheduled CI |
+| Qwen3-Omni-30B-A3B-Instruct | `Intel/Qwen3-Omni-30B-A3B-Instruct-int4-AutoRound` | Thinker language-model stage | W4A16 | AutoRound |
 | Qwen3-TTS | Not listed | TTS language-model stage | W4A16 | Not validated |
 AutoRound support is checkpoint-driven. A model is supported when its
 checkpoint uses a compatible INC/AutoRound config and the target stage maps to
 vLLM-Omni's runtime module names.
 For Qwen-Omni, only the thinker language model uses the AutoRound checkpoint.
 Audio and vision encoders, the talker, and waveform-decoder stages stay in
 BF16.
 ### Multi-Stage Diffusion Model (BAGEL, GLM-Image)
-| Model | Checkpoint | Scope | Validation |
+| Model | Scope | Status | Notes |
-|-------|------------|-------|------------|
+|-------|-------|--------|-------|
-| GLM-Image | `Intel/GLM-Image-int4-AutoRound` | Diffusion transformer | CI-backed |
+| GLM-Image | Diffusion transformer  | ✅ | `Intel/GLM-Image-int4-AutoRound` |
-| BAGEL | Not listed | Checkpoint-defined diffusion or transformer stage | Not validated |
+| BAGEL | Checkpoint-defined diffusion or transformer stage | Not validated | Requires a compatible AutoRound checkpoint |
 ## Configuration
@@ -134,14 +119,6 @@ The checkpoint should contain a config like:
 At load time, vLLM-Omni builds an `OmniINCConfig`, remaps checkpoint block names
 to runtime module names, and selects the matching vLLM compute backend.
 Checkpoint auto-detection and successful layer construction establish the
 **Integrated** level. Promote a model to **Validated** only after the named
 checkpoint completes an end-to-end generation or multimodal request and its
 output passes a model-appropriate sanity or quality check. Compare against the
 BF16 baseline for quality-sensitive image and video paths. Promote it to
 **CI-backed** only when that named checkpoint is selected by scheduled hardware
 CI.
 Example checkpoint creation:
 ```bash
@@ -9,10 +9,6 @@ stay in BF16/FP16. No pre-quantized checkpoint is required.
 This is an online (dynamic) path: load the normal HuggingFace checkpoint and
 quantize during model loading via the `bitsandbytes` CUDA kernels.
 The validation terms on this page follow the
 [support levels](overview.md#support-levels). BitsAndBytes kernel support does
 not imply that every diffusion transformer has been quality-validated.
 ## Hardware Support
 | Device | Support |
@@ -25,7 +21,7 @@ not imply that every diffusion transformer has been quality-validated.
 | Intel XPU | ❌ |
 | Ascend NPU | ❌ |
-Legend: `✅` backend available, `❌` unsupported. Non-CUDA platforms raise in
+Legend: `✅` supported, `❌` unsupported. Non-CUDA platforms raise in
 `get_quant_method()`; CUDA requires compute capability 7.5+ (`SM 75+`).
 Requires the optional `bitsandbytes` package (`pip install bitsandbytes`).
@@ -34,11 +30,11 @@ Requires the optional `bitsandbytes` package (`pip install bitsandbytes`).
 ### Diffusion Model (Qwen-Image, Wan2.2)
-| Model | HF models | Scope | Level | Recommendation |
+| Model | HF models | CUDA | Mode | Recommendation |
-|-------|-----------|-------|-------|----------------|
+|-------|-----------|:----:|------|----------------|
-| Z-Image | `Tongyi-MAI/Z-Image-Turbo` | Diffusion transformer | Validated | All heavy linear layers; sensitive embedders stay BF16 |
+| Z-Image | `Tongyi-MAI/Z-Image-Turbo` | Yes | Online W4 weight-only | All heavy linear layers; sensitive embedders stay BF16 |
-| Qwen-Image | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Diffusion transformer | Not validated | Compare with BF16 before enabling |
+| Qwen-Image | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Not validated | Online W4 weight-only | Compare vs BF16 before enabling |
-| Wan2.2 | Wan2.2 diffusion pipelines | One or both diffusion transformers | Not validated | Validate quality and cascade behavior before enabling |
+| Wan2.2 | Wan2.2 diffusion pipelines | Not validated | Online W4 weight-only | Validate before enabling in docs |
 Other diffusion models may work if their transformer uses supported linear
 layers, but they are not validated in this guide.
@@ -108,10 +104,6 @@ Multi-GPU tensor parallelism (`tensor_parallel_size` > 1) is not validated for
 BitsAndBytes in diffusion models; each rank quantizes its own weight shard
 independently.
 Z-Image-Turbo is validated outside scheduled full-model BitsAndBytes CI. The
 real CUDA kernel smoke tests cover quantization and matrix multiplication, but
 do not establish model-level quality for unlisted architectures.
 If quality regresses, use `ignored_layers` to keep sensitive projections in BF16
 (for example `to_out` or `w2`).
@@ -13,11 +13,6 @@ components, such as the tokenizer, scheduler, text encoder, vision/audio
 encoder, and VAE, on the base checkpoint unless a model-specific guide says
 otherwise.
 The validation terms on this page follow the
 [support levels](overview.md#support-levels) defined in the overview. ModelOpt
 format detection and checkpoint adaptation are shared runtime integrations;
 each model and stage still requires its own end-to-end validation.
 !!! note
    ModelOpt checkpoints are pre-quantized checkpoints. Do not pass
    `--quantization fp8` for these checkpoints. The checkpoint
@@ -63,11 +58,11 @@ stay unquantized unless the model-specific recipe validates otherwise.
 | Intel XPU | ⭕ |
 | Ascend NPU | ❌ |
-Legend: `✅` backend available, `❌` unsupported, `⭕` not verified in this
+Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this guide.
-guide. The optional CUTLASS FP8 runtime override requires CUDA SM89+. ModelOpt
+The optional CUTLASS FP8 runtime override requires CUDA SM89+. ModelOpt NVFP4
-NVFP4 and mixed FP8/NVFP4 diffusion checkpoints are currently validated on
+and mixed FP8/NVFP4 diffusion checkpoints are currently validated on Blackwell
-Blackwell CUDA systems in the recipes below; other CUDA generations require
+CUDA systems in the recipes below; other CUDA generations require separate
-separate backend and quality validation.
+backend and quality validation.
 ## Model Type Support
@@ -84,9 +79,6 @@ separate backend and quality validation.
 | HunyuanImage-3.0 | `feizhai123/hunyuan-image3-modelopt-mixed-experts-nvfp4-dense-fp8` | MoE diffusion transformer | Validated for ModelOpt mixed FP8/NVFP4 checkpoints |
 | Wan2.2 | Not available | Diffusion transformer | Not validated |
 The validated diffusion entries above have named checkpoint and recipe
 evidence. They are not all exercised by scheduled full-model CI.
 For full serving commands and benchmark context, see
 [`recipes/Qwen/Qwen-Image.md`](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen-Image.md)
 and
@@ -96,8 +88,8 @@ and
 | Model | Scope | Status |
 |-------|-------|--------|
-| Qwen3-Omni | Thinker language-model stage (FP8) | Validated outside scheduled CI |
+| Qwen3-Omni | Thinker language-model stage | ModelOpt FP8 checkpoint path |
-| Qwen3-Omni | Thinker language-model stage (W4A4 NVFP4) | CI-backed for the H100 Marlin fallback; native Blackwell path is recipe-validated |
+| Qwen3-Omni | Thinker language-model stage (W4A4 NVFP4) | Validated; see [Qwen3-Omni NVFP4 W4A4](#qwen3-omni-nvfp4-w4a4-thinker) below |
 | Qwen3-TTS | TTS language-model stage | Not validated |
 Audio encoder, vision encoder, talker, and code2wav stages stay in BF16 unless
@@ -110,9 +102,9 @@ Qwen3-Omni-30B-A3B-Instruct thinker language model. The thinker text body
 (attention + MoE experts) is quantized to NVFP4 with FP8 per-tensor input
 scales; the audio encoder, vision encoder, talker, and code2wav stay in BF16.
-| Variant | HF checkpoint | Hardware and validation |
+| Variant | HF checkpoint | Hardware |
-|---------|---------------|-------------------------|
+|---------|---------------|----------|
-| W4A4 NVFP4 (full thinker) | `YihongJin/Qwen3-Omni-30B-A3B-Instruct-NVFP4-W4A4-full-thinker-awqclip` | sm_100+ native FlashInfer FP4 GEMM; H100 Marlin fallback is CI-backed |
+| W4A4 NVFP4 (full thinker) | `YihongJin/Qwen3-Omni-30B-A3B-Instruct-NVFP4-W4A4-full-thinker-awqclip` | sm_100+ (Blackwell, FlashInfer FP4 GEMM) |
 Calibration uses ModelOpt `mtq.NVFP4_DEFAULT_CFG` with the `awq_clip` algorithm
 on 1024 ultrachat samples chat-templated through the Qwen3-Omni tokenizer.
@@ -240,6 +232,3 @@ omni = Omni(
 5. Keep `--quantization fp8` for online FP8 from BF16 checkpoints; use this
   ModelOpt path only when the checkpoint already contains ModelOpt quantized
   weights and scales.
 6. The Qwen3-Omni full-model CI smoke exercises the H100 Marlin fallback. A
   scheduled Blackwell test is still required before treating the native
   FlashInfer FP4 kernel path as CI-backed.
@@ -3,17 +3,12 @@
 ## Overview
 [msModelSlim](https://github.com/Ascend/msmodelslim) is an Ascend compression
-toolkit for producing pre-quantized model checkpoints. vLLM-Omni consumes its
+toolkit for producing pre-quantized model checkpoints. In vLLM-Omni, these
-output either through a compatible generic Ascend checkpoint path or through
+checkpoints run through the Ascend/NPU path with `--quantization ascend`.
 the native MXFP8 and MXFP4 merge workflows documented in this guide.
 msModelSlim is static quantization: quantized weights are generated offline
 before vLLM-Omni inference starts.
 The validation terms on this page follow the
 [support levels](overview.md#support-levels). A producer recipe or hardware
 backend alone is an integration point, not model-level validation.
 ## Hardware Support
 | Device | Support |
@@ -25,28 +20,23 @@ backend alone is an integration point, not model-level validation.
 | Intel XPU | ❌ |
 | Ascend NPU | ✅ |
-Legend: `✅` backend available, `❌` unsupported, `⭕` not verified in this
+Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
 guide.
 ## Model Type Support
 ### Diffusion Model (Qwen-Image, Wan2.2)
-| Model | Base model | Scope | Level | Notes |
+| Model | Base model | Scope | Hardware | Notes |
-|-------|------------|-------|-------|-------|
+|-------|------------|-------|----------|-------|
-| Wan2.2 | Wan2.2 diffusion weights | DiT or diffusion stage | Integrated | Upstream msModelSlim provides a producer recipe; use the [MXFP8](mxfp8.md) or [MXFP4](mxfp4.md) workflow for vLLM-Omni's native merged formats |
+| Wan2.2 | Wan2.2 diffusion weights | DiT or diffusion stage | Ascend NPU | Upstream msModelSlim provides a Wan2.2 quantization recipe; vLLM-Omni inference validation is not listed |
-| Qwen-Image | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | DiT or diffusion stage | Not validated | No supported checkpoint workflow is documented |
+| Qwen-Image | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | DiT or diffusion stage | Ascend NPU | Not validated in this guide |
-| HunyuanImage-3.0 | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | DiT or diffusion stage | Integrated | Generate quantized weights with the HunyuanImage-3.0 msModelSlim adaptation and validate against BF16 |
+| HunyuanImage-3.0 | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | DiT or diffusion stage | Ascend A2/A3 NPU | Generate quantized weights with the HunyuanImage-3.0 msModelSlim adaptation |
 Public Hugging Face quantized weights are not available yet. Use the
 [HunyuanImage-3.0 msModelSlim adaptation](https://gitcode.com/betta18/msmodelslim/tree/hyimage3_mxfp8)
 to generate the checkpoint manually.
 msModelSlim is also the producer used by the native offline MXFP8 and MXFP4
 DualScale workflows. Those workflows preprocess and annotate each transformer
 checkpoint and are auto-detected at inference; follow their method pages rather
 than passing `--quantization ascend`.
 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
 | Model | Scope | Status | Notes |
@@ -63,32 +53,25 @@ than passing `--quantization ascend`.
 ## Configuration
 For a compatible generic Ascend checkpoint, select the Ascend quantization
 method explicitly:
 Offline inference:
 ```bash
-python text_to_image.py --model <compatible-ascend-checkpoint> --quantization ascend
+python text_to_image.py --model <quantized-model-path> --quantization ascend
 ```
 Online serving:
 ```bash
-vllm serve <compatible-ascend-checkpoint> --omni --quantization ascend
+vllm serve <quantized-model-path> --omni --quantization ascend
 ```
 ## Parameters
 | Parameter | Type | Default | Description |
 |-----------|------|---------|-------------|
-| `--quantization` | str | - | Use `ascend` only for a compatible generic Ascend-format checkpoint; omit it for native merged MXFP8/MXFP4 checkpoints |
+| `--quantization` | str | - | Use `ascend` for msModelSlim-produced checkpoints |
 | `model` | str | - | Path to the quantized checkpoint generated by Ascend tooling |
 Do not pass `--quantization ascend` for a Wan checkpoint processed by the native
 MXFP8 or MXFP4 merge tools. Their transformer metadata selects `mxfp8` or
 `mxfp4_dualscale` automatically.
 Example msModelSlim command for a Wan2.2 W8A8 checkpoint:
 ```bash
@@ -110,6 +93,3 @@ For HunyuanImage-3.0, use the Hunyuan-specific adaptation linked above.
   tooling; it is not a load-time CUDA quantizer.
 3. Keep the quantized checkpoint aligned with the same model architecture and
   stage config used for BF16 inference.
 4. No named full-model msModelSlim checkpoint is currently scheduled in NPU
   CI. Treat new producer outputs as **Integrated** until an end-to-end BF16
   comparison establishes **Validated** support.
@@ -13,10 +13,6 @@ vLLM-Omni provides two quantization methods with different scale structures:
 | `mxfp4` | Single-scale (per-32 fine only) | Online only | Quick accuracy baseline; no checkpoint prep needed |
 | `mxfp4_dualscale` | Dual-scale (fine per-32 + coarse per-512 + per-channel `mul_scale`) | Online + Offline | Production; better accuracy; offline recommended |
 The validation terms on this page follow the
 [support levels](overview.md#support-levels). Current model guidance covers
 Wan diffusion transformers only; the text encoder and VAE stay in BF16.
 !!! tip "Recommended: `mxfp4_dualscale` offline"
    For production deployments, use the `mxfp4_dualscale` offline mode with a
    pre-quantized checkpoint produced by msModelSlim. Offline checkpoints load
@@ -38,45 +34,26 @@ Wan diffusion transformers only; the text encoder and VAE stay in BF16.
 ## Hardware Support
-| Device | Online `mxfp4` | Online `mxfp4_dualscale` | Offline `mxfp4_dualscale` |
+| Device | Support |
-|--------|-----------------|---------------------------|----------------------------|
+|--------|---------|
-| NVIDIA Blackwell GPU (SM 100+) | ❌ | ❌ | ❌ |
+| NVIDIA Blackwell GPU (SM 100+) | ⭕ |
-| NVIDIA Ada/Hopper GPU (SM 89+) | ❌ | ❌ | ❌ |
+| NVIDIA Ada/Hopper GPU (SM 89+) | ⭕ |
-| NVIDIA Ampere GPU (SM 80+) | ❌ | ❌ | ❌ |
+| NVIDIA Ampere GPU (SM 80+) | ⭕ |
-| AMD ROCm (gfx950 / MI355X) | ✅ | ❌ | ❌ |
+| AMD ROCm (gfx950 / MI355X) | ✅ |
-| Intel XPU | ❌ | ❌ | ❌ |
+| Intel XPU | ⭕ |
-| Ascend NPU (Atlas 950 A5) | ✅ | ✅ | ✅ |
+| Ascend NPU (Atlas 950 A5) | ✅ |
-Legend: `✅` backend available, `❌` unsupported, `⭕` not verified in this
+Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this guide.
 guide. Backend availability does not imply full-model checkpoint validation.
 !!! note "Why NVIDIA is marked unsupported"
    This table describes the methods currently wired into vLLM-Omni, not every
    low-precision format NVIDIA hardware could execute. `DiffusionMXFP4Config`
    has runtime implementations only for Ascend and ROCm gfx950; on CUDA it
    raises `NotImplementedError` because no CUDA MXFP4 linear method or
    checkpoint loader is registered.
    On NVIDIA, use [ModelOpt](modelopt.md) NVFP4 for the documented 4-bit
    checkpoint path. NVFP4 and OCP MXFP4 have different checkpoint metadata,
    scale layouts, and kernel contracts, so they are not interchangeable. A
    native CUDA MXFP4 path would require a CUDA linear implementation, loader
    contract, and model-level output validation.
 The ROCm path dispatches online single-scale `mxfp4` to AITER on gfx950. It has
 kernel and dispatch coverage, but this guide does not yet claim end-to-end
 quality validation for a named Wan checkpoint on ROCm. DualScale and
 pre-quantized MXFP4 checkpoints are not supported on ROCm.
 ## Model Type Support
 ### Diffusion Model (Wan2.2)
-| Model | Online (Ascend) | Offline (Ascend) | Validation and notes |
+| Model | Online | Offline | Notes |
-|-------|-----------------|------------------|----------------------|
+|-------|--------|---------|-------|
-| Wan2.2-T2V-A14B | `mxfp4` / `mxfp4_dualscale` | `mxfp4_dualscale` | Validated outside scheduled full-model offline NPU CI; both cascade transformers are quantized |
+| Wan2.2-T2V-A14B | `mxfp4` / `mxfp4_dualscale` | `mxfp4_dualscale` | MoE cascade (`transformer` + `transformer_2`); both transformers quantized with the same config |
-| Wan2.2-I2V-A14B | `mxfp4` / `mxfp4_dualscale` | `mxfp4_dualscale` | Validated outside scheduled full-model offline NPU CI; same cascade scheme as T2V-A14B |
+| Wan2.2-I2V-A14B | `mxfp4` / `mxfp4_dualscale` | `mxfp4_dualscale` | MoE cascade; same scheme as T2V-A14B |
-| Wan2.2-TI2V-5B | ❌ | ❌ | Unsupported because W4A4 causes unacceptable accuracy loss |
+| Wan2.2-TI2V-5B | ❌ | ❌ | Parameter count too small; W4A4 causes unacceptable accuracy loss |
 The choice between `mxfp4` and `mxfp4_dualscale` in **online mode** is about
 quantization quality, not model compatibility — both work on cascade (A14B) and
@@ -106,13 +83,12 @@ by the merge script); there is no offline `mxfp4` single-scale format.
 ### `mxfp4` — Single-Scale Online Mode
-Online mode requires no pre-processing. On Ascend, vLLM-Omni quantizes BF16
+Online mode requires no pre-processing. vLLM-Omni quantizes BF16 weights to
-weights at load time using `npu_dynamic_mx_quant`. On ROCm gfx950, the same
+MXFP4 at load time using `npu_dynamic_mx_quant`. A single block scale
 `mxfp4` config dispatches to the AITER online kernel. A single block scale
 (`float8_e8m0fnu`, one per 32 K elements) is computed on the fly; no
-calibration `mul_scale` is available. The config applies equally to
+calibration `mul_scale` is available. Applies equally to single-transformer
-single-transformer and cascade (A14B) models, but named Wan model validation is
+and cascade (A14B) models — both transformers in a cascade receive the same
-currently limited to the documented Ascend path.
+quantization config automatically.
 ```python
 from vllm_omni import Omni
@@ -135,10 +111,10 @@ vllm serve <your-model> --omni --quantization mxfp4
 ### `mxfp4_dualscale` — DualScale Online Mode
-Ascend-only online DualScale mode computes both fine and coarse scales on the
+Online DualScale mode computes both fine and coarse scales on the fly from BF16
-fly from BF16 weights using `npu_dynamic_dual_level_mx_quant`. It applies
+weights using `npu_dynamic_dual_level_mx_quant`. Applies equally to
-equally to single-transformer and cascade (A14B) models. Compared to `mxfp4`
+single-transformer and cascade (A14B) models. Compared to `mxfp4` online,
-online, DualScale provides better quantization accuracy at higher compute cost.
+DualScale provides better quantization accuracy at higher compute cost.
 The default configuration keeps the leading 5 transformer blocks in BF16
 (`num_bf16_fallback_layers=5`). Accuracy evaluation on Wan2.2-A14B shows this
@@ -365,11 +341,6 @@ unless they appear in `ignored_layers`.
   quantization. Use `ignored_layers` and `num_bf16_fallback_layers` to trade
   off compression vs. accuracy for precision-sensitive layers.
 8. The native methods, merge mappings, and per-transformer cascade metadata
   have unit coverage. The current Buildkite configuration does not schedule a
   named full-model offline MXFP4 checkpoint on NPU, so this path is
   **Validated**, not **CI-backed**.
 ## Adapting MXFP4 for a New Model
 This section is aimed at developers who want to add MXFP4 support to a model
@@ -15,45 +15,30 @@ This method supports three modes:
 | **Offline (Native)** | msModelSlim-exported MXFP8 weights converted to diffusers format via `merge_mxfp8_checkpoint.py` — weights and scales are loaded directly from the preprocessed checkpoint |
 | **Offline (AutoRound)** | AutoRound MXFP8 checkpoints with `data_type="mx_fp"` — auto-detected from `config.json` |
 The validation terms on this page follow the
 [support levels](overview.md#support-levels). Current model guidance covers
 Wan diffusion transformers only; the text encoder and VAE stay in BF16.
 ## Hardware Support
 | Device | Online | Offline (Native) | Offline (AutoRound) |
 |--------|--------|------------------|---------------------|
-| NVIDIA Blackwell GPU (SM 100+) | ❌ | ❌ | ⭕ |
+| NVIDIA Blackwell GPU (SM 100+) | ⭕ | ⭕ | ⭕ |
-| NVIDIA Ada/Hopper GPU (SM 89+) | ❌ | ❌ | ⭕ |
+| NVIDIA Ada/Hopper GPU (SM 89+) | ⭕ | ⭕ | ⭕ |
-| NVIDIA Ampere GPU (SM 80+) | ❌ | ❌ | ⭕ |
+| NVIDIA Ampere GPU (SM 80+) | ⭕ | ⭕ | ⭕ |
-| AMD ROCm | ❌ | ❌ | ⭕ |
+| AMD ROCm | ⭕ | ⭕ | ⭕ |
 | Intel XPU | ✅ | ❌ | ✅ |
 | Ascend NPU (Atlas 950 A5) | ✅ | ✅ | ⭕ |
-Legend: `✅` backend available, `❌` unsupported, `⭕` not verified in this
+Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this guide.
 guide. Backend availability does not imply full-model checkpoint validation.
-The native `DiffusionMXFP8Config` currently dispatches only to Intel XPU and
+**Note**: Intel XPU only supports AutoRound MXFP8 for offline mode. Use AutoRound quantized checkpoints or online mode for XPU.
 Ascend NPU implementations. CUDA reaches `NotImplementedError`, so the NVIDIA
 rows are marked unsupported even though this is an implementation gap rather
 than a general claim about NVIDIA hardware capabilities. On NVIDIA, use
 [FP8](fp8.md) for load-time quantization or [ModelOpt](modelopt.md) FP8 for
 pre-quantized checkpoints. Adding native CUDA MXFP8 requires a registered CUDA
 linear method, a compatible checkpoint contract, and model-level validation.
 Native `mxfp8` is available online on Intel XPU and online or offline on
 Ascend. Intel XPU only supports AutoRound MXFP8 for offline mode. The AutoRound
 column is a separate checkpoint format and compute path.
 ## Model Type Support
 ### Diffusion Model (Wan2.2)
-| Model | Mode | Scope | Validation |
+| Model | Mode | Notes |
-|-------|------|-------|------------|
+|-------|------|-------|
-| Wan2.2-T2V-A14B | Online + Offline | MoE cascade: `transformer` + `transformer_2` | Validated outside scheduled full-model offline NPU CI |
+| Wan2.2-T2V-A14B | Online + Offline | MoE cascade; quantizes two transformers (`transformer` + `transformer_2`) |
-| Wan2.2-I2V-A14B | Online + Offline | MoE cascade: `transformer` + `transformer_2` | Validated outside scheduled full-model offline NPU CI |
+| Wan2.2-I2V-A14B | Online + Offline | MoE cascade; quantizes two transformers (`transformer` + `transformer_2`) |
-| Wan2.2-TI2V-5B | Online + Offline | Single diffusion transformer | Validated outside scheduled full-model offline NPU CI |
+| Wan2.2-TI2V-5B | Online + Offline | Single transformer |
 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
@@ -233,7 +218,3 @@ omni = Omni(model="<autoround-mxfp8-model>")
   `--output-dir`), regenerate it with the current script. The old script
   wrote a separate `quantization_config.json` that is not read by vLLM-Omni;
   the current script injects the config directly into `transformer/config.json`.
 4. The native loader, merge mappings, and Wan cascade config propagation have
   unit coverage. The current Buildkite configuration does not schedule a
   named full-model offline MXFP8 checkpoint on NPU, so this path is
   **Validated**, not **CI-backed**.
@@ -6,7 +6,7 @@ Online quantization means vLLM-Omni computes quantized weights and scales while
 loading the model. Use it when you want memory savings without preparing a
 separate quantized checkpoint.
-This mode is different from pre-quantized checkpoint formats such as ModelOpt,
+This mode is different from pre-quantized checkpoint formats such as GGUF,
 AutoRound, msModelSlim, or serialized Int8 checkpoints. Those formats are
 prepared before serving and are documented in their method-specific guides.
 For MXFP8 and MXFP4, use this page for load-time quantization from BF16
@@ -15,22 +15,18 @@ by msModelSlim and the merge tools.
 ## Hardware Support
-| Device | FP8 W8A8 | Int8 W8A8 | BitsAndBytes W4 | MXFP8 W8A8 | MXFP4 W4A4 |
+| Device | FP8 W8A8 | Int8 W8A8 | MXFP8 W8A8 | MXFP4 W4A4 |
-|--------|----------|-----------|------------------|------------|------------|
+|--------|----------|-----------|------------|------------|
-| NVIDIA Blackwell GPU (SM 100+) | ✅ | ✅ | ✅ | ❌ | ❌ |
+| NVIDIA Blackwell GPU (SM 100+) | ✅ | ✅ | ⭕ | ⭕ |
-| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ | ✅ | ✅ | ❌ | ❌ |
+| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ | ✅ | ⭕ | ⭕ |
-| NVIDIA Ampere GPU (SM 80+) | ✅ | ✅ | ✅ | ❌ | ❌ |
+| NVIDIA Ampere GPU (SM 80+) | ✅ | ✅ | ⭕ | ⭕ |
-| AMD ROCm (gfx950) | ⭕ | ⭕ | ❌ | ❌ | ✅ |
+| AMD ROCm | ⭕ | ⭕ | ⭕ | ⭕ |
-| Intel XPU | ⭕ | ⭕ | ❌ | ✅ | ❌ |
+| Intel XPU | ⭕ | ⭕ | ✅ | ⭕ |
-| Ascend NPU | ❌ | ✅ | ❌ | ✅ | ✅ |
+| Ascend NPU | ❌ | ✅ | ✅ | ✅ |
-Legend: `✅` backend available, `❌` unsupported, `⭕` not verified in this
+Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
-guide. These entries describe method-level backend availability, not validation
+guide. FP8 on Ampere may use a weight-only path where available. MXFP8 and
-for every model. FP8 on Ampere may use a weight-only path where available.
+MXFP4 are documented for the Ascend NPU path.
 On ROCm gfx950, MXFP4 means online single-scale `mxfp4`; online and offline
 `mxfp4_dualscale` are Ascend-only. The native MXFP8/MXFP4 configs do not
 currently register CUDA implementations; use FP8 or ModelOpt FP8/NVFP4 on
 NVIDIA instead.
 ## Model Type Support
@@ -40,9 +36,8 @@ NVIDIA instead.
 |--------|-------|----------------|--------|
 | FP8 W8A8 | [FP8](fp8.md) | Qwen-Image; Wan2.2 is not validated | Validated for Qwen-Image family and other DiT models |
 | Int8 W8A8 | [Int8](int8.md) | Qwen-Image; Wan2.2 is not validated | Validated for Qwen-Image and Z-Image |
 | BitsAndBytes W4 | [BitsAndBytes](bitsandbytes.md) | Z-Image-Turbo; Qwen-Image and Wan2.2 are not validated | Validated for Z-Image-Turbo |
 | MXFP8 W8A8 | [MXFP8](mxfp8.md) | Wan2.2-T2V-A14B, Wan2.2-I2V-A14B, Wan2.2-TI2V-5B | Validated on Ascend NPU and Intel XPU |
-| MXFP4 W4A4 | [MXFP4](mxfp4.md) | Wan2.2-T2V-A14B, Wan2.2-I2V-A14B | Validated on Ascend NPU; the ROCm gfx950 backend is integrated but not model-validated; TI2V-5B is unsupported |
+| MXFP4 W4A4 | [MXFP4](mxfp4.md) | Wan2.2-T2V-A14B, Wan2.2-I2V-A14B | Ascend NPU only; TI2V-5B is not supported |
 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
@@ -64,7 +59,6 @@ from vllm_omni import Omni
 omni_fp8 = Omni(model="<your-model>", quantization="fp8")
 omni_int8 = Omni(model="<your-model>", quantization="int8")
 omni_bnb = Omni(model="Tongyi-MAI/Z-Image-Turbo", quantization="bitsandbytes")
 omni_mxfp8 = Omni(model="<your-model>", quantization="mxfp8")
 omni_mxfp4 = Omni(model="<your-model>", quantization="mxfp4")
 omni_mxfp4_dualscale = Omni(model="<your-model>", quantization="mxfp4_dualscale")
@@ -75,7 +69,6 @@ CLI:
 ```bash
 vllm serve <your-model> --omni --quantization fp8
 vllm serve <your-model> --omni --quantization int8
 vllm serve Tongyi-MAI/Z-Image-Turbo --omni --quantization bitsandbytes
 vllm serve <your-model> --omni --quantization mxfp8
 vllm serve <your-model> --omni --quantization mxfp4
 vllm serve <your-model> --omni --quantization mxfp4_dualscale
@@ -92,19 +85,14 @@ config = build_quant_config({
 })
 ```
 Component routing is an extension mechanism, not automatic support for every
 pipeline component. See [Quantization Scope](overview.md#quantization-scope)
 before applying it to an encoder, VAE, or decoder.
 ## Parameters
 | Parameter | Methods | Description |
 |-----------|---------|-------------|
-| `method` | FP8, Int8, BitsAndBytes, MXFP8, MXFP4 | Quantization method: `"fp8"`, `"int8"`, `"bitsandbytes"`, `"mxfp8"`, `"mxfp4"`, or `"mxfp4_dualscale"` |
+| `method` | FP8, Int8, MXFP8, MXFP4 | Quantization method: `"fp8"`, `"int8"`, `"mxfp8"`, `"mxfp4"`, or `"mxfp4_dualscale"` |
-| `ignored_layers` | FP8, Int8, BitsAndBytes, MXFP8, MXFP4 | Layer name patterns to keep in BF16/FP16 |
+| `ignored_layers` | FP8, Int8, MXFP8, MXFP4 | Layer name patterns to keep in BF16/FP16 |
 | `activation_scheme` | FP8, Int8 | The runtime value `"dynamic"` selects online activation scaling |
 | `weight_block_size` | FP8 | Optional block-wise FP8 weight quantization size |
 | `quant_type` | BitsAndBytes | Weight format: `"nf4"` (default) or `"fp4"` |
 | `num_bf16_fallback_layers` | MXFP4 DualScale | Leading transformer blocks to keep in BF16 for online `mxfp4_dualscale`; defaults to `5` |
 ## Validation and Notes
@@ -16,123 +16,72 @@ For the internal architecture and backend extension points, see the
 | Runtime attention quantization | [Quantized KV Cache](quantized_kvcache.md) | vLLM-Omni dynamically quantizes eligible diffusion Flash Attention tensors during inference. | FP8 FA |
 | Pre-quantized checkpoints | Method-specific guides | The checkpoint or an offline quantizer provides quantized weights and scales before serving. | ModelOpt, AutoRound, msModelSlim, serialized Int8, offline MXFP8, offline MXFP4 DualScale |
-Online quantization starts from a normal BF16/FP16 checkpoint and repeats the
+## Hardware Support
 conversion on each fresh model load. It is the simplest path for experiments.
 Pre-quantized checkpoints store the packed weights and scales produced by a
 separate tool; they are easier to reuse across deployments and can carry
 calibrated or mixed-precision policies that load-time conversion cannot
 reproduce.
-## Support Levels
+| Device | FP8 W8A8 | Int8 W8A8 | BitsAndBytes W4 | ModelOpt | MXFP8 W8A8 | MXFP4 W4A4 | AutoRound | msModelSlim |
 |--------|----------|-----------|------------------|----------|------------|------------|-----------|-------------|
 | NVIDIA Blackwell GPU (SM 100+) | ✅ | ✅ | ✅ | ✅ | ⭕ | ⭕ | ✅ | ❌ |
 | NVIDIA Ada/Hopper GPU (SM 89+) | ✅ | ✅ | ✅ | ✅ | ⭕ | ⭕ | ✅ | ❌ |
 | NVIDIA Ampere GPU (SM 80+) | ✅ | ✅ | ✅ | ⭕ | ⭕ | ⭕ | ✅ | ❌ |
 | AMD ROCm | ⭕ | ⭕ | ❌ | ⭕ | ⭕ | ✅ | ⭕ | ❌ |
 | Intel XPU | ⭕ | ⭕ | ❌ | ⭕ | ⭕ | ⭕ | ✅ | ❌ |
 | Ascend NPU | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ |
-Quantization support is specific to a model, component, checkpoint format, and
+Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
-hardware backend. The tables in this guide use the following levels:
+guide. FP8 on Ampere may use a weight-only path where available.
-| Level | Meaning |
+## Model Type Support
 |-------|---------|
 | **CI-backed** | A named model and quantized checkpoint are exercised by scheduled full-model hardware CI. |
 | **Validated** | A named model and checkpoint have end-to-end or quality-validation evidence, but the check is not necessarily scheduled in CI. |
 | **Integrated** | The config, loader, quantized layers, or model wiring exists, but this guide has no completed end-to-end validation for the named model and checkpoint. |
 | **Not validated** | There is no current support recommendation for this model/component combination. |
 | **Unsupported** | The combination is intentionally rejected or known not to work. |
-A method being available on a device does not make every model on that device
+### Diffusion Model (Qwen-Image, Wan2.2)
 supported. Likewise, successfully detecting a checkpoint's
 `quantization_config` establishes loader compatibility, not output quality.
-## Hardware × Quantization Method
+These models run a diffusion transformer as the primary inference module. The
 default quantization target is the transformer; tokenizer, scheduler, text
 encoder, and VAE stay on the base checkpoint unless a method guide says
 otherwise.
-This matrix uses concrete inference paths as columns. Producer toolkits are not
+| Method | Guide | Mode | Example models | Status |
-separate methods: NVIDIA ModelOpt produces the ModelOpt FP8/NVFP4 formats,
+|--------|-------|------|----------------|--------|
-Intel AutoRound produces AutoRound checkpoints, and Ascend msModelSlim produces
+| FP8 W8A8 | [FP8](fp8.md) | Online W8A8 or checkpoint FP8 | Qwen-Image; Wan2.2 is not validated | Validated for Qwen-Image family and other DiT models |
-checkpoints consumed by the native MXFP or compatible Ascend paths.
+| Int8 W8A8 | [Int8](int8.md) | Online or serialized W8A8 | Qwen-Image; Wan2.2 is not validated | Validated for Qwen-Image and Z-Image |
 | BitsAndBytes W4 | [BitsAndBytes](bitsandbytes.md) | Online W4 weight-only | Z-Image-Turbo; Qwen-Image/Wan2.2 not validated | Validated for Z-Image-Turbo |
 | ModelOpt | [ModelOpt](modelopt.md) | Pre-quantized FP8 checkpoints | Qwen-Image, Z-Image, FLUX.2, HunyuanImage-3.0 | Validated for ModelOpt FP8 diffusion checkpoints |
 | MXFP8 W8A8 | [MXFP8](mxfp8.md) | Online W8A8 or offline pre-quantized | Wan2.2-T2V-A14B, I2V-A14B, TI2V-5B | Ascend NPU only; validated for Wan2.2 |
 | MXFP4 W4A4 | [MXFP4](mxfp4.md) | `mxfp4`: online single-scale only; `mxfp4_dualscale`: online or offline dual-scale (offline recommended) | Wan2.2-T2V-A14B, I2V-A14B | Ascend NPU only; validated for Wan2.2 A14B cascade models; TI2V-5B not supported; offline `mxfp4_dualscale` uses calibrated `mul_scale` for best accuracy |
 | AutoRound | [AutoRound](autoround.md) | Pre-quantized W4A16 checkpoints | FLUX.1-dev; Qwen-Image/Wan2.2 not validated | Checkpoint-driven |
 | msModelSlim | [msModelSlim](msmodelslim.md) | Pre-quantized Ascend checkpoints | Wan2.2 recipe; HunyuanImage-3.0 inference target | Ascend/NPU path |
-| Hardware | [Online FP8](fp8.md) | [Int8](int8.md) | [BitsAndBytes W4](bitsandbytes.md) | [ModelOpt FP8](modelopt.md) | [ModelOpt NVFP4](modelopt.md#supported-modelopt-checkpoint-formats) | [AutoRound W4A16](autoround.md) | [OCP MXFP8](mxfp8.md) | [OCP MXFP4](mxfp4.md) |
+### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
 |----------|-----------------------|------------------|------------------------------------------|--------------------------------|-------------------------------------------------------------------------|--------------------------------------|--------------------------|--------------------------|
 | NVIDIA Blackwell (SM 100+) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 | NVIDIA Hopper (SM 90) | ✅ | ✅ | ✅ | ✅ | ✅ (Marlin fallback) | ✅ | ❌ | ❌ |
 | NVIDIA Ada (SM 89) | ✅ | ✅ | ✅ | ✅ | ⭕ | ✅ | ❌ | ❌ |
 | NVIDIA Ampere (SM 80+) | ✅ | ✅ | ✅ | ⭕ | ⭕ | ✅ | ❌ | ❌ |
 | AMD ROCm (gfx950 for MXFP4) | ⭕ | ⭕ | ❌ | ⭕ | ⭕ | ⭕ | ❌ | ✅ (online only) |
 | Intel XPU | ⭕ | ⭕ | ❌ | ⭕ | ⭕ | ✅ | ✅ | ❌ |
 | Ascend NPU | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ |
-Legend: `✅` backend available, `❌` rejected or no implementation, `⭕` not
+These models combine an AR language model with audio, vision, talker, or TTS
-verified in this guide. This matrix does not establish model-level support; use
+stages. Quantization is scoped to the AR language-model stage when the
-the model-and-stage tables below for that.
+checkpoint contains a supported `quantization_config`; the non-AR stages stay
 in BF16 unless the model guide explicitly adds support.
-The Int8 column covers online and serialized Int8 paths. OCP MXFP8 is online on
+| Method | Guide | Scope | Example models | Status |
-Intel XPU and online/offline on Ascend; Intel's separate AutoRound MXFP8
+|--------|-------|-------|----------------|--------|
-checkpoint path is described in the [MXFP8 guide](mxfp8.md). OCP MXFP4 on AMD
+| ModelOpt | [ModelOpt](modelopt.md) | Thinker or language-model checkpoint config | Qwen3-Omni thinker | ModelOpt checkpoint path |
-means online single-scale `mxfp4` on gfx950, while MXFP4 DualScale is
+| Int8 | [Int8](int8.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
-Ascend-only. NVIDIA's NVFP4 and OCP MXFP4 have different checkpoint metadata,
+| BitsAndBytes | [BitsAndBytes](bitsandbytes.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
-scale layouts, and kernel contracts. Mixed ModelOpt FP8/NVFP4 checkpoints also
+| MXFP8 | [MXFP8](mxfp8.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
-require a model-specific mixed-precision policy and validation. FP8 on Ampere
+| MXFP4 | [MXFP4](mxfp4.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
-may use a weight-only path where available.
+| AutoRound | [AutoRound](autoround.md) | Thinker or language-model checkpoint config | Qwen2.5-Omni, Qwen3-Omni | Supported through AutoRound checkpoints |
 | msModelSlim | [msModelSlim](msmodelslim.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
-## Current Model and Stage Support
+### Multi-Stage Diffusion Model (BAGEL, GLM-Image)
-The current offline checkpoint guidance is transformer-first. A listed
+These models split generation across multiple stages. Quantization must be
-checkpoint quantizes the stage named in the **Scope** column; it does not imply
+attached to the intended stage rather than applied globally.
 that the model's encoders, VAE, decoder, or other stages are also quantized.
-### Vendor and Runtime Ecosystems
+| Method | Guide | Scope | Example models | Status |
-
+|--------|-------|-------|----------------|--------|
-The producer and runtime backend are related, but they are not the same support
+| FP8 | [FP8](fp8.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Requires model-specific validation |
-claim. A producer can emit several checkpoint formats, and each format still
+| Int8 | [Int8](int8.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Requires model-specific validation |
-needs a compatible vLLM-Omni loader, model-stage wiring, and validation. A
+| BitsAndBytes | [BitsAndBytes](bitsandbytes.md) | Stage-specific transformer or DiT module | BAGEL, GLM-Image | Not validated |
-runtime may also provide online quantization without an offline producer path.
+| ModelOpt | [ModelOpt](modelopt.md) | Checkpoint-defined diffusion stage | BAGEL, GLM-Image | Requires model-specific validation |
-
+| MXFP8 | [MXFP8](mxfp8.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Not validated |
-| Ecosystem | Current paths | Current vLLM-Omni scope | Model evidence |
+| MXFP4 | [MXFP4](mxfp4.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Not validated |
-|-----------|---------------|-------------------------|----------------|
+| AutoRound | [AutoRound](autoround.md) | Checkpoint-defined stage | BAGEL, GLM-Image | No validated checkpoint listed |
-| NVIDIA | ModelOpt FP8, NVFP4, and mixed FP8/NVFP4 | Diffusion transformer or Qwen-Omni thinker | Qwen-Image and other named diffusion checkpoints are validated; Qwen3-Omni NVFP4 has H100 CI for the Marlin fallback |
+| msModelSlim | [msModelSlim](msmodelslim.md) | Ascend-generated stage weights | GLM-Image | Requires model-specific adaptation |
 | Intel | AutoRound W4A16 and AutoRound MXFP8 | Diffusion/world-model transformer or Qwen-Omni thinker | Wan A14B and GLM-Image W4A16 are CI-backed; Qwen-Image, Qwen-Omni, and Cosmos3-Super have validation outside scheduled CI |
 | AMD ROCm | Online OCP MXFP4 through AITER on gfx950; no pre-quantized MXFP4 or AMD Quark workflow is integrated | Diffusion transformer linear layers | Integrated backend with dispatch and weight-allocation unit coverage; no named-model quality validation or scheduled gfx950 full-model CI |
 | Ascend | msModelSlim output through native MXFP8/MXFP4 merge workflows or a compatible generic Ascend format | Wan diffusion transformers | Native Wan MXFP8/MXFP4 paths are validated outside scheduled full-model NPU CI; other model families remain integrated or unvalidated |
 ### Pre-Quantized Model Matrix
 | Model family | Method | Scope | Level | Notes |
 |--------------|--------|-------|-------|-------|
 | Qwen-Image | [ModelOpt](modelopt.md) FP8 or mixed FP8/NVFP4 | Diffusion transformer | Validated | Named Qwen-Image 2512 checkpoints and recipes are available. Text encoder and VAE stay BF16. |
 | Qwen-Image | [AutoRound](autoround.md) W4A16 | Diffusion transformer | Validated | Uses `INC4AI/Qwen-Image-AutoRound-W4A16`; text encoder and VAE stay BF16. |
 | Wan2.2 I2V/T2V A14B | [AutoRound](autoround.md) W4A16 | Both diffusion transformers | CI-backed | The high-noise and low-noise DiTs are quantized; text encoder and VAE stay BF16. |
 | Wan2.2 TI2V 5B | [AutoRound](autoround.md) W4A16 | Diffusion transformer | Validated | Named checkpoint available; not covered by the scheduled Wan AutoRound job. |
 | Wan2.2 | [MXFP8](mxfp8.md) | One or both diffusion transformers | Validated | Native Ascend and AutoRound checkpoint formats are integrated; no scheduled full-model offline NPU job. |
 | Wan2.2 I2V/T2V A14B | [MXFP4 DualScale](mxfp4.md) | Both diffusion transformers | Validated | Offline calibrated checkpoints are recommended; no scheduled full-model offline NPU job. |
 | Qwen2.5-Omni | [AutoRound](autoround.md) W4A16 | Thinker language model | Validated | Encoders and output/audio stages stay BF16. |
 | Qwen3-Omni | [ModelOpt](modelopt.md) FP8 | Thinker language model | Validated | Tested outside scheduled CI. |
 | Qwen3-Omni | [ModelOpt](modelopt.md) NVFP4 | Thinker language model | CI-backed | H100 CI exercises the Marlin fallback; the native Blackwell FP4 path has separate recipe validation. |
 | Qwen3-Omni | [AutoRound](autoround.md) W4A16 | Thinker language model | Validated | Audio encoder, vision encoder, talker, and Code2Wav stay BF16. |
 | GLM-Image | [AutoRound](autoround.md) W4A16 | Diffusion transformer stage | CI-backed | Text-to-image and image-to-image full-model smoke coverage. |
 | Cosmos3-Super | [AutoRound](autoround.md) W4A16 | World-model transformer | Validated | Manually validated serving path; VAE and other components stay BF16. |
 | Cosmos3-Nano | [AutoRound](autoround.md) W4A16 | World-model transformer | Integrated | Named checkpoint is listed, but no model-specific end-to-end test is maintained in-tree. |
 | MiniMax H3 | None | None | Not validated | No offline vendor checkpoint path is documented; online FP8 targets the DiT only. |
 The method pages contain the complete checkpoint lists, including FLUX,
 Z-Image, and HunyuanImage-3.0 variants.
 ### Online Quantization
 | Model family | Methods | Scope | Level |
 |--------------|---------|-------|-------|
 | Qwen-Image | FP8, Int8 | Diffusion transformer | Validated |
 | Z-Image | Int8, BitsAndBytes W4 | Diffusion transformer | Validated |
 | Wan2.2 | MXFP8 | One or both diffusion transformers | Validated on the documented XPU/NPU paths |
 | Wan2.2 A14B | MXFP4 | Both diffusion transformers | Validated on Ascend; the ROCm gfx950 online backend is integrated but not model-validated |
 | MiniMax H3 | FP8 | DiT and optional reference DiT | Validated |
 | Qwen3-Omni | Generic online methods, such as FP8 | Thinker language model only | Integrated; not model-validated as an online recommendation |
 See [Online Quantization](online.md) for configuration and hardware details.
 ### Unlisted and Long-Tail Models
 For models not listed above, generic quantized linear layers or compatible
 checkpoint metadata are only an integration starting point. The default support
 target is the model's primary transformer, DiT, or AR language-model stage.
 Text/audio/vision encoders, VAEs, vocoders, decoders, and output stages remain
 BF16 until a model-specific integration supplies quant-aware layers, checkpoint
 mapping, end-to-end output validation, and backend coverage.
 !!! note
    "Online quantization" means vLLM-Omni computes the quantization data while
@@ -143,8 +92,8 @@ mapping, end-to-end output validation, and backend coverage.
 ### Diffusion Model (Qwen-Image, Wan2.2)
-The default target is the diffusion transformer. `build_quant_config()` can
+The default target is the diffusion transformer. Component routing is available
-construct a component router:
+through `build_quant_config()`:
 ```python
 from vllm_omni.quantization import build_quant_config
@@ -155,19 +104,11 @@ config = build_quant_config({
 })
 ```
 !!! warning "Routing syntax is not a support claim"
    A component dictionary only selects a quantization config by module prefix.
    It does not make an encoder or VAE quantizable. The model must construct
    that component from quantization-aware layers, and an offline checkpoint
    must contain matching packed weights and scales. Current user guidance
    keeps diffusion text encoders and VAEs in BF16.
 | Component | Default quantized? | Notes |
 |-----------|--------------------|-------|
-| Diffusion transformer | Yes, for listed model/method pairs | Primary target for FP8, Int8, BitsAndBytes, ModelOpt, MXFP8, MXFP4, AutoRound, and msModelSlim |
+| Diffusion transformer | Yes | Primary target for FP8, Int8, BitsAndBytes, ModelOpt, MXFP8, MXFP4, AutoRound, and msModelSlim |
 | Second diffusion transformer | Yes, for listed Wan2.2 A14B methods | `transformer` and `transformer_2` read their own offline checkpoint metadata; arbitrary mixed methods are not currently validated |
 | Text encoder | No | Keep BF16 unless a method-specific guide documents support |
-| VAE | No | Keep BF16 |
+| VAE | No | Keep BF16; storage-only paths are method-specific |
 | Scheduler/tokenizer | No | Loaded from the base model repository |
 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
@@ -180,18 +121,6 @@ config = build_quant_config({
 | Talker or TTS stage | No | BF16 unless model-specific support is documented |
 | Code2Wav | No | BF16 |
 An explicit `ComponentQuantizationConfig` can be used by model developers to
 route different configs to `audio_tower`, `visual`, and `language_model`, but
 the supported Qwen-Omni checkpoints currently quantize only the thinker
 language model.
 ### World Model (Cosmos3)
 | Component | Default quantized? | Notes |
 |-----------|--------------------|-------|
 | World-model transformer | AutoRound checkpoints only | Cosmos3-Super is validated; Cosmos3-Nano is integrated but not end-to-end tested in-tree |
 | VAE and guardrail components | No | BF16 |
 ### Multi-Stage Diffusion Model (BAGEL, GLM-Image)
 | Component | Default quantized? | Notes |
@@ -212,7 +141,6 @@ build_quant_config("fp8")
 build_quant_config({"method": "fp8", "activation_scheme": "static"})
 build_quant_config("bitsandbytes")
 build_quant_config("auto-round", bits=4, group_size=128)
 # Component routing syntax; model-specific support is still required.
 build_quant_config({"transformer": {"method": "fp8"}, "vae": None})
 build_quant_config(None)
 ```
@@ -131,7 +131,7 @@ def build_sampling_params() -> list[SamplingParams]:
 def save_stage_output(stage_outputs, output_dir: str) -> None:
-    output = stage_outputs
+    output = stage_outputs.request_output
    request_id = output.request_id
    if stage_outputs.final_output_type == "text":
        text_output = output.outputs[0].text
@@ -104,7 +104,7 @@ def main(args):
    os.makedirs(output_dir, exist_ok=True)
    for stage_outputs in omni_outputs:
-        output = stage_outputs
+        output = stage_outputs.request_output
        if stage_outputs.final_output_type == "text":
            request_id = output.request_id
            text_output = output.outputs[0].text
@@ -228,7 +228,7 @@ async def main():
    if not outputs:
        raise ValueError("No output produced from omni.generate()")
-    first_out = outputs[0]
+    first_out = outputs[0].request_output
    req_out: OmniRequestOutput = first_out
    # Verify trajectory data (from custom pipeline)
@@ -741,7 +741,7 @@ def make_mmu_prompt(
 def iter_mm_outputs(outputs: list[Any]):
    for omni_out in outputs:
-        req_out = omni_out
+        req_out = getattr(omni_out, "request_output", None)
        req_list = req_out if isinstance(req_out, list) else [req_out]
        for item in req_list:
            if item is None:
@@ -303,7 +303,7 @@ def main():
    )
    img_idx = 0
    for req_output in omni_outputs:
-        ro = req_output
+        ro = getattr(req_output, "request_output", None)
        stage_id = getattr(req_output, "stage_id", None)
        # AR stage text — each CompletionOutput.text is already a delta when
@@ -621,7 +621,7 @@ def main():
        images = getattr(output, "images", None)
        if images:
            break
-        req_out = output
+        req_out = getattr(output, "request_output", None)
        images = getattr(req_out, "images", None) if req_out is not None else None
        if images:
            break
@@ -646,8 +646,8 @@ def main():
        if frames.multimodal_output and "audio" in frames.multimodal_output:
            audio = frames.multimodal_output["audio"]
            audio_sample_rate = frames.multimodal_output.get("audio_sample_rate", audio_sample_rate)
-        if frames.is_pipeline_output and frames is not None:
+        if frames.is_pipeline_output and frames.request_output is not None:
-            inner_output = frames
+            inner_output = frames.request_output
            if isinstance(inner_output, OmniRequestOutput):
                if inner_output.multimodal_output and "audio" in inner_output.multimodal_output:
                    audio = inner_output.multimodal_output["audio"]
@@ -1,117 +0,0 @@
 """
 Offline inference example: MammothModa2 image summarization (single AR stage).
 Example:
  uv run python examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py \
    --model path/to/MammothModa2-Preview \
    --deploy-config vllm_omni/deploy/mammoth_moda2_ar.yaml \
    --image /path/to/input.jpg \
    --question "Please summarize the content of this image."
 """
 from __future__ import annotations
 import argparse
 import os
 from PIL import Image
 from vllm import SamplingParams
 from vllm.multimodal.image import convert_image_mode
 from vllm_omni import Omni
 DEFAULT_SYSTEM = "You are a helpful assistant."
 DEFAULT_QUESTION = "Please summarize the content of this image."
 def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="MammothModa2 image summarization (offline, AR only).")
    parser.add_argument("--model", type=str, required=True, help="Path to model directory or model id.")
    parser.add_argument("--deploy-config", type=str, required=True, help="Path to deploy config yaml.")
    parser.add_argument("--image", type=str, required=True, help="Path to input image.")
    parser.add_argument("--question", type=str, default=DEFAULT_QUESTION, help="Question/instruction for the model.")
    parser.add_argument("--system", type=str, default=DEFAULT_SYSTEM, help="System prompt.")
    parser.add_argument(
        "--max-tokens",
        type=int,
        default=512,
        help="Max new tokens to generate.",
    )
    parser.add_argument("--temperature", type=float, default=0.2)
    parser.add_argument("--top-p", type=float, default=0.9)
    parser.add_argument("--seed", type=int, default=42)
    parser.add_argument("--trust-remote-code", action="store_true")
    parser.add_argument(
        "--enable-diffusion-pipeline-profiler",
        action="store_true",
        help="Enable diffusion pipeline profiler to display stage durations.",
    )
    return parser.parse_args()
 def build_prompt(system: str, question: str) -> str:
    return (
        f"<|im_start|>system\n{system}<|im_end|>\n"
        "<|im_start|>user\n"
        "<|vision_start|><|image_pad|><|vision_end|>"
        f"{question}<|im_end|>\n"
        "<|im_start|>assistant\n"
    )
 def main() -> None:
    args = parse_args()
    if not os.path.exists(args.image):
        raise FileNotFoundError(f"Image file not found: {args.image}")
    pil_image = Image.open(args.image)
    image_data = convert_image_mode(pil_image, "RGB")
    prompt = build_prompt(args.system, args.question)
    omni = Omni(
        model=args.model,
        deploy_config=args.deploy_config,
        trust_remote_code=args.trust_remote_code,
        enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
    )
    try:
        sp = SamplingParams(
            temperature=float(args.temperature),
            top_p=float(args.top_p),
            top_k=-1,
            max_tokens=int(args.max_tokens),
            seed=int(args.seed),
            detokenize=True,
        )
        # NOTE: omni.generate() returns a Generator[OmniRequestOutput, None, None].
        # Consume it inside the try block so the worker isn't closed early.
        outputs = list(
            omni.generate(
                [
                    {
                        "prompt": prompt,
                        "multi_modal_data": {"image": image_data},
                        "additional_information": {"omni_task": ["chat"]},
                    }
                ],
                [sp],
            )
        )
    finally:
        omni.close()
    lines: list[str] = []
    for stage_outputs in outputs:
        ro = stage_outputs
        text = ro.outputs[0].text if getattr(ro, "outputs", None) else str(ro)
        lines.append(f"request_id: {getattr(ro, 'request_id', 'unknown')}\n")
        lines.append("answer:\n")
        lines.append(text.strip() + "\n")
        lines.append("\n")
    print("\n".join(lines))
 if __name__ == "__main__":
    main()
@@ -297,7 +297,7 @@ def main(args):
    os.makedirs(output_dir, exist_ok=True)
    for stage_outputs in omni_outputs:
-        output = stage_outputs
+        output = stage_outputs.request_output
        if stage_outputs.final_output_type == "text":
            request_id = output.request_id
            text_output = output.outputs[0].text
@@ -347,7 +347,7 @@ def main(args):
        omni.start_profile(stages=args.profiler_stages)
    for stage_outputs in omni.generate(prompts, sampling_params_list):
-        output = stage_outputs
+        output = stage_outputs.request_output
        if stage_outputs.final_output_type == "text":
            request_id = output.request_id
            text_output = output.outputs[0].text
@@ -1,229 +0,0 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 """Offline speech-to-speech with NVIDIA-NemotronLabs-VoiceChat-11B.
 Feeds a 16 kHz user utterance (plus a spoken-style system prompt) through the
 3-stage nemotron_voicechat pipeline and writes the agent's reply as text and a
 22.05 kHz WAV.
 Usage:
    python end2end.py \
        --checkpoint /path/to/NVIDIA-NemotronLabs-VoiceChat-11B \
        --wav /path/to/user_question_16k.wav \
        --output-dir results/nemotron_voicechat
 The system prompt defaults to the NeMo reference default. The text tokenizer
 resolves from the ``nvidia/NVIDIA-Nemotron-Nano-9B-v2`` HF id (or the
 ``NEMOTRON_VOICECHAT_LLM_PATH`` env var for air-gapped runs).
 Sizing the input WAV: the timeline is frame-locked, so the reply budget IS the
 input duration — the acoustic channel trails the text channel, and a reply
 that has not finished when the frames run out is truncated SILENTLY (no error;
 ASR of the WAV will just disagree with the emitted text). Leave generous
 trailing silence after the question (a question ending at ~4.5 s truncated in
 an 8 s WAV but completed cleanly in a 16 s one). The script warns when the
 text channel is still emitting non-PAD tokens near the last frame.
 """
 from __future__ import annotations
 import argparse
 import json
 import os
 from pathlib import Path
 import numpy as np
 import soundfile as sf
 # Warn if the text channel is still speaking within this many final frames
 # (12 frames = ~1 s) — the frame-locked reply likely got cut off.
 TRUNCATION_GUARD_FRAMES = 12
 # The NeMo reference offline script's default system prompt.
 DEFAULT_SYSTEM_PROMPT = (
    "You are an AI voice assistant developed by NVIDIA. "
    "Your name is NVIDIA Voice Chat. "
    "Answer in a spoken, conversational style rather than a written one. "
    "Do not repeat the same sentence over and over again. "
    "Start the conversation by greeting the user."
 )
 def load_wav_16k_mono(path: str) -> np.ndarray:
    wav, sr = sf.read(path, dtype="float32")
    if wav.ndim > 1:
        wav = wav.mean(axis=1)
    if sr != 16000:
        from vllm.multimodal.audio import resample_audio_scipy
        wav = resample_audio_scipy(wav, orig_sr=sr, target_sr=16000)
    return wav.astype(np.float32)
 def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--checkpoint", required=True, help="NVIDIA-NemotronLabs-VoiceChat-11B directory")
    parser.add_argument("--wav", required=True, help="user speech (any sr; resampled to 16 kHz mono)")
    parser.add_argument("--output-dir", default="results/nemotron_voicechat")
    parser.add_argument("--system-prompt", default=DEFAULT_SYSTEM_PROMPT)
    parser.add_argument(
        "--stage-configs-path",
        default=None,
        help=(
            "Deploy yaml override. Pass vllm_omni/deploy/nemotron_labs_voicechat_streaming.yaml "
            "for the async-chunk streaming pipeline (stages overlap; audio starts decoding "
            "before the thinker finishes). Default: the bundled non-streaming yaml (bit-exact "
            "parity path)."
        ),
    )
    args = parser.parse_args()
    if not Path(args.wav).is_file():
        raise FileNotFoundError(f"Input WAV not found: {args.wav}")
    ckpt = Path(args.checkpoint)
    raw_cfg = json.loads((ckpt / "config.json").read_text())
    stt_cfg = raw_cfg["model"]["stt"]["model"]
    from transformers import AutoTokenizer
    from vllm.sampling_params import SamplingParams
    from vllm_omni.entrypoints.omni import Omni
    from vllm_omni.model_executor.models.nemotron_voicechat.nemotron_voicechat_thinker import (
        compute_acoustic_frame_count,
    )
    tok_ref = os.environ.get("NEMOTRON_VOICECHAT_LLM_PATH") or stt_cfg.get(
        "pretrained_llm", "nvidia/NVIDIA-Nemotron-Nano-9B-v2"
    )
    tokenizer = AutoTokenizer.from_pretrained(tok_ref, trust_remote_code=False)
    bos_id = tokenizer.convert_tokens_to_ids(stt_cfg.get("bos_token", "<s>"))
    eos_id = tokenizer.convert_tokens_to_ids(stt_cfg.get("eos_token", "</s>"))
    pad_id = tokenizer.convert_tokens_to_ids(stt_cfg.get("pad_token", "<SPECIAL_12>"))
    # NeMo encode_system_prompt: [bos] + tokens + [eos].
    prompt_ids = [bos_id] + tokenizer.encode(args.system_prompt, add_special_tokens=False) + [eos_id]
    wav = load_wav_16k_mono(args.wav)
    n_frames = compute_acoustic_frame_count(stt_cfg, int(wav.shape[0]))
    print(f"input: {wav.shape[0]} samples @16kHz -> {n_frames} acoustic frames; prompt {len(prompt_ids)} tokens")
    inputs = {
        # vLLM prompt = system prompt + one placeholder position (acoustic
        # frame 0); the thinker replaces every position with fused embeddings.
        "prompt_token_ids": prompt_ids + [pad_id],
        "additional_information": {
            # Plain float list: raw ndarrays do not survive the engine-core
            # message serialization.
            "nvc_audio": wav.astype(np.float32).tolist(),
            "nvc_sr": 16000,
            "nvc_prompt_token_ids": prompt_ids,
            "nvc_expected_frames": n_frames,
        },
    }
    sampling_params_list = [
        # thinker: greedy, frame-locked generation length. NO min_tokens: the
        # tokenizer's eos_token is <SPECIAL_12> — the frame-locked PAD/silence
        # token — and vLLM's min_tokens masks "EOS" to -inf, which would forbid
        # silence and force the agent to babble through the whole utterance.
        SamplingParams(temperature=0.0, max_tokens=n_frames, ignore_eos=True, seed=0),
        # talker: placeholder loop, stops on the stage's stop token. 16383:
        # the 1-token placeholder prompt + max_tokens must fit the stage's
        # max_model_len (16384).
        SamplingParams(temperature=0.0, max_tokens=16383, stop_token_ids=[1], detokenize=False, seed=0),
        # code2wav: single decode step.
        SamplingParams(temperature=0.0, max_tokens=1, detokenize=False, seed=0),
    ]
    omni_kwargs: dict = {}
    if args.stage_configs_path:
        omni_kwargs["stage_configs_path"] = args.stage_configs_path
    omni = Omni(model=str(ckpt), trust_remote_code=True, **omni_kwargs)
    outputs = omni.generate(inputs, sampling_params_list)
    out_dir = Path(args.output_dir)
    out_dir.mkdir(parents=True, exist_ok=True)
    stem = Path(args.wav).stem
    agent_text = None
    audio = None
    sr = 22050
    for stage_outputs in outputs:
        # One entry per final stage (text from the thinker, audio from code2wav);
        # mirrors the qwen2_5_omni offline example's collection loop.
        output = getattr(stage_outputs, "request_output", stage_outputs)
        final_type = getattr(stage_outputs, "final_output_type", None)
        completions = getattr(output, "outputs", None) or []
        if not completions:
            continue
        completion = completions[0]
        if final_type in (None, "text"):
            text = getattr(completion, "text", None)
            if text:
                agent_text = text
            token_ids = getattr(completion, "token_ids", None)
            if token_ids:
                (out_dir / f"{stem}_text_tokens.json").write_text(json.dumps(list(map(int, token_ids))))
                # Frame lock: the reply budget IS the input length. If the text
                # channel is still mid-sentence when the frames run out, the
                # spoken answer is silently truncated — warn instead.
                tail = list(map(int, token_ids))[-TRUNCATION_GUARD_FRAMES:]
                if any(t != pad_id for t in tail):
                    print(
                        f"WARNING: the text channel emits non-PAD tokens within the last "
                        f"{TRUNCATION_GUARD_FRAMES} frames ({TRUNCATION_GUARD_FRAMES * 80} ms) — the reply "
                        "likely ran out of acoustic frames and is cut off mid-sentence. "
                        "Pad the input WAV with more trailing silence and rerun."
                    )
            # Best-effort: dump the function-channel debug timeline if the
            # engine surfaced the request info on this output.
            info = getattr(output, "additional_information", None)
            func_timeline = info.get("nvc_function_tokens") if isinstance(info, dict) else None
            if func_timeline is not None:
                values = func_timeline.tolist() if hasattr(func_timeline, "tolist") else list(func_timeline)
                (out_dir / f"{stem}_function_tokens.json").write_text(json.dumps([int(v) for v in values]))
                print(f"function-channel timeline: {len(values)} entries")
        if final_type in (None, "audio"):
            # multimodal_output is a MultimodalPayload (Mapping) — duck-type it.
            mm = getattr(completion, "multimodal_output", None)
            get = getattr(mm, "get", None)
            if not callable(get):
                continue
            candidate = None
            for key in ("audio", "model_outputs", "wav", "waveform"):
                value = get(key)
                if value is not None:
                    candidate = value
                    break
            if isinstance(candidate, list):
                candidate = candidate[0] if candidate else None
            if candidate is not None:
                arr = np.asarray(
                    candidate.float().cpu() if hasattr(candidate, "cpu") else candidate, dtype=np.float32
                ).reshape(-1)
                if arr.size > 0:
                    audio = arr
                sr_val = get("sr")
                if sr_val is not None:
                    sr_item = sr_val[0] if isinstance(sr_val, list) else sr_val
                    sr = int(sr_item.item() if hasattr(sr_item, "item") else sr_item)
    if agent_text is not None:
        # The frame-locked text channel emits PAD tokens on silent frames;
        # strip them (NeMo's post-inference detokenizer does the same).
        pad_token = stt_cfg.get("pad_token", "<SPECIAL_12>")
        agent_text = " ".join(agent_text.replace(pad_token, " ").split())
        text_path = out_dir / f"{stem}_output.txt"
        text_path.write_text(agent_text + "\n")
        print(f"agent text -> {text_path}\n{agent_text}")
    else:
        print("WARNING: no agent text in outputs")
    if audio is not None and audio.size > 0:
        wav_path = out_dir / f"{stem}_output.wav"
        sf.write(wav_path, audio, sr)
        print(f"agent audio -> {wav_path} ({audio.size / sr:.2f}s @ {sr} Hz)")
    else:
        raise RuntimeError("No audio produced by the code2wav stage.")
 if __name__ == "__main__":
    main()
@@ -392,7 +392,7 @@ def main(args):
    total_requests = len(prompts)
    processed_count = 0
    for stage_outputs in omni_generator:
-        output = stage_outputs
+        output = stage_outputs.request_output
        if stage_outputs.final_output_type == "text":
            request_id = output.request_id
            text_output = output.outputs[0].text
@@ -367,7 +367,7 @@ def main(args):
    print(f"query type: {args.query_type}")
    for stage_outputs in omni_generator:
-        output = stage_outputs
+        output = stage_outputs.request_output
        if stage_outputs.final_output_type == "text":
            request_id = output.request_id
            text_output = output.outputs[0].text
@@ -237,7 +237,7 @@ async def run_single_request(
            sampling_params_list=sampling_params_list,
            output_modalities=output_modalities,
        ):
-            output = omni_output
+            output = omni_output.request_output
            if omni_output.final_output_type == "text":
                if stage_0_first_output_ts is None:
                    stage_0_first_output_ts = time.perf_counter()
@@ -26,6 +26,7 @@ import torch
 from vllm_omni.diffusion.data import DiffusionParallelConfig
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 from vllm_omni.outputs import OmniRequestOutput
 from vllm_omni.platforms import current_omni_platform
@@ -351,8 +352,8 @@ def main():
        else:
            print("[Profiler] No valid profiling data returned.")
-    # omni.generate() returns a list for sync calls; unwrap single-result list.
+    # Extract output from result
-    output = result[0] if isinstance(result, list) else result
+    output = OmniRequestOutput.unwrap_result(result)
    if not output.images:
        raise ValueError("No video frames found in OmniRequestOutput.")
@@ -345,7 +345,7 @@ def main(args):
    # Process outputs from each stage
    for stage_idx, stage_outputs in enumerate(omni_outputs):
        # request_output may be a single object or a list
-        raw_output = stage_outputs
+        raw_output = stage_outputs.request_output
        outputs_list = raw_output if isinstance(raw_output, list) else [raw_output]
        if stage_outputs.final_output_type == "text":
@@ -43,7 +43,6 @@ from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 from vllm_omni.model_extras import get_extra_body_params, get_model_class_name
 from vllm_omni.outputs import OmniRequestOutput
 from vllm_omni.platforms import current_omni_platform
 AUDIOX_TASKS = ("t2a", "t2m", "v2a", "v2m", "tv2a", "tv2m")
@@ -389,9 +388,9 @@ def main():
        raise ValueError("No output generated from omni.generate()")
    output = outputs[0]
-    if not isinstance(output, OmniRequestOutput) or not output:
+    if not hasattr(output, "request_output") or not output.request_output:
        raise ValueError("No request_output found in OmniRequestOutput")
-    request_output = output
+    request_output = output.request_output
    if not hasattr(request_output, "multimodal_output"):
        raise ValueError("No multimodal_output found in request_output")
@@ -115,7 +115,7 @@ def build_demo(args: argparse.Namespace) -> gr.Blocks:
        )
        images_outputs = []
        for output in outputs:
-            req_out = output
+            req_out = output.request_output
            if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"):
                raise ValueError("Invalid request_output structure or missing 'images' key")
            images = req_out.images
@@ -656,7 +656,7 @@ def main():
        images = getattr(output, "images", None)
        if images:
            break
-        req_out = output
+        req_out = getattr(output, "request_output", None)
        images = getattr(req_out, "images", None) if req_out is not None else None
        if images:
            break
@@ -372,10 +372,18 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
 Streaming requires `async_chunk: true` in the stage config.
 ### Word Timestamps
-Word-level timestamps are currently a serving-path feature: launch
+Generate a WAV offline and a JSON sidecar with word-level timestamps from
-`vllm-omni serve` with `--forced-aligner` and request `word_timestamps`
+`Qwen/Qwen3-ForcedAligner-0.6B`:
-(see `examples/online_serving/text_to_speech/README.md`). An offline
+```bash
-example is not provided in this release.
+python examples/offline_inference/text_to_speech/qwen3_tts/word_timestamps.py \
    --model Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
    --forced-aligner Qwen/Qwen3-ForcedAligner-0.6B \
    --text "Hello world." \
    --output-dir /tmp/qwen3_tts_timestamps
 ```
 The script writes `qwen3_tts_word_timestamps.wav` and
 `qwen3_tts_word_timestamps.json`. On machines without a local CUDA toolkit,
 set `VLLM_USE_FLASHINFER_SAMPLER=0` to avoid FlashInfer sampler JIT.
 ### Batched decoding
 The Code2Wav stage supports batched decoding through the SpeechTokenizer. Pass multiple prompts via `--txt-prompts` and set `--batch-size` accordingly. To raise `max_num_seqs` on either stage, point `--stage-configs-path` at a stage configs YAML with the desired values (see `vllm_omni/model_executor/stage_configs/` for templates):
@@ -168,7 +168,7 @@ def run_e2e():
    print(f"Received {len(outputs)} outputs.")
    for i, output in enumerate(outputs):
        try:
-            ro = output
+            ro = output.request_output
            if ro is None:
                print("No request_output found.")
                continue
@@ -179,7 +179,7 @@ def main(args):
    t_start = time.perf_counter()
    for stage_outputs in omni.generate(inputs):
-        request_output = stage_outputs
+        request_output = stage_outputs.request_output
        if request_output is None or not request_output.outputs:
            continue
        _save_wav(
@@ -215,7 +215,7 @@ async def main_streaming(args):
    sample_rate = None
    async for stage_output in omni.generate(prompt, request_id=request_id):
-        mm = stage_output.outputs[0].multimodal_output
+        mm = stage_output.request_output.outputs[0].multimodal_output
        if not stage_output.finished:
            t_now = time.perf_counter()
            sample_rate, n = _collect_audio_chunk(mm, all_audio_chunks, sample_rate)
@@ -137,7 +137,7 @@ def main(args) -> None:
    params_list = sampling_params
    for stage_outputs in omni.generate(inputs, params_list):
-        for i, req_output in enumerate(stage_outputs):
+        for i, req_output in enumerate(stage_outputs.request_output):
            for j, out in enumerate(req_output.outputs):
                mm = out.multimodal_output
                if mm is None:
@@ -135,7 +135,7 @@ def run_e2e():
    print(f"Received {len(outputs)} outputs.")
    for i, output in enumerate(outputs):
        try:
-            ro = output
+            ro = output.request_output
            if ro is None:
                print("No request_output found.")
                continue
@@ -391,7 +391,7 @@ def main(args):
    for batch_start in range(0, len(inputs), batch_size):
        batch = inputs[batch_start : batch_start + batch_size]
        for stage_outputs in omni.generate(batch):
-            output = stage_outputs
+            output = stage_outputs.request_output
            _save_wav(output_dir, output.request_id, output.outputs[0].multimodal_output)
@@ -412,7 +412,7 @@ async def main_streaming(args):
        t_prev = t_start
        chunk_idx = 0
        async for stage_output in omni.generate(prompt, request_id=request_id):
-            mm = stage_output.outputs[0].multimodal_output
+            mm = stage_output.request_output.outputs[0].multimodal_output
            if not stage_output.finished:
                t_now = time.perf_counter()
                audio = mm.get("audio")
@@ -116,7 +116,7 @@ def main(args: Any) -> None:
    prompt = _build_custom_voice_input(args)
    final_output = None
    for stage_outputs in omni.generate([prompt]):
-        final_output = stage_outputs
+        final_output = stage_outputs.request_output
    if final_output is None:
        raise RuntimeError("Qwen3-TTS did not produce an output.")
@@ -38,7 +38,7 @@ def _stage_durations_from_output(outputs) -> dict[str, float]:
    durations = getattr(omni_out, "stage_durations", None) or {}
    if durations:
        return dict(durations)
-    ro = omni_out
+    ro = getattr(omni_out, "request_output", None)
    if ro is not None:
        return dict(getattr(ro, "stage_durations", None) or {})
    return {}
@@ -99,7 +99,7 @@ def extract_audio(outputs) -> tuple[np.ndarray, int]:
    out = outputs[0]
    mm = getattr(out, "multimodal_output", None)
    if mm is None:
-        ro = out
+        ro = getattr(out, "request_output", None)
        if ro is not None:
            mm = getattr(ro, "multimodal_output", None)
            if mm is None and ro.outputs:
@@ -8,7 +8,6 @@ A unified script for text-to-video generation. Supports multiple models with mod
 |---|---|---|---|---|---|
 | `Wan-AI/Wan2.1-VACE-1.3B-diffusers` | 480x832 | 81 | 30 | 5.0 | ~20 GiB (RTX 5090, VAE tiling) |
 | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB |
 | `robbyant/lingbot-video-dense-1.3b` / `robbyant/lingbot-video-moe-30b-a3b` | 192x320 | 9 | 2 | 3.0 | ~68 GiB (MoE smoke) |
 | `Lightricks/LTX-2` | 512x768 | 121 | 40 | video 3.0 / audio 7.0 | Model-dependent |
 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB |
 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required |
@@ -58,31 +57,18 @@ python text_to_video.py \
 ### LingBot-Video
-The shared runner recognizes the official dense and MoE checkpoint IDs, selects
+The dedicated LingBot example explicitly selects the video output modality:
 `LingBotVideoPipeline`, and builds the canonical video request envelope:
 ```bash
-python text_to_video.py \
+python examples/offline_inference/text_to_video/text_to_video_lingbot.py \
  --model robbyant/lingbot-video-dense-1.3b \
  --prompt "a robotic arm picks up a red block" \
  --height 192 --width 320 --num-frames 9 --num-inference-steps 2 \
  --guidance-scale 3.0 --flow-shift 3.0 --fps 24 \
  --output lingbot_t2v.mp4
 ```
 Use the same command with `robbyant/lingbot-video-moe-30b-a3b` for the MoE
 checkpoint. Requested frame counts are rounded upward to the causal
-VAE's `4n+1` grid. For a local checkpoint path whose name does not contain
+VAE's `4n+1` grid.
 `lingbot`, pass `--model-class-name LingBotVideoPipeline`.
 LingBot-only options such as `batch_cfg` and `output_type` use the shared
 model-extra channel, for example:
 ```bash
 python text_to_video.py \
  --model robbyant/lingbot-video-dense-1.3b \
  --extra-body '{"batch_cfg": true, "output_type": "np"}'
 ```
 ### LTX-2
@@ -16,7 +16,7 @@ from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 from vllm_omni.lora.request import LoRARequest
 from vllm_omni.lora.utils import stable_lora_int_id
-from vllm_omni.model_extras import get_extra_body_params, get_model_class_name, get_output_tensor_range
+from vllm_omni.model_extras import get_extra_body_params, get_model_class_name
 from vllm_omni.outputs import OmniRequestOutput
 from vllm_omni.platforms import current_omni_platform
@@ -80,17 +80,6 @@ _MODEL_PRESETS = {
        "fps": 16,
        "output": "helios_output.mp4",
    },
    "lingbot": {
        "model_class_name": "LingBotVideoPipeline",
        "height": 192,
        "width": 320,
        "num_frames": 9,
        "num_inference_steps": 2,
        "guidance_scale": 3.0,
        "fps": 24,
        "flow_shift": 3.0,
        "output": "lingbot_video_output.mp4",
    },
    "ltx2": {
        "height": 512,
        "width": 768,
@@ -121,51 +110,27 @@ _MODEL_PRESETS = {
 def _detect_preset(model: str, model_class_name: str | None = None) -> dict:
    model_lower = model.lower()
    class_lower = (model_class_name or "").lower()
-    if "lingbot" in model_lower or "lingbotvideo" in class_lower:
+    if "distilled" in class_lower or "distilled" in model_lower:
-        return _MODEL_PRESETS["lingbot"]
+        return _MODEL_PRESETS["ltx2_distilled"]
-    if "ltx" in class_lower or "ltx" in model_lower:
+    if "ltx23" in class_lower or "ltx-2.3" in model_lower or "ltx_2.3" in model_lower:
-        if "distilled" in class_lower or "distilled" in model_lower:
+        return _MODEL_PRESETS["ltx23"]
-            return _MODEL_PRESETS["ltx2_distilled"]
+    if "ltx2" in class_lower or "ltx-2" in model_lower or "ltx_2" in model_lower:
        if "ltx23" in class_lower or "ltx-2.3" in model_lower or "ltx_2.3" in model_lower:
            return _MODEL_PRESETS["ltx23"]
        return _MODEL_PRESETS["ltx2"]
-    if "vace" in model_lower or "vace" in class_lower:
+    if "vace" in model_lower:
        return _MODEL_PRESETS["vace"]
    # Edge must be matched before the generic cosmos branch (its "cosmos" substring would
    # otherwise pick up the Nano/Super 720p / gs 6.0 / flow_shift 10.0 preset).
    if ("cosmos" in model_lower or "cosmos" in class_lower) and ("edge" in model_lower or "edge" in class_lower):
        return _MODEL_PRESETS["cosmos3_edge"]
-    if "cosmos" in model_lower or "cosmos" in class_lower:
+    if "cosmos" in model_lower:
        return _MODEL_PRESETS["cosmos"]
-    if "hunyuan" in model_lower or "hunyuan" in class_lower:
+    if "hunyuan" in model_lower:
        return _MODEL_PRESETS["hunyuan"]
-    if "helios" in model_lower or "helios" in class_lower:
+    if "helios" in model_lower:
        return _MODEL_PRESETS["helios"]
    return _MODEL_PRESETS["wan"]
 def build_text_to_video_prompt(prompt: str, negative_prompt: str | None) -> dict[str, Any]:
    """Build the canonical request envelope for the shared T2V example."""
    result: dict[str, Any] = {
        "prompt": prompt,
        "modalities": ["video"],
    }
    if negative_prompt is not None:
        result["negative_prompt"] = negative_prompt
    return result
 def _normalize_float_tensor(tensor: torch.Tensor, source_range: str) -> torch.Tensor:
    """Normalize decoded frames according to the pipeline's output contract."""
    if not tensor.is_floating_point():
        return tensor
    if source_range == "negative_one_to_one":
        return tensor.clamp(-1, 1) * 0.5 + 0.5
    if source_range == "zero_to_one":
        return tensor.clamp(0, 1)
    raise ValueError(f"Unsupported floating-point tensor range: {source_range!r}")
 def parse_profiler_config(value: str) -> dict[str, Any]:
    try:
        config = json.loads(value)
@@ -195,7 +160,7 @@ def parse_extra_body(value: str) -> dict[str, Any]:
 def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Generate a video from a text prompt. "
-        "Supports Wan2.2, HunyuanVideo-1.5, Helios, LingBot-Video, and other text-to-video models."
+        "Supports Wan2.2, HunyuanVideo-1.5, Helios, and other text-to-video models."
    )
    parser.add_argument(
        "--model",
@@ -425,7 +390,7 @@ def _extract_peak_memory_mb(result: Any) -> float:
        return 0.0
    val = getattr(result, "peak_memory_mb", 0.0)
    if not val:
-        inner = result
+        inner = getattr(result, "request_output", None)
        if isinstance(inner, list):
            inner = inner[0] if inner else None
        val = getattr(inner, "peak_memory_mb", 0.0)
@@ -443,7 +408,6 @@ def main():
    for key, default_val in preset.items():
        if getattr(args, key.replace("-", "_"), None) is None:
            setattr(args, key.replace("-", "_"), default_val)
    model_class_name = args.model_class_name
    generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
    # Cache-dit config (Wan2.2 only)
@@ -516,7 +480,6 @@ def main():
    omni = Omni(**omni_kwargs)
    model_class_name = get_model_class_name(omni) or model_class_name
    declared_extra_body_params = get_extra_body_params(model_class_name)
    output_tensor_range = get_output_tensor_range(model_class_name)
    if profiler_enabled:
        print("[Profiler] Starting profiling...")
@@ -550,11 +513,12 @@ def main():
            lora_path=lora_path,
        )
-    negative_prompt = args.negative_prompt
+    prompt_dict = {"prompt": args.prompt}
-    if negative_prompt is None and all(preset is not _MODEL_PRESETS[name] for name in ("lingbot", "ltx2", "ltx23")):
+    if args.negative_prompt is not None:
        prompt_dict["negative_prompt"] = args.negative_prompt
    elif preset not in (_MODEL_PRESETS["ltx2"], _MODEL_PRESETS["ltx23"]):
        # Preserve the historical empty-prompt behavior for non-LTX examples.
-        negative_prompt = ""
+        prompt_dict["negative_prompt"] = ""
    prompt_dict = build_text_to_video_prompt(args.prompt, negative_prompt)
    extra_args = {}
    if lora_request:
@@ -611,8 +575,8 @@ def main():
        if frames.multimodal_output and "audio" in frames.multimodal_output:
            audio = frames.multimodal_output["audio"]
            audio_sample_rate = frames.multimodal_output.get("audio_sample_rate", audio_sample_rate)
-        if frames.is_pipeline_output and frames is not None:
+        if frames.is_pipeline_output and frames.request_output is not None:
-            inner_output = frames
+            inner_output = frames.request_output
            if isinstance(inner_output, OmniRequestOutput):
                if inner_output.multimodal_output and "audio" in inner_output.multimodal_output:
                    audio = inner_output.multimodal_output["audio"]
@@ -666,7 +630,8 @@ def main():
                frame_tensor = frame_tensor[0]
            if frame_tensor.dim() == 3 and frame_tensor.shape[0] in (3, 4):
                frame_tensor = frame_tensor.permute(1, 2, 0)
-            frame_tensor = _normalize_float_tensor(frame_tensor, output_tensor_range)
+            if frame_tensor.is_floating_point():
                frame_tensor = frame_tensor.clamp(-1, 1) * 0.5 + 0.5
            return frame_tensor.float().numpy()
        if isinstance(frame, np.ndarray):
            frame_array = frame
@@ -718,7 +683,8 @@ def main():
                video_tensor = video_tensor[0]
        elif video_tensor.dim() == 4 and video_tensor.shape[0] in (3, 4):
            video_tensor = video_tensor.permute(1, 2, 3, 0)
-        video_tensor = _normalize_float_tensor(video_tensor, output_tensor_range)
+        if video_tensor.is_floating_point():
            video_tensor = video_tensor.clamp(-1, 1) * 0.5 + 0.5
        video_array = video_tensor.float().numpy()
    elif isinstance(frames, np.ndarray):
        video_array = frames
@@ -0,0 +1,131 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 from __future__ import annotations
 import argparse
 import time
 from pathlib import Path
 from typing import Any
 import numpy as np
 import torch
 from vllm_omni.diffusion.data import DiffusionParallelConfig
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 from vllm_omni.outputs import OmniRequestOutput
 from vllm_omni.platforms import current_omni_platform
 def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Generate text-to-video with a LingBot-Video checkpoint.")
    parser.add_argument("--model", default="robbyant/lingbot-video-dense-1.3b")
    parser.add_argument("--prompt", default="a robotic arm picks up a red block")
    parser.add_argument("--negative-prompt", default=None)
    parser.add_argument("--output", default="lingbot_video_output.mp4")
    parser.add_argument("--height", type=int, default=192)
    parser.add_argument("--width", type=int, default=320)
    parser.add_argument("--num-frames", type=int, default=9)
    parser.add_argument("--num-inference-steps", type=int, default=2)
    parser.add_argument("--guidance-scale", type=float, default=3.0)
    parser.add_argument("--flow-shift", type=float, default=3.0)
    parser.add_argument("--seed", type=int, default=42)
    parser.add_argument("--fps", type=int, default=24)
    parser.add_argument("--output-type", choices=["pt", "np"], default="pt")
    parser.add_argument("--batch-cfg", action="store_true")
    return parser.parse_args()
 def _peak_memory_mb(output: Any) -> float:
    result = output[0] if isinstance(output, list) and output else output
    return float(getattr(result, "peak_memory_mb", 0.0) or 0.0)
 def _extract_video(output: Any) -> Any:
    if isinstance(output, list):
        output = output[0] if output else None
    if isinstance(output, OmniRequestOutput):
        if output.final_output_type != "image":
            raise ValueError(f"Unexpected output type {output.final_output_type!r}.")
        if not output.images:
            raise ValueError("No video frames found in OmniRequestOutput.")
        return output.images[0] if len(output.images) == 1 else output.images
    return output
 def _to_frame_array(video: Any) -> np.ndarray:
    if isinstance(video, torch.Tensor):
        tensor = video.detach().cpu()
        if tensor.dim() == 5:
            tensor = tensor[0]
        if tensor.dim() == 4 and tensor.shape[-1] not in (3, 4):
            if tensor.shape[0] in (3, 4):
                tensor = tensor.permute(1, 2, 3, 0)
            elif tensor.shape[1] in (3, 4):
                tensor = tensor.permute(0, 2, 3, 1)
        if tensor.is_floating_point():
            if float(tensor.min()) < 0.0:
                tensor = tensor.clamp(-1, 1) * 0.5 + 0.5
            else:
                tensor = tensor.clamp(0, 1)
        return tensor.float().numpy()
    if isinstance(video, np.ndarray):
        array = video[0] if video.ndim == 5 else video
        if np.issubdtype(array.dtype, np.integer):
            return array.astype(np.float32) / 255.0
        return array.astype(np.float32, copy=False)
    if isinstance(video, list):
        return np.stack([_to_frame_array(frame) for frame in video], axis=0)
    return np.asarray(video, dtype=np.float32)
 def main() -> None:
    args = parse_args()
    generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
    omni = Omni(
        model=args.model,
        model_class_name="LingBotVideoPipeline",
        flow_shift=args.flow_shift,
        parallel_config=DiffusionParallelConfig(),
    )
    prompt: dict[str, Any] = {
        "prompt": args.prompt,
        "modalities": ["video"],
        "num_frames": args.num_frames,
    }
    if args.negative_prompt is not None:
        prompt["negative_prompt"] = args.negative_prompt
    sampling_params = OmniDiffusionSamplingParams(
        height=args.height,
        width=args.width,
        num_frames=args.num_frames,
        num_inference_steps=args.num_inference_steps,
        guidance_scale=args.guidance_scale,
        generator=generator,
        output_type=args.output_type,
        extra_args={"batch_cfg": args.batch_cfg, "shift": args.flow_shift},
    )
    start = time.perf_counter()
    output = omni.generate(prompt, sampling_params)
    elapsed = time.perf_counter() - start
    print(f"Total generation time: {elapsed:.4f} seconds ({elapsed * 1000:.2f} ms)")
    peak_mb = _peak_memory_mb(output)
    if peak_mb:
        print(f"Worker peak GPU memory (reserved): {peak_mb:.2f} MiB ({peak_mb / 1024:.2f} GiB)")
    video = _to_frame_array(_extract_video(output))
    output_path = Path(args.output)
    output_path.parent.mkdir(parents=True, exist_ok=True)
    from diffusers.utils import export_to_video
    export_to_video(list(video), str(output_path), fps=args.fps)
    print(f"Saved generated video to {output_path}")
    omni.close()
 if __name__ == "__main__":
    main()
@@ -138,7 +138,7 @@ def _extract_peak_memory_mb(result: Any) -> float:
        return 0.0
    val = getattr(result, "peak_memory_mb", 0.0)
    if not val:
-        inner = result
+        inner = getattr(result, "request_output", None)
        if isinstance(inner, list):
            inner = inner[0] if inner else None
        val = getattr(inner, "peak_memory_mb", 0.0)
@@ -605,13 +605,6 @@ python qwen3_tts/streaming_speech_client.py \
 The client writes one PCM file per sentence and a matching
 `sentence_XXX_timestamps.json` sidecar.
 Non-streaming requests can also ask for timestamps: pass
 `"word_timestamps": true` to `POST /v1/audio/speech` and read the
 `X-Word-Timestamps` response header (JSON list of `{word, start_ms, end_ms}`,
 ASCII-escaped). Past 4 KB the header is replaced by
 `X-Word-Timestamps-Omitted: oversize; bytes=<n>; limit=4096` and the audio
 still returns — use the WebSocket path for long transcripts.
 To *see* the alignment instead of reading a JSON sidecar, run the
 word-timestamp Gradio demo (server must be launched with `--forced-aligner`):
 ```bash
@@ -111,7 +111,6 @@ plugins:
        - "vllm_omni.entrypoints.openai"  # avoid importing vllm in mkdocs building
        - "vllm_omni.entrypoints.openai.protocol"  # avoid importing vllm in mkdocs building
        - "vllm_omni.experimental"  # avoid importing optional serving deps (openai, fastapi, uvicorn, httpx) in mkdocs building
        - "vllm_omni.model_executor.models.nemotron_voicechat.nemo_vendored"  # vendored NeMo code; upstream docstrings trip griffe strict warnings
  - mkdocstrings:
      handlers:
        python:
@@ -206,12 +206,6 @@ ignore = [
 # These files keep upstream conventions (star imports, long lines, math-symbol
 # argument names) so they stay easy to diff against upstream releases.
 "vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py" = ["E402", "E501", "F403", "F405", "N803"]
 # NemotronVoiceChat inference modules vendored from NVIDIA-NeMo/Speech (branch
 # nemotron-labs-voicechat). Numerics-bearing code is kept byte-close to
 # upstream to stay diffable and parity-safe, so upstream style (long lines,
 # legacy typing, whitespace, unused-import shims) is exempted. TID251 and the
 # F-correctness rules still apply.
 "vllm_omni/model_executor/models/nemotron_voicechat/nemo_vendored/**" = ["E501", "E721", "E741", "F401", "F841", "I001", "N803", "N814", "UP006", "UP007", "UP008", "UP011", "UP032", "UP035", "W291", "W293"]
 "vllm_omni/experimental/fullduplex/joyvl/decision/prompts.py" = ["E501"]  # System/summarizer prompts are long prose lines
 [tool.ruff.lint.flake8-tidy-imports.banned-api]
@@ -320,10 +314,6 @@ extend-ignore-identifiers-re = [
    ".*[Nn][Dd]3.*",
    ".*updator.*",
    "byt5.*",
    # NemotronVoiceChat vendored triton kernel: dout/DOUT abbreviate d_out
    # (output feature dim), not "doubt".
    ".*dout.*",
    ".*DOUT.*",
 ]
 extend-ignore-re = [
    "\\.\\.\\.n,d->\\.\\.\\.nd",
@@ -344,9 +334,6 @@ byt5 = "byt5"
 # "seeked" is the standard HTML media element DOM event name (fires when a
 # seek completes), used by the word-timestamps Gradio demo. Not a typo.
 seeked = "seeked"
 # Ryan Leary: author credited in the NemotronVoiceChat vendored NeMo
 # features.py copyright header. A surname, not a typo of "leery".
 Leary = "Leary"
 # IndexTTS 2.5 language codes and speech-emotion-recognition event token.
 fo = "fo"
 ba = "ba"
@@ -1,113 +0,0 @@
 # NemotronLabs VoiceChat 11B
 > Offline speech-to-speech: a 16 kHz user utterance in, the agent's spoken reply
 > (text + 22.05 kHz WAV) out, on a 3-stage vLLM-Omni pipeline.
 ## Summary
 - Vendor: NVIDIA
 - Model: [`nvidia/NVIDIA-NemotronLabs-VoiceChat-11B`](https://huggingface.co/nvidia/NVIDIA-NemotronLabs-VoiceChat-11B)
 - Task: Speech-to-speech voice chat. The model runs a frame-locked 12.5 Hz
  timeline: a Conformer + NemotronH hybrid-Mamba thinker emits one text token
  per acoustic frame, a Gemma3-1B EAR-TTS talker turns the text timeline into
  31-quantizer RVQ code stacks, and an RVQ-VAE codec decodes them to audio.
 - Mode: Offline single-turn inference (batch=1). Online/duplex serving,
  function calling, and batch>1 are documented follow-ups.
 - Maintainer: [`@yuekaizhang`](https://github.com/yuekaizhang)
 ## When to use this recipe
 Use this recipe to run a single-turn voice-chat exchange with
 `NVIDIA-NemotronLabs-VoiceChat-11B` on one GPU: you provide a user utterance as
 a WAV file (any sample rate; it is resampled to 16 kHz mono) plus an optional
 spoken-style system prompt, and get back the agent's reply as text and a
 22.05 kHz WAV. The integration is NeMo-free at runtime — the perception
 Conformer, EAR-TTS talker, and RVQ-VAE codec are vendored, dependency-stripped
 NeMo modules (`nemo_vendored/`), so no `nemo_toolkit` install is needed.
 ## References
 - Offline example:
  [`examples/offline_inference/nemotron_voicechat/end2end.py`](../../examples/offline_inference/nemotron_voicechat/end2end.py)
 - Model modules (thinker / talker / code2wav / vendored NeMo):
  [`vllm_omni/model_executor/models/nemotron_voicechat/`](../../vllm_omni/model_executor/models/nemotron_voicechat/)
 - Staged pipeline config:
  [`vllm_omni/deploy/nemotron_labs_voicechat.yaml`](../../vllm_omni/deploy/nemotron_labs_voicechat.yaml)
 - Upstream model card:
  [`nvidia/NVIDIA-NemotronLabs-VoiceChat-11B`](https://huggingface.co/nvidia/NVIDIA-NemotronLabs-VoiceChat-11B)
 - Reference implementation: NVIDIA-NeMo/Speech, branch `nemotron-labs-voicechat`
 ## Pipeline
 | stage | arch | dtype | role |
 |---|---|---|---|
 | 0 thinker | `NemotronVoiceChatThinkerForConditionalGeneration` (LLM_AR) | fp32 | WAV + system prompt -> frame-locked text-token timeline (+ function channel) |
 | 1 talker | `NemotronVoiceChatTalker` (LLM_AR) | fp32 | text timeline -> 31-quantizer RVQ code stacks (one per 80 ms frame) |
 | 2 code2wav | `NemotronVoiceChatCode2Wav` (LLM_GENERATION) | fp32 | RVQ-VAE decode -> 22.05 kHz PCM |
 Stages 0/1 default to fp32 for exact parity with the NeMo reference
 implementation (greedy decoding matches it token for token on the acceptance
 fixture). The deploy yaml documents a ~2x-faster bf16 thinker option whose
 output stayed within one word of the reference in testing.
 ## Hardware Support
 ## GPU
 ### 1x H100 80GB
 #### Environment
 - OS: Linux
 - Python: 3.12
 - vLLM version: 0.26.0
 - vLLM-Omni version or commit: this PR / current `main`
 #### Command
 ```bash
 # Tokenizer: the checkpoint ships no HF tokenizer; it resolves from the
 # nvidia/NVIDIA-Nemotron-Nano-9B-v2 HF id automatically. For air-gapped runs,
 # point NEMOTRON_VOICECHAT_LLM_PATH at a local snapshot of that repo instead.
 python examples/offline_inference/nemotron_voicechat/end2end.py \
    --checkpoint /path/to/NVIDIA-NemotronLabs-VoiceChat-11B \
    --wav /path/to/user_question.wav \
    --output-dir results/nemotron_voicechat
 ```
 #### Verification
 ```bash
 ls results/nemotron_voicechat
 # <stem>_output.txt          the agent reply as text
 # <stem>_output.wav          the agent reply as 22.05 kHz audio
 # <stem>_text_tokens.json    the frame-locked text-token timeline
 ```
 The reply text should read as a coherent spoken-style answer to the question in
 the input WAV, and the WAV should transcribe to (approximately) the same text
 with any ASR model.
 #### Notes
 - Memory usage: the shipped yaml runs all three stages on one GPU
  (`gpu_memory_utilization` 0.62 / 0.12 / 0.06); peak usage is dominated by the
  fp32 thinker. The fp32 default has a hard floor of roughly 43 GB of thinker
  weights alone (9B backbone + 587M `embed_tokens` + 587M `function_head` +
  0.6B Conformer), so 48 GB cards cannot run it — use the bf16 thinker option
  documented in the deploy yaml on anything smaller than an 80 GB part.
 - Input sizing: the timeline is frame-locked, so the reply budget IS the input
  duration. The acoustic channel trails the text channel; if the WAV does not
  carry enough trailing silence for the reply to finish, the spoken answer is
  truncated silently. Leave generous trailing silence (a question ending at
  ~4.5 s truncated in an 8 s WAV but completed cleanly in 16 s); the offline
  example warns when the text channel is still speaking near the last frame.
 - Key flags: sampling is greedy end to end. The thinker is frame-locked —
  `max_tokens` equals the acoustic frame count with `ignore_eos=True`. Do NOT
  set `min_tokens` on the thinker: the tokenizer's EOS token is also the
  frame-locked PAD/silence token, so masking it forces the model to babble
  instead of pausing.
 - The talker's `max_tokens` is 16383 (its stage prompt is one placeholder
  token, and the stage context is 16384).
 - Known limitations: batch=1 only; offline single turn (no online/duplex
  serving yet); the function-call channel is decoded but not yet acted on.
@@ -66,7 +66,6 @@ recipes/
 | [`OpenBMB/VoxCPM2.md`](./OpenBMB/VoxCPM2.md) | Online + offline TTS with native AR pipeline (48 kHz, 30+ languages) | 1x RTX 4090 24GB |
 | [`OpenMOSS/MOSS-TTS.md`](./OpenMOSS/MOSS-TTS.md) | Online + offline multilingual TTS (MOSS-TTS family, 8B) | 1x H100 80GB |
 | [`NVIDIA/Nemotron-Labs-Audex.md`](./NVIDIA/Nemotron-Labs-Audex.md) | TTS / text-to-audio / audio understanding / cascaded S2S (Audex 2B + 30B-A3B) | 1x H100 80GB |
 | [`NVIDIA/NemotronLabs-VoiceChat.md`](./NVIDIA/NemotronLabs-VoiceChat.md) | Offline speech-to-speech voice chat (11B, frame-locked 12.5 Hz, 3-stage thinker/talker/code2wav) | 1x H100 80GB |
 | [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt mixed FP8/NVFP4 | 1x A100 80GB / 2x B200 |
 | [`Qwen/Qwen-Image-2512.md`](./Qwen/Qwen-Image-2512.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt FP8 / mixed FP8/NVFP4 | 1x A800 80GB / 2x B200 |
 | [`Qwen/Qwen-Image-Edit.md`](./Qwen/Qwen-Image-Edit.md) | Text-guided single-image editing | 1x or 2x H200 141GB |
@@ -27,7 +27,7 @@ parallel, cache, quantized, or expert-kernel backends.
 - Dense checkpoint: <https://huggingface.co/robbyant/lingbot-video-dense-1.3b>
 - MoE checkpoint: <https://huggingface.co/robbyant/lingbot-video-moe-30b-a3b>
 - Upstream project: <https://github.com/Robbyant/lingbot-video>
- Related offline example: [`examples/offline_inference/text_to_video/text_to_video.py`](../../examples/offline_inference/text_to_video/text_to_video.py)
+- Related offline example: [`examples/offline_inference/text_to_video/text_to_video_lingbot.py`](../../examples/offline_inference/text_to_video/text_to_video_lingbot.py)
 - Related online video API docs: [`docs/serving/videos_api.md`](../../docs/serving/videos_api.md)
 ## Hardware Support
@@ -52,7 +52,7 @@ The MoE path is validated with BF16 expert weights.
 ```bash
 CUDA_VISIBLE_DEVICES=0 \
-python examples/offline_inference/text_to_video/text_to_video.py \
+python examples/offline_inference/text_to_video/text_to_video_lingbot.py \
  --model robbyant/lingbot-video-dense-1.3b \
  --prompt "a robotic arm picks up a red block" \
  --output lingbot_t2v.mp4 \
@@ -70,7 +70,7 @@ python examples/offline_inference/text_to_video/text_to_video.py \
 ```bash
 CUDA_VISIBLE_DEVICES=0 \
-python examples/offline_inference/text_to_video/text_to_video.py \
+python examples/offline_inference/text_to_video/text_to_video_lingbot.py \
  --model robbyant/lingbot-video-moe-30b-a3b \
  --prompt "a robotic arm picks up a red block" \
  --output lingbot_moe_t2v.mp4 \
@@ -82,7 +82,7 @@ example and the DiT sections below.
 - Optional environment variables:
 ```bash
-export CACHE_DIT_VERSION=1.5.0
+export CACHE_DIT_VERSION=1.3.0
 ```
 HunyuanImage-3.0 sets the diffusion attention backend to `TORCH_SDPA`
@@ -16,7 +16,7 @@ diffusers==0.38.0
 safetensors>=0.8.0
 accelerate==1.12.0
 soundfile>=0.13.1
-cache-dit==1.5.0
+cache-dit==1.3.0
 tqdm>=4.66.0
 torchsde>=0.2.6
 openai-whisper>=20250625
@@ -34,5 +34,3 @@ pydantic-settings>=2.0.0
 # base image does not ship it, so declare it explicitly. The loader also guards
 # this import so a missing gguf no longer breaks all diffusion model loading.
 gguf>=0.10.0
 # Required for running the Cosmos3 model with guardrails enabled (default behavior).
 cosmos-guardrail>=0.3.1
@@ -1,92 +0,0 @@
 # SPDX-License-Identifier: Apache-2.0
 """Offline Daily-Omni sources: local mirrors must not fall back to the Hub id."""
 import json
 import os
 import tarfile
 from pathlib import Path
 import pytest
 from vllm_omni.benchmarks.data_modules.daily_omni_dataset import (
    daily_omni_local_qa_json,
    daily_omni_local_videos_dir,
    resolve_daily_omni_local_root,
 )
 pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.benchmark]
 def _write_qa_json(root: Path) -> Path:
    qa = root / "qa.json"
    qa.write_text(json.dumps([{"Question": "q", "Answer": "A", "video_id": "vid0"}]), encoding="utf-8")
    return qa
 def test_resolve_local_root_returns_none_for_hub_id() -> None:
    assert resolve_daily_omni_local_root("liarliar/Daily-Omni") is None
    assert resolve_daily_omni_local_root("") is None
    assert resolve_daily_omni_local_root(None) is None
 def test_plain_local_mirror(tmp_path: Path) -> None:
    _write_qa_json(tmp_path)
    (tmp_path / "Videos" / "vid0").mkdir(parents=True)
    root = resolve_daily_omni_local_root(str(tmp_path))
    assert root == tmp_path.resolve()
    assert daily_omni_local_qa_json(root) == (tmp_path / "qa.json").resolve()
    assert daily_omni_local_videos_dir(root) == root / "Videos"
 def test_hf_cache_dir_resolves_to_snapshot(tmp_path: Path) -> None:
    cache_dir = tmp_path / "datasets--liarliar--Daily-Omni"
    snapshot = cache_dir / "snapshots" / "deadbeef"
    snapshot.mkdir(parents=True)
    _write_qa_json(snapshot)
    (cache_dir / "refs").mkdir()
    (cache_dir / "refs" / "main").write_text("deadbeef", encoding="utf-8")
    root = resolve_daily_omni_local_root(str(cache_dir))
    assert root == snapshot.resolve()
    assert daily_omni_local_qa_json(root) == (snapshot / "qa.json").resolve()
 def test_hf_cache_dir_without_refs_picks_newest_snapshot(tmp_path: Path) -> None:
    cache_dir = tmp_path / "datasets--liarliar--Daily-Omni"
    older = cache_dir / "snapshots" / "aaa"
    newer = cache_dir / "snapshots" / "bbb"
    older.mkdir(parents=True)
    newer.mkdir(parents=True)
    os.utime(older, (1, 1))
    os.utime(newer, (10, 10))
    assert resolve_daily_omni_local_root(str(cache_dir)) == newer.resolve()
 def test_videos_tar_is_extracted(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setenv("HF_HOME", str(tmp_path / "hf_home"))
    root = tmp_path / "mirror"
    (root / "Videos" / "vid0").mkdir(parents=True)
    (root / "Videos" / "vid0" / "vid0_video.mp4").write_bytes(b"fake")
    with tarfile.open(root / "Videos.tar", "w") as tf:
        tf.add(root / "Videos", arcname="Videos")
    # Only the tarball is left in the mirror.
    for child in (root / "Videos").rglob("*"):
        if child.is_file():
            child.unlink()
    (root / "Videos" / "vid0").rmdir()
    (root / "Videos").rmdir()
    videos_dir = daily_omni_local_videos_dir(root)
    assert videos_dir is not None
    assert (videos_dir / "vid0" / "vid0_video.mp4").is_file()
    # Second call reuses the extraction instead of redoing it.
    assert daily_omni_local_videos_dir(root) == videos_dir
 def test_missing_qa_json_reports_none(tmp_path: Path) -> None:
    root = resolve_daily_omni_local_root(str(tmp_path))
    assert daily_omni_local_qa_json(root) is None
    assert daily_omni_local_videos_dir(root) is None
@@ -2,8 +2,9 @@
 The ``minicpm-interleave`` mode is an accuracy-critical protocol: MiniCPM-o only reaches
 OpenBMB's reported Daily-Omni score when frames and 1s audio segments arrive strictly
-interleaved, with ``max_slice_nums=1`` / ``use_image_id=False`` and no Qwen system prompt.
+interleaved, with ``max_slice_nums=1`` / ``use_image_id=False`` and an empty MiniCPM
-These tests pin that contract so a refactor cannot silently regress the score.
+system message (not the Qwen system prompt). These tests pin that contract so a refactor
 cannot silently regress the score.
 vllm stubs are installed by tests/benchmarks/conftest.py before collection.
 """
@@ -98,13 +99,22 @@ def _stub_extract(monkeypatch, dataset: DailyOmniDataset, counter: list[int]) ->
 # ---------------------------------------------------------------------------
-@pytest.mark.parametrize(("n", "k"), [(10, 4), (100, 64), (7, 7), (3, 5), (1, 1)])
+@pytest.mark.parametrize(("n", "k"), [(10, 4), (100, 64), (1000, 64), (7, 7), (3, 5), (1, 1)])
-def test_uniform_sample_indices_matches_numpy_linspace(n: int, k: int) -> None:
+def test_uniform_sample_indices_matches_openbmb_midpoint(n: int, k: int) -> None:
    """Pin OpenBMB midpoint-bin sampling (not np.linspace endpoints)."""
    got = _uniform_sample_indices(n, k)
    if n <= k:
        assert got == list(range(n))
    else:
-        assert got == [int(i) for i in np.linspace(0, n - 1, k, dtype=int)]
+        gap = n / k
        expected = [int(i * gap + gap / 2) for i in range(k)]
        assert got == expected
        # Endpoints differ from linspace for typical Daily-Omni downsample sizes.
        linspace = [int(i) for i in np.linspace(0, n - 1, k, dtype=int)]
        if n == 1000 and k == 64:
            assert got[0] == 7 and got[-1] == 992
            assert linspace[0] == 0 and linspace[-1] == 999
            assert got != linspace
 def test_uniform_sample_indices_non_positive_k() -> None:
@@ -213,14 +223,16 @@ def test_minicpm_interleave_missing_local_video_returns_none(qa_json, tmp_path)
 # ---------------------------------------------------------------------------
-def test_minicpm_interleave_omits_qwen_system_message(monkeypatch, qa_json, video_dir) -> None:
+def test_minicpm_interleave_keeps_empty_system_message(monkeypatch, qa_json, video_dir) -> None:
    """Official MiniCPM omni mode still sends role=system with empty content."""
    ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
    _stub_extract(monkeypatch, ds, [])
    parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
    messages = ds._build_daily_omni_openai_messages(parts, "q?", {"A": "a"})
-    assert [m["role"] for m in messages] == ["user"]
+    assert [m["role"] for m in messages] == ["system", "user"]
    assert messages[0]["content"][0]["text"] == ""
 def test_qwen_pack_mode_keeps_system_message(qa_json, video_dir) -> None:
@@ -1,71 +0,0 @@
 import pytest
 import vllm_omni.config.pipeline_registry  # noqa: F401  (populate registry)
 from vllm_omni.config.pipeline_registry import OMNI_PIPELINES as _PIPELINE_REGISTRY
 from vllm_omni.config.stage_config import (
    DeployConfig,
    StageExecutionType,
 )
 from vllm_omni.model_executor.stage_input_processors.forced_aligner import (
    POOLING_OUTPUT_DECODER_PATH,
    TIMESTAMPS_MODALITY,
 )
 from vllm_omni.utils.forced_aligner import _DEFAULT_CONFIG_PATH, inject_forced_aligner_stage
 pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
 def test_inject_appends_pooling_aligner_stage():
    pipeline = _PIPELINE_REGISTRY["qwen3_tts"]
    deploy = DeployConfig()
    n0 = len(pipeline.stages)
    ext_pipeline, ext_deploy = inject_forced_aligner_stage(
        pipeline, deploy, {"forced_aligner": "/models/Qwen3-ForcedAligner-0.6B"}
    )
    # A new pooling stage is appended; the original pipeline is unchanged.
    assert len(pipeline.stages) == n0
    assert len(ext_pipeline.stages) == n0 + 1
    # The original deploy config is not mutated either (double-injection safe).
    assert len(deploy.stages) == 0
    assert len(ext_deploy.stages) == 1
    aligner = ext_pipeline.stages[-1]
    # Pooling stage reuses LLM_AR (+ runner=pooling), no dedicated LLM_POOLING.
    assert aligner.execution_type == StageExecutionType.LLM_AR
    assert aligner.final_output is True
    assert aligner.final_output_type == TIMESTAMPS_MODALITY
    assert aligner.input_sources == (n0 - 1,)
    assert aligner.requires_multimodal_data is True
    assert aligner.owns_tokenizer is True
    # The deploy stage carries the separate aligner checkpoint + decoder hook.
    ds = ext_deploy.stages[-1]
    assert ds.engine_extras["model"] == "/models/Qwen3-ForcedAligner-0.6B"
    assert ds.engine_extras["runner"] == "pooling"
    assert ds.engine_extras["pooling_output_decoder"] == POOLING_OUTPUT_DECODER_PATH
 def test_inject_from_config_only():
    # --forced-aligner-config alone (model in the YAML) must also inject.
    pipeline = _PIPELINE_REGISTRY["qwen3_tts"]
    deploy = DeployConfig()
    n0 = len(pipeline.stages)
    ext_pipeline, ext_deploy = inject_forced_aligner_stage(
        pipeline, deploy, {"forced_aligner": None, "forced_aligner_config": str(_DEFAULT_CONFIG_PATH)}
    )
    assert len(ext_pipeline.stages) == n0 + 1
    assert ext_pipeline.stages[-1].final_output_type == TIMESTAMPS_MODALITY
    assert ext_deploy.stages[-1].engine_extras["model"] == "Qwen/Qwen3-ForcedAligner-0.6B"
 def test_inject_noop_without_forced_aligner():
    pipeline = _PIPELINE_REGISTRY["qwen3_tts"]
    deploy = DeployConfig()
    ext_pipeline, ext_deploy = inject_forced_aligner_stage(pipeline, deploy, {})
    assert ext_pipeline is pipeline
    assert len(ext_deploy.stages) == 0
@@ -513,7 +513,6 @@ def test_sub_config_fields_match_structured_scopes():
        "enable_prefix_caching",
        "disable_hybrid_kv_cache_manager",
        "mm_processor_cache_gb",
        "mamba_ssm_cache_dtype",
    }
    assert {f.name for f in fields(OmniStageSchedulerConfig)} == {
        "max_num_seqs",
@@ -27,7 +27,6 @@ _MIXIN_UPDATE_HELPERS = (
    "_attach_finished_request_sets",
    "_attach_scheduler_stats",
    "_capture_omni_connector_output",
    "_maybe_decode_pooling_output",
 )
@@ -139,7 +138,6 @@ def _make_scheduler_stub(requests: list[_Request]) -> SimpleNamespace:
        pending_stop_after_extraction=set(),
        waiting_for_transfer_free=set(),
        _new_prompt_len_snapshot={},
        _pooling_output_decoder=None,
        finished_req_ids=set(),
        finished_req_ids_dict=defaultdict(set),
        kv_cache_manager=SimpleNamespace(take_events=lambda: None),
@@ -269,72 +267,3 @@ def test_invalid_logprobs_finish_only_the_affected_scheduler_request() -> None:
    assert output_by_id["bad"].new_token_ids == []
    assert output_by_id["good"].new_token_ids == [8]
    np.testing.assert_array_equal(output_by_id["good"].new_logprobs.logprob_token_ids[:, 0], [8])
 def _pooling_model_runner_output(pooler_tensor) -> SimpleNamespace:
    return SimpleNamespace(
        sampled_token_ids=[[]],
        logprobs=None,
        prompt_logprobs_dict={},
        pooler_output=[pooler_tensor],
        num_nans_in_logits=None,
        kv_connector_output=None,
        cudagraph_stats=None,
        req_id_to_index={"req": 0},
        routed_experts=None,
    )
 def _make_pooling_stub(decoder) -> tuple[_Request, SimpleNamespace]:
    request = _Request("req")
    request.sampling_params = SimpleNamespace(num_logprobs=None)
    request.pooling_params = SimpleNamespace()
    scheduler = _make_scheduler_stub([request])
    scheduler._pooling_output_decoder = decoder
    scheduler.vllm_config = SimpleNamespace(model_config=SimpleNamespace(hf_config=None))
    _bind_request_lifecycle(scheduler, update_request=lambda req, tokens: (tokens, False))
    return request, scheduler
 def test_pooling_decode_success_emits_decoded_payload() -> None:
    import torch
    decoded = torch.tensor([[1, 2]], dtype=torch.int32)
    request, scheduler = _make_pooling_stub(lambda logits, req, hf: decoded)
    scheduler_output = SimpleNamespace(
        num_scheduled_tokens={"req": 1},
        scheduled_spec_decode_tokens={},
        num_invalid_spec_tokens=0,
    )
    outputs = OmniARScheduler.update_from_output(
        scheduler, scheduler_output, _pooling_model_runner_output(torch.zeros(2, 4))
    )
    (output,) = outputs[0].outputs
    assert output.finish_reason is FinishReason.STOP
    assert output.pooling_output is decoded
 def test_pooling_decode_failure_finishes_request_with_error() -> None:
    import torch
    def raising_decoder(logits, req, hf):
        raise ValueError("boom")
    request, scheduler = _make_pooling_stub(raising_decoder)
    scheduler_output = SimpleNamespace(
        num_scheduled_tokens={"req": 1},
        scheduled_spec_decode_tokens={},
        num_invalid_spec_tokens=0,
    )
    outputs = OmniARScheduler.update_from_output(
        scheduler, scheduler_output, _pooling_model_runner_output(torch.zeros(2, 4))
    )
    (output,) = outputs[0].outputs
    # Decoder failure is a request error (500), not an empty success.
    assert output.finish_reason is FinishReason.ERROR
    assert output.pooling_output is None
    assert "pooling output decode failed" in str(request.stop_reason)
@@ -120,9 +120,6 @@ class TestFullPayloadCoordinatorSelection(unittest.TestCase):
            ("MiMoAudioModel", "code2wav"),
            ("Qwen3TTSCode2Wav", "code2wav"),
            ("MiniCPMO45Code2Wav", "code2wav"),
            # nemotron_voicechat: sync-path RVQ-VAE decode waits for the
            # talker's full [frames, 31] code-stack payload.
            ("NemotronVoiceChatCode2Wav", "code2wav"),
            ("CosyVoice3Model", "cosyvoice3_code2wav"),
            # audex: sync-path TTS decoder (async_chunk: false deploys) and
            # the TTA XCodec1 stage (always sync full-payload).
@@ -1,80 +0,0 @@
 [
  {
    "test_name": "test_minicpmo_4_5_ready",
    "mark": [
      {
        "hardware_marks": {
          "res": {
            "cuda": "H100",
            "npu": "A3"
          },
          "num_cards": 1
        }
      },
      "core_model",
      "omni"
    ],
    "server_params": {
      "model": "openbmb/MiniCPM-o-4_5",
      "extra_cli_args": [
        "--deploy-config",
        "vllm_omni/deploy/minicpmo_4_5.yaml",
        "--trust-remote-code"
      ]
    },
    "benchmark_params": [
      {
        "dataset_name": "seed-tts",
        "dataset_path": "zhaochenyang20/seed-tts-eval",
        "backend": "openai-chat-omni",
        "endpoint": "/v1/chat/completions",
        "num_prompts": [
          10,
          40
        ],
        "max_concurrency": [
          1,
          4
        ],
        "no_oversample": true,
        "trust_remote_code": true,
        "seed_tts_locale": "en",
        "extra_body": {
          "modalities": [
            "text",
            "audio"
          ],
          "chat_template_kwargs": {
            "enable_thinking": false,
            "use_tts_template": true
          }
        },
        "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
        "baseline": {
          "H100": {
            "request_throughput": [
              0.7803,
              1.1008
            ],
            "mean_ttft_ms": [
              166.9764,
              307.7133
            ],
            "mean_e2el_ms": [
              1361.1682,
              3800.6332
            ],
            "mean_audio_ttfp_ms": [
              635.8099,
              1615.6591
            ],
            "mean_audio_rtf": [
              0.3151,
              0.8568
            ]
          }
        }
      }
    ]
  }
 ]
@@ -125,7 +125,7 @@ def _extract_images(output: OmniRequestOutput) -> list:
        return output.images
    # When the output comes from the orchestrator pipeline, images may be
    # nested inside request_output.
-    inner = output
+    inner = getattr(output, "request_output", None)
    if inner is not None and hasattr(inner, "images") and inner.images:
        return inner.images
    return []
@@ -56,10 +56,10 @@ def test_teacache(model_name: str):
        # Extract images from request_output['images']
        first_output = outputs[0]
        assert first_output.final_output_type == "image"
-        if not isinstance(first_output, OmniRequestOutput) or not first_output:
+        if not hasattr(first_output, "request_output") or not first_output.request_output:
            raise ValueError("No request_output found in OmniRequestOutput")
-        req_out = first_output
+        req_out = first_output.request_output
        if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"):
            raise ValueError("Invalid request_output structure or missing 'images' key")
@@ -131,7 +131,7 @@ def _run_inference(
            elapsed_ms = (time.time() - start) * 1000
            return InferenceResult(
-                images=outputs[0].images,
+                images=outputs[0].request_output.images,
                elapsed_ms=elapsed_ms,
            )
    finally:
@@ -49,10 +49,10 @@ def _diff_metrics(a: Image.Image, b: Image.Image) -> tuple[float, float]:
 def _extract_single_image(outputs) -> Image.Image:
    first_output = outputs[0]
    assert first_output.final_output_type == "image"
-    if not isinstance(first_output, OmniRequestOutput) or not first_output:
+    if not hasattr(first_output, "request_output") or not first_output.request_output:
        raise ValueError("No request_output found in OmniRequestOutput")
-    req_out = first_output
+    req_out = first_output.request_output
    if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"):
        raise ValueError("Invalid request_output structure or missing 'images' key")
@@ -231,128 +231,6 @@ def test_reshard_from_trimmed_width_pads_invalid_columns(monkeypatch: pytest.Mon
    assert torch.equal(out.flatten(), torch.tensor([4.0, 0.0]))
@pytest.mark.core_model
@pytest.mark.cpu
@pytest.mark.parametrize("split_dim", ["height", "width"])
@pytest.mark.parametrize(
    ("full_extent", "world_size"),
    [
        pytest.param(5, 4, id="four-ranks-empty-tail"),
        pytest.param(20, 8, id="production-eight-ranks-empty-tail"),
        pytest.param(39, 2, id="two-ranks-partial-tail"),
    ],
 )
 def test_reshard_from_trimmed_extent_clamps_and_reconstructs(
    monkeypatch: pytest.MonkeyPatch,
    split_dim: str,
    full_extent: int,
    world_size: int,
 ):
    local_extent = (full_extent + world_size - 1) // world_size
    shape = (1, 1, 1, full_extent, 1) if split_dim == "height" else (1, 1, 1, 1, full_extent)
    x = torch.arange(full_extent, dtype=torch.float32).reshape(shape)
    outputs = []
    for rank in range(world_size):
        monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group, rank=rank: (rank, world_size))
        token = wan_spatial_shard._SPATIAL_SHARD_CONTEXT.set(
            wan_spatial_shard.SpatialShardContext(
                input_extent=full_extent,
                local_input_extent=local_extent,
                split_dim=split_dim,
                rank=rank,
                world_size=world_size,
            )
        )
        try:
            out = wan_spatial_shard.reshard_from_trimmed_extent(
                x,
                local_extent=local_extent,
                split_dim=split_dim,
                group=object(),
            )
        finally:
            wan_spatial_shard._SPATIAL_SHARD_CONTEXT.reset(token)
        start = rank * local_extent
        valid_extent = min(local_extent, max(0, full_extent - start))
        expected = torch.zeros(local_extent, dtype=torch.float32)
        if valid_extent:
            expected[:valid_extent] = torch.arange(start, start + valid_extent, dtype=torch.float32)
        assert out.shape[_spatial_dim_for_test(split_dim)] == local_extent
        assert torch.equal(out.flatten(), expected)
        outputs.append(out)
    reconstructed = torch.cat(outputs, dim=_spatial_dim_for_test(split_dim))
    reconstructed = reconstructed.narrow(_spatial_dim_for_test(split_dim), 0, full_extent)
    assert torch.equal(reconstructed, x)
 def _spatial_dim_for_test(split_dim: str) -> int:
    return -2 if split_dim == "height" else -1
@pytest.mark.core_model
@pytest.mark.cpu
 def test_reshard_from_trimmed_extent_rejects_rank_mismatch(monkeypatch: pytest.MonkeyPatch):
    monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (1, 2))
    x = torch.zeros((1, 1, 1, 1, 4), dtype=torch.float32)
    token = wan_spatial_shard._SPATIAL_SHARD_CONTEXT.set(
        wan_spatial_shard.SpatialShardContext(
            input_extent=4,
            local_input_extent=2,
            split_dim="width",
            rank=0,
            world_size=2,
        )
    )
    try:
        with pytest.raises(RuntimeError, match="group_rank=1, context_rank=0"):
            wan_spatial_shard.reshard_from_trimmed_extent(
                x,
                local_extent=2,
                split_dim="width",
                group=object(),
            )
    finally:
        wan_spatial_shard._SPATIAL_SHARD_CONTEXT.reset(token)
@pytest.mark.core_model
@pytest.mark.cpu
 def test_attention_wrapper_rejects_spatial_extent_change(monkeypatch: pytest.MonkeyPatch):
    class ShrinkingWanAttentionBlock(torch.nn.Module):
        def forward(self, x):
            return x[..., :-1]
    monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (0, 2))
    monkeypatch.setattr(
        wan_spatial_shard,
        "all_gather_along_dim",
        lambda x, *, group, dim, dst=None: torch.cat([x, x], dim=dim),
    )
    module = ShrinkingWanAttentionBlock()
    wan_spatial_shard._patch_attention_block(module, group=object(), split_dim="width")
    x = torch.zeros((1, 1, 1, 1, 2), dtype=torch.float32)
    token = wan_spatial_shard._SPATIAL_SHARD_CONTEXT.set(
        wan_spatial_shard.SpatialShardContext(
            input_extent=4,
            local_input_extent=2,
            split_dim="width",
            rank=0,
            world_size=2,
        )
    )
    try:
        with pytest.raises(
            RuntimeError,
            match=r"local_extent=2, gathered_extent=4, trimmed_extent=4, output_extent=3",
        ):
            module(x)
    finally:
        wan_spatial_shard._SPATIAL_SHARD_CONTEXT.reset(token)
@pytest.mark.core_model
@pytest.mark.cpu
 def test_halo_exchange_single_rank_noop(monkeypatch: pytest.MonkeyPatch):
@@ -29,10 +29,10 @@ def test_diffusion_model(tmp_path: Path, omni_runner):
            raise ValueError("Empty outputs from Omni.generate()")
        first_output = outputs[0]
        assert first_output.final_output_type == "image"
-        if not isinstance(first_output, OmniRequestOutput) or not first_output:
+        if not hasattr(first_output, "request_output") or not first_output.request_output:
            raise ValueError("No request_output found in OmniRequestOutput")
-        req_out = first_output
+        req_out = first_output.request_output
        if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"):
            raise ValueError("Invalid request_output structure or missing 'images' key")
        return req_out.images
@@ -1,26 +0,0 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 import torch
 from vllm_omni.diffusion.layers.norm import RMSNorm
 from vllm_omni.diffusion.models.minimax_h3.encoder import MiniMaxH3Qwen3VLRMSNorm
 def test_qwen3_vl_rmsnorm_uses_common_fused_rmsnorm_contract() -> None:
    """Qwen keeps BF16 gamma while native fallback accumulates in FP32."""
    eps = 1e-6
    norm = MiniMaxH3Qwen3VLRMSNorm(hidden_size=4, eps=eps, dtype=torch.bfloat16)
    norm.weight.data.copy_(torch.tensor([1.0, 0.5, 1.5, 2.0], dtype=torch.bfloat16))
    x = torch.tensor(
        [[1.0, -2.0, 3.0, -4.0], [-5.0, 6.0, -7.0, 8.0]],
        dtype=torch.bfloat16,
    )
    x_fp32 = x.float()
    expected = (x_fp32 * torch.rsqrt(x_fp32.square().mean(-1, keepdim=True) + eps) * norm.weight.float()).to(x.dtype)
    assert isinstance(norm, RMSNorm)
    assert norm.weight.dtype == torch.bfloat16
    assert set(norm.state_dict()) == {"weight"}
    torch.testing.assert_close(norm.forward_native(x), expected, atol=0, rtol=0)
@@ -122,8 +122,8 @@ def test_cpu_offload_diffusion_model(model_name: str):
    print(f"No offload peak memory: {no_offload_peak_memory} MB")
    if model_name == "stabilityai/stable-audio-open-1.0":
-        audio_offload = output_offload[0].multimodal_output.get("audio")
+        audio_offload = output_offload[0].request_output.multimodal_output.get("audio")
-        audio_no_offload = output_no_offload[0].multimodal_output.get("audio")
+        audio_no_offload = output_no_offload[0].request_output.multimodal_output.get("audio")
        check_audio_determinism(audio_offload, audio_no_offload, atol=1e-2)
    # Set platform-specific VRAM saving thresholds to account
@@ -20,10 +20,18 @@ IMAGE_VIDEO_MODELS = {
 MODELS = {**AUDIO_MODEL, **IMAGE_VIDEO_MODELS}
-MODEL_MARKS = {
+LAYERWISE_MODEL_CASES = [
-    "riverclouds/qwen_image_random": pytest.mark.core_model,
+    pytest.param(
-    "stabilityai/stable-audio-open-1.0": pytest.mark.full_model,
+        "stabilityai/stable-audio-open-1.0",
-}
+        marks=[pytest.mark.full_model, pytest.mark.diffusion],
        id="stable_audio",
    ),
    pytest.param(
        "riverclouds/qwen_image_random",
        marks=[pytest.mark.core_model, pytest.mark.diffusion],
        id="qwen_image_random",
    ),
 ]
 AUDIO_MODEL_PARAMS = {
    "runner_params": {},
@@ -93,9 +101,8 @@ def run_inference(
    return peak, output
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325"})
-@pytest.mark.parametrize("model_name", list(MODELS.keys()))
+@pytest.mark.parametrize("model_name", LAYERWISE_MODEL_CASES)
 def test_layerwise_offload_diffusion_model(model_name: str):
    """Test that layerwise offloading reduces GPU memory usage.
@@ -127,8 +134,8 @@ def test_layerwise_offload_diffusion_model(model_name: str):
    print(f"No offload peak memory: {no_offload_peak_memory} MB")
    if model_name == "stabilityai/stable-audio-open-1.0":
-        audio_offload = output_offload[0].multimodal_output.get("audio")
+        audio_offload = output_offload[0].request_output.multimodal_output.get("audio")
-        audio_no_offload = output_no_offload[0].multimodal_output.get("audio")
+        audio_no_offload = output_no_offload[0].request_output.multimodal_output.get("audio")
        # Match the sibling cpu-offload test's tolerance: layerwise offload moves
        # blocks across the PCIe bus on a side stream, which can perturb cuBLAS
        # algorithm selection and produce ~ULP-level drift larger than 1e-3.
@@ -94,8 +94,8 @@ def _generate_single_stage_image(
    if hasattr(first_output, "images") and first_output.images:
        images = first_output.images
    else:
-        assert isinstance(first_output, OmniRequestOutput) and first_output
+        assert hasattr(first_output, "request_output") and first_output.request_output
-        request_output = first_output
+        request_output = first_output.request_output
        if isinstance(request_output, list):
            req_out = request_output[0]
        else:
@@ -88,8 +88,8 @@ def _generate_single_stage_image(
        if hasattr(first_output, "images") and first_output.images:
            images = first_output.images
        else:
-            assert isinstance(first_output, OmniRequestOutput) and first_output
+            assert hasattr(first_output, "request_output") and first_output.request_output
-            request_output = first_output
+            request_output = first_output.request_output
            if isinstance(request_output, list):
                req_out = request_output[0]
            else:
@@ -147,10 +147,10 @@ def _generate_single_stage_video(
        first = outputs[0]
-        # Unwrap pipeline-style outputs (multi-stage / OmniRequestOutput).
+        # Unwrap pipeline-style outputs (multi-stage / OmniRequestOutput.request_output).
        frames: Any = None
-        if isinstance(first, OmniRequestOutput) and isinstance(first, list):
+        if hasattr(first, "request_output") and isinstance(first.request_output, list):
-            inner = first[0]
+            inner = first.request_output[0]
            if isinstance(inner, OmniRequestOutput) and inner.images:
                frames = inner.images[0]
        if frames is None and hasattr(first, "images") and first.images:
@@ -189,7 +189,7 @@ def _generate_single_stage_video(
 def _generate_bagel_image(
-    diffusion_quantization_config: str | None = None,
+    quantization_config: str | None = None,
    num_inference_steps: int = 15,
 ) -> tuple[Any, float]:
    """Generate an image with BAGEL (multi-stage: LLM + Diffusion).
@@ -202,8 +202,8 @@ def _generate_bagel_image(
        "stage_configs_path": config_path,
        "stage_init_timeout": 300,
    }
-    if diffusion_quantization_config:
+    if quantization_config:
-        omni_kwargs["diffusion_quantization_config"] = diffusion_quantization_config
+        omni_kwargs["quantization_config"] = quantization_config
    model_name = omni_kwargs.pop("model")
    with OmniRunner(model_name, **omni_kwargs) as runner:
@@ -234,8 +234,8 @@ def _generate_bagel_image(
            if images := getattr(req_output, "images", None):
                generated_image = images[0]
                break
-            if isinstance(req_output, OmniRequestOutput) and req_output:
+            if hasattr(req_output, "request_output") and req_output.request_output:
-                stage_outputs = req_output
+                stage_outputs = req_output.request_output
                if not isinstance(stage_outputs, list):
                    stage_outputs = [stage_outputs]
                for stage_out in stage_outputs:
@@ -250,8 +250,8 @@ def _generate_bagel_image(
        # Check LLM stage output — should have finish_reason=stop (not length)
        for req_output in omni_outputs:
-            if isinstance(req_output, OmniRequestOutput) and req_output:
+            if hasattr(req_output, "request_output") and req_output.request_output:
-                stage_outputs = req_output
+                stage_outputs = req_output.request_output
                if not isinstance(stage_outputs, list):
                    stage_outputs = [stage_outputs]
                for stage_out in stage_outputs:
@@ -380,21 +380,21 @@ def test_single_stage_ltx2_fp8_uses_less_memory():
@hardware_test(res={"cuda": "H100"})
 def test_bagel_fp8_generates_image():
-    """BAGEL with diffusion-stage FP8 generates a valid image.
+    """BAGEL with FP8 quantization_config generates a valid image.
    FP8 should only apply to the diffusion stage (Stage-1), not the
    LLM stage (Stage-0). We verify this by checking:
      1. Image is generated successfully
      2. LLM stage finish_reason is 'stop' (not 'length' from garbled output)
    """
-    image, _ = _generate_bagel_image(diffusion_quantization_config="fp8")
+    image, _ = _generate_bagel_image(quantization_config="fp8")
    image.save("test_bagel_fp8.png")
@hardware_test(res={"cuda": "H100"})
 def test_bagel_bf16_generates_image():
    """BAGEL without quantization generates a valid image (baseline)."""
-    image, _ = _generate_bagel_image(diffusion_quantization_config=None)
+    image, _ = _generate_bagel_image(quantization_config=None)
    image.save("test_bagel_bf16.png")
@@ -310,7 +310,7 @@ def _generate_image(omni, config: QualityTestConfig):
    first = outputs[0]
    if hasattr(first, "images") and first.images:
        return first.images[0], peak_mem
-    inner = first
+    inner = first.request_output
    if inner is not None and hasattr(inner, "images") and inner.images:
        return inner.images[0], peak_mem
    raise ValueError("Could not extract image from output.")
@@ -341,15 +341,15 @@ def _generate_video(omni, config: QualityTestConfig):
    )
    peak_mem = torch.accelerator.max_memory_allocated() / (1024**3)
-    if isinstance(outputs, list) and isinstance(outputs[0], OmniRequestOutput):
+    first = outputs[0]
-        first = outputs[0]
+    if hasattr(first, "request_output") and isinstance(first.request_output, list):
-    elif isinstance(outputs, OmniRequestOutput):
+        inner = first.request_output[0]
-        first = outputs
+        if isinstance(inner, OmniRequestOutput) and hasattr(inner, "images"):
-    else:
+            frames = inner.images[0] if inner.images else None
-        raise ValueError("Could not extract video frames from output.")
+        else:
-
+            frames = inner
-    if hasattr(first, "images"):
+    elif hasattr(first, "images") and first.images:
-        frames = first.images[0] if first.images else None
+        frames = first.images[0]
    else:
        raise ValueError("Could not extract video frames from output.")
@@ -444,11 +444,11 @@ def test_benchmark_generate_image_unwraps_nested_omni_request_output(monkeypatch
    image = Image.new("RGB", (2, 2))
    inner = OmniRequestOutput.from_diffusion(request_id="req", images=[image])
-    outer = OmniRequestOutput.from_stage_output(
+    outer = OmniRequestOutput(
        inner,
        request_id="req",
        stage_id=0,
        final_output_type="image",
        request_output=inner,
        finished=True,
    )
@@ -73,9 +73,11 @@ def test_run_summary_reports_worker_peak_memory():
    assert summary["per_run_peak_memory_mb"] == [100.0, 150.0]
-def test_request_peak_memory_reads_peak_memory_mb_directly():
+def test_request_peak_memory_prefers_inner_diffusion_output():
-    result = type("Output", (), {"peak_memory_mb": 123.0})()
+    inner = type("InnerOutput", (), {"peak_memory_mb": 321.0})()
-    assert _request_peak_memory_mb(result) == 123.0
+    outer = type("OuterOutput", (), {"request_output": inner, "peak_memory_mb": 123.0})()
    assert _request_peak_memory_mb(outer) == 321.0
 def test_metric_guidance_describes_thresholds():
@@ -95,7 +95,6 @@ def test_formatter_preserves_single_video_audio_actions_and_metadata(
        "resolution": 512,
        "width": None,
        "height": None,
        "num_inference_steps": 1,
    }
@@ -273,7 +273,7 @@ class TestPipelineStreamingOutputToEntrypoint:
            await _wait_for(lambda: len(pipeline.requests) == 1)
            assert pipeline.requests[0].request_id.startswith("req-omni-")
            assert [output.multimodal_output["metadata"]["stream"]["chunk"] for output in outputs] == [0, 1]
-            assert [output.finished for output in outputs] == [False, True]
+            assert [getattr(output.request_output, "finished", output.finished) for output in outputs] == [False, True]
        finally:
            await self._shutdown_pipeline_omni_harness(omni, fixture, inline_client)
@@ -25,7 +25,6 @@ from tests.helpers.mark import hardware_test
 from tests.helpers.runtime import OmniRunner
 from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.outputs import OmniRequestOutput
 pytestmark = [pytest.mark.usefixtures("clean_gpu_memory_between_tests")]
@@ -96,8 +95,8 @@ def _extract_generated_image(omni_outputs: list) -> Image.Image | None:
    for req_output in omni_outputs:
        if images := getattr(req_output, "images", None):
            return images[0]
-        if isinstance(req_output, OmniRequestOutput) and req_output:
+        if hasattr(req_output, "request_output") and req_output.request_output:
-            stage_out = req_output
+            stage_out = req_output.request_output
            if hasattr(stage_out, "images") and stage_out.images:
                return stage_out.images[0]
    return None
@@ -24,7 +24,6 @@ from tests.helpers.mark import hardware_test
 from tests.helpers.runtime import OmniRunner
 from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.outputs import OmniRequestOutput
 pytestmark = [pytest.mark.usefixtures("clean_gpu_memory_between_tests")]
@@ -123,8 +122,8 @@ def _extract_generated_image(omni_outputs: list) -> Image.Image | None:
    for req_output in omni_outputs:
        if images := getattr(req_output, "images", None):
            return images[0]
-        if isinstance(req_output, OmniRequestOutput) and req_output:
+        if hasattr(req_output, "request_output") and req_output.request_output:
-            stage_out = req_output
+            stage_out = req_output.request_output
            if hasattr(stage_out, "images") and stage_out.images:
                return stage_out.images[0]
    return None
@@ -3,8 +3,8 @@
 Daily-Omni settings follow the MiniCPM interleaved AV recipe that reaches
 ~78% overall accuracy on Daily-Omni (``minicpm-interleave``, ``temperature=0``,
-text modalities, and server ``--interleave-mm-strings`` + 1fps / 128-frame
+text modalities with ``use_tts_template``, and server ``--interleave-mm-strings``
-media-io kwargs, also pinned in ``minicpmo_4_5.yaml``).
+ 1fps / 128-frame media-io kwargs, also pinned in ``minicpmo_4_5.yaml``).
 """
 from __future__ import annotations
@@ -40,7 +40,10 @@ _MAX_SEED_TTS_MEAN_WER = 0.05
 # Match the validated Daily-Omni client body from daily_omni_bench.sh.
 _DAILY_EXTRA_BODY = {
    "modalities": ["text"],
-    "chat_template_kwargs": {"enable_thinking": False},
+    "chat_template_kwargs": {
        "enable_thinking": False,
        "use_tts_template": True,
    },
 }
 _SEED_EXTRA_BODY = {
    "modalities": ["text", "audio"],
@@ -328,7 +328,7 @@ def _run_offline(deploy_config_path: str, output_path: Path) -> tuple[Image.Imag
    images = None
    cot_text = ""
    for out in outputs:
-        ro = out
+        ro = getattr(out, "request_output", None)
        if ro and getattr(ro, "outputs", None):
            cot_text = "".join(getattr(o, "text", "") or "" for o in ro.outputs)
        if not cot_text:
@@ -467,7 +467,7 @@ def _extract_image(outputs) -> Image.Image:
    assert outputs, "Pipeline produced no outputs"
    for output in outputs:
        images = getattr(output, "images", None)
-        request_output = output
+        request_output = getattr(output, "request_output", None)
        if not images and request_output is not None:
            images = getattr(request_output, "images", None)
        if images:
@@ -141,8 +141,8 @@ def _extract_frames(result) -> list[Image.Image]:
    if isinstance(result, list):
        result = result[0] if result else None
-    if isinstance(result, OmniRequestOutput) and result.is_pipeline_output and result is not None:
+    if isinstance(result, OmniRequestOutput) and result.is_pipeline_output and result.request_output is not None:
-        result = result
+        result = result.request_output
    images = getattr(result, "images", None)
    if not images:
        raise AssertionError("Pipeline output had no images")
@@ -171,11 +171,11 @@ def _build_text_output(text: str = "This is a test response.") -> OmniRequestOut
        metrics=None,
        lora_request=None,
    )
-    return OmniRequestOutput.from_stage_output(
+    return OmniRequestOutput(
        request_output,
        request_id="test_req_text",
        finished=True,
        final_output_type="text",
        request_output=request_output,
    )
@@ -200,11 +200,11 @@ def _build_audio_chat_output(num_samples: int = 24000) -> OmniRequestOutput:
        metrics=None,
        lora_request=None,
    )
-    return OmniRequestOutput.from_stage_output(
+    return OmniRequestOutput(
        request_output,
        request_id="test_req_audio_chat",
        finished=True,
        final_output_type="audio",
        request_output=request_output,
    )
@@ -240,11 +240,11 @@ def _build_diffusion_image_output_for_chat_endpoint() -> OmniRequestOutput:
        images=[_build_image_output(color="blue")],
        finished=True,
    )
-    return OmniRequestOutput.from_stage_output(
+    return OmniRequestOutput(
        request_output,
        request_id="test_req_img_chat",
-        images=[_build_image_output(color="blue")],
+        finished=True,
        final_output_type="image",
        request_output=request_output,
    )
@@ -515,19 +515,8 @@ def mock_async_omni(
@pytest.fixture
-def api_server(unused_tcp_port_factory, server_case: ServerCase, mock_async_omni, tmp_path):
+def api_server(unused_tcp_port_factory, server_case: ServerCase, mock_async_omni):
    """Set up a API server in background process from command line with parametrized model name and mocked AsyncOmni."""
    # Override the STORAGE_MANAGER path to a writable temp directory before
    # forking the server subprocess.  The default /tmp/storage may be owned
    # by a different user (e.g. from a previous CI run) and cause
    # PermissionError when the server tries to save generated video files.
    from vllm_omni.entrypoints.openai.storage import STORAGE_MANAGER
    old_storage_path = STORAGE_MANAGER.storage_path
    new_storage_path = tmp_path / "omni_storage"
    new_storage_path.mkdir(exist_ok=True)
    STORAGE_MANAGER.storage_path = str(new_storage_path)
    parser = TrackingArgumentParser()
    subparsers = parser.add_subparsers(dest="command")
    cmd = OmniServeCommand()
@@ -566,9 +555,6 @@ def api_server(unused_tcp_port_factory, server_case: ServerCase, mock_async_omni
    yield f"http://127.0.0.1:{port}/v1"
    # Restore the original storage path after the test.
    STORAGE_MANAGER.storage_path = old_storage_path
    if server_process.is_alive():
        server_process.terminate()
    server_process.join(timeout=10)
@@ -294,7 +294,7 @@ class vLLMOmniHttpServerLocal:
            "global_steps": self.global_steps,
        }
-        req_output = final_res
+        req_output = final_res.request_output
        if req_output is not None and hasattr(req_output, "finish_reason"):
            finish_reason = req_output.finish_reason or "stop"
        else:
@@ -53,9 +53,9 @@ def test_audiox_model(omni_runner: OmniRunner) -> None:
    assert outputs is not None
    first_output = outputs[0]
    assert first_output.final_output_type == "audio"
-    assert isinstance(first_output, OmniRequestOutput) and first_output
+    assert hasattr(first_output, "request_output") and first_output.request_output
-    req_out = first_output
+    req_out = first_output.request_output
    assert isinstance(req_out, OmniRequestOutput)
    assert req_out.final_output_type == "audio"
    assert hasattr(req_out, "multimodal_output") and req_out.multimodal_output
@@ -55,7 +55,7 @@ pytestmark = [
 def _extract_text(omni_outputs: list) -> str:
    """Extract generated text from OmniRequestOutput list."""
    for req_output in omni_outputs:
-        ro = req_output
+        ro = getattr(req_output, "request_output", None)
        if ro and getattr(ro, "outputs", None):
            return "".join(getattr(o, "text", "") or "" for o in ro.outputs)
    return ""