Merge branch 'main' into dev/vllm-align

ci: use vLLM 0.29 release image and align installation docs
Drop the old-base wheel reinstall and dependency repair block. Update source installation and configuration references to vLLM 0.29 while retaining explicitly matched 0.28 prebuilt Omni examples until its next release. Validation: official image and CUDA/ROCm release artifacts checked; 6 documentation tests passed; diff whitespace checks passed. Markdown lint reports existing snippet/anchor issues. Container build not run: no builder available. Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
2026-09-10 11:23:08 +08:00 · 2026-09-10 11:11:42 +08:00 · 2026-09-10 03:07:35 +00:00 · 2026-09-10 02:16:14 +00:00 · 2026-09-10 10:09:39 +08:00 · 2026-09-10 09:54:20 +08:00
612 changed files with 43289 additions and 12278 deletions
@@ -14,6 +14,8 @@ set -euo pipefail
 # shellcheck disable=SC1091
 source .buildkite/common/scripts/resolve_skip_ci.sh

+AMD_SUITE_SELECTOR=".buildkite/amd/scripts/select_test_suites.py"
+
 if [[ -z "${RUN_ALL:-}" ]]; then
    RUN_ALL=0
 fi
@@ -30,31 +32,20 @@ if [[ -z "${AMD_MIRROR_HW:-}" ]]; then
    AMD_MIRROR_HW="amdproduction"
 fi

-fail_fast() {
-    DISABLE_LABEL="ci-no-fail-fast"
-    # If BUILDKITE_PULL_REQUEST != "false", then we check the PR labels using curl and jq
-    if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then
-        PR_LABELS=$(curl -s "https://api.github.com/repos/vllm-project/vllm-omni/pulls/$BUILDKITE_PULL_REQUEST" | jq -r '.labels[].name')
-        if [[ $PR_LABELS == *"$DISABLE_LABEL"* ]]; then
-            echo false
-        else
-            echo true
-        fi
-    else
-        echo false  # not a PR or BUILDKITE_PULL_REQUEST not set
+PR_LABELS=""
+# Explicit debug suites do not depend on PR labels.
+if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" && -z "${DEBUG_TEST_YAML:-}" ]]; then
+    if ! PR_LABELS=$(curl -fsSL \
+        "https://api.github.com/repos/vllm-project/vllm-omni/pulls/$BUILDKITE_PULL_REQUEST" \
+        | jq -r '.labels[].name'); then
+        echo "ERROR: Could not read PR labels; refusing to select an incorrect AMD test tier." >&2
+        exit 1
    fi
-}
+fi

-check_run_all_label() {
-    RUN_ALL_LABEL="ready-run-all-tests"
-    # If BUILDKITE_PULL_REQUEST != "false", then we check the PR labels using curl and jq
-    if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then
-        PR_LABELS=$(curl -s "https://api.github.com/repos/vllm-project/vllm-omni/pulls/$BUILDKITE_PULL_REQUEST" | jq -r '.labels[].name')
-        if [[ $PR_LABELS == *"$RUN_ALL_LABEL"* ]]; then
-            echo true
-        else
-            echo false
-        fi
+fail_fast() {
+    if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
+        echo true
    else
        echo false  # not a PR or BUILDKITE_PULL_REQUEST not set
    fi
@@ -64,6 +55,57 @@ if [[ -z "${COV_ENABLED:-}" ]]; then
    COV_ENABLED=0
 fi

+resolve_test_specs() {
+    local selected_specs
+    if ! selected_specs=$(python3 "$AMD_SUITE_SELECTOR" \
+        --branch "$BUILDKITE_BRANCH" \
+        --labels "$PR_LABELS" \
+        --debug-test-yaml "${DEBUG_TEST_YAML:-}"); then
+        exit 1
+    fi
+
+    TEST_SPECS=()
+    while IFS= read -r suite_spec; do
+        [[ -z "$suite_spec" ]] && continue
+        TEST_SPECS+=("$suite_spec")
+    done <<< "$selected_specs"
+}
+
+filter_test_specs_by_skip_ci() {
+    local suite_spec level decision
+    local skip_all=0
+    local -a runnable_specs=()
+
+    for suite_spec in "${TEST_SPECS[@]}"; do
+        case "$suite_spec" in
+            READY_TESTS:*) level="l2" ;;
+            MERGE_TESTS:*) level="l3" ;;
+            *)
+                echo "ERROR: unknown AMD test suite spec '$suite_spec'" >&2
+                exit 1
+                ;;
+        esac
+
+        if decision=$(python3 "$SKIP_CI_PY" gate amd "$level"); then
+            echo "Skipping AMD $level suite due to $decision decision."
+            [[ "$decision" == "skip-all" ]] && skip_all=1
+        else
+            runnable_specs+=("$suite_spec")
+        fi
+    done
+
+    TEST_SPECS=("${runnable_specs[@]}")
+    if [[ ${#TEST_SPECS[@]} -eq 0 ]]; then
+        if [[ $skip_all -eq 1 ]]; then
+            buildkite-agent annotate \
+                ":memo: CI skipped — docs or pytest skip-mark changes only" \
+                --style "info" 2>/dev/null || true
+        fi
+        echo "No AMD L2/L3 suites remain after skip-ci filtering."
+        exit 0
+    fi
+}
+
 upload_pipeline() {
    echo "Uploading pipeline..."
    # Install minijinja
@@ -84,7 +126,7 @@ upload_pipeline() {
    # (WIP) Use pipeline generator instead of jinja template
    if [ -e ".buildkite/amd/pipeline_generator/pipeline_generator.py" ]; then
        python -m pip install click pydantic
-        python .buildkite/amd/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW"
+        python .buildkite/amd/pipeline_generator/pipeline_generator.py --run_all="$RUN_ALL" --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW"
        buildkite-agent pipeline upload .buildkite/amd/pipeline.yaml
        exit 0
    fi
@@ -97,49 +139,12 @@ upload_pipeline() {

    cd .buildkite/amd

-    # Select test definition file: merge suite for main, ready suite for PRs.
-    # For debugging, DEBUG_TEST_YAML accepts a comma-separated list containing
-    # "merge" and/or "ready" (case-insensitive). Multiple suites are combined
-    # before rendering so they share one amd-build step.
-    if [[ -n "${DEBUG_TEST_YAML:-}" ]]; then
-        declare -a DEBUG_TEST_SPECS=()
-        declare -A SEEN_DEBUG_TESTS=()
-        IFS=',' read -ra REQUESTED_DEBUG_TESTS <<< "${DEBUG_TEST_YAML,,}"
-
-        for requested_test in "${REQUESTED_DEBUG_TESTS[@]}"; do
-            # Permit readable values such as "ready, merge," and ignore the
-            # empty item produced by a trailing comma.
-            requested_test="${requested_test//[[:space:]]/}"
-            [[ -z "$requested_test" ]] && continue
-
-            if [[ -n "${SEEN_DEBUG_TESTS[$requested_test]:-}" ]]; then
-                echo "ERROR: duplicate DEBUG_TEST_YAML suite '$requested_test'" >&2
-                exit 1
-            fi
-            SEEN_DEBUG_TESTS[$requested_test]=1
-
-            case "$requested_test" in
-                ready)
-                    DEBUG_TEST_SPECS+=("READY_TESTS:test-amd-ready.yml")
-                    ;;
-                merge)
-                    DEBUG_TEST_SPECS+=("MERGE_TESTS:test-amd-merge.yml")
-                    ;;
-                *)
-                    echo "ERROR: DEBUG_TEST_YAML entries must be 'merge' or 'ready', got '$requested_test'" >&2
-                    exit 1
-                    ;;
-            esac
-        done
-
-        if [[ ${#DEBUG_TEST_SPECS[@]} -eq 0 ]]; then
-            echo "ERROR: DEBUG_TEST_YAML did not contain a test suite" >&2
-            exit 1
-        elif [[ ${#DEBUG_TEST_SPECS[@]} -eq 1 ]]; then
-            TEST_YAML="${DEBUG_TEST_SPECS[0]#*:}"
-        else
-            TEST_YAML=$(mktemp "${TMPDIR:-/tmp}/amd-debug-tests.XXXXXX.yml")
-            python - "$TEST_YAML" "${DEBUG_TEST_SPECS[@]}" <<'PY'
+    # Multiple label-selected or debug-selected suites share one image build.
+    if [[ ${#TEST_SPECS[@]} -eq 1 ]]; then
+        TEST_YAML="${TEST_SPECS[0]#*:}"
+    else
+        TEST_YAML=$(mktemp "${TMPDIR:-/tmp}/amd-selected-tests.XXXXXX.yml")
+        python - "$TEST_YAML" "${TEST_SPECS[@]}" <<'PY'
 import sys

 import yaml
@@ -172,13 +177,8 @@ for suite_spec in suite_specs:
 with open(output_path, "w", encoding="utf-8") as output_file:
    yaml.safe_dump(combined, output_file, sort_keys=False)
 PY
-        fi
-        echo "DEBUG_TEST_YAML override: using ${DEBUG_TEST_SPECS[*]}"
-    elif [[ $BUILDKITE_BRANCH == "main" ]]; then
-        TEST_YAML="test-amd-merge.yml"
-    else
-        TEST_YAML="test-amd-ready.yml"
    fi
+    echo "AMD test suites: ${TEST_SPECS[*]}"

    (
        set -x
@@ -198,7 +198,7 @@ PY
            > pipeline.yaml
    )
    cat pipeline.yaml
-    if [[ "$TEST_YAML" == "${TMPDIR:-/tmp}/amd-debug-tests."*.yml ]]; then
+    if [[ "$TEST_YAML" == "${TMPDIR:-/tmp}/amd-selected-tests."*.yml ]]; then
        rm -f -- "$TEST_YAML"
    fi
    buildkite-agent artifact upload pipeline.yaml
@@ -219,12 +219,10 @@ if [[ $BUILDKITE_BRANCH == "main" ]]; then
    file_diff=$(get_diff_main)
 fi

-# Early exit: unified skip-ci (docs / skip marks) and CI-yaml-only level targeting.
-if [[ $BUILDKITE_BRANCH == "main" ]]; then
-    gate_bootstrap_ci amd l3
-else
-    gate_bootstrap_ci amd l2
-fi
+# Resolve PR tier labels before skip-ci so ready and merge suites can be
+# filtered independently for CI-YAML-only changes.
+resolve_test_specs
+filter_test_specs_by_skip_ci

 patterns=(
    "docker/Dockerfile"
@@ -274,14 +272,6 @@ for file in $file_diff; do
    fi
 done

-# Check for ready-run-all-tests label
-LABEL_RUN_ALL=$(check_run_all_label)
-if [[ $LABEL_RUN_ALL == true ]]; then
-    RUN_ALL=1
-    NIGHTLY=1
-    echo "Found 'ready-run-all-tests' label. Running all tests including optional tests."
-fi
-
 # Decide whether to use precompiled wheels
 # Relies on existing patterns array as a basis.
 if [[ -n "${VLLM_USE_PRECOMPILED:-}" ]]; then
@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Select AMD L2/L3 suites after Buildkite has created an AMD build.
+
+This repository-side selector does not decide which GitHub label events start
+the external ``vllm-omni-amd-ci`` pipeline. Its Buildkite PR build condition
+must admit the relevant label event before this code can inspect the labels.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Iterable
+
+SUITE_SPECS = {
+    "ready": "READY_TESTS:test-amd-ready.yml",
+    "merge": "MERGE_TESTS:test-amd-merge.yml",
+}
+
+
+def _parse_debug_suites(value: str) -> tuple[str, ...]:
+    suites = tuple(item.strip().lower() for item in value.split(",") if item.strip())
+    if not suites:
+        raise ValueError("DEBUG_TEST_YAML did not contain a test suite")
+
+    seen: set[str] = set()
+    for suite in suites:
+        if suite not in SUITE_SPECS:
+            raise ValueError(f"DEBUG_TEST_YAML entries must be 'merge' or 'ready', got {suite!r}")
+        if suite in seen:
+            raise ValueError(f"duplicate DEBUG_TEST_YAML suite {suite!r}")
+        seen.add(suite)
+    return suites
+
+
+def select_amd_test_suites(
+    *,
+    branch: str,
+    labels: Iterable[str],
+    debug_test_yaml: str = "",
+) -> tuple[str, ...]:
+    """Return ordered suite names while preserving legacy unlabeled PRs."""
+
+    if debug_test_yaml:
+        return _parse_debug_suites(debug_test_yaml)
+    if branch == "main":
+        return ("merge",)
+
+    label_set = {label.strip() for label in labels if label.strip()}
+    selected = tuple(suite for label, suite in (("ready", "ready"), ("merge-test", "merge")) if label in label_set)
+    # AMD historically uploaded L2 for every PR build. Keep that behavior for
+    # triggers such as amd-test while adding tier selection for ready/merge-test.
+    return selected or ("ready",)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--branch", required=True)
+    parser.add_argument("--labels", default="")
+    parser.add_argument("--debug-test-yaml", default="")
+    args = parser.parse_args()
+
+    labels = tuple(args.labels.splitlines())
+    try:
+        suites = select_amd_test_suites(
+            branch=args.branch,
+            labels=labels,
+            debug_test_yaml=args.debug_test_yaml,
+        )
+    except ValueError as exc:
+        parser.error(str(exc))
+
+    label_set = {label.strip() for label in labels if label.strip()}
+    if not args.debug_test_yaml and args.branch != "main" and not ({"ready", "merge-test"} & label_set):
+        print(
+            "No AMD L2/L3 tier label found; preserving the legacy ready-suite fallback.",
+            file=sys.stderr,
+        )
+    if "nightly-test" in label_set:
+        print(
+            "AMD nightly-test has no suite yet and does not affect L2/L3 selection.",
+            file=sys.stderr,
+        )
+
+    for suite in suites:
+        print(SUITE_SPECS[suite])
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -107,6 +107,11 @@ steps:
        - export VLLM_LOGGING_LEVEL=DEBUG
        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
        - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
+        - >-
+          timeout 20m pytest -s -v
+          tests/diffusion/models/ltx2/test_ltx2_transformer_ulysses.py
+          -m 'core_model and rocm and cards_2'
+          --run-level "core_model"

    - label: "Diffusion GPU Worker Test"
      agent_pool: mi300_2
@@ -122,7 +127,13 @@ steps:
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
-        - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
+        # Preserve legacy one-GPU tests without cards_1 while excluding every
+        # registered multi-GPU marker from this one-GPU job.
+        - >-
+          timeout 15m pytest -sv tests/diffusion/models/
+          -m 'core_model and cuda and not
+          (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
+          --run-level "core_model"

    - label: "Diffusion · Wan22 Test"
      agent_pool: mi300_1
@@ -17,6 +17,8 @@
 #   a2b3_npu_1  — ascend-a2b3, resource_class npu-1 (A2/B3 CI image)
 #   a2b3_npu_4  — ascend-a2b3, resource_class npu-4
 #   a3_npu_2    — ascend-a3, resource_class npu-2 (A3 CI image tag …:$BUILDKITE_COMMIT-a3)
+#   a5_npu_2    — ascend-a5, resource_class npu-2
+#   310p_npu_1  — ascend-310p, resource_class npu-1
 #
 # B200 (b200-k8s): matches vLLM device ``b200-k8s`` from ci-infra
 # (AgentQueue.B200_K8S + b200_plugin_template). Queue routes the K8s pool;
@@ -26,6 +28,9 @@
 # packed onto g6.12xlarge L4 nodes (``vllm.ci/gpu-pool=l4x4``). GPU / CPU /
 # memory / shm requests scale with card count (1 GPU = 10 CPU / 40Gi / 24Gi shm).
 # Retry covers lost agents / killed pods, not pytest or OOM (exit 1 / 137).
+# The Kubernetes HF secret deliberately uses VLLM_CI_HF_TOKEN. Scheduled
+# build-level HF_TOKEN values can override pod env entries with the same name;
+# upload_pipeline.py restores the secret-backed value inside the job shell.

 x-npu-k8s-plugins: &npu_k8s_plugins
  - kubernetes:
@@ -84,7 +89,7 @@ mirror_hardwares:
                env:
                  - name: HF_HOME
                    value: /fsx/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
@@ -135,7 +140,7 @@ mirror_hardwares:
                env:
                  - name: HF_HOME
                    value: /fsx/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
@@ -186,7 +191,7 @@ mirror_hardwares:
                env:
                  - name: HF_HOME
                    value: /fsx/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
@@ -237,7 +242,7 @@ mirror_hardwares:
                env:
                  - name: HF_HOME
                    value: /fsx/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
@@ -271,7 +276,7 @@ mirror_hardwares:
                env:
                  - name: HF_HOME
                    value: /root/.cache/huggingface
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
@@ -306,7 +311,7 @@ mirror_hardwares:
                env:
                  - name: HF_HOME
                    value: /root/.cache/huggingface
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
@@ -341,7 +346,7 @@ mirror_hardwares:
                env:
                  - name: HF_HOME
                    value: /root/.cache/huggingface
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
@@ -376,7 +381,7 @@ mirror_hardwares:
                env:
                  - name: HF_HOME
                    value: /root/.cache/huggingface
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
@@ -426,7 +431,7 @@ mirror_hardwares:
                    value: "0"
                  - name: HF_HOME
                    value: /raid/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
@@ -477,7 +482,7 @@ mirror_hardwares:
                    value: "0"
                  - name: HF_HOME
                    value: /raid/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
@@ -528,7 +533,7 @@ mirror_hardwares:
                    value: "0"
                  - name: HF_HOME
                    value: /raid/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
@@ -579,7 +584,7 @@ mirror_hardwares:
                    value: "0"
                  - name: HF_HOME
                    value: /raid/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
@@ -646,3 +651,52 @@ mirror_hardwares:
      resource_class: "npu-16"
    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
    plugins: *npu_k8s_plugins
+
+  a5_npu_2:
+    agents:
+      queue: "ascend-a5"
+      resource_class: "npu-2"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a5"
+    plugins: *npu_k8s_plugins
+
+  a5_npu_4:
+    agents:
+      queue: "ascend-a5"
+      resource_class: "npu-4"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a5"
+    plugins: *npu_k8s_plugins
+
+  a5_npu_8:
+    agents:
+      queue: "ascend-a5"
+      resource_class: "npu-8"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a5"
+    plugins: *npu_k8s_plugins
+
+  a5_npu_16:
+    agents:
+      queue: "ascend-a5"
+      resource_class: "npu-16"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a5"
+    plugins: *npu_k8s_plugins
+
+  310p_npu_1:
+    agents:
+      queue: "ascend-310p"
+      resource_class: "npu-1"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-310p"
+    plugins: *npu_k8s_plugins
+
+  310p_npu_2:
+    agents:
+      queue: "ascend-310p"
+      resource_class: "npu-2"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-310p"
+    plugins: *npu_k8s_plugins
+
+  310p_npu_4:
+    agents:
+      queue: "ascend-310p"
+      resource_class: "npu-4"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-310p"
+    plugins: *npu_k8s_plugins
@@ -70,7 +70,15 @@ from tests.helpers.mark import (  # noqa: E402

 LOG = "upload_pipeline"
 BOOTSTRAP_STEPS_FILENAME = "bootstrap-upload-steps.yml"
-BOOTSTRAP_IMAGE_BUILD_KEYS = frozenset({"image-build", "image-build-a2", "image-build-a3"})
+BOOTSTRAP_IMAGE_BUILD_KEYS = frozenset(
+    {
+        "image-build",
+        "image-build-a2",
+        "image-build-a3",
+        "image-build-a5",
+        "image-build-310p",
+    }
+)
 BOOTSTRAP_UPLOAD_IF_KEYS = {
    "upload-ready-pipeline": "ready",
    "upload-merge-pipeline": "merge",
@@ -79,6 +87,8 @@ BOOTSTRAP_UPLOAD_IF_KEYS = {
 }
 E2E_GROUP_MARKER = "E2E Test"
 CI_MIRROR_HARDWARES_PATH = ROOT / ".buildkite/common/ci_mirror_hardwares.yml"
+CUDA_HF_TOKEN_ENV = "VLLM_CI_HF_TOKEN"
+CUDA_HF_TOKEN_EXPORT = f'if [ -n "$${{{CUDA_HF_TOKEN_ENV}:-}}" ]; then export HF_TOKEN="$${{{CUDA_HF_TOKEN_ENV}}}"; fi'

 # Bootstrap Buildkite ``if`` expressions.
 # ``*_MAIN_IF``: main + env schedule. ``*_LABEL_IF``: PR label (and/or composed with MAIN).
@@ -501,6 +511,17 @@ def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any] | None:

    expanded = copy.deepcopy(preset)
    merged = {key: value for key, value in step.items() if key != "mirror_hardwares"} | expanded
+    # A scheduled build's HF_TOKEN may override a pod env entry with the same
+    # name. The preset injects the Kubernetes secret under a collision-proof
+    # alias; restore the conventional name after Buildkite has merged env.
+    if any(preset_name == chip or preset_name.startswith(f"{chip}_") for chip in _cuda_mirror_chips()):
+        commands = merged.get("commands")
+        if isinstance(commands, list):
+            merged["commands"] = [CUDA_HF_TOKEN_EXPORT, *commands]
+        elif commands is None:
+            merged["commands"] = [CUDA_HF_TOKEN_EXPORT]
+        else:
+            merged["commands"] = [CUDA_HF_TOKEN_EXPORT, commands]
    # Preset retry (K8S_RETRY on l4_*) must not clobber a step that opted out.
    if "retry" in step:
        merged["retry"] = step["retry"]
@@ -79,6 +79,8 @@ steps:
    steps:
      - label: "TTS · Qwen3-TTS CustomVoice Test"
        source_file_dependencies:
+          - tests/e2e/features/prefill_phase/
+          - vllm_omni/worker/gpu_model_runner.py
          - tests/helpers/media.py
          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
          - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
@@ -96,7 +98,7 @@ steps:
            timeout 40m bash -c "
              export VLLM_LOGGING_LEVEL=DEBUG
              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
-              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py tests/e2e/features/prefill_phase/test_chunked_prefill_tail.py -m 'advanced_model and cuda' --run-level 'advanced_model'
            "
        mirror_hardwares: l4_1

@@ -76,7 +76,7 @@ steps:

      - label: "Diffusion · Other Test"
        commands:
-          - timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"
+          - timeout 15m pytest -sv tests/diffusion/*.py tests/diffusion/ar_diffusion -m 'core_model and cuda' --run-level "core_model"
        mirror_hardwares: l4_1

  - group: ":card_index_dividers: Tiny Model Tests [Single GPU]"
@@ -93,7 +93,7 @@ steps:
    commands:
      - |
        timeout 15m bash -c "
-                  pytest -sv tests/engine/ tests/model_executor/ -m 'core_model and cuda' --run-level 'core_model'
+                  pytest -sv tests/engine/ tests/model_executor/ tests/worker/test_batched_omni_output.py -m 'core_model and cuda' --run-level 'core_model'
        "
    mirror_hardwares: l4_1

@@ -3,10 +3,10 @@
 steps:
  - label: ":buildkit: Build and Push NPU Test Image - A2"
    key: image-build-a2
-    agents:
+    agents: &npu-image-build-agents
      queue: "ascend-a2b3"
      resource_class: "npu-1"
-    plugins:
+    plugins: &npu-image-build-plugins
      - kubernetes:
          metadata:
            annotations:
@@ -46,123 +46,47 @@ steps:
      IMAGE_NAME: "vllm-omni-ci-npu"
      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
-    command: |
-      set -ex
-
-      echo "--- Building and pushing NPU Test Image"
-      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-      echo "buildkitd address: $${BUILDKITD_ADDR}"
-
-      if ! command -v buildctl &> /dev/null; then
-        echo "Installing buildctl..."
-        mkdir -p /tmp/buildkit
-        BUILDKIT_VERSION="v0.29.0"
-        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
-        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
-        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
-      fi
-
-      export DOCKER_CONFIG=/home/user/.docker
-      buildctl \
-        --addr="$${BUILDKITD_ADDR}" \
-        --tlscacert=/home/user/.docker/ca.pem \
-        --tlscert=/home/user/.docker/cert.pem \
-        --tlskey=/home/user/.docker/key.pem \
-        build \
-        --frontend dockerfile.v0 \
-        --local context=. \
-        --local dockerfile=./docker \
-        --opt filename=Dockerfile.npu.ci \
-        --secret id=dockerconfig,src=/home/user/.docker/config.json \
-        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
-        --progress=plain
-
-      echo "--- Image pushed successfully"
-      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+    command: bash .buildkite/npu/scripts/build-npu-image.sh Dockerfile.npu.ci "$${VLLM_IMAGE_TAG}"

  - label: ":buildkit: Build and Push NPU Test Image - A3"
    key: image-build-a3
-    agents:
-      queue: "ascend-a2b3"
-      resource_class: "npu-1"
-    plugins:
-      - kubernetes:
-          metadata:
-            annotations:
-              vault.hashicorp.com/agent-init-first: "true"
-              vault.hashicorp.com/agent-inject: "true"
-              vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
-              vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
-              vault.hashicorp.com/agent-inject-perms-config.json: "0400"
-              vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
-              vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
-              vault.hashicorp.com/agent-pre-populate-only: "true"
-              vault.hashicorp.com/agent-run-as-group: "1000"
-              vault.hashicorp.com/agent-run-as-user: "1000"
-              vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
-              vault.hashicorp.com/role: ascend-gha-runners
-              vault.hashicorp.com/secret-volume-path: /home/user/.docker/
-              vault.hashicorp.com/tls-skip-verify: "true"
-          podSpecPatch:
-            volumes:
-            - name: token-vol
-              projected:
-                defaultMode: 420
-                sources:
-                - serviceAccountToken:
-                    audience: api
-                    expirationSeconds: 600
-                    path: token
+    agents: *npu-image-build-agents
+    plugins: *npu-image-build-plugins
    env:
      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-a3"
      IMAGE_NAME: "vllm-omni-ci-npu"
      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
-    command: |
-      set -ex
+    command: bash .buildkite/npu/scripts/build-npu-image.sh Dockerfile.npu.ci "$${VLLM_IMAGE_TAG}" -a3

-      echo "--- Building and pushing NPU Test Image"
-      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-      echo "buildkitd address: $${BUILDKITD_ADDR}"
+  - label: ":buildkit: Build and Push NPU Test Image - A5"
+    key: image-build-a5
+    agents: *npu-image-build-agents
+    plugins: *npu-image-build-plugins
+    env:
+      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-a5"
+      IMAGE_NAME: "vllm-omni-ci-npu"
+      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
+      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
+    command: bash .buildkite/npu/scripts/build-npu-image.sh Dockerfile.npu.ci "$${VLLM_IMAGE_TAG}" -a5

-      if ! command -v buildctl &> /dev/null; then
-        echo "Installing buildctl..."
-        mkdir -p /tmp/buildkit
-        BUILDKIT_VERSION="v0.29.0"
-        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
-        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
-        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
-      fi
-
-      export DOCKER_CONFIG=/home/user/.docker
-      buildctl \
-        --addr="$${BUILDKITD_ADDR}" \
-        --tlscacert=/home/user/.docker/ca.pem \
-        --tlscert=/home/user/.docker/cert.pem \
-        --tlskey=/home/user/.docker/key.pem \
-        build \
-        --frontend dockerfile.v0 \
-        --local context=. \
-        --local dockerfile=./docker \
-        --opt filename=Dockerfile.npu.ci.a3 \
-        --secret id=dockerconfig,src=/home/user/.docker/config.json \
-        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
-        --progress=plain
-
-      echo "--- Image pushed successfully"
-      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+  - label: ":buildkit: Build and Push NPU Test Image - 310P"
+    key: image-build-310p
+    agents: *npu-image-build-agents
+    plugins: *npu-image-build-plugins
+    env:
+      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-310p"
+      IMAGE_NAME: "vllm-omni-ci-npu"
+      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
+      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
+    command: bash .buildkite/npu/scripts/build-npu-image.sh Dockerfile.npu.ci "$${VLLM_IMAGE_TAG}" -310p

  - label: "Upload Ready Pipeline"
    depends_on:
      - image-build-a2
      - image-build-a3
+      - image-build-a5
+      - image-build-310p
    key: upload-ready-pipeline
    agents:
      queue: "cpu_queue_premerge"
@@ -174,6 +98,8 @@ steps:
    depends_on:
      - image-build-a2
      - image-build-a3
+      - image-build-a5
+      - image-build-310p
    key: upload-nightly-pipeline
    commands:
      - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/test-npu-nightly.yml
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -ex
+
+dockerfile=${1:?Dockerfile is required}
+image_tag=${2:?Image tag is required}
+device_suffix=${3:-}
+
+image_name=${IMAGE_NAME:-vllm-omni-ci-npu}
+image_registry=${IMAGE_REGISTRY:-swr.cn-southwest-2.myhuaweicloud.com/modelfoundry}
+buildkitd_addr=${BUILDKITD_ADDR:-tcp://buildkitd-service.buildkitd:1234}
+
+echo "--- Building and pushing NPU Test Image"
+echo "Image: ${image_registry}/${image_name}:${image_tag}"
+echo "Dockerfile: ${dockerfile}"
+
+if ! command -v buildctl &> /dev/null; then
+  mkdir -p /tmp/buildkit
+  buildkit_version=v0.29.0
+  wget -q \
+    "https://github.com/moby/buildkit/releases/download/${buildkit_version}/buildkit-${buildkit_version}.linux-arm64.tar.gz" \
+    -O /tmp/buildkit.tar.gz
+  tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
+  cp /tmp/buildkit/bin/buildctl /usr/local/bin/
+fi
+
+build_args=()
+if [[ -n "${device_suffix}" ]]; then
+  build_args+=(--opt "build-arg:VLLM_ASCEND_DEVICE_SUFFIX=${device_suffix}")
+fi
+
+export DOCKER_CONFIG=/home/user/.docker
+buildctl \
+  --addr="${buildkitd_addr}" \
+  --tlscacert=/home/user/.docker/ca.pem \
+  --tlscert=/home/user/.docker/cert.pem \
+  --tlskey=/home/user/.docker/key.pem \
+  build \
+  --frontend dockerfile.v0 \
+  --local context=. \
+  --local dockerfile=./docker \
+  --opt "filename=${dockerfile}" \
+  "${build_args[@]}" \
+  --secret id=dockerconfig,src=/home/user/.docker/config.json \
+  --output "type=image,name=${image_registry}/${image_name}:${image_tag},push=true" \
+  --progress=plain
+
+echo "--- Image pushed successfully"
+echo "${image_registry}/${image_name}:${image_tag}"
@@ -1,26 +1,59 @@
 steps:
-  - label: "Platforms Test"
-    key: ready-platforms-test
+  - group: ":card_index_dividers: Simple Test"
    depends_on: upload-ready-pipeline
-    mirror_hardwares: a3_npu_2
-    commands:
-      - >-
-        pytest -s -v
-        tests/platforms/npu/test_minicpmo_code2wav_npugraph.py
-        -m 'core_model and npu and A3 and omni'
-        --run-level 'core_model'
+    steps:
+      - label: "Platforms Test"
+        key: ready-platforms-test
+        mirror_hardwares: a3_npu_2
+        commands:
+          - >-
+            pytest -s -v
+            tests/platforms/npu/test_minicpmo_code2wav_npugraph.py
+            -m 'core_model and npu and A3 and omni'
+            --run-level 'core_model'

-  - label: "Diffusion NPU layer tests"
-    key: ready-diffusion-npu-layer-tests
-    depends_on: upload-ready-pipeline
-    mirror_hardwares: a3_npu_2
-    commands:
-      - >-
-        pytest -s -v
-        tests/diffusion/layers/test_activation.py
-        tests/diffusion/layers/test_fused_qk_norm_rope_npu.py
-        -m 'core_model and npu and A3'
-        --run-level 'core_model'
+      - label: "Diffusion NPU layer tests - A2"
+        key: ready-diffusion-npu-layer-tests-a2
+        mirror_hardwares: a2b3_npu_1
+        commands:
+          - >-
+            pytest -s -v
+            tests/diffusion/layers/test_activation.py
+            -m 'core_model and npu and A2'
+            --run-level 'core_model'
+
+      - label: "Diffusion NPU layer tests - A3"
+        key: ready-diffusion-npu-layer-tests
+        mirror_hardwares: a3_npu_2
+        commands:
+          - >-
+            pytest -s -v
+            tests/diffusion/layers/test_activation.py
+            tests/diffusion/layers/test_fused_qk_norm_rope_npu.py
+            tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_swiglu.py
+            tests/diffusion/models/minimax_h3/test_minimax_h3_modulation.py
+            -m 'core_model and npu and A3'
+            --run-level 'core_model'
+
+      - label: "Diffusion NPU layer tests - A5"
+        key: ready-diffusion-npu-layer-tests-a5
+        mirror_hardwares: a5_npu_2
+        commands:
+          - >-
+            pytest -s -v
+            tests/diffusion/layers/test_activation.py
+            -m 'core_model and npu and A5'
+            --run-level 'core_model'
+
+      - label: "Diffusion NPU layer tests - 310P"
+        key: ready-diffusion-npu-layer-tests-310p
+        mirror_hardwares: 310p_npu_1
+        commands:
+          - >-
+            pytest -s -v
+            tests/diffusion/layers/test_activation.py
+            -m 'core_model and npu and 310P'
+            --run-level 'core_model'

  - group: ":card_index_dividers: Omni Model Test"
    key: ready-omni-group
@@ -123,10 +123,11 @@ it changes again, report the churn and wait for a stable target.

 For a trusted PR head, materialize the pinned head in an isolated detached
 worktree. A worktree freezes identity but is not a security sandbox. Treat fork
-heads as untrusted unless the user and environment policy explicitly establish
-otherwise: execute them only in a disposable, secret-free sandbox with restricted
-filesystem, network, and resources; without one, use static SHA-addressed reads
-and CI evidence only. For a local review, freeze the committed, index, worktree,
+heads as untrusted until the user and environment policy explicitly establish
+trust: execute their code only after that trust is recorded for this host,
+with the trusted SHA noted in the review state and secrets kept out of the
+execution scope; without recorded trust, use static SHA-addressed reads and CI
+evidence only. For a local review, freeze the committed, index, worktree,
 and NUL-safe in-scope untracked contents. Follow
 [review-execution.md](references/process/review-execution.md) for trust gates,
 state fingerprints, and byte-for-byte staleness checks.
@@ -196,7 +197,7 @@ style preferences as simplification findings.

 Before each validation group, verify the frozen SHA plus the tracked, index,
 untracked, and ignored-file fingerprint, or recreate a pristine snapshot. On a
-trusted head or inside the required sandbox, run an import/version preflight,
+head the user explicitly trusted for this host, run an import/version preflight,
 then the narrowest relevant tests and low-cost static checks. Bind every result
 to the head SHA, snapshot fingerprint, and environment fingerprint. Never run
 imports, tests, builds, hooks, or repo-configurable tooling from an untrusted
@@ -22,6 +22,19 @@ Keep exact commands and report variability rather than a cherry-picked run.
 Use the repository's benchmark when it covers the claim; otherwise use the
 smallest reproducible workload that reaches the changed production path.

+## Isolation per claim
+
+Isolate one claim per comparison. When a PR stacks several changes (a fused or
+restructured load path, a distilled step schedule, a new cache), request one
+column per claim at equal steps/workload instead of one blended base-vs-head
+delta, and name the unchanged stage or metric that pins equivalence.
+
+When the changed path is runtime-switchable (feature flag or environment
+variable), run every column on the same head and toggle the switch rather than
+diffing against a base commit that cannot separate the stacked changes:
+base, head with the switch off, head with the switch on, then head with the
+switch on plus each further claim.
+
 ## Evidence table

 | Dimension | Typical evidence |
@@ -34,6 +47,100 @@ smallest reproducible workload that reaches the changed production path.
 Every optimization needs correctness or quality evidence. A faster result on a
 different workload, precision, seed, or topology is not a valid comparison.

+## E2e stage-attribution table
+
+For an end-to-end latency claim, request the stage-attribution table the
+repository's perf PRs use:
+
+- Rows carry the benchmark's own field names: total e2e, per-stage execution,
+  postprocess, response encoding, and a request-to-output roll-up. Add a stage
+  handoff/idle row when the pipeline spans stages — connector transfer and
+  scheduling gaps are invisible in per-stage latency deltas.
+- Sanity rows pin comparability: inference steps, output frames/pixels/samples,
+  and generated tokens held identical across columns.
+- Report P50 and P100 from the stated warmup and measured counts. One
+  population per number: never mix cold-start and warm samples into one value.
+- Name the baseline and candidate commit SHAs, and close with an isolation
+  statement naming the metric that stayed unchanged ("X remained effectively
+  unchanged, isolating the gain to Y").
+
+When the serving path caches, captures, or regenerates at runtime (CUDA-graph
+bucket growth, cache rebuilds, recompilation), medians hide the resulting tail.
+Also request the event count and the maximum observed stall.
+
+Component-level tables pair with, not replace, the e2e table: per-op reference
+vs candidate timings (with the numerical-equality result per op) plus one
+complete-pipeline check under the production shapes.
+
+## Proportionality
+
+When a PR adds a model and explicitly labels its numbers as smoke observations
+rather than claims, do not demand the attribution machinery. Ask only for
+evidence protecting the model's stated operating contract — for a
+realtime/duplex model that is sustained cadence over a realistic session
+(enough ticks to leave warmup), with any mid-session capture/regeneration
+event and its stall reported, since a cadence that collapses mid-session is a
+functional failure, not a perf claim.
+
+## Realtime/duplex world models
+
+Request the operating-contract evidence as serving metrics, not claims:
+
+- Sustained per-tick latency over one realistic session: median, max, and the
+  argmax tick.
+- Stall events with their causes.
+- Peak memory at first versus last tick — session-owned state must not grow
+  per tick.
+- Control latency from a submitted action to the tick that applies it.
+- Realtime-versus-offline parity under the same controls, using the
+  repository's video-similarity helpers. Parity plotted over tick index is the
+  drift-over-horizon curve and the honest measure of session lifetime;
+  distribution-level suites (FVD, VBench) belong to model evaluation, not the
+  serving integration.
+
+Watch the contract boundaries the model documents: if a control input is
+tick-API-only, the parity pair must use a horizon both paths can run.
+
+## Speech-generation (TTS) models
+
+The streaming contract outranks the average — a synthesis that pauses
+mid-stream is broken for realtime use regardless of its averages. Request:
+
+- The benchmark's own speech metrics over one long utterance (enough audio to
+  leave warmup, tens of seconds): `audio_ttfp` (time to first packet),
+  `audio_rtf` (real-time factor; above 1.0 cannot keep up), `e2el`, and
+  `ttft`/`tpot` where tokens precede audio.
+- Mid-utterance stalls as the benchmark's continuity fields, not prose:
+  `audio_underrun` percentiles and the `audio_continuity_ok` rate
+  (`vllm_omni/benchmarks/metrics/metrics.py`), computed by
+  `vllm_omni/benchmarks/audio_continuity.py` — the companion to `audio_ttfp`
+  and `audio_rtf`. A synthesis that bursts chunks fast enough on average but
+  underruns mid-stream fails the streaming contract even when the averages
+  pass.
+- Quality as a paired comparison against the reference implementation under
+  the same text, speaker prompt, and seed; report sample rate and frame
+  count, and assert output completeness — truncated or dropped trailing audio
+  is a known failure class, not a quality nuance.
+- For realtime or duplex TTS: interruption (barge-in) latency and the
+  per-session memory trend across turns.
+
+## Omni models
+
+Apply the e2e stage-attribution table directly to the voice-to-voice path of
+omni (speech-to-speech and multimodal AR) models:
+
+- TTFT for text output, `audio_ttfp` for audio output, and per-stage
+  ITL/TPOT cadence across the perception-understanding-generation chain.
+- Cross-modal parity — the omni analogue of realtime-versus-offline parity:
+  the same semantic request through text and audio inputs, compared on output
+  content or timing.
+- For duplex omni sessions: turn-taking latency, interruption handling, the
+  concurrent session count actually validated, and a per-turn memory trend —
+  session-owned KV state must not grow per turn. When either turn produces
+  audio, continuity is judged with the same `audio_underrun` and
+  `audio_continuity_ok` fields as TTS, since turn-taking latency alone does
+  not expose mid-utterance underruns.
+
 ## Reviewer verification ladder

 1. Run base/head A/B on suitable hardware when affordable.
@@ -121,6 +121,19 @@ Search bounded callers and sibling implementations before reporting dead code,
 duplication, or a missed consumer. Keep new abstractions only when they have a
 distinct live caller, invariant, or compatibility need.

+Verify absence and equality claims with whole-file evidence. A line window, a
+grep for one symbol, or a commit message proves presence, never absence: to
+assert that a file matches another version or lacks a change, diff or hash the
+full files (for example `git diff <sha1>:<path> <sha2>:<path>` or the contents
+API's size and SHA fields) before anchoring the finding. A verification
+shortcut that would itself be wrong to run is not evidence.
+
+For a value written or finalized in more than one place, establish which site
+executes first on each entry path — and whether the earlier site makes the
+later ones unreachable — before naming where a fix belongs. Enumerating all
+call sites correctly still misses the bug when a path the enumeration ranked
+last actually finalizes first and clears the state the proposed fix checks.
+
 ## Finding bar

 Anchor each finding to a changed `path:line`; name the trigger or call path,
@@ -39,13 +39,13 @@ isolated detached worktree. Run every source read, `rg` search, import, and test
 from that snapshot, not the caller's checkout. A detached worktree provides
 snapshot isolation, not a security boundary.

-Treat a fork head as untrusted unless the user and environment policy explicitly
-establish trust. Never run its imports, tests, builds, hooks, package setup, or
-repo-configurable linters/plugins on the reviewer host. Execute them only in a
-disposable sandbox or VM with no credentials or inherited secrets, no host agent
-or service sockets, minimal read-only host mounts, disabled or explicitly
-allowlisted network, resource/time limits, and destruction after the run. If
-that boundary is unavailable, limit the review to the remote diff, SHA-addressed
+Treat a fork head as untrusted until the user and environment policy
+explicitly establish trust. Without recorded trust, never run its imports,
+tests, builds, hooks, package setup, or repo-configurable linters/plugins on
+the reviewer host. Once the user explicitly trusts this pinned commit for
+this host, record the trusted SHA with the review state and execute with
+credentials, agent sockets, and other secrets out of scope. Without recorded
+trust, limit the review to the remote diff, SHA-addressed
 reads such as `git show <head_sha>:<path>`, and existing CI evidence; report all
 executable validation as a gap.

@@ -2,6 +2,8 @@

 You may find information about contributing to vLLM-Omni on [Contributing](https://vllm-omni.readthedocs.io/en/latest/contributing/).

+AI-assisted contributions must follow the [AI Usage Policy](docs/contributing/README.md#ai-usage-policy), including disclosure, human review, and validation requirements.
+
 Local `pre-commit` gates (SPDX, forbidden imports, `torch.cuda`, shellcheck,
 mypy, test marks, TTS adapter ratchet, markdownlint, Buildkite schema, and
 how to extend allowlists) are documented in the
@@ -17,18 +17,18 @@ The main entrypoint is:
 vllm serve Qwen/Qwen-Image --omni --port 8099
 ```

-2. Run a minimal benchmark:
+1. Run a minimal benchmark:

 ```bash
 python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-	--base-url http://localhost:8099 \
-	--model Qwen/Qwen-Image \
-	--task t2i \
-	--dataset vbench \
-	--num-prompts 5
+ --base-url http://localhost:8099 \
+ --model Qwen/Qwen-Image \
+ --task t2i \
+ --dataset vbench \
+ --num-prompts 5
 ```

-**Notes**
+### Notes

 - By default, image tasks talk to `http://<host>:<port>/v1/chat/completions`; video tasks talk to `/v1/videos`.
 - If you run the server on another host or port, pass `--base-url` accordingly.
@@ -52,14 +52,14 @@ Example (`t2v`):

 ```bash
 python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-	--base-url http://localhost:8099 \
-	--model Wan-AI/Wan2.2-T2V-A14B-Diffusers \
-	--task t2v \
-	--dataset vbench \
-	--num-prompts 50 \
-	--width 640 --height 480 \
-	--num-frames 81 --fps 16 \
-	--num-inference-steps 40
+ --base-url http://localhost:8099 \
+ --model Wan-AI/Wan2.2-T2V-A14B-Diffusers \
+ --task t2v \
+ --dataset vbench \
+ --num-prompts 50 \
+ --width 640 --height 480 \
+ --num-frames 81 --fps 16 \
+ --num-inference-steps 40
 ```

 Note: `vbench` can also be used for other tasks such as `t2i` / `i2v` (and `i2i`). For `t2i`, the loader reuses VBench t2v text prompts; for `i2v` / `i2i`, it loads the VBench i2v dataset (with image paths).
@@ -140,6 +140,21 @@ Traffic / concurrency flags:
 - `--request-rate`: Target request rate (requests/second). If set to `inf`, the script sends all requests immediately.
 - `--max-concurrency`: Max number of in-flight requests (default: `1`). This can hard-cap the achieved QPS: if it is too small, requests will queue behind the semaphore, and both achieved throughput and observed SLO attainment can be skewed.

+### Video job timeout and failures
+
+`--video-job-timeout` controls how long each `/v1/videos` job may be polled,
+including time queued on the server (default: `900` seconds). For long video
+jobs at high concurrency, use a larger budget, for example
+`--video-job-timeout 1800`. This also applies to warmup jobs. The polling budget
+starts after job creation; it excludes time waiting for the client concurrency
+semaphore. Individual HTTP calls still use the aiohttp session timeout.
+
+An expired job is counted as failed and deleted, which cancels queued or running
+server generation. The benchmark continues processing the remaining requests.
+The progress bar counts both successful and failed requests. The final report
+shows the failure count and the first 10 errors; `--output-file` saves all errors
+with their request IDs in `request_errors`.
+
 ### Batched warmup note

 For batched serving runs, warm the same in-flight shape you plan to measure.
@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 import asyncio
 import base64
 import json
@@ -43,6 +46,7 @@ class RequestFuncInput:
    video_paths: list[str] | None = None
    request_id: str = field(default_factory=lambda: str(uuid.uuid4()))
    default_bot_task: str | None = DEFAULT_EDITS_BOT_TASK
+    video_job_timeout: float = 900.0


@dataclass
@@ -331,64 +335,64 @@ async def async_request_v1_videos(
    output = RequestFuncOutput()
    output.start_time = time.perf_counter()

-    files = dict(input.extra_body)
-    if input.prompt:
-        files.setdefault("prompt", input.prompt)
-    if input.width and input.height:
-        files.setdefault("height", input.height)
-        files.setdefault("width", input.width)
-    if input.num_frames:
-        files.setdefault("num_frames", input.num_frames)
-    if input.num_inference_steps:
-        files.setdefault("num_inference_steps", input.num_inference_steps)
-    if input.seed is not None:
-        files.setdefault("seed", input.seed)
-    if input.fps:
-        files.setdefault("fps", input.fps)
-
-    form = aiohttp.FormData()
-    for k, v in files.items():
-        form.add_field(k, str(v))
-
    image_file = None
-    if input.image_paths and input.video_paths:
-        output.error = "Only one of image_paths or video_paths can be provided"
-        output.success = False
-        return output
+    job_id = None
+    try:
+        files = dict(input.extra_body)
+        if input.prompt:
+            files.setdefault("prompt", input.prompt)
+        if input.width and input.height:
+            files.setdefault("height", input.height)
+            files.setdefault("width", input.width)
+        if input.num_frames:
+            files.setdefault("num_frames", input.num_frames)
+        if input.num_inference_steps:
+            files.setdefault("num_inference_steps", input.num_inference_steps)
+        if input.seed is not None:
+            files.setdefault("seed", input.seed)
+        if input.fps:
+            files.setdefault("fps", input.fps)

-    if input.image_paths and len(input.image_paths) > 0:
-        image_path = input.image_paths[0]
-        image_file = open(image_path, "rb")
-        form.add_field(
-            "input_reference",
-            image_file,
-            filename=os.path.basename(image_path),
-            content_type=_guess_mime_type(image_path),
-        )
-    elif input.video_paths and len(input.video_paths) > 0:
-        video_path = input.video_paths[0]
-        if _is_minimax_h3_mixed_reference(input.model):
-            with open(video_path, "rb") as reference_file:
-                video_b64 = base64.b64encode(reference_file.read()).decode("ascii")
-            form.add_field(
-                "video_reference",
-                json.dumps([{"video_url": f"data:video/mp4;base64,{video_b64}"}]),
-            )
-        else:
-            image_file = open(video_path, "rb")
+        form = aiohttp.FormData()
+        for k, v in files.items():
+            form.add_field(k, str(v))
+
+        if input.image_paths and input.video_paths:
+            output.error = "Only one of image_paths or video_paths can be provided"
+            output.success = False
+            return output
+
+        if input.image_paths and len(input.image_paths) > 0:
+            image_path = input.image_paths[0]
+            image_file = open(image_path, "rb")
            form.add_field(
                "input_reference",
                image_file,
-                filename=os.path.basename(video_path),
-                content_type=_guess_mime_type(video_path),
+                filename=os.path.basename(image_path),
+                content_type=_guess_mime_type(image_path),
            )
+        elif input.video_paths and len(input.video_paths) > 0:
+            video_path = input.video_paths[0]
+            if _is_minimax_h3_mixed_reference(input.model):
+                with open(video_path, "rb") as reference_file:
+                    video_b64 = base64.b64encode(reference_file.read()).decode("ascii")
+                form.add_field(
+                    "video_reference",
+                    json.dumps([{"video_url": f"data:video/mp4;base64,{video_b64}"}]),
+                )
+            else:
+                image_file = open(video_path, "rb")
+                form.add_field(
+                    "input_reference",
+                    image_file,
+                    filename=os.path.basename(video_path),
+                    content_type=_guess_mime_type(video_path),
+                )

-    job_id = None
-    job_status = None
-    poll_json = {}
-    resp_json = {}
+        job_status = None
+        poll_json = {}
+        resp_json = {}

-    try:
        # invoke a post request (POST /v1/videos)
        async with session.post(input.api_url, data=form) as response:
            if response.status == 200:
@@ -406,7 +410,7 @@ async def async_request_v1_videos(

        # invoke a poll request (GET /v1/videos/{video_id})
        poll_interval = 2.0  # Unit(s)
-        timeout_seconds = 600.0
+        timeout_seconds = input.video_job_timeout
        deadline = time.perf_counter() + timeout_seconds
        job_url = f"{input.api_url}/{job_id}"

@@ -422,8 +426,11 @@ async def async_request_v1_videos(
                poll_json = await poll_response.json()
                job_status = poll_json.get("status")

-                if time.perf_counter() >= deadline:
-                    output.error = f"Timed out waiting for video job {job_id} to complete."
+                if job_status not in {"completed", "failed"} and time.perf_counter() >= deadline:
+                    output.error = (
+                        f"Timed out after {timeout_seconds:g}s waiting for video job {job_id} to complete. "
+                        "Increase --video-job-timeout for long-running or queued video jobs."
+                    )
                    output.success = False
                    return output

@@ -452,7 +459,7 @@ async def async_request_v1_videos(
            elif "peak_memory_mb" in resp_json:
                output.peak_memory_mb = resp_json["peak_memory_mb"]
    except Exception as e:
-        output.error = str(e)
+        output.error = f"{type(e).__name__}: {e}"
        output.success = False
    finally:
        if image_file is not None:
@@ -465,13 +472,13 @@ async def async_request_v1_videos(
            except Exception as e:
                print(f"Failed to clean up video job {job_id}: {e}")

-    output.latency = time.perf_counter() - output.start_time
+        output.latency = time.perf_counter() - output.start_time
+        if pbar is not None:
+            pbar.update(1)

    if output.success and input.slo_ms is not None:
        output.slo_achieved = (output.latency * 1000.0) <= float(input.slo_ms)

-    if pbar:
-        pbar.update(1)
    return output


@@ -1,6 +1,6 @@
 # adapted from fastvideo
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

 """
 Benchmark online serving for diffusion models (Image/Video Generation).
@@ -87,6 +87,7 @@ import base64
 import glob
 import json
 import logging
+import math
 import os
 import random
 import tempfile
@@ -1116,6 +1117,11 @@ def calculate_metrics(
        "duration": total_duration,
        "completed_requests": num_success,
        "failed_requests": len(error_outputs),
+        "request_errors": [
+            {"request_id": req.request_id, "error": out.error}
+            for req, out in zip(requests_list, outputs)
+            if not out.success
+        ],
        "throughput_qps": num_success / total_duration if total_duration > 0 else 0,
        "latency_mean": np.mean(latencies) if latencies else 0,
        "latency_median": np.median(latencies) if latencies else 0,
@@ -1261,6 +1267,9 @@ def _default_endpoint_for_task(task: str) -> str:


 async def benchmark(args):
+    if not math.isfinite(args.video_job_timeout) or args.video_job_timeout <= 0:
+        raise ValueError("--video-job-timeout must be a positive finite number of seconds.")
+
    # Construct base_url if not provided
    if args.base_url is None:
        args.base_url = f"http://{args.host}:{args.port}"
@@ -1312,6 +1321,8 @@ async def benchmark(args):
    print("Loading requests...")
    requests_list = dataset.get_requests()
    print(f"Prepared {len(requests_list)} requests from {args.dataset} dataset.")
+    for req in requests_list:
+        req.video_job_timeout = args.video_job_timeout

    if args.return_stage_metrics and args.endpoint in _STAGE_METRICS_ENDPOINTS:
        for req in requests_list:
@@ -1376,6 +1387,8 @@ async def benchmark(args):
    metrics["model"] = args.model
    metrics["dataset"] = args.dataset
    metrics["task"] = args.task
+    if args.endpoint == "/v1/videos":
+        metrics["video_job_timeout"] = args.video_job_timeout
    if args.endpoint == "/v1/images/edits":
        metrics["bot_task"] = args.bot_task

@@ -1398,6 +1411,11 @@ async def benchmark(args):
        )
    )
    print("{:<40} {}/{:<15}".format("Successful requests:", metrics["completed_requests"], len(requests_list)))
+    print("{:<40} {:<15}".format("Failed requests:", metrics["failed_requests"]))
+    for failure in metrics["request_errors"][:10]:
+        print(f"Request {failure['request_id']} failed: {failure['error']}")
+    if metrics["failed_requests"] > 10:
+        print("Showing first 10 failures; use --output-file to save all request errors.")

    # Section 3: Performance Metrics
    print(f"{'-' * 50}")
@@ -1481,6 +1499,13 @@ if __name__ == "__main__":
        help="Path to local dataset file (optional).",
    )
    parser.add_argument("--num-prompts", type=int, default=10, help="Number of prompts to benchmark.")
+    parser.add_argument(
+        "--video-job-timeout",
+        type=float,
+        default=900.0,
+        help="Maximum polling time in seconds per video job, including server queue time (default: 900). "
+        "Increase this for long-running video generation or high concurrency.",
+    )
    parser.add_argument(
        "--max-concurrency",
        type=int,
@@ -85,6 +85,25 @@ vllm bench serve --omni \
    --save-result --result-dir ./results
 ```

+#### Set the sample rate for models that are not 24 kHz
+
+Streamed PCM carries no header, so the harness derives `audio_duration` from the
+byte count and a **default 24000 Hz** assumption
+(`vllm_omni/metrics/definitions.py:DEFAULT_AUDIO_SAMPLE_RATE`). For a model that
+emits another rate, export the real one or every duration-derived metric is
+wrong by `real_sr / 24000`:
+
+```bash
+# Audio8 TTS, Fish Speech S2 Pro, Ming-omni-tts, ... all decode at 44.1 kHz
+export VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100
+```
+
+Without it, a 44.1 kHz model reports `audio_duration` 1.84x too long, so
+`audio_rtf` looks 1.84x *better* than reality and `audio_underrun` /
+`Streaming continuity OK rate` are measured against inflated chunk budgets.
+A quick sanity check: compare `Mean AUDIO_DURATION (s)` against the actual
+length of a saved WAV for the same prompt.
+
 #### Streaming continuity (`audio_underrun`)

 `audio_underrun` (seconds) is the per-request worst-case buffer deficit
@@ -17,6 +17,17 @@ models:
        extra_params:
          lang: en
          text_normalization: true
+  # Audio8 TTS Preview: DualAR, 44.1 kHz. trust_remote_code stays false.
+  # Bench at 44.1 kHz: export VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100, otherwise
+  # the streamed-PCM duration defaults to 24 kHz and RTF looks 1.84x too good.
+  Audio8/Audio8-TTS-Preview-0.6b:
+    supported_tasks: [default_voice, voice_clone]
+    backend: openai-audio-speech
+    endpoint: /v1/audio/speech
+    trust_remote_code: false
+    task_extra_body:
+      default_voice: {}
+      voice_clone: {}

  # -CustomVoice checkpoints lack speaker_encoder weights, so voice_clone is
  # NOT supported (an attempt raises ValueError from _extract_speaker_embedding
@@ -1,27 +1,10 @@
-# Target vllm version: v0.28.0 (tag v0.28.0 @ 2cf0a6915ce5).
-# Keep VLLM_BASE_TAG aligned with the target vLLM release on every rebase.
-#
-# Release-boundary form: there is no precompiled-wheel pin here, because the
-# base image already ships exactly the target vLLM. Everything the runtime
-# needs — torch, flashinfer, nvJitLink, numpy — comes from that image as one
-# self-consistent set.
-#
-# This block reappears only when the rebase targets a vLLM commit that is NOT a
-# released tag. During the 0.27 rebase, targeting v0.27.0rc1 on top of the
-# v0.26.0 base image required reinstalling vLLM from wheels.vllm.ai and then
-# hand-repairing the fallout: pinning torch ahead of flashinfer (installing it
-# after bound flashinfer's TVM FFI runtime to the old torch and aborted engine
-# core init), registering libnvJitLink.so.13 with ldconfig, and re-pinning numpy
-# under numba's ceiling. All of it was compensation for a base image and a vLLM
-# that disagreed. At a real release boundary they agree, so none of it is
-# needed. The 0.28 rebase repeated the cycle: the v0.28.0rc / final-tag wheels
-# rode on the v0.27.0 base image with the full reinstall block until the
-# official v0.28.0 image shipped, at which point this file dropped back to the
-# form you are reading. If you pin an untagged commit again, restore the
-# reinstall block — agent/tasks/20_update_wheel_url.sh fails the run rather
-# than letting the Dockerfile silently ship the wrong vLLM.
+# Target vLLM version: v0.29.0 (tag @ 98dff2a81d74).
+# Use the matching release image and its coherent CUDA/PyTorch/FlashInfer
+# runtime. Do not reinstall a commit wheel or repair dependencies on top of
+# an older base image at a release boundary. For a future unreleased target,
+# restore an explicit wheel pin and validate its complete runtime stack.
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai
-ARG VLLM_BASE_TAG=v0.28.0
+ARG VLLM_BASE_TAG=v0.29.0

 # We use registry-based layer caching for the dependencies,
 # where the tag is determined based on the hash of the contents
@@ -1,20 +1,7 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.28.0
-FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
-
-# WORKDIR /vllm-workspace/vllm
-# RUN git fetch origin --tags && git checkout v0.18.0
-
-# WORKDIR /vllm-workspace/vllm-ascend
-# RUN git fetch origin releases/v0.18.0 && git checkout d781902ce9dbda8ab1e11bb0f2f0c1bc508fee7a
-# # Install vllm-ascend
-# # Append `libascend_hal.so` path (devlib) to LD_LIBRARY_PATH
-# RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \
-#     source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-#     source /usr/local/Ascend/nnal/atb/set_env.sh && \
-#     export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-#     python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \
-#     python3 -m pip cache purge
+ARG VLLM_ASCEND_VERSION=v0.28.0
+ARG VLLM_ASCEND_DEVICE_SUFFIX=
+FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_VERSION}${VLLM_ASCEND_DEVICE_SUFFIX}

 ARG APP_DIR=/vllm-workspace/vllm-omni
 WORKDIR ${APP_DIR}
@@ -24,6 +11,7 @@ RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi
    source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
    source /usr/local/Ascend/nnal/atb/set_env.sh && \
    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
+    python3 -m pip install -v "mindiesd==3.1.0" && \
    python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation

 RUN python3 -m pip install -v --no-deps \
@@ -1,36 +0,0 @@
-ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.28.0-a3
-FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
-
-# WORKDIR /vllm-workspace/vllm
-# RUN git fetch origin --tags && git checkout v0.18.0
-
-# WORKDIR /vllm-workspace/vllm-ascend
-# RUN git fetch origin releases/v0.18.0 && git checkout d781902ce9dbda8ab1e11bb0f2f0c1bc508fee7a
-# # Install vllm-ascend
-# # Append `libascend_hal.so` path (devlib) to LD_LIBRARY_PATH
-# RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \
-#     source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-#     source /usr/local/Ascend/nnal/atb/set_env.sh && \
-#     export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-#     python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \
-#     python3 -m pip cache purge
-
-ARG APP_DIR=/vllm-workspace/vllm-omni
-WORKDIR ${APP_DIR}
-COPY . .
-
-RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \
-    source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-    source /usr/local/Ascend/nnal/atb/set_env.sh && \
-    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-    python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation
-
-RUN python3 -m pip install -v --no-deps \
-      "stepaudio2-minicpmo>=0.1.1" \
-      "step-audio2>=1.0.0"
-
-ENV VLLM_TARGET_DEVICE=npu
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-ENTRYPOINT []
@@ -1,10 +1,10 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.28.0
-FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
+ARG VLLM_ASCEND_VERSION=v0.28.0
+ARG VLLM_ASCEND_DEVICE_SUFFIX=
+FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_VERSION}${VLLM_ASCEND_DEVICE_SUFFIX}

 ARG APP_DIR=/vllm-workspace/vllm-omni
 WORKDIR ${APP_DIR}
-COPY . .

 RUN sed -i "s@^\(deb.*\)https\?://[a-z0-9.-]*\.ubuntu\.com@\1http://cache-service.nginx-pypi-cache.svc.cluster.local:8081@g" /etc/apt/sources.list

@@ -22,11 +22,9 @@ RUN apt-get update && \
    rm -rf /var/cache/apt/* && \
    rm -rf /var/lib/apt/lists/*

-RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-    source /usr/local/Ascend/nnal/atb/set_env.sh && \
-    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-    python3 -m pip install -U pip && \
+RUN python3 -m pip install -U pip && \
    python3 -m pip install -v \
+      "mindiesd==3.1.0" \
      "pytest>=7.0.0" \
      "pytest-asyncio>=0.21.0" \
      "pytest-cov>=4.0.0" \
@@ -48,7 +46,13 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
      "datasets>=3.0.0" \
      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
-      --default-timeout=60 && \
+      --default-timeout=60
+
+COPY . .
+
+RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
+    source /usr/local/Ascend/nnal/atb/set_env.sh && \
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
    python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation \
      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
@@ -1,72 +0,0 @@
-ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.28.0-a3
-FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
-
-ARG APP_DIR=/vllm-workspace/vllm-omni
-WORKDIR ${APP_DIR}
-COPY . .
-
-RUN sed -i "s@^\(deb.*\)https\?://[a-z0-9.-]*\.ubuntu\.com@\1http://cache-service.nginx-pypi-cache.svc.cluster.local:8081@g" /etc/apt/sources.list
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-        ffmpeg \
-        libgl1 \
-        libglib2.0-0 \
-        libgomp1 \
-        libsm6 \
-        libxext6 \
-        libxrender1 \
-        libxcb1 \
-        espeak-ng && \
-    rm -rf /var/cache/apt/* && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-    source /usr/local/Ascend/nnal/atb/set_env.sh && \
-    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-    python3 -m pip install -U pip && \
-    python3 -m pip install -v \
-      "pytest>=7.0.0" \
-      "pytest-asyncio>=0.21.0" \
-      "pytest-cov>=4.0.0" \
-      "pytest-mock>=3.10.0" \
-      "openpyxl>=3.0.0" \
-      "psutil>=7.2.0" \
-      "soundfile>=0.13.1" \
-      "pyttsx3>=2.99" \
-      "opencc==1.4.1" \
-      "opencv-python>=4.12.0.88" \
-      "imageio[ffmpeg]>=0.6.0" \
-      "jiwer>=3.0.0" \
-      "zhon>=2.0.0" \
-      "zhconv>=1.4.2" \
-      "funasr>=1.0.0" \
-      "hyperpyyaml>=1.2.0" \
-      "onnx>=1.16.0" \
-      "librosa>=0.10.0" \
-      "torchmetrics>=1.9.0" \
-      "datasets>=3.0.0" \
-      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
-      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
-      --default-timeout=60 && \
-    python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation \
-      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
-      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
-      --default-timeout=60
-
-# MiniCPM-o 4.5 Stage-2 vocoder assets: s3tokenizer plus flashcosyvoice, which
-# is shipped by step-audio2. --no-deps keeps pip from replacing the Ascend torch
-# build and from shadowing onnxruntime-cann with the vanilla onnxruntime wheel;
-# the deps these two actually need are installed above.
-RUN python3 -m pip install -v --no-deps \
-      "s3tokenizer>=0.3.0" \
-      "step-audio2>=1.0.0" \
-      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
-      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
-      --default-timeout=60
-
-ENV VLLM_TARGET_DEVICE=npu
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-ENTRYPOINT []
@@ -127,6 +127,7 @@ nav:
        - Async Chunk: design/feature/async_chunk.md
        - Async Diffusion Output: design/feature/async_diffusion_output.md
        - Async Omni Output Materialization: design/feature/omni_async_output_materialization.md
+        - Prefill/Decode Phase Contract: design/feature/preprocess_phase_contract.md
        - Automatic Prefix Caching: design/feature/prefix_caching.md
        - Realtime AR-Diffusion: design/feature/realtime_ar_diffusion.md
      - Communication:
@@ -436,6 +436,76 @@ This should be seen as an edge case, and if this behavior can be avoided by sett

 </details>

+### Image / video generation
+
+For `/v1/images/generations`, `/v1/images/edits`, and `/v1/videos`, set `--endpoint` to that path and omit `--backend`. The endpoint path is registered as the bench request adapter, so a separate named backend is not required.
+
+Example (`/v1/images/generations`):
+
+```bash
+vllm bench serve --omni \
+  --endpoint /v1/images/generations \
+  --dataset-name random \
+  --model ~/models/Qwen/Qwen-Image \
+  --tokenizer ~/models/Qwen/Qwen-Image/tokenizer \
+  --max-concurrency 1 \
+  --num-warmups 2 \
+  --num-prompts 10 \
+  --random-input-len 64 \
+  --random-output-len 1 \
+  --ignore-eos \
+  --percentile-metrics e2el \
+  --extra-body '{
+    "num_inference_steps": 20,
+    "seed": 42,
+    "true_cfg_scale": 4.0
+  }'
+```
+
+If successful, following output is like:
+
+```text
+============ Serving Benchmark Result ============
+Successful requests:                     3  
+Failed requests:                         0  
+Maximum request concurrency:             1  
+Benchmark duration (s):                  8.42  
+Request throughput (req/s):              0.36  
+Peak concurrent requests:                2.00  
+-------------------Peak Memory--------------------
+Mean PEAK_MEMORY_MB (MB):                58832.00  
+Median PEAK_MEMORY_MB (MB):              58832.00  
+P99 PEAK_MEMORY_MB (MB):                 58832.00  
+----------------End-to-end Latency----------------
+Mean E2EL (ms):                          2798.16  
+Median E2EL (ms):                        2799.83  
+P99 E2EL (ms):                           2800.03  
+================== Image Result ==================
+Total images generated:                  3  
+Image throughput (img/s):                0.36  
+Average pixels per image:                1048576.00
+Mean denoise step latency (ms):          136.59  
+---------------- Image Generation ----------------
+Mean IMAGE_GENERATION (ms):              2731.84  
+Median IMAGE_GENERATION (ms):            2732.02  
+P99 IMAGE_GENERATION (ms):               2734.65  
+==================================================
+```
+
+Use the same pattern with `--endpoint /v1/images/edits` or `--endpoint /v1/videos` (and model-specific `--extra-body` as needed). Pure image/video runs omit the Text Result section when there is no generated text.
+
+`/v1/images/edits` defaults to **non-streaming JSON** (`stream=false`) so single-stage edit models are not rejected by the server. For multi-stage pipelines that need SSE (AR TTFT / image chunks), pass `"stream": true` in `--extra-body`.
+
+`/v1/videos` is an async job API: the client creates a job, then polls until `completed`/`failed`. Measured **e2el** therefore includes client poll sleep and any overshoot after the job actually finishes. Tune polling via `--extra-body`:
+
+- `poll_interval_s` (default `2.0`): sleep between status polls
+- `poll_timeout_s` (default `21600`, i.e. 6 hours): give up waiting for the job
+
+**VIDEO_RTF** prefers server-reported generation time when available:
+
+- `video_rtf = video_generation_time_ms / 1000 / video_duration`, where `video_generation_time_ms` comes from response `stage_durations` (or `inference_time_s` as fallback)
+- If generation time is missing, falls back to `e2el / video_duration` (this fallback *does* include poll overhead)
+
 ### Multi-Stage Benchmark

 <details class="admonition abstract" markdown="1">
@@ -2,7 +2,7 @@

 This section lists the most common options for running vLLM-Omni.

-For options within a vLLM Engine, please refer to the [vLLM 0.28 configuration guide](https://docs.vllm.ai/en/v0.28.0/configuration/index.html).
+For options within a vLLM Engine, please refer to the [vLLM 0.29 configuration guide](https://docs.vllm.ai/en/v0.29.0/configuration/index.html).

 Each model defines fixed topology in a registered `PipelineConfig` and runtime
 overrides in a deploy YAML.
@@ -151,7 +151,7 @@ their keys only.
 ## Inherited vLLM variables

 vLLM-Omni also reads variables through its aligned vLLM dependency. Refer to
-the [vLLM 0.28 environment-variable reference](https://docs.vllm.ai/en/v0.28.0/configuration/env_vars.html)
+the [vLLM 0.29 environment-variable reference](https://docs.vllm.ai/en/v0.29.0/configuration/env_vars.html)
 for their definitions. This includes vLLM launch, cache, logging, plugin, ROCm,
 XPU, ModelScope, and FlashInfer workspace settings.

@@ -262,6 +262,28 @@ or serving, explain the required L2/L3 environment and provide the exact command

 Repository skills guide the agent, but they do not replace contributor judgment, the accepted issue or RFC, required test evidence, or maintainer review. Review all generated changes before submitting them.

+## AI Usage Policy
+
+AI tools are welcome in vLLM-Omni contributions. Contributors, including maintainers, remain responsible for everything they submit. These requirements apply to pull requests, issues, discussions, and review comments:
+
+- **Disclose AI assistance.** Name the tools used and describe their role in the submission, including assistance with code, tests, documentation, or writing.
+- **Understand your changes.** Review every generated change and be able to explain its behavior, design choices, and interaction with vLLM-Omni without relying on AI to answer for you.
+- **Validate before submitting.** Run relevant checks and report actual results. For model, kernel, or performance changes, include the hardware, model, configuration, and commands needed to reproduce accuracy or benchmark evidence. Clearly identify checks you could not run; generated claims are not test results.
+- **Keep communication useful.** Verify technical claims, search for existing reports, and edit AI-assisted text for accuracy and brevity. Submit only content you have personally reviewed, and take responsibility for responding to review feedback.
+- **Label generated media.** Model-generated images, video, and audio are valid examples or test artifacts for this multimodal project. Identify them as generated output and provide the model, inputs, and reproduction settings. Do not present them as ground-truth references or independent evidence of correctness.
+
+For example, a PR description could include:
+
+```text
+AI assistance: Used Codex to draft the regression test and PR description.
+I reviewed the changes and verified the test reproduces the reported failure.
+Validation: <commands, results, hardware, and any checks not run>.
+```
+
+Maintainer review time is limited. Contributions that are unverified, misleading, or beyond the author's understanding may be closed. Repeated violations may lead to contribution restrictions. New contributors are welcome to ask for help and submit small, well-understood changes.
+
+This policy is adapted from [Ghostty's AI Usage Policy](https://github.com/ghostty-org/ghostty/blob/main/AI_POLICY.md) for vLLM-Omni's contribution and multimodal validation workflows. The [coding-agent workflow](#using-repository-skills-with-coding-agents) provides practical guidance for applying it.
+
 ## Issues

 If you encounter a bug or have a feature request, please search existing issues first to see if it has already been reported. If not, please file a new issue, providing as much relevant information as possible.
@@ -129,7 +129,7 @@ Canonical layout (prefer these paths for new changes):

 === "NPU"

-    **Bootstrap:** [`npu/pipeline-npu.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/npu/pipeline-npu.yml) + [`npu/bootstrap-upload-steps.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/npu/bootstrap-upload-steps.yml)—same split as CUDA; builds A2/B3 and A3 CI images, then uploads child test pipelines.
+    **Bootstrap:** [`npu/pipeline-npu.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/npu/pipeline-npu.yml) + [`npu/bootstrap-upload-steps.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/npu/bootstrap-upload-steps.yml)—same split as CUDA; builds A2/B3, A3, A5, and 310P CI images, then uploads child test pipelines.

    **Test YAML:** `npu/test-npu-ready.yml` (L2), `test-npu-nightly.yml` (L4).

@@ -137,7 +137,12 @@ Canonical layout (prefer these paths for new changes):

    **Upload:** same as CUDA—`upload_pipeline.py --upload`.

-    **Hardware in YAML:** `mirror_hardwares` preset (string), expanded to `agents`, top-level `image`, and `plugins`. Presets: `a2b3_npu_1`, `a2b3_npu_4`, `a3_npu_2` in `common/ci_mirror_hardwares.yml`.
+    **Hardware in YAML:** `mirror_hardwares` preset (string), expanded to `agents`, top-level `image`, and `plugins`. NPU presets in `common/ci_mirror_hardwares.yml` are:
+
+    - A2: `a2b3_npu_1`, `a2b3_npu_4`, `a2b3_npu_8`
+    - A3: `a3_npu_2`, `a3_npu_4`, `a3_npu_8`, `a3_npu_16`
+    - A5: `a5_npu_2`, `a5_npu_4`, `a5_npu_8`, `a5_npu_16`
+    - 310P: `310p_npu_1`, `310p_npu_2`, `310p_npu_4`

    **Conventions**

@@ -161,7 +166,9 @@ Canonical layout (prefer these paths for new changes):

    **Bootstrap:** [`amd/scripts/bootstrap-amd-omni.sh`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/amd/scripts/bootstrap-amd-omni.sh)—skip-ci, diff filtering, Jinja render, then `buildkite-agent pipeline upload`.

-    **Test YAML (data):** `amd/test-amd-ready.yml` (L2 / PR), `test-amd-merge.yml` (L3 / main).
+    **Test YAML (data):** `amd/test-amd-ready.yml` (L2 / `ready`), `test-amd-merge.yml` (L3 / `merge-test` and `main`). On a PR carrying both labels, AMD combines both suites behind one image build. PR builds without either tier label retain the legacy L2 fallback for compatibility with the `amd-test` trigger. `DEBUG_TEST_YAML` remains an explicit override. AMD has no L4 file yet, so `nightly-test` does not select an additional suite.
+
+    **Trigger boundary:** AMD label handling has two layers. First, the external `vllm-omni-amd-ci` Buildkite pipeline condition decides whether a GitHub label event creates a build. Only after that build starts does this repository's bootstrap inspect all current PR labels and select L2, L3, or both. Repository-side selection therefore cannot make a `merge-test` event start AMD CI by itself. The external condition must admit both `ready` and `merge-test`, while preserving the legacy `amd-test` trigger and non-PR `main` / scheduled builds. For PR builds without a `DEBUG_TEST_YAML` override, an unreadable label set fails the bootstrap instead of silently selecting the wrong tier. Debug overrides skip label lookup and are validated directly; invalid values, including whitespace-only values, fail the bootstrap. Keep `nightly-test` out of the AMD condition until an AMD L4 suite exists; otherwise it would start a build without providing nightly coverage.

    **Rendering:** [`test-template-amd-omni.j2`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/amd/test-template-amd-omni.j2) wraps data steps with `amd-build` image build and `amd_<agent_pool>` queues. Do **not** hand-edit generated `pipeline.yaml`.

@@ -48,6 +48,8 @@ Defined in `pyproject.toml`:
 | `MI325`            | Tests that require MI325 GPU (AMD/ROCm) *                                                               |
 | `A2`               | Tests that require A2 NPU *                                                                             |
 | `A3`               | Tests that require A3 NPU *                                                                             |
+| `A5`               | Tests that require A5 NPU *                                                                             |
+| `310P`             | Tests that require 310P NPU *                                                                           |
 | `cards_{n}`        | Tests that require *n* accelerator cards (`cards_1` … `cards_8`); auto-added from `num_cards` *         |
 | `slow`             | Slow tests (may skip in quick CI)                                                                       |
 | `benchmark`        | Benchmark tests (decorator on runner test functions; perf JSON uses `full_model` + type marker instead) |
@@ -76,7 +78,7 @@ def test_video_to_audio()
 This decorator is intended to make hardware-aware, cross-platform test authoring easier and more robust for CI/CD environments. The `hardware_test` decorator in `vllm-omni/tests/helpers/mark.py` performs the following actions:

 1. **Applies platform and resource markers**  
-   Adds the appropriate pytest markers for each specified hardware platform (e.g., `cuda`, `rocm`, `xpu`, `npu`) and resource type (e.g., `L4`, `H100`, `H200`, `B200`, `MI325`, `B60`, `A2`, `A3`).
+   Adds the appropriate pytest markers for each specified hardware platform (e.g., `cuda`, `rocm`, `xpu`, `npu`) and resource type (e.g., `L4`, `H100`, `H200`, `B200`, `MI325`, `B60`, `A2`, `A3`, `A5`, `310P`).
   ```python
   @pytest.mark.cuda
   @pytest.mark.L4
@@ -779,6 +779,31 @@ turning them on. For Qwen-Image-style serving examples, document
 `--step-execution` as the feature gate and `--max-num-seqs N` as the
 companion batching knob.

+### Streaming Output and Mid-Stream Interactions
+
+To add streaming generation with optional mid-stream prompt updates to a new diffusion pipeline (often a video generation pipeline):
+
+0. **Model should internally support** chunked video emission and bounded VAE decode of a chunk.
+1. **Implement stepwise execution** so interaction RPCs can be consumed at chunk
+   boundaries (see [Step Execution](#step-execution) above).
+2. **Mix in** ``InteractionMixin`` on the pipeline class.
+3. **Implement** ``peek_chunk_media(StepRequestState) -> ChunkMediaSpec`` to offer information
+   about the next chunk to generate.
+   This is called whenever one supported interaction modality needs frame-level calculation,
+   e.g., per-frame camera trajectory. If only chunk-level calculation is involved, peeking is skipped.
+4. **Optionally implement** ``prepare_next_chunk(StepRequestState)`` if the
+   pipeline must rebuild something (e.g., per-chunk state) after an interaction is applied.
+   This function is called at chunk boundary.
+5. **Register** supported modalities and handlers in ``vllm_omni/diffusion/interaction/registry.py``
+   (keyed by ``od_config.model_class_name`` → modality key as in ``req["multi_modal_data"]``).
+
+Serve with ``--diffusion-streaming-output`` (auto-enables step execution). See
+[Diffusion Execution Modes](../../user_guide/diffusion/execution_modes.md#streaming-output)
+and the example script at `examples/online_serving/streaming_video_generation/`.
+
+**Interaction payload contract:** Although ``OmniInteractionPrompt`` type annocation requires ``event_id``,
+the WebSocket API payload may omit ``event_id`` and let the API layer assigns one.
+
 ### Cache Acceleration

 #### TeaCache
@@ -222,6 +222,16 @@ __all__ = ["Qwen3OmniMoeForConditionalGeneration"]

 ## Key Components

+### Preprocess phase metadata
+
+Models with `has_preprocess=True`, including out-of-tree plugins, can rely on
+the runner's stable `_omni_is_prefill`, `_omni_prompt_len`, and
+`_omni_num_computed_tokens` fields. Use request progress to distinguish prompt
+prefill from decode: a one-token span can still be a prefill tail. See the
+[runner-to-model phase contract](../../design/feature/preprocess_phase_contract.md)
+for types, batched-hook semantics, MTP routing guarantees, and the correctness
+workaround for vLLM-Omni 0.20.x and earlier.
+
 ### 1. Model Interfaces

 Your model should implement the appropriate interfaces:
@@ -22,11 +22,15 @@ A pipeline opts in by implementing `SupportsARDiffusionPipeline`. Its immutable
 - named fixed-length cross-attention KV, model-specific scratch geometry, and
  the pipeline's requested session-capacity upper bound.

-The runner binds an `ARDiffusionKVState` only around one `forward()` call. The
-pipeline may use it during that context but must not retain it. The session
-survives subsequent requests with the same `session_id`. Request reset, explicit
-close, LRU eviction, and forward exceptions all release runner-owned KV and call
-the pipeline's reset/close lifecycle hook. Cross-attention KV is allocated lazily
+The runner binds an `ARDiffusionKVState` only around one runner invocation
+(`execute_model` or `execute_stepwise`). The pipeline may use it during that
+context but must not retain it. Scratch and committed pages stay on the
+runner-owned session object, so unbinding does not drop an in-flight stepwise
+chunk. Tick-mode sessions survive subsequent requests with the same
+`session_id`. On the stepwise path the session id is the request id and is
+released when that request finishes or fails. Request reset, explicit close,
+LRU eviction, and forward exceptions all release runner-owned KV and call the
+pipeline's reset/close lifecycle hook. Cross-attention KV is allocated lazily
 per session and logical KV branch, and is released on the same path. A pipeline
 publishes one named cache transactionally by yielding exactly one `(k, v)` pair
 per layer:
@@ -48,11 +52,12 @@ successful forward. The runner also exposes `reset_session()` and

 ## Current execution limits

-The experimental runner supports request-mode execution with
-`max_num_seqs=1`. It rejects step execution and request-batch execution at load
-time, and its inherited batch/step entry points also fail explicitly. Supporting
-those paths requires a batch-aware state-binding contract rather than silently
-bypassing `bind_ar_diffusion_state()`.
+The experimental runner supports `max_num_seqs=1`. Request-batch execution is
+still rejected. Step execution is allowed only when the loaded pipeline
+implements `SupportsStepExecution`; other AR-Diffusion pipelines still fail at
+load if `step_execution=True`. Capable pipelines bind KV around each
+`execute_stepwise` invocation instead of inheriting the generic runner entry
+point unwrapped.

 The implementation may retain multiple sessions per model instance. It selects
 the largest feasible capacity up to the pipeline's `session_capacity`, based on
@@ -323,18 +323,24 @@ inputs, fields, and ownership rules are described below.
 ```mermaid
 flowchart TB
    layer1["Layer 1 · Authoring inputs<br/>PipelineConfig + DeployConfig"]
-    layer2["Layer 2 · Resolve once<br/>StageConfigFactory.create_from_model()<br/>VllmOmniConfig.from_pipeline_config()"]
-    layer3["Layer 3 · Transport-safe control plane<br/>VllmOmniConfig"]
+    layer2["Layer 2 · Production resolution boundary<br/>resolve_omni_config()<br/>(StageConfigFactory + VllmOmniConfig internally)"]
+    layer3["Layer 3 · Startup hand-off<br/>OmniConfigResolution<br/>PipelineConfig + temporary OmegaConf stage bridge"]
    layer4["Layer 4 · Runtime launch planning<br/>StageRuntime"]
    layer5["Layer 5 · Engine materialization<br/>VllmConfig / OmniDiffusionConfig"]

    layer1 --> layer2 --> layer3 --> layer4 --> layer5
 ```

-The single resolution boundary is implemented by
-`StageConfigFactory.create_from_model()` and
-`VllmOmniConfig.from_pipeline_config()` in
+The production resolution boundary is `resolve_omni_config()` in
 [`vllm_omni/config`](https://github.com/vllm-project/vllm-omni/tree/main/vllm_omni/config).
+It delegates typed construction to `StageConfigFactory.create_from_model()` and
+`VllmOmniConfig.from_pipeline_config()`, then returns an
+`OmniConfigResolution` consumed by both `AsyncOmniEngine` and headless startup.
+Until `StageRuntime` consumes typed stage configs directly, this envelope carries
+the effective `PipelineConfig` alongside OmegaConf-compatible `stage_configs` as
+a temporary runtime bridge; both views describe the same resolved topology,
+including injected stages.
+
 The legacy `stage_args` YAML path has been removed. Model topology now resolves
 through `PipelineConfig`, with runtime overrides supplied by `DeployConfig`.

@@ -354,9 +360,12 @@ The important ownership rules are:
   deploy-time defaults; it does not redefine the model graph.
 3. CLI and Python overrides are applied at the resolution boundary, with
   per-stage overrides taking precedence over global values where supported.
-4. `StageRuntime` owns launch planning and replica lifecycle. `ReplicaInitPlan`
+4. `OmniConfigResolution` is the sole startup hand-off. Its `pipeline_config`
+   and temporary `stage_configs` compatibility view must describe the same
+   topology.
+5. `StageRuntime` owns launch planning and replica lifecycle. `ReplicaInitPlan`
   is runtime-private state, not a user configuration object.
-5. `VllmConfig` and the enriched `OmniDiffusionConfig` are materialized in the
+6. `VllmConfig` and the enriched `OmniDiffusionConfig` are materialized in the
   process that owns the corresponding engine.

 ## Main features
@@ -53,6 +53,11 @@ must use distinct hook names and remove only hooks they own.

 ## Discovery and topology

+`resolve_offload_plan(pipeline, config)` is the single topology entry point.
+Every backend calls it once when `enable()` starts and then consumes the
+resolved components, block stacks, resident heads, and staged-residency flags
+it returns. Backends do not re-read model declarations.
+
 Pipeline component discovery prefers `SupportsComponentDiscovery` declarations:

 - `_dit_modules`;
@@ -61,16 +66,26 @@ Pipeline component discovery prefers `SupportsComponentDiscovery` declarations:
 - `_resident_modules`.

 Dotted paths are supported. Legacy pipelines may use the fallback scan of
-well-known attribute names, but new integrations should declare components
-explicitly.
+well-known attribute names; it now warns once per pipeline class, and new
+integrations must declare components explicitly.

-Both layerwise backends first consume pipeline `OffloadPlan` metadata. A
-plan's `block_attrs` maps each DiT path to its ordered block containers, while
-`encoder_block_attrs` declares streamable encoder stacks. DiTs absent from the
-plan fall back to `_layerwise_offload_blocks_attrs` (including the deprecated
-singular-name compatibility path). Discovery metadata describes structure
-only; the backend remains responsible for transfer, synchronization, and
-storage ownership.
+Block topology comes from the pipeline `OffloadPlan` first: `block_attrs` maps
+each DiT path to its ordered block containers, `encoder_block_attrs` declares
+streamable encoder stacks, `on_demand_component_paths` marks pipeline-managed
+residency, `resident_dit_paths` marks the DiTs that may hold resident layers,
+and `encoder_dlo_weight_replication` marks the encoders whose loader-produced
+weights are safe for AllGather. DiTs absent from the plan fall back to
+`_layerwise_offload_blocks_attrs` (including the deprecated singular-name
+compatibility path).
+
+The resolver is pure: it moves no tensor, installs no hook, writes no module
+attribute, and reads no process group — multi-rank facts come from
+`OffloadConfig`. It owns topology validation, so a configuration the model
+cannot serve fails before the first component is placed. Explicit component
+selection turns every declaration problem into an error, while the
+compatibility topology (no component selector) keeps the historical
+warn-and-skip behavior. Metadata still describes structure only; the backend
+remains responsible for transfer, synchronization, and storage ownership.

 ## Cross-strategy invariants

@@ -0,0 +1,172 @@
+# Runner-to-model prefill/decode phase contract
+
+## Overview
+
+Autoregressive models with a custom `preprocess` hook need to distinguish
+prompt processing from decoding. A one-token span can be either a complete
+one-token prompt, the last chunk of a longer prompt, or a decode step. Using
+span length as the phase signal can replace a prompt embedding with a generated
+audio embedding and advance teacher-forced text too early without an exception.
+
+[Issue #4382](https://github.com/vllm-project/vllm-omni/issues/4382) requests a
+stable contract for the metadata and routing guards introduced by
+[PR #3662](https://github.com/vllm-project/vllm-omni/pull/3662).
+The shared runner already implements those guards. This design documents the
+existing interface and adds runner-level regression coverage rather than
+introducing a second phase detector or changing scheduling policy.
+
+## Scope and objectives
+
+- Make the three existing phase keys safe to depend on in out-of-tree models.
+- Preserve prompt embeddings and model state on a one-token prefill tail.
+- Keep normal and batched decode routing consistent in mixed batches.
+- Explain the correctness workaround required on older releases.
+
+This contract does not enable chunked prefill for models whose other state
+handling does not support it. It does not change the scheduler, introduce new
+kwargs, or standardize other `_omni_*` fields. The implementation adds
+documentation and regression coverage; no hot-path allocation is added.
+
+## Design
+
+### Ownership and data flow
+
+```text
+Scheduler / current input batch
+  prompt_token_ids + num_computed_tokens_cpu (before this step)
+                         |
+            OmniGPUModelRunner._preprocess
+                         |
+            per-request phase metadata
+                  /                 \
+       prefill / other spans     single-token decode
+          preprocess             preprocess or preprocess_decode_batch
+              |                              |
+       prompt embedding              runner-managed talker_mtp
+                  \                 /
+                    model forward
+```
+
+The producer is `OmniGPUModelRunner._preprocess` in
+`vllm_omni/worker/gpu_model_runner.py`. `GPUARModelRunner` and
+`GPUGenerationModelRunner` inherit it. The current `OmniNPUModelRunner` also
+inherits this method; device-specific execution still needs its own hardware
+validation. A plugin runner that overrides preprocessing is responsible for
+preserving this interface.
+
+### Stable model-facing fields
+
+The following names, types, and meanings are a supported interface for in-tree
+and out-of-tree model hooks despite the leading underscore. Models should treat
+them as read-only runner-owned values. Changes to their meanings or removal
+require an explicit compatibility/migration plan.
+
+| Key | Python type | Meaning |
+| --- | --- | --- |
+| `_omni_prompt_len` | `int` | Length of the current request state's `prompt_token_ids`. This is the stage's prompt length, not the current chunk size or raw text length. |
+| `_omni_num_computed_tokens` | `int` | The row's `input_batch.num_computed_tokens_cpu` value **before** processing the scheduled span. It includes already-computed/cached progress, not just tokens computed in the current call. |
+| `_omni_is_prefill` | `bool` | Exactly `_omni_num_computed_tokens < _omni_prompt_len`. Equality and greater progress mean decode. |
+
+The runner refreshes the fields after current request state has been gathered,
+immediately before selecting the per-row hook. Existing values in intermediate
+model state cannot override the current phase. Row indices follow the current
+input batch (including any reordering), not request arrival order. A resumed
+or streaming request is classified from its **current** prompt/progress values;
+the phase is not latched for the lifetime of the request.
+
+The normal hook receives these fields as keyword arguments:
+
+```python
+def preprocess(self, input_ids, input_embeds, **info):
+    if info["_omni_is_prefill"]:
+        offset = info["_omni_num_computed_tokens"]
+        # Select the scheduled slice of the model's prompt embeddings.
+        ...
+    else:
+        # Advance decode-only text/audio state.
+        ...
+```
+
+For `preprocess_decode_batch(input_ids=..., req_infos=...)`, each `req_infos`
+entry contains the same fields, aligned with its corresponding input row.
+Every such row has `_omni_is_prefill == False`. The optional earlier
+`preprocess_batch` hook is a separate whole-batch preparation hook; these freshly
+computed per-row fields are not promised at that earlier boundary.
+
+### Routing guarantee
+
+With `has_preprocess=True`, the batched decode hook is eligible only when it is
+callable, the runner has `talker_mtp`, the scheduled span is one token, and
+`_omni_is_prefill` is false. Otherwise the normal `preprocess` hook runs.
+
+Runner-managed MTP is likewise restricted to single-token decode rows. A prefill
+row, including a one-token tail, is never added to that MTP batch: the runner
+does not consume its `mtp_inputs`, generate codes for it, or overwrite its
+preprocessed embedding with an MTP embedding. Decode rows in the same step
+continue to receive their MTP outputs at their own token offsets. Models may
+still implement their own model-specific work inside `preprocess`.
+
+| Prompt length | Computed before step | Scheduled span | Phase | Runner MTP eligible |
+| --- | --- | --- | --- | --- |
+| 5 | 0 | 4 | Prefill | No |
+| 5 | 4 | 1 | Prefill tail | No |
+| 5 | 5 | 1 | Decode | Yes |
+| 5 | 6 | 1 | Decode | Yes |
+| 1 | 0 | 1 | Prefill | No |
+| 5 | 5 | 2 | Decode | No (not a single-token span) |
+
+### Compatibility
+
+!!! warning "vLLM-Omni 0.20.x and earlier"
+    AR talker models with custom preprocessing that infer phase from span length
+    must use `enable_chunked_prefill: false` for correctness on these releases.
+    They lack the phase metadata and the runner's prefill-tail MTP protection.
+    A model-only heuristic cannot prevent the older runner from overwriting a
+    one-token prompt embedding. Disabling chunking avoids split tails; it does
+    not make span length a reliable phase detector for every possible prompt.
+
+The mechanism landed in #3662 (v0.21.0rc2 / v0.22.0 and later); this document
+formalizes that existing behavior. Prefer upgrading the matching vLLM and
+vLLM-Omni release pair and consuming the explicit phase keys. Do not silently
+fall back to `input_ids.numel() > 1` when the model requires chunking correctness.
+For a custom or backported runtime, verify both metadata and MTP guards.
+
+## Correctness and testing plan
+
+Runner tests call the production `_preprocess` and `_talker_mtp_forward` methods
+with CPU buffers and a deterministic MTP model. They inspect kwargs at the model
+boundary, resulting embeddings, and per-request generated codes. Coverage must
+include a multi-step prompt split with a one-token tail followed by decode,
+mixed prefill/decode batches in different orders, normal/batched decode hooks,
+adjacent decode requests in the same hook call with exact per-request codes,
+single-token prompts, cached progress, and decode spans longer than one token.
+No test should pass merely because MTP was disabled for every row.
+
+Real-weight validation uses Qwen3-TTS CustomVoice through the full talker and
+code2wav pipeline. Record the actual scheduled prompt progress, assert that at
+least one one-token prefill tail was observed, check its hook metadata and MTP
+exclusion, and require finite non-empty output audio. Compare to an unchunked
+run with the same real input, greedy sampling, and `VLLM_BATCH_INVARIANT=1`.
+Greedy sampling alone does not guarantee identical codes across different
+prefill shapes: floating-point reduction differences can change an argmax and
+diverge in subsequent autoregressive steps. Invariant kernels keep this exact
+comparison focused on the phase contract. The two engines must be initialized
+separately because their chunking settings and token budgets differ. GPU tests belong under
+`tests/e2e/features/`; existing model tests remain useful adjacent regression
+coverage. Functional audio checks do not establish throughput or all-model
+semantic parity; this change makes no performance claim.
+
+## Open questions and discussions
+
+There are no new field names or execution modes to choose. Future runner
+implementations, including Model Runner V2, should preserve these semantics or
+provide an explicit migration path. Hardware validation beyond the tested
+platform remains separate from the shared Python routing tests.
+
+## References
+
+- [Issue #4382](https://github.com/vllm-project/vllm-omni/issues/4382)
+- [Original metadata and guards, #3662](https://github.com/vllm-project/vllm-omni/pull/3662)
+- [Qwen2.5-Omni consumer fix, #5019](https://github.com/vllm-project/vllm-omni/pull/5019)
+- [Adding an Omni model](../../contributing/model/adding_omni_model.md)
+- [Test writing guide](../../contributing/ci/test_writing_guide.md)
@@ -66,7 +66,8 @@ The checkpoint keeps these verified contracts:
 - scheduler data-plane append over a resumable request;
 - stale epoch/turn/response fencing;
 - `/v1/duplex` and OpenAI Realtime projection;
- optional OpenAI `server_vad` turn detection backed by per-session Silero VAD;
+- optional OpenAI `server_vad` turn detection backed by a process-shared Silero
+  model and per-session VAD state;
 - existing JoyVL behavior.

 The checkpoint does not claim:
@@ -652,7 +653,7 @@ EOS decisions advance the model conversation.

 Explicit Realtime clients may still use `input_audio_buffer.commit` to create a
 conversation item. The translator validates that wire input before producing a
-commit carrying `realtime_item_id`. Native auto-response may already have
+commit carrying `item_id`. Native auto-response may already have
 streamed those PCM samples into the runtime, so the runner accepts a validated
 commit even when no runtime-side chunk remains. A truly empty explicit buffer
 continues to return `input_audio_buffer_empty`.
@@ -677,8 +678,7 @@ multi-turn runtime evidence.

 During auto-response overlap, `preserve_realtime_input` distinguishes "do not
 append this silent chunk to the native buffer" from "clear the open Realtime
-item". Silent overlap no longer discards earlier user PCM. A separate explicit
-`server_vad` policy is documented in the Realtime contract above.
+item". Silent overlap no longer discards earlier user PCM.

 The first chunk of one overlapping input item also reserves its target model
 turn. A later Realtime commit uses that reserved identity even if response EOS
@@ -24,6 +24,7 @@ implementation contract; it is not, by itself, a general support claim.
 - [Async Chunk](feature/async_chunk.md)
 - [Async Diffusion Output](feature/async_diffusion_output.md)
 - [Async Omni Output Materialization](feature/omni_async_output_materialization.md)
+- [Runner-to-model Prefill/Decode Phase Contract](feature/preprocess_phase_contract.md)
 - [Automatic Prefix Caching in Omni Models](feature/prefix_caching.md)
 - [Realtime AR-Diffusion Sessions](feature/realtime_ar_diffusion.md)

@@ -8,22 +8,44 @@ owners:
  - "@SamitHuang"
  - "@fhfuih"
 primary_code_paths:
+  - vllm_omni/diffusion/diffusion_engine.py
+  - vllm_omni/diffusion/request.py
+  - vllm_omni/diffusion/data.py
  - vllm_omni/diffusion/sched/**
  - vllm_omni/diffusion/executor/**
  - vllm_omni/diffusion/worker/**
 related_code_paths:
+  - vllm_omni/diffusion/stage_diffusion_client.py
+  - vllm_omni/diffusion/stage_diffusion_proc.py
+  - vllm_omni/diffusion/inline_stage_diffusion_client.py
+  - vllm_omni/diffusion/ipc.py
  - vllm_omni/diffusion/hooks/**
  - vllm_omni/diffusion/postprocess/**
 depends_on:
  - ../engine_orchestration.md
+  - ../stage_runtime.md
  - ../input_output_modality_contracts.md
  - diffusion_model_integration.md
 validation_paths:
-  - tests/diffusion/diffusion_backend/**
-  - tests/diffusion/hooks/**
+  - tests/diffusion/test_diffusion_engine.py
+  - tests/diffusion/test_diffusion_engine_cleanup.py
+  - tests/diffusion/test_diffusion_engine_dummy_run.py
+  - tests/diffusion/test_diffusion_engine_rpc_routing.py
+  - tests/diffusion/test_diffusion_scheduler.py
+  - tests/diffusion/test_diffusion_model_runner.py
+  - tests/diffusion/test_diffusion_worker.py
+  - tests/diffusion/test_multiproc_engine_concurrency.py
+  - tests/diffusion/test_uniproc_executor.py
+  - tests/diffusion/test_result_pump.py
+  - tests/diffusion/test_async_output_worker.py
+  - tests/diffusion/test_diffusion_ipc.py
+  - tests/diffusion/test_diffusion_streaming_output.py
+  - tests/diffusion/test_stage_diffusion_proc.py
+  - tests/diffusion/test_inline_stage_diffusion_client.py
 upstream_refs:
  - diffusers.DiffusionPipeline
-last_reviewed: 2026-07-16
+last_reviewed: 2026-09-02
+last_verified_commit: e51fe6ec1b9a9a0e14bb1fdb296d61b6593b93c6
 ---

 # Diffusion runtime
@@ -31,6 +53,380 @@ last_reviewed: 2026-07-16
 The diffusion runtime owns request admission, scheduling, execution, progress,
 output, cancellation, and cleanup inside a diffusion stage.

+It is easiest to think of the runtime as a small control loop:
+
+1. the **scheduler** chooses ready requests;
+2. the **executor** sends that choice to the worker layer;
+3. each **worker** manages its device and delegates model work to a **runner**;
+4. the runner calls the model pipeline and returns per-request results; and
+5. the engine feeds those results back to the scheduler and output streams.
+
+!!! note
+
+    This page describes the runtime plumbing. Model loading and pipeline
+    contracts belong to [Diffusion model integration](diffusion_model_integration.md).
+    Batch compatibility and step-state details belong to
+    [Diffusion continuous batching](continuous_batching.md).
+
+## Goals and non-goals
+
+The runtime aims to keep request policy separate from device execution. It
+also keeps request identity stable from admission through the final output,
+including cancellation and failures.
+
+This module does **not** choose the next Omni stage, place stage replicas, or
+define a model's denoising algorithm:
+
+- cross-stage routing belongs to [Engine orchestration](../engine_orchestration.md);
+- process placement and replica lifecycle belong to
+  [Stage runtime](../stage_runtime.md); and
+- model-specific work belongs to
+  [Diffusion model integration](diffusion_model_integration.md).
+
+## Runtime at a glance
+
+The stage owns `DiffusionEngine` and a `DiffusionExecutor` backend. The inline
+stage client keeps that stack in the caller process; a process-backed stage
+runs it inside `StageDiffusionProc`.
+
+The executor then chooses how the worker runs:
+
+- **`uni`** (default when `num_gpus == 1`): `UniProcDiffusionExecutor` builds
+  one in-process worker. There is no worker subprocess and no shared-memory
+  RPC.
+- **`mp`** (default when `num_gpus > 1`, or when set explicitly):
+  `MultiprocDiffusionExecutor` starts one `WorkerProc` per device.
+
+Either way, each worker owns its device, distributed state,
+`DiffusionWorker`, `DiffusionModelRunner`, and pipeline instance.
+
+```mermaid
+flowchart TB
+    client["Stage client"]
+
+    subgraph stage["Diffusion stage process"]
+        stageProc["StageDiffusionProc<br/>stage transport and lifecycle"]
+        engine["DiffusionEngine<br/>admission and control loop"]
+        scheduler["RequestScheduler or StepScheduler<br/>request state and policy"]
+        executor["DiffusionExecutor<br/>uni or mp backend"]
+        streams["Per-request output streams"]
+
+        stageProc --> engine
+        engine --> scheduler
+        scheduler -->|"DiffusionSchedulerOutput"| engine
+        engine --> executor
+        engine --> streams
+    end
+
+    subgraph workers["Worker layer"]
+        workerProc["WorkerProc<br/>mp IPC loop only"]
+        worker["DiffusionWorker<br/>device and distributed setup"]
+        runner["DiffusionModelRunner<br/>model state and execution"]
+        pipeline["Diffusion pipeline<br/>model-specific computation"]
+
+        workerProc --> worker
+        worker --> runner --> pipeline
+    end
+
+    client -->|"process-backed: ZMQ"| stageProc
+    client -.->|"inline: direct call"| engine
+    executor -->|"mp: broadcast / result queues"| workerProc
+    executor -.->|"uni: in-process call"| worker
+    streams -->|"process-backed"| stageProc
+    stageProc -->|"ZMQ results"| client
+    streams -.->|"inline"| client
+```
+
+!!! info "Stage process vs worker process"
+
+    A deployed stage may already run in a `StageDiffusionProc` child process.
+    That is separate from the optional `mp` worker processes. With `uni`, the
+    worker lives in the same process as the engine.
+
+## Component responsibilities
+
+| Component | Owns | Does not own |
+| --- | --- | --- |
+| `DiffusionEngine` | Admission, the busy loop, scheduler/executor coordination, RPC ordering, output streams, warmup, abort routing | Batch policy details, device setup, model computation |
+| `BaseScheduler` | Request states, waiting/running sets, capacity, compatibility, terminal transitions | IPC, worker calls, output formatting |
+| `RequestScheduler` | Complete-request waves and optional admission delay for request-level batching | Denoising-step progress |
+| `StepScheduler` | Per-request denoising progress and step-wise completion | Pipeline tensors and model state |
+| `DiffusionExecutor` | Execution backend contract, worker RPC, health, shutdown | Admission and request-state transitions |
+| `UniProcDiffusionExecutor` | Single in-process worker for `num_gpus == 1`; no IPC or async output pump | Multi-GPU execution |
+| `MultiprocDiffusionExecutor` | Worker processes, shared-memory message queues, result dispatch, worker monitoring | In-process single-GPU path |
+| `WorkerProc` | One `mp` worker process's IPC loop and reply rules | Scheduling policy; unused by `uni` |
+| `DiffusionWorker` | Device/distributed setup, LoRA activation, profiling, sleep/wake, runner delegation | Request admission |
+| `DiffusionModelRunner` | Pipeline loading, request-local model state, cache/compile setup, request or step execution | Queueing and cross-stage routing |
+
+The boundary between policy and execution is
+`DiffusionSchedulerOutput`. It contains newly admitted request payloads,
+IDs for requests whose runner state is already cached, finished IDs,
+optional KV-prefetch work, and per-request `diffusion_kv_metadata` on
+`NewRequestData` envelopes when paged Diffusion KV is enabled. Executors and
+workers consume this output; they do not decide what should run next.
+
+`BaseScheduler.initialize()` sets `max_num_running_reqs` from
+`od_config.max_num_seqs` (default 1). The engine may override that capacity
+for distributed layerwise offload with AllGather DP concurrency.
+
+## One scheduling tick
+
+Async requests share one engine busy loop. Worker calls and control RPCs pass
+through that loop, so they cannot race on the executor transport.
+
+```mermaid
+sequenceDiagram
+    participant Caller
+    participant Engine as DiffusionEngine
+    participant Scheduler
+    participant Executor
+    participant Worker
+    participant Runner
+
+    Caller->>Engine: add request
+    Engine->>Scheduler: add_request(request)
+    Engine->>Scheduler: schedule()
+    Scheduler-->>Engine: DiffusionSchedulerOutput
+    Engine->>Executor: execute_batch() or execute_step()
+    Executor->>Worker: run worker method
+    Worker->>Runner: execute model or one step
+    Runner-->>Worker: RunnerOutput(s)
+    Worker-->>Executor: result
+    Executor-->>Engine: BaseRunnerOutput
+    Engine->>Scheduler: update_from_output(...)
+    Scheduler-->>Engine: finished request IDs
+    Engine-->>Caller: chunk or final output
+```
+
+The engine catches request execution errors and turns them into per-request
+error outputs. A dead worker group is different: the executor marks itself
+failed, health checks raise `EngineDeadError`, and the owning stage client
+handles the stage-level failure.
+
+## Execution modes
+
+The engine resolves one mode at startup and binds the matching scheduler and
+executor call.
+
+| Mode | Scheduler | Executor call | Runner path |
+| --- | --- | --- | --- |
+| Request batch | `RequestScheduler` | `execute_batch()` | `execute_model()` for one request, or `execute_model_batch()` for a fused batch |
+| Step batch | `StepScheduler` | `execute_step()` | `execute_stepwise()` |
+
+**Request mode** runs a complete pipeline forward for each scheduled wave. A
+single-request wave is the conservative path. A multi-request wave usually
+uses fused `execute_model_batch()` and requires the pipeline to declare
+request-batch support.
+
+**Distributed layerwise offload with AllGather (DLO DP concurrency)** is a
+separate multiproc dispatch path. It activates when
+`data_parallel_size > 1`, `enable_distributed_layerwise_offload` is set, and
+`dlo_use_allgather` is true. The engine then sets `dp_concurrent = True` and
+raises `scheduler.max_num_running_reqs` to `dp_size`, overriding the
+`max_num_seqs` value from `initialize()`. The scheduler still emits a
+multi-request `DiffusionSchedulerOutput`; the multiproc executor routes that
+wave through `execute_request()` instead of fused `execute_model_batch()`.
+
+Optional admission coalescing can fill a `dp_size` wave: when
+`request_batch_max_wait_ms > 0`, `RequestScheduler` may wait briefly before
+the first schedule of a wave (under `dp_concurrent`, the stable window is
+`min(0.3s, wait/2)`). With the default `request_batch_max_wait_ms == 0`,
+there is no wait.
+
+Before dispatch, `MultiprocDiffusionExecutor.execute_request()` rejects waves
+whose requests differ in sampling-parameter compatibility or `extra_args`
+(AllGather requires every DP rank to follow the same forward schedule). Each
+worker rank then picks one envelope from the wave:
+`req[dp_rank % len(req)]`, so every rank enters the same collective while
+computing different requests.
+
+**Step mode** keeps `StepRequestState` in the runner. New requests run
+`prepare_encode()` once; every tick runs `denoise_step()` and
+`step_scheduler()`; completed requests run `post_decode()`. The scheduler keeps
+only lifecycle and progress metadata—it does not hold model tensors.
+
+**Step mode is also the path for streaming diffusion output**. Enabling
+`streaming_output` turns on step execution if needed. Chunk-capable pipelines
+can emit intermediate outputs through the same request stream; final-only step
+pipelines still emit only the final result. If the pipeline does not implement
+step execution, initialization fails.
+
+!!! tip
+
+    For user-facing flags and examples, see
+    [Diffusion execution modes](../../../user_guide/diffusion/execution_modes.md).
+    That guide explains how to select a mode; this page explains what happens
+    after the choice is made.
+
+## Request state and identity
+
+The scheduler is the source of truth for an admitted request. Its normal state
+flow is:
+
+```mermaid
+stateDiagram-v2
+    [*] --> WAITING: add_request
+    WAITING --> RUNNING: schedule
+    WAITING --> FINISHED_ABORTED: abort
+    RUNNING --> FINISHED_ABORTED: abort
+    RUNNING --> FINISHED_COMPLETED: successful output
+    RUNNING --> FINISHED_ERROR: failed or missing output
+    FINISHED_COMPLETED --> [*]: deliver and remove state
+    FINISHED_ABORTED --> [*]: deliver and remove state
+    FINISHED_ERROR --> [*]: deliver and remove state
+```
+
+`PREEMPTED` and `BaseScheduler.preempt_request()` exist on the scheduler API,
+but the engine loop does not call them today—only scheduler unit tests do.
+
+`request_id` ties together scheduler state, runner state, executor results, and
+the output queue. Batch positions are temporary and must never replace request
+identity when results are mapped back.
+
+Only compatible requests enter the same running wave. The scheduler compares a
+mode-specific sampling-parameter key and stops at the first incompatible
+waiting request. This is intentionally conservative and can cause
+head-of-line blocking.
+
+## Executor and IPC
+
+`DiffusionExecutor` is the backend interface. Built-in backends are selected by
+`distributed_executor_backend`:
+
+| Backend | When chosen | Worker layout |
+| --- | --- | --- |
+| `uni` | Default for `num_gpus == 1`, or set explicitly | One in-process worker |
+| `mp` | Default for `num_gpus > 1`, or set explicitly | One worker process per device |
+
+Ray and external-launcher diffusion backends are not implemented. A custom
+`DiffusionExecutor` subclass or import path is also accepted.
+
+### Uniproc backend
+
+`UniProcDiffusionExecutor` constructs `WorkerWrapperBase` in the engine process
+and calls worker methods directly. It avoids a second model load, MessageQueue
+rings, ZMQ IPC sockets, and `/dev/shm` tensor packing. RPC timeouts are
+accepted for interface parity but are not enforced: a hung worker blocks the
+calling thread. Sticky accelerator faults mark the executor dead through
+failure callbacks; ordinary request errors stay per-request.
+
+### Multiproc backend
+
+`MultiprocDiffusionExecutor`:
+
+1. creates a broadcast queue shared by all workers;
+2. spawns one worker process per configured device;
+3. waits until every worker reports ready;
+4. sends generation and control calls as RPC messages;
+5. applies rank-aware reply rules so only expected ranks respond; and
+6. monitors worker process sentinels and fails the executor if a worker dies.
+
+In request mode on this backend, each async output produces two correlated
+messages keyed by `async_output_id`:
+
+- `COMPUTE_DONE` — the forward finished and the device can start the next
+  request.
+- `OUTPUT_READY` — background D2H/SHM packing finished and the final output
+  is ready.
+
+The worker queues background packing work before it enqueues `COMPUTE_DONE`,
+but either message may reach the executor first. The result pump and
+`execute_batch()` must handle both orderings. Step mode keeps the synchronous
+result path and does not start those pumps.
+
+See [Async diffusion output](../../feature/async_diffusion_output.md) for the
+multiproc request-mode timeline.
+
+## Worker and runner boundary
+
+On the `mp` path, `WorkerProc` receives messages and calls methods through
+`WorkerWrapperBase`. On the `uni` path, the executor owns that wrapper
+directly. In both cases, `DiffusionWorker` handles the parts tied to a device:
+distributed initialization, model-runner construction, LoRA activation,
+profiling, and memory sleep/wake. It delegates actual model work to
+`DiffusionModelRunner`.
+
+The runner owns long-lived model-side state:
+
+- the loaded pipeline and compilation setup;
+- cache and offload integration;
+- random-generator setup;
+- request batches for complete forwards; and
+- `StepRequestState` plus `InputBatch` for step execution.
+
+This split keeps infrastructure out of model pipelines. A pipeline receives a
+request batch or step state and performs model work; it should not inspect
+engine queues or change scheduler state.
+
+## Lifecycle and cleanup
+
+### Startup
+
+`DiffusionEngine.make_engine()` resolves the engine class, constructs the
+executor and scheduler, then normally runs a small dummy request to warm up the
+model.
+
+Warmup is skipped only when distributed layerwise offload with AllGather
+is enabled (via `enable_distributed_layerwise_offload` and `dlo_use_allgather`)
+and `max(data_parallel_size, sequence_parallel_size) > 1`, because
+the dummy run sends one request while AllGather needs every shard rank to
+enter the same collective. If initialization or warmup fails, the runtime
+closes scheduler state and worker resources before returning the error.
+
+### Cancellation
+
+`abort()` places request IDs on an abort queue. The busy loop marks those
+requests `FINISHED_ABORTED`; finalization removes scheduler state and emits an
+aborted output when a consumer still exists. The runner removes cached
+step-mode state when it receives a scheduler output that reports the finished
+request ID.
+
+On the multiproc request-mode async path, abort or consumer drop while output
+is still materializing should release the associated async-output bookkeeping
+so late results are not retained after the request ends
+*(pending #6253/#6439/#6580)*. Today, an unconsumed `OUTPUT_READY` is cached
+by design until `wait_output_ready()` or executor teardown. Dropping an
+output-stream consumer removes that consumer's queue. It does not take
+ownership of scheduler cleanup.
+
+### Shutdown and worker failure
+
+`DiffusionEngine.close()` is idempotent. It stops the busy loop, wakes pending
+streams with an error, closes scheduler state, and shuts down the executor.
+
+- **`mp`**: the executor asks workers to stop, waits for them, then terminates
+  workers that miss the grace period. Shutdown marks unfinished RPC and
+  async-output futures as failed, then clears them. Completed async outputs
+  cached for later `wait_output_ready()` calls are dropped with the executor
+  object.
+- **`uni`**: the executor shuts down the in-process worker, drops the final
+  model reference, and empties the accelerator cache so a later engine can reuse
+  the device.
+
+!!! warning
+
+    Do not continue using a worker group after a fatal collective timeout or
+    unexpected worker exit. Distributed state may be incomplete. The
+    multiprocess executor deliberately fails closed so the stage can restart.
+    The uniproc executor fails closed when the accelerator context itself is
+    poisoned.
+
+## Extension boundaries
+
+- A custom engine may set `default_diffusion_model_runner_cls`; an explicit
+  `diffusion_model_runner_cls` configuration still wins.
+- Tests and custom engine integrations may inject a `BaseScheduler` subclass.
+  `SchedulerInterface` remains only as a deprecated compatibility name.
+- `distributed_executor_backend` may be `"uni"`, `"mp"`, a custom `DiffusionExecutor`
+  subclass, or an import path. A backend must preserve scheduler output,
+  request identity, health, and cleanup contracts. Use `"mp"` when you need
+  process isolation or multi-GPU; `"uni"` is the single-GPU default.
+- Worker extensions go through `WorkerWrapperBase`. They add worker methods;
+  they do not become a second scheduler or request lifecycle.
+
+These are advanced Python integration points, not stable end-user CLI
+customization surfaces.
+
 ## Candidate invariants

 ### DIFF-RUNTIME-INV-001: One lifecycle owner
@@ -41,19 +437,50 @@ until completion, cancellation, or failure.
 ### DIFF-RUNTIME-INV-002: Execution follows scheduler output

 **Rule:** Executors and workers MUST execute scheduler decisions without
-admitting, reordering, or forwarding requests independently.
+admitting, reordering, or forwarding requests independently. Distributed
+layerwise offload with AllGather is an exception at dispatch time: the
+scheduler may schedule a multi-request wave, but each DP rank executes one
+envelope from that wave so every rank enters the same collective schedule.

 ### DIFF-RUNTIME-INV-003: Terminal cleanup is complete

 **Rule:** Every terminal path MUST release request state, temporary tensors,
-hooks, and runtime-owned resources.
+hooks, and runtime-owned resources. Async-output bookkeeping on abort or
+consumer drop before consumption is also required *(pending #6253/#6439/#6580)*.

 ### DIFF-RUNTIME-INV-004: Optional features use runtime hooks

 **Rule:** Cache, profiling, offload, and parallel features SHOULD integrate at
 defined hooks instead of creating another request lifecycle.

+### DIFF-RUNTIME-INV-005: Results keep request identity
+
+**Rule:** Batched and asynchronous results MUST be mapped by stable request ID,
+not by assumed completion order.
+
 ## Safe-change guide

-Test success, cancellation, failure, shutdown, multiple requests, and resource
-cleanup.
+Test the smallest affected slice, then cover its neighboring boundary:
+
+| Change area | Minimum evidence |
+| --- | --- |
+| Admission or state transitions | `test_diffusion_scheduler.py`, including duplicate IDs, compatibility, abort, and missing output |
+| Engine loop or output delivery | `test_diffusion_engine.py`, `test_diffusion_engine_cleanup.py`, `test_diffusion_engine_rpc_routing.py` |
+| Executor or IPC | `test_multiproc_engine_concurrency.py`, `test_uniproc_executor.py`, `test_result_pump.py`, `test_diffusion_ipc.py`, `test_async_output_worker.py` |
+| Worker or runner | `test_diffusion_worker.py`, `test_diffusion_model_runner.py` |
+| Stage boundary | `test_stage_diffusion_proc.py`, `test_inline_stage_diffusion_client.py` |
+| Warmup or streaming | `test_diffusion_engine_dummy_run.py`, `test_diffusion_streaming_output.py` |
+
+Always exercise success, cancellation, per-request failure, fatal worker
+failure, repeated shutdown, one request, and multiple compatible requests.
+For step-mode changes, also test partial progress and cleanup of runner state.
+
+## Related documents
+
+- [Diffusion module overview](index.md)
+- [Diffusion model integration](diffusion_model_integration.md)
+- [Diffusion continuous batching](continuous_batching.md)
+- [Diffusion execution modes](../../../user_guide/diffusion/execution_modes.md)
+- [Async diffusion output](../../feature/async_diffusion_output.md)
+- [Engine orchestration](../engine_orchestration.md)
+- [Stage runtime](../stage_runtime.md)
@@ -1,7 +1,7 @@
 # Installation

 !!! important
-    vLLM-Omni is released against the matching upstream vLLM major/minor version. vLLM-Omni 0.28.x requires vLLM 0.28.x; the stable v0.28.0 instructions pin vLLM 0.28.0.
+    vLLM-Omni is released against the matching upstream vLLM major/minor version. The 0.29 development line requires vLLM 0.29.x; the source-install instructions pin vLLM 0.29.0. Published vLLM-Omni 0.28.0 wheels and images still require vLLM 0.28.x.

 vLLM-Omni supports the following hardware platforms:

@@ -34,6 +34,8 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr

 ### Pre-built wheels

+The source-install instructions target vLLM 0.29.0. Until vLLM-Omni 0.29.0 artifacts are published, the CUDA/ROCm pre-built examples below remain on the matching 0.28.0 release pair.
+
 Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, 0.25.0, 0.26.0, and 0.28.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).

 === "NVIDIA CUDA"
@@ -5,7 +5,7 @@
 # --8<-- [end:requirements]
 # --8<-- [start:set-up-using-python]

-vLLM-Omni depends on the matching major/minor release of vLLM. The vLLM-Omni 0.28.x release line uses vLLM 0.28.x.
+vLLM-Omni depends on the matching major/minor release of vLLM. The 0.29 development line uses vLLM 0.29.x. Published 0.28.0 wheels and images use vLLM 0.28.x.

 !!! note
    PyTorch installed via `conda` will statically link `NCCL` library, which can cause issues when vLLM tries to use `NCCL`. See <gh-issue:8420> for more details.
@@ -18,6 +18,8 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new**

 #### Installation of vLLM

+These pre-built wheel instructions install the published vLLM-Omni 0.28.0 release. For the 0.29 development line, use the source-install instructions below.
+
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
 uv pip install vllm==0.28.0 --torch-backend=auto
@@ -26,12 +28,12 @@ uv pip install vllm==0.28.0 --torch-backend=auto
 #### Installation of vLLM-Omni

 ```bash
-uv pip install vllm-omni
+uv pip install vllm-omni==0.28.0
 ```

 To run Gradio demos, also install the optional extras:
 ```bash
-uv pip install 'vllm-omni[demo]'
+uv pip install 'vllm-omni[demo]==0.28.0'
 ```

 # --8<-- [end:pre-built-wheels]
@@ -39,13 +41,13 @@ uv pip install 'vllm-omni[demo]'
 # --8<-- [start:build-wheel-from-source]

 #### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.28.0 release version of the library
+If you do not need to modify source code of vLLM, you can directly install the stable 0.29.0 release version of the library

 ```bash
-uv pip install vllm==0.28.0 --torch-backend=auto
+uv pip install vllm==0.29.0 --torch-backend=auto
 ```

-The 0.28.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The 0.29.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.

 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -66,12 +68,12 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.28.0
+git checkout v0.29.0
 ```
 Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
 ```bash
-#For CUDA 13.0 (the default for v0.28.0; the wheel filename has no `+cu130` suffix)
-export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.28.0/vllm-0.28.0-cp38-abi3-manylinux_2_28_x86_64.whl
+#For CUDA 13.0 (the default for v0.29.0; the wheel filename has no `+cu130` suffix)
+export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.29.0/vllm-0.29.0-cp38-abi3-manylinux_2_28_x86_64.whl
 ```
 Install vllm with command below (If you have no existing PyTorch).
 ```bash
@@ -131,7 +133,7 @@ If you want to specify the base vLLM version:
 ```bash
 DOCKER_BUILDKIT=1 docker build \
  -f docker/Dockerfile.cuda \
-  --build-arg BASE_IMAGE=vllm/vllm-openai:v0.28.0 \
+  --build-arg BASE_IMAGE=vllm/vllm-openai:v0.29.0 \
  -t vllm-omni-cuda .
 ```

@@ -11,6 +11,8 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.

 #### Installation of vLLM

+These pre-built wheel instructions install the published vLLM-Omni 0.28.0 release. For the 0.29 development line, use the source-install instructions below.
+
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
 uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.28.0/rocm723
@@ -22,7 +24,7 @@ uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/roc
 # we need to add --no-build-isolation as the torch
 # is not obtained from pypi, we have to install using the
 # torch installed in our environment
-uv pip install vllm-omni
+uv pip install vllm-omni==0.28.0

 # Optional if want to run Qwen3 TTS
 uv pip uninstall onnxruntime # should be removed before we can install onnxruntime-rocm
@@ -34,13 +36,13 @@ uv pip install onnxruntime-rocm
 # --8<-- [start:build-wheel-from-source]

 #### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.28.0 release version of the library
+If you do not need to modify source code of vLLM, you can directly install the stable 0.29.0 release version of the library

 ```bash
-uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.28.0/rocm723
+uv pip install vllm==0.29.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.29.0/rocm723
 ```

-The pre-built 0.28.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The pre-built 0.29.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.

 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -58,7 +60,7 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.28.0
+git checkout v0.29.0
 python3 -m pip install -r requirements/rocm.txt
 python3 setup.py develop
 ```
@@ -7,13 +7,34 @@ For detailed hardware and software requirements, please refer to the [vLLM-Ascen

 The recommended way to use vLLM-Omni on NPU is through the vLLM-Ascend pre-built Docker images:

+=== "A2"
+
+    ```bash
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.28.0
+    ```
+
+=== "A3"
+
+    ```bash
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.28.0-a3
+    ```
+
+=== "A5"
+
+    ```bash
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.28.0-a5
+    ```
+
+=== "310P (Experimental)"
+
+    ```bash
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.28.0-310p
+    ```
+
 ```bash
-# vLLM-Ascend image
-# Atlas A2: quay.io/atlas-ci/vllm-ascend:v0.28.0
-# Atlas A3: quay.io/atlas-ci/vllm-ascend:v0.28.0-a3
 docker run --rm \
    --name vllm-omni-npu \
-    --shm-size=1g \
+    --shm-size=64g \
    --device /dev/davinci0 \
    --device /dev/davinci1 \
    --device /dev/davinci2 \
@@ -27,8 +48,8 @@ docker run --rm \
    -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
    -v /etc/ascend_install.info:/etc/ascend_install.info \
    -v /root/.cache:/root/.cache \
-    -p 8000:8000 \
-    -it quay.io/atlas-ci/vllm-ascend:v0.28.0-a3 bash
+    -p 8091:8091 \
+    -it "$IMAGE" bash

 # Inside the container, install vLLM-Omni from source
 cd /vllm-workspace
@@ -72,53 +93,146 @@ Supported images as following.

 | image name | Hardware | OS |
 |-|-|-|
-| image-tag | Atlas A2 | Ubuntu |
-| image-tag-a3 | Atlas A3 | Ubuntu |
+| image-tag | A2 | Ubuntu |
+| image-tag-a3 | A3 | Ubuntu |
+| image-tag-a5 | A5 | Ubuntu |
+| image-tag-310p | 310P (experimental) | Ubuntu |

 Here's an example deployment command that has been verified on 4 x NPUs:

-```bash
-# Atlas A2: quay.io/ascend/vllm-omni:v0.28.0
-# Atlas A3: quay.io/ascend/vllm-omni:v0.28.0-a3
-docker run --rm \
-    --name vllm-omni-a3 \
-    --shm-size=4g \
-    --device /dev/davinci0 \
-    --device /dev/davinci1 \
-    --device /dev/davinci2 \
-    --device /dev/davinci3 \
-    --device /dev/davinci_manager \
-    --device /dev/devmm_svm \
-    --device /dev/hisi_hdc \
-    -v /usr/local/dcmi:/usr/local/dcmi \
-    -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
-    -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
-    -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
-    --device /dev/davinci_manager \
-    --device /dev/devmm_svm \
-    --device /dev/hisi_hdc \
-    -v /usr/local/dcmi:/usr/local/dcmi \
-    -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
-    -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
-    -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
-    -v /etc/ascend_install.info:/etc/ascend_install.info \
-    -v ~/.cache:/root/.cache \
-    -p 8091:8091 \
-    -it quay.io/ascend/vllm-omni:v0.28.0-a3 bash
-```
+=== "A2"
+
+    ```bash
+    export IMAGE=quay.io/ascend/vllm-omni:v0.28.0
+    docker run --rm \
+        --name vllm-omni-a2 \
+        --shm-size=64g \
+        --device /dev/davinci0 \
+        --device /dev/davinci1 \
+        --device /dev/davinci2 \
+        --device /dev/davinci3 \
+        --device /dev/davinci_manager \
+        --device /dev/devmm_svm \
+        --device /dev/hisi_hdc \
+        -v /usr/local/dcmi:/usr/local/dcmi \
+        -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
+        -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
+        -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
+        -v /etc/ascend_install.info:/etc/ascend_install.info \
+        -v ~/.cache:/root/.cache \
+        -p 8091:8091 \
+        -it "$IMAGE" bash
+    ```
+
+=== "A3"
+
+    ```bash
+    export IMAGE=quay.io/ascend/vllm-omni:v0.28.0-a3
+    docker run --rm \
+        --name vllm-omni-a3 \
+        --shm-size=64g \
+        --device /dev/davinci0 \
+        --device /dev/davinci1 \
+        --device /dev/davinci2 \
+        --device /dev/davinci3 \
+        --device /dev/davinci_manager \
+        --device /dev/devmm_svm \
+        --device /dev/hisi_hdc \
+        -v /usr/local/dcmi:/usr/local/dcmi \
+        -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
+        -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
+        -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
+        -v /etc/ascend_install.info:/etc/ascend_install.info \
+        -v ~/.cache:/root/.cache \
+        -p 8091:8091 \
+        -it "$IMAGE" bash
+    ```
+
+=== "A5"
+
+    ```bash
+    export IMAGE=quay.io/ascend/vllm-omni:v0.28.0-a5
+    docker run --rm \
+        --name vllm-omni-a5 \
+        --shm-size=64g \
+        --device /dev/davinci0 \
+        --device /dev/davinci1 \
+        --device /dev/davinci2 \
+        --device /dev/davinci3 \
+        --device /dev/davinci_manager \
+        --device /dev/devmm_svm \
+        --device /dev/hisi_hdc \
+        -v /usr/local/dcmi:/usr/local/dcmi \
+        -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
+        -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
+        -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
+        -v /etc/ascend_install.info:/etc/ascend_install.info \
+        -v ~/.cache:/root/.cache \
+        -p 8091:8091 \
+        -it "$IMAGE" bash
+    ```
+
+=== "310P (Experimental)"
+
+    ```bash
+    export IMAGE=quay.io/ascend/vllm-omni:v0.28.0-310p
+    docker run --rm \
+        --name vllm-omni-310p \
+        --shm-size=64g \
+        --device /dev/davinci0 \
+        --device /dev/davinci1 \
+        --device /dev/davinci2 \
+        --device /dev/davinci3 \
+        --device /dev/davinci_manager \
+        --device /dev/devmm_svm \
+        --device /dev/hisi_hdc \
+        -v /usr/local/dcmi:/usr/local/dcmi \
+        -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
+        -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
+        -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
+        -v /etc/ascend_install.info:/etc/ascend_install.info \
+        -v ~/.cache:/root/.cache \
+        -p 8091:8091 \
+        -it "$IMAGE" bash
+    ```

 !!! tip
    You can use this docker image to serve models the same way you would with in vLLM! To do so, make sure you overwrite the default entrypoint (`vllm serve --omni`) which works only for models supported in the vLLM-Omni project.

-Or build IMAGE from **source code**:
+Or build the image from **source code**:

 ```bash
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
-# A2
-# docker build -t vllm-omni-dev-image:latest -f ./docker/Dockerfile.npu .
-# A3
-docker build -t vllm-omni-dev-image:latest -f ./docker/Dockerfile.npu.a3 .
 ```

+All hardware variants use `docker/Dockerfile.npu`. Choose the device suffix for your hardware:
+
+=== "A2"
+
+    ```bash
+    docker build -t vllm-omni-dev-image:latest -f docker/Dockerfile.npu .
+    ```
+
+=== "A3"
+
+    ```bash
+    docker build -t vllm-omni-dev-image:latest -f docker/Dockerfile.npu \
+        --build-arg VLLM_ASCEND_DEVICE_SUFFIX=-a3 .
+    ```
+
+=== "A5"
+
+    ```bash
+    docker build -t vllm-omni-dev-image:latest -f docker/Dockerfile.npu \
+        --build-arg VLLM_ASCEND_DEVICE_SUFFIX=-a5 .
+    ```
+
+=== "310P (Experimental)"
+
+    ```bash
+    docker build -t vllm-omni-dev-image:latest -f docker/Dockerfile.npu \
+        --build-arg VLLM_ASCEND_DEVICE_SUFFIX=-310p .
+    ```
+
 # --8<-- [end:pre-built-images]
@@ -19,10 +19,10 @@ uv venv --python 3.12 --seed
 source .venv/bin/activate

 # On CUDA
-uv pip install vllm==0.28.0 --torch-backend=auto
+uv pip install vllm==0.29.0 --torch-backend=auto

 # On ROCm
-uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.28.0/rocm723
+uv pip install vllm==0.29.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.29.0/rocm723

 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
@@ -34,7 +34,7 @@ For additional installation methods — please see the [installation guide](inst
 !!! note
    It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.

-    If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.28.0` and vLLM Omni `0.28.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
+    If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.29.0` and vLLM Omni `0.29.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.

 ## Offline Inference

@@ -124,6 +124,7 @@ def extract_omni_serve_subparser_init():
                        "make_arg_parser": lambda parser: parser,  # no-op for doc
                        "_ensure_vllm_platform": lambda: None,  # no-op for doc
                        "_nonneg_finite_float": float,
+                        "_parse_stage_overrides": __import__("json").loads,
                        "VLLM_SUBCMD_PARSER_EPILOG": "",
                        "logger": logger,
                        "DummySubparsers": DummySubparsers,
@@ -64,6 +64,7 @@ th {
 | `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `LongCatVideoAvatarPipeline` | LongCat-Video-Avatar-1.5 A2V/AI2V (native single-speaker and multi-speaker AI2V/AVC) | `meituan-longcat/LongCat-Video-Avatar-1.5` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/meituan-longcat/LongCat-Video-Avatar-1.5.md) |
 | `BooguImagePipeline` | Boogu-Image | `Boogu/Boogu-Image-0.1-Base`, `Boogu/Boogu-Image-0.1-Edit`, `Boogu/Boogu-Image-0.1-Edit-Turbo` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Boogu/Boogu-Image.md) |
+| `BooguImageTurboPipeline` | Boogu-Image Turbo distilled T2I | `Boogu/Boogu-Image-0.1-Turbo` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Boogu/Boogu-Image.md) |
 | `StableDiffusionXLPipeline` | Stable-Diffusion-XL | `stabilityai/stable-diffusion-xl-base-1.0` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ | — |
@@ -96,6 +97,7 @@ th {
 | `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | | — |
 | `Flux2Pipeline` | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | ✅︎ | ✅︎ | | | — |
 | `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | ✅︎ | — |
+| `Audio8TTSSlowARForConditionalGeneration` | Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | ✅︎ | | | | — |
 | `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only), SenseNova-U1.5 | `SenseNova/SenseNova-U1-8B-MoT`, `sensenova/SenseNova-U1.5-8B-MoT` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SenseNova/SenseNova-U1.5.md) |
 | `LancePipeline` | Lance | `bytedance-research/Lance` | ✅︎ | | | | — |
 | `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | | — |
@@ -64,7 +64,7 @@ connect
  <- msgpack(policy_server_config)

 infer
-  -> msgpack({"endpoint": "infer", "session_id": "...", ...observation})
+  -> msgpack({"endpoint": "infer", "session_id": "...", "seed": <optional int>, ...observation})
  <- msgpack(ndarray | dict[str, ndarray])

 reset
@@ -92,6 +92,12 @@ frames are not observation messages.
 - The policy pipeline owns observation transforms and persistent model state,
  normally keyed by `session_id`; the API layer forwards the raw observation
  dictionary.
+- An optional integer `seed` in an inference message becomes the engine
+  request's `sampling_params.seed`. A policy whose sampling consumes the
+  request generator (GR00T-N1.7) returns the same action chunk for the same
+  observation and seed. If omitted, the engine assigns a random per-request
+  seed, so unseeded inferences are independent of each other. The API layer
+  consumes `seed`; it is not forwarded inside the observation.

 ## Limits and Errors

@@ -1,17 +1,17 @@
 # Realtime Duplex API

 vLLM-Omni serves full-duplex speech models over a WebSocket endpoint,
-`/v1/realtime?duplex=1`, that speaks the OpenAI Realtime vocabulary plus a
+`/v1/realtime`, that speaks the OpenAI Realtime vocabulary plus a
 set of duplex extensions: the model decides when to listen and when to speak,
 the user can talk over the assistant, playback progress is reported back so
 history stays honest, and a dropped connection can resume the same session.
 This page covers how to run a duplex deployment, how to drive it from Python
 with `vllm_omni.clients.duplex.DuplexClient`, and the complete wire contract.

-The endpoint is served for models that ship a duplex plugin — currently
-MiniCPM-o 4.5, PersonaPlex, and Nemotron VoiceChat — and only for deploy
-configurations that declare `session_mode: duplex`. The runtime architecture
-is described in [Full-Duplex Runtime (MiniCPM-o 4.5)](../design/fullduplex.md).
+The handler is enabled by either a model-native `session_mode: duplex`
+deployment or an explicit `duplex_session` configuration for turn-based
+Server VAD. The runtime architecture is described in
+[Full-Duplex Runtime (MiniCPM-o 4.5)](../design/fullduplex.md).

 ## Quick Start

@@ -27,7 +27,7 @@ vllm-omni serve openbmb/MiniCPM-o-4_5 \

 `vllm_omni/deploy/minicpmo_4_5.yaml` declares `session_mode: duplex` and
 `duplex_session.max_sessions: 4`, which mounts the WebSocket routes
-`ws://<host>:8099/v1/realtime?duplex=1` (this page) and `ws://<host>:8099/v1/duplex`
+`ws://<host>:8099/v1/realtime` (this page) and `ws://<host>:8099/v1/duplex`
 (the native dialect for raw-protocol clients) next to the usual HTTP API.
 PersonaPlex and Nemotron VoiceChat use `vllm_omni/deploy/personaplex.yaml` and
 `vllm_omni/deploy/nemotron_labs_voicechat_duplex.yaml` in the same way.
@@ -240,7 +240,7 @@ on the same helpers.

 ## API Reference

-This section is the complete wire contract of `/v1/realtime?duplex=1`: how
+This section is the complete wire contract of `/v1/realtime`: how
 the dialect relates to the OpenAI Realtime protocol, which surfaces each
 model's capability flags gate, a catalogue of every client-to-server and
 server-to-client event, and a JSON example for each. `DuplexClient` speaks
@@ -253,10 +253,11 @@ its protocol tests.

 ### Endpoint and transport

-The endpoint is `ws(s)://<host>/v1/realtime?duplex=1`; the `duplex` query
-parameter (`1`, `true`, or `on`) selects the duplex session handler, and the
-route is mounted only for deployments whose deploy configuration declares
-`session_mode: duplex`. Optional query parameters are `model`, `session_id`,
+The endpoint is `ws(s)://<host>/v1/realtime`. Bare connections select the
+session handler when the deployment declares either `session_mode: duplex`
+or `duplex_session`. `?duplex=1` remains a compatibility alias, while
+`?duplex=0` selects the legacy Realtime handler. Optional query parameters
+are `model`, `session_id`,
 `autostart` (`0` means resume-only), `resume`, and `native_duplex`
 (`minicpmo45_native_duplex` is accepted as a deprecated alias and folded
 into the canonical name). Every message is one JSON object per WebSocket
@@ -342,9 +343,14 @@ Semantic divergences hidden behind shared names:
  before any commit; a commit may end in `response.listen` and no
  response, and the model may open a response with no commit at all.
  OpenAI: commit ⇒ item, `response.create` ⇒ exactly one response.
- `turn_detection.interrupt_response=false` is rejected;
-  `create_response` is ignored; `semantic_vad` is unsupported; the VAD
-  runs per session (Silero) and implies `overlap_policy=barge_in_on_speech`.
+- With `turn_detection={"type":"server_vad"}`, omitted `interrupt_response`
+  defaults to `false` on turn-based models such as Qwen3-Omni, enabling
+  endpointing and automatic responses without interruption. Explicit `true`
+  is rejected there. Model-native duplex keeps the `true` default and rejects
+  `false`; its VAD implies `overlap_policy=barge_in_on_speech`.
+  This capability-dependent default differs from OpenAI's `true` default;
+  the effective configuration is returned in `session.created` / `session.updated`.
+  `semantic_vad` is unsupported.
 - `rate_limits.updated` is always an empty list (compatibility only).
 - Cancellation never reuses a `response_id`; truncation is driven by
  `playback.ack` as well as `conversation.item.truncate`.
@@ -1201,9 +1207,10 @@ Realtime projector renames and fans out these events before they reach a
  barge-in, or audio truncation; Nemotron VoiceChat does not support barge-in
  or audio truncation; camera frames are consumed only by MiniCPM-o 4.5; tool
  calls are produced only by Nemotron VoiceChat.
- `turn_detection` supports only `server_vad` with `interrupt_response=true`;
-  `semantic_vad`, `interrupt_response=false`, and `create_response` are not
-  supported. Server VAD requires a model-native session.
+- `turn_detection` supports `server_vad` or `null`, not `semantic_vad`.
+  Turn-based models support endpointing without interruption, including
+  `create_response=false` for explicit response creation. Model-native duplex
+  requires `interrupt_response=true` and `create_response=true`.
 - `input_audio_transcription` and `input_audio_noise_reduction` are accepted
  and echoed but no separate transcription or noise-reduction stage runs;
  transcripts come from the model.
@@ -279,10 +279,15 @@ curl -X POST http://localhost:8091/v1/audio/voices \

 ## Streaming Text Input (WebSocket)

-The `/v1/audio/speech/stream` WebSocket endpoint accepts text incrementally and generates audio per sentence as boundaries are detected.
+The `/v1/audio/speech/stream` WebSocket endpoint accepts text incrementally.
+By default (`split_granularity=none`) it buffers until `input.done` and
+synthesizes that flush as **one** TTS request, which keeps long-form timbre
+stable. Set `split_granularity` to `sentence` or `clause` to emit a request
+at each detected boundary (lower time-to-first-audio for STT/LLM pipelines).

-> Note: text input is always streamed incrementally. Audio output remains sentence-scoped:
-> use `stream_audio=false` for one binary frame per sentence, or `stream_audio=true` for one or more PCM chunks per sentence.
+> Note: `stream_audio` only changes how **audio bytes** are framed (one WAV/PCM
+> payload vs chunked PCM). Text segmentation is controlled separately by
+> `split_granularity`.

 ### WebSocket Protocol

@@ -314,14 +319,16 @@ upstream LLM) pays the WebSocket handshake once instead of once per utterance.

 - The session config is sticky. Send `input.text` again straight after
  `session.done` to reuse it, or send another `session.config` first to change
-  voice, format, or reference audio. A `session.config` sent while text is
-  still buffered is rejected so no pending input is silently dropped.
- An utterance is the flush unit, not a linguistic one: it is whatever text was
-  buffered when `input.done` arrived, of any length, synthesized as one request.
-  `utterance_index` counts those flushes across the connection, so it tells you
-  which `input.done` a frame belongs to. `sentence_index` counts within one
-  flush and so pairs with `total_sentences`, which means every utterance reports
-  `sentence_index: 0` of `total_sentences: 1` (or `0` for an empty buffer).
+  voice, format, or reference audio. A `session.config` sent in the middle of
+  an utterance is rejected so no pending input is silently dropped and so a
+  split utterance cannot end up half in one voice and half in another.
+- An utterance is the flush unit, not a linguistic one: it is one `input.done`
+  cycle. `utterance_index` counts those flushes across the connection, so it
+  tells you which `input.done` a frame belongs to. `sentence_index` counts the
+  TTS requests inside one flush and so pairs with `total_sentences`: with the
+  default `split_granularity=none` that is always `sentence_index: 0` of
+  `total_sentences: 1` (or `0` for an empty buffer), while `sentence` or
+  `clause` counts the linguistic units actually synthesized.
 - End the connection with `session.close`, or by closing the socket. An idle
  connection is still closed after the server's idle timeout, which now also
  applies to the gap between utterances.
@@ -332,7 +339,15 @@ All REST API parameters are supported, plus:

 | Parameter | Type | Default | Description |
 | ----------- | ------ | --------- | ------------- |
-| `stream_audio` | bool | false | Stream one or more PCM chunks for the buffered input over WebSocket |
+| `stream_audio` | bool | false | Stream one or more PCM chunks for each TTS request over WebSocket |
+| `split_granularity` | string | `"none"` | `"none"`: one request per `input.done`. `"sentence"`: split on `.!?` plus CJK `。！？…`, Indic danda `।॥`, and Arabic `؟`. `"clause"`: also split on `,;，；،؛`. |
+| `seed` | integer | null | Forwarded to the speech engine for this session |
+
+ASCII punctuation only ends a unit when whitespace or `input.done` follows it,
+and decimals (`3.14`), thousands separators (`1,000`), abbreviations (`Dr.`,
+`e.g.`) and initials (`J. R.`) are not treated as boundaries. A punctuation run
+and any closing quote or bracket stay with the unit they close, so `Wait...`
+and `He said "Hello."` are one request each.

 ```bash
 DELETE /v1/audio/voices/{name}
@@ -752,6 +767,10 @@ Fish Speech uses `ref_audio` and `ref_text` for voice cloning (no `task_type` ne
 | ------- | ------------- |
 | `k2-fsa/OmniVoice` | Pure-diffusion TTS. Supports voice cloning via `ref_audio` (with optional `ref_text`); no built-in voice presets. |

+OmniVoice uses packed variable-length attention for batched generator execution. The attention operator accepts FP16 and BF16 inputs, so its query,
+key, and value tensors are evaluated in BF16 even when the stage is configured with `dtype: float32`; the attention output is converted back to the model's
+hidden-state dtype before the output projection. Consequently, a float32 stage configuration does not imply FP32 attention arithmetic.
+
 ### VoxCPM2

 | Model | Description |
@@ -68,6 +68,8 @@ curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4
 | Parameter | Type | Default | Description |
 | ----------- | ------ | --------- | ------------- |
 | `input_reference` | file | null | Uploaded reference image or video for image-to-video/video-to-video requests |
+| `control_reference` | file | null | Optional uploaded image/video control, up to 512 MiB, for models that declare control-upload support |
+| `control_type` | string | null | Model control name associated with `control_reference`; currently Cosmos3 supports `edge`, `blur`, `depth`, `seg`, and `wsm` |
 | `image_reference` | string | null | JSON-encoded reference image payload; do not combine with `input_reference` or `video_reference` |
 | `video_reference` | string | null | JSON-encoded reference video payload; do not combine with `input_reference` or `image_reference` |
 | `audio_reference` | string | null | JSON-encoded audio reference for speech-to-video: `{"audio_url": "..."}` — supports HTTP(s) URLs or base64 data URLs |
@@ -177,6 +179,23 @@ corresponding addition. `negative_metadata_mode` accepts `same`, `inverse`, or
 `none` and defaults to `same` for transfer. See the Cosmos3 recipe for complete
 examples.

+A client that cannot place the control on the server filesystem can upload one
+control with `control_reference` and identify it with `control_type`. The API
+streams the upload to request-scoped storage, supplies its path to the model,
+and removes it after synchronous or asynchronous generation completes. Other
+options for the selected control can remain in `extra_params`; do not also set
+`control` or `control_path` there. Uploads larger than 512 MiB are rejected.
+
+```bash
+curl -s http://localhost:8091/v1/videos/sync \
+  -F "prompt=Preserve the scene while following the world-state control" \
+  -F "input_reference=@input.mp4;type=video/mp4" \
+  -F "control_reference=@wsm.mp4;type=video/mp4" \
+  -F "control_type=wsm" \
+  -F 'extra_params={"wsm":{"control_weight":1.0}}' \
+  -o output.mp4
+```
+
 HTTP redirects for `image_reference.image_url` follow vLLM's
 `VLLM_MEDIA_URL_ALLOW_REDIRECTS` setting. Before starting the server, set it to
 `1` (the default) to allow redirects or `0` to reject them. A redirect target
@@ -12,7 +12,7 @@ implementations to reduce numerical drift and sampling divergence.
 ## Installation

 ```bash
-pip install kernels==0.15.2
+pip install kernels==0.16.1
 ```

 If `kernels` is unavailable, an explicit `FLASH_ATTN_HUB` or
@@ -8,7 +8,7 @@ are exposed.
 ## Choose a Mode

 | Goal | CLI configuration |
-|---|---|
+| --- | --- |
 | Serial request execution | `--max-num-seqs 1` |
 | Fused request-level batching | `--max-num-seqs N` |
 | Single-request step execution | `--step-execution --max-num-seqs 1` |
@@ -177,8 +177,10 @@ wait
 The scheduler may batch these requests when their sampling parameters are
 compatible. See the
 [Image Generation API](../../serving/image_generation_api.md) for response
-formats and additional client examples. Streaming models can use
-model-specific streaming endpoints documented by their serving guide.
+formats and additional client examples.
+
+Streaming models can use model-specific streaming endpoints documented at
+[Streaming Video Output API](../../serving/streaming_video_output_api.md).

 ## Python API

@@ -241,7 +243,7 @@ For step execution, set `step_execution: true` and remove
 ## CLI Reference

 | Flag | Default | Effect |
-|---|---:|---|
+| --- | ---: | --- |
 | `--step-execution` | disabled | Select step-wise scheduling |
 | `--max-num-seqs` | `1` for diffusion stages | Set request- or step-scheduler capacity |
 | `--request-batch-max-wait-ms` | `0` | Wait for burst coalescing in request mode |
@@ -249,8 +249,14 @@ MiniMax-H3 supports two few-step mechanisms that must not be conflated:
 - **Checkpoint-pinned schedule**: a merged release writes `base_schedule` into
  `model_index.json`. Requests must pass `num_inference_steps` as the interval
  count (for example `4` for `[1.0, 0.7, 0.4, 0.15, 0.0]`).
- **Runtime Turbo LoRA**: LightX2V Turbo artifacts request five sigma points and
-  enforce `flow_shift=6`, `audio_flow_shift=3`.
+- **Runtime Turbo LoRA**: each LightX2V Turbo artifact carries its own contract,
+  read from its filename. A four-step artifact requests five sigma points and an
+  eight-step one requests nine; the 768p retrains enforce `flow_shift=6` and the
+  544p artifacts `flow_shift=12`. `audio_flow_shift=3` across the family. A
+  request that does not match the loaded artifact is rejected by name. Alpha
+  comes from the artifact's metadata, or 8 when it declares none. `ref2v`
+  artifacts require `--task-type ref2va`; `fl2v` artifacts serve T2VA and FL2VA.
+  The ComfyUI exports use a fused-QKV layout and are not supported.
 - **Runtime native LoRA**: FlashGen-style artifacts declare
  `key_format=minimax-h3-native` and embed `base_schedule` in safetensors
  metadata. When active, the adapter schedule overrides the base checkpoint and
@@ -162,9 +162,9 @@ The following tables show which models support each feature:
 | **Wan2.2**                   |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ✅         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ❌        |        ❌         |
 | **Wan2.2-S2V**               |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ❌        |        ❌         |
 | **Wan2.1-VACE**              |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
-| **LTX-2**                    |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
-| **LTX-2.3**                  |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
-| **LTX-2.5**                  |     ❌     | ❓ (one-stage) | ❓ (Ulysses only) | ✅ (Full only) |         ❓         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |    ❓ (FP8)     |        ❌         |
+| **LTX-2**                    |     ❌     |     ✅      | ✅ (Ulysses only) |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
+| **LTX-2.3**                  |     ❌     |     ✅      | ✅ (Ulysses only) |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
+| **LTX-2.5**                  |     ❌     | ❓ (one-stage) | ✅* (Ulysses only) | ✅ (Full only) |         ❓         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |    ❓ (FP8)     |        ❌         |
 |  **SANA-Video-2B T2V I2V**   |     ❌     |     ❌      |   ✅ (`--usp`, frame-sharded)   |       ✅       |  ✅ (TP=2 only)   |         ❌         |   ❌   |            ❌             |          ❌          |       ❌       |        ❌        |
 | **Helios**                   |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ✅*        |
 | **HunyuanVideo-1.5 T2V I2V** |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
@@ -126,11 +126,7 @@ python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
 ## Fish Speech S2 Pro

 4B dual-AR text-to-speech model from FishAudio with the DAC codec at 44.1 kHz.
-
-### Prerequisites
-```bash
-pip install fish-speech
-```
+No extra packages are required; the DAC codec is vendored in vLLM-Omni.

 ### Quick start
 ```bash
@@ -155,13 +155,8 @@ The client supports `--api-base`, `--model`, `--text`, `--ref-audio`, `--ref-tex

 ## Fish Speech S2 Pro

-4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec.
-
-### Prerequisites
-
-```bash
-pip install fish-speech
-```
+4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec, which is vendored in
+vLLM-Omni (no extra packages required).

 ### Launch

@@ -199,12 +199,21 @@ curl -X POST http://localhost:8091/v1/videos \
 | `boundary_ratio`      | float  | None    | Boundary split ratio for low/high DiT (Wan2.2)   |
 | `flow_shift`          | float  | None    | Scheduler flow shift (Wan2.2)                    |
 | `seed`                | int    | None    | Random seed (reproducible)                       |
+| `num_outputs_per_prompt` | int  | 1       | Number of videos to generate (1-10, MiniMax H3); only the first is returned, see below |
 | `lora`                | object | None    | LoRA configuration                               |
 | `enable_frame_interpolation` | bool | false | Enable RIFE frame interpolation before MP4 encoding |
 | `frame_interpolation_exp` | int | 1 | Interpolation exponent; 1=2x temporal resolution, 2=4x |
 | `frame_interpolation_scale` | float | 1.0 | RIFE inference scale; use 0.5 for high-resolution inputs |
 | `frame_interpolation_model_path` | str | None | Local directory or Hugging Face repo ID with `flownet.pkl`; defaults to `elfgum/RIFE-4.22.lite` |

+!!! note "Only the first output is returned"
+    `num_outputs_per_prompt` is forwarded to the pipeline, so the model does
+    generate that many videos and you pay the generation cost for all of them.
+    `/v1/videos` and `/v1/videos/sync` then return only the **first** one,
+    because an async video job stores a single file per video id. The server
+    logs a warning when it discards the extra outputs. Keep this at `1`
+    unless you have a specific reason to generate videos you will not receive.
+
 ## Frame Interpolation

 Frame interpolation is an optional post-processing step for `/v1/videos` and
@@ -30,7 +30,6 @@ from typing import Any
 import numpy as np

 _MODEL = "robbyant/lingbot-world-v2-14b-causal-fast-diffusers"
-_NUM_INFERENCE_STEPS = 4
 _MAX_SEQUENCE_LENGTH = 512
 _MAX_PIXEL_AREA = 480 * 832
 _MAX_RAW_FRAMES = 117
@@ -209,7 +208,6 @@ def build_request(
        "height": args.height,
        "width": args.width,
        "num_frames": args.num_frames,
-        "num_inference_steps": _NUM_INFERENCE_STEPS,
        "max_sequence_length": _MAX_SEQUENCE_LENGTH,
        "seed": args.seed,
        "fps": args.fps,
@@ -155,7 +155,6 @@ async def run(argv: Sequence[str] | None = None) -> Path:
        height=args.height,
        width=args.width,
        num_frames=9,
-        num_inference_steps=4,
        max_sequence_length=512,
        seed=args.seed,
        output_type="latent",
@@ -5,10 +5,12 @@ This example shows how to use vLLM-Omni for running offline inference
 with the correct prompt format on Qwen2.5-Omni
 """

+import argparse
+import functools
 import json
 import os
 import time
-from typing import NamedTuple
+from typing import Any, NamedTuple

 import numpy as np
 import soundfile as sf
@@ -26,6 +28,20 @@ from vllm_omni.utils.tracking_parser import TrackingArgumentParser
 SEED = 42


+def parse_json_object(value: str, flag_name: str = "argument") -> dict[str, Any]:
+    """Parse a CLI value as a JSON object, attributing errors to ``flag_name``."""
+    try:
+        config = json.loads(value)
+    except json.JSONDecodeError as e:
+        raise argparse.ArgumentTypeError(f"{flag_name} must be valid JSON: {e}") from e
+    if not isinstance(config, dict):
+        raise argparse.ArgumentTypeError(f"{flag_name} must be a JSON object")
+    return config
+
+
+parse_profiler_config = functools.partial(parse_json_object, flag_name="--profiler-config")
+
+
 class QueryResult(NamedTuple):
    inputs: dict
    limit_mm_per_prompt: dict[str, int]
@@ -378,11 +394,10 @@ def main(args):
        for i, prompt in enumerate(prompts):
            prompt["modalities"] = output_modalities

-    profiler_enabled = bool(os.getenv("VLLM_TORCH_PROFILER_DIR"))
-    if profiler_enabled and hasattr(omni, "start_profile"):
+    profiler_enabled = args.profiler_config is not None
+    if profiler_enabled:
+        print("[Profiler] Starting profiling...")
        omni.start_profile(stages=[0])
-    elif profiler_enabled:
-        print("[Warn] VLLM_TORCH_PROFILER_DIR is set, but current engine does not support profiler controls.")
    omni_generator = omni.generate(prompts, sampling_params_list, py_generator=args.py_generator)

    # Determine output directory: prefer --output-dir; fallback to --output-wav
@@ -418,7 +433,7 @@ def main(args):
            print(f"Request ID: {request_id}, Saved audio to {output_wav}")

        processed_count += 1
-        if profiler_enabled and hasattr(omni, "stop_profile") and processed_count >= total_requests:
+        if profiler_enabled and processed_count >= total_requests:
            print(f"[Info] Processed {processed_count}/{total_requests}. Stopping profiler inside active loop...")
            # Stop the profiler while workers are still alive
            omni.stop_profile()
@@ -476,6 +491,12 @@ def parse_args():
        default=300,
        help="Timeout for initializing stages in seconds (default: 300)",
    )
+    parser.add_argument(
+        "--profiler-config",
+        type=parse_profiler_config,
+        default=None,
+        help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
+    )
    parser.add_argument(
        "--shm-threshold-bytes",
        type=int,
@@ -14,6 +14,7 @@ list of supported architectures across all modalities, see

 | Model | HuggingFace repo | Stages | Voice cloning | Streaming | Special modes | Sample rate |
 |---|---|---|---|---|---|---|
+| Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | dual-AR | ✓ | ✓ | 11 languages | 44.1 kHz |
 | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ | — | 24 kHz |
 | Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ | — | 44.1 kHz |
 | Gepard-1.0 | `nineninesix/gepard-1.0` | single (native AR + NanoCodec) | — (zero-shot; cloning WIP) | — (serving WIP) | zero-shot | 22.05 kHz |
@@ -44,6 +45,57 @@ python examples/offline_inference/text_to_speech/<model>/end2end.py \

 ---

+## Audio8 TTS Preview
+
+0.6B DualAR text-to-speech model (Audio8) with its own 44.1 kHz neural audio
+codec: a slow AR transformer predicts one semantic token per codec frame, and a
+4-layer fast AR predicts that frame's remaining 9 codebooks.
+
+### Prerequisites
+No extra packages: the codec architecture ships in tree and its weights
+(`codec.pth`) come with the checkpoint.
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
+    --text "Welcome to Audio8 TTS, a compact multilingual text to speech model."
+```
+
+### Voice cloning
+```bash
+python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
+    --text "Welcome to Audio8 TTS." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text  "The exact transcript of the reference recording."
+```
+The reference transcript must match what is actually spoken in the clip;
+a mismatch degrades both stability and speaker similarity.
+
+### Streaming
+```bash
+python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
+    --text "Welcome to Audio8 TTS." --streaming
+```
+
+### Notes
+- Output: 44.1 kHz mono WAV; the codec emits ~21.5 frames/s (2048 samples per frame).
+- Context is 2048 packed text+audio positions, so `max_model_len` is 2048.
+- Greedy decoding (`temperature: 0`) is degenerate for this checkpoint — it
+  ends the utterance almost immediately. Keep the deploy defaults
+  (`temperature 0.7`, `top_k 50`, `top_p 0.9`), which also match
+  `generation_config.json`.
+- Repetition-Aware Sampling is applied to the semantic token: a repeat inside
+  the last 10 frames is re-drawn from a flatter distribution instead of being
+  masked out.
+- `--num-prompts 4` smoke-tests concurrency (stage 0 runs `max_num_seqs: 4`).
+
+### Measured behaviour (1x H20, shared GPU)
+`--streaming` on a 58-character English sentence: TTFA ~0.74 s (4-frame first
+chunk), 7 chunks, 1.55 s wall for 3.16 s of audio (RTF ~0.49). Treat these as
+smoke-test magnitudes, not a benchmark.
+
+---
+
 ## CosyVoice3

 2-stage TTS pipeline (`talker` + `code2wav`) at 24 kHz.
@@ -130,11 +182,7 @@ Text → [Stage 0: AR] → Speech Tokens → [Stage 1: DiT + HiFT] → Audio (24
 ## Fish Speech S2 Pro

 4B dual-AR text-to-speech model from FishAudio with the DAC codec at 44.1 kHz.
-
-### Prerequisites
-```bash
-pip install fish-speech
-```
+No extra packages are required; the DAC codec is vendored in vLLM-Omni.

 ### Quick start
 ```bash
@@ -0,0 +1,237 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Offline inference demo for Audio8 TTS Preview 0.6B via vLLM Omni.
+
+Usage:
+    # Text-only synthesis
+    python end2end.py --text "Welcome to Audio8 TTS."
+
+    # Zero-shot voice cloning (the transcript must match the reference audio)
+    python end2end.py --text "Welcome to Audio8 TTS." \
+        --ref-audio reference.wav --ref-text "The exact transcript of the reference recording."
+
+    # Streaming (per-chunk TTFA / inter-chunk timings)
+    python end2end.py --text "Welcome to Audio8 TTS." --streaming
+"""
+
+import asyncio
+import os
+import time
+
+import numpy as np
+import soundfile as sf
+import torch
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+
+from vllm_omni import AsyncOmni, Omni
+from vllm_omni.model_executor.models.audio8_tts.codec_utils import estimate_reference_code_frames
+from vllm_omni.model_executor.models.audio8_tts.prompt_utils import (
+    build_text_only_prompt_ids,
+    estimate_voice_clone_prompt_len,
+    normalize_text,
+)
+from vllm_omni.utils.tracking_parser import TrackingArgumentParser
+
+DEFAULT_MODEL = "Audio8/Audio8-TTS-Preview-0.6b"
+
+
+def build_prompt(
+    text: str,
+    ref_audio_path: str | None = None,
+    ref_text: str | None = None,
+    model_name: str = DEFAULT_MODEL,
+) -> dict:
+    """Build an engine prompt, using the same protocol as online serving."""
+    from transformers import AutoTokenizer
+
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    if ref_audio_path is None and ref_text is None:
+        prompt_ids, normalized_text = build_text_only_prompt_ids(tokenizer, text)
+        return {
+            "prompt_token_ids": prompt_ids,
+            "additional_information": {"text": [normalized_text]},
+        }
+
+    if not ref_audio_path or not ref_text:
+        raise ValueError("Audio8 TTS voice cloning requires both --ref-audio and --ref-text")
+
+    normalized_text = normalize_text(text)
+    normalized_ref_text = normalize_text(ref_text, add_default_speaker=True)
+    ref_audio_wav, ref_audio_sr = sf.read(ref_audio_path, dtype="float32", always_2d=False)
+    if ref_audio_wav.ndim > 1:
+        ref_audio_wav = ref_audio_wav.mean(axis=-1)
+    ref_frames = estimate_reference_code_frames(int(ref_audio_wav.shape[0]), int(ref_audio_sr))
+    placeholder_len = estimate_voice_clone_prompt_len(tokenizer, normalized_text, normalized_ref_text, ref_frames)
+
+    # The clone prompt embeds the reference audio's own codec codes, so the
+    # model builds it in preprocess(); we only reserve the exact length here.
+    return {
+        "prompt_token_ids": [1] * placeholder_len,
+        "additional_information": {
+            "text": normalized_text,
+            "ref_text": normalized_ref_text,
+            "ref_audio_wav": torch.from_numpy(np.asarray(ref_audio_wav, dtype=np.float32)),
+            "ref_audio_sr": int(ref_audio_sr),
+            "audio8_structured_voice_clone": True,
+        },
+    }
+
+
+def extract_audio(multimodal_output: dict) -> tuple[torch.Tensor, int]:
+    """Pull the waveform and sample rate out of a multimodal output.
+
+    The output processor concatenates the per-step delta tensors under
+    ``model_outputs``; ``audio`` is the older alias. Never use ``a or b`` on
+    these values -- truthiness on a tensor raises.
+    """
+    audio = multimodal_output.get("model_outputs")
+    if audio is None:
+        audio = multimodal_output.get("audio")
+    if audio is None:
+        raise ValueError(f"No audio in multimodal_output: {sorted(multimodal_output)}")
+    if isinstance(audio, list):
+        audio = torch.cat([torch.as_tensor(chunk).reshape(-1) for chunk in audio], dim=0)
+    audio = torch.as_tensor(audio).reshape(-1)
+
+    sr_raw = multimodal_output.get("sr")
+    if isinstance(sr_raw, list):
+        sr_raw = sr_raw[-1] if sr_raw else None
+    if sr_raw is None:
+        raise ValueError("Missing sample rate in multimodal_output")
+    sample_rate = int(sr_raw.item()) if hasattr(sr_raw, "item") else int(sr_raw)
+    return audio, sample_rate
+
+
+def _save_wav(output_dir: str, request_id: str, audio: torch.Tensor, sample_rate: int) -> None:
+    out_wav = os.path.join(output_dir, f"output_{request_id}.wav")
+    sf.write(out_wav, audio.float().cpu().numpy(), samplerate=sample_rate, format="WAV")
+    duration = audio.numel() / sample_rate
+    # print(), not logging: vLLM reconfigures logging on engine start and
+    # disables loggers created before that, which would swallow these lines.
+    print(f"Request {request_id}: saved {out_wav} (sr={sample_rate}, {duration:.2f}s)")
+
+
+def _omni_kwargs(args, model_name: str) -> dict:
+    kwargs = {
+        "model": model_name,
+        "log_stats": args.log_stats,
+        "stage_init_timeout": args.stage_init_timeout,
+    }
+    if args.deploy_config:
+        kwargs["deploy_config"] = args.deploy_config
+    return kwargs
+
+
+def main(args) -> None:
+    os.makedirs(args.output_dir, exist_ok=True)
+    model_name = args.model or DEFAULT_MODEL
+    prompts = [
+        build_prompt(text, args.ref_audio, args.ref_text, model_name) for text in ([args.text] * args.num_prompts)
+    ]
+
+    omni = Omni(**_omni_kwargs(args, model_name))
+    t_start = time.perf_counter()
+    total_audio_seconds = 0.0
+    for stage_outputs in omni.generate(prompts):
+        request_output = stage_outputs
+        if request_output is None or not request_output.outputs:
+            continue
+        audio, sample_rate = extract_audio(request_output.outputs[0].multimodal_output)
+        assert audio.numel() > 0, "Audio8 TTS produced an empty waveform"
+        total_audio_seconds += audio.numel() / sample_rate
+        _save_wav(args.output_dir, request_output.request_id, audio, sample_rate)
+    elapsed = time.perf_counter() - t_start
+    rtf = elapsed / max(total_audio_seconds, 1e-6)
+    print(f"Total {elapsed * 1000:.1f} ms for {total_audio_seconds:.2f} s of audio (RTF={rtf:.3f})")
+
+
+async def main_streaming(args) -> None:
+    os.makedirs(args.output_dir, exist_ok=True)
+    model_name = args.model or DEFAULT_MODEL
+    prompt = build_prompt(args.text, args.ref_audio, args.ref_text, model_name)
+
+    omni = AsyncOmni(**_omni_kwargs(args, model_name))
+    request_id = "0"
+    chunks: list[torch.Tensor] = []
+    sample_rate = None
+    t_start = time.perf_counter()
+    t_prev = t_start
+    chunk_index = 0
+
+    async for stage_output in omni.generate(prompt, request_id=request_id):
+        multimodal_output = stage_output.outputs[0].multimodal_output
+        if not multimodal_output:
+            continue
+        audio = multimodal_output.get("model_outputs")
+        if audio is None:
+            audio = multimodal_output.get("audio")
+        if audio is not None:
+            if isinstance(audio, list):
+                chunks.extend(torch.as_tensor(chunk).reshape(-1) for chunk in audio)
+            else:
+                chunks.append(torch.as_tensor(audio).reshape(-1))
+        sr_raw = multimodal_output.get("sr")
+        if isinstance(sr_raw, list):
+            sr_raw = sr_raw[-1] if sr_raw else None
+        if sr_raw is not None:
+            sample_rate = int(sr_raw.item()) if hasattr(sr_raw, "item") else int(sr_raw)
+
+        if stage_output.finished:
+            break
+        now = time.perf_counter()
+        if chunk_index == 0:
+            print(f"chunk 0: TTFA={(now - t_start) * 1000:.1f} ms")
+        else:
+            print(f"chunk {chunk_index}: inter_chunk={(now - t_prev) * 1000:.1f} ms")
+        t_prev = now
+        chunk_index += 1
+
+    if not chunks or sample_rate is None:
+        raise RuntimeError("Audio8 TTS streaming produced no audio")
+    _save_wav(args.output_dir, request_id, torch.cat(chunks, dim=0), sample_rate)
+    print(f"Streaming done: total={(time.perf_counter() - t_start) * 1000:.1f} ms chunks={chunk_index}")
+
+
+def parse_args():
+    parser = TrackingArgumentParser(description="Audio8 TTS Preview offline inference with vLLM Omni")
+    parser.add_argument("--model", type=str, default=DEFAULT_MODEL, help="Model path or HuggingFace repo ID.")
+    parser.add_argument(
+        "--text",
+        type=str,
+        default="Welcome to Audio8 TTS, a compact multilingual text to speech model.",
+        help="Text to synthesize.",
+    )
+    parser.add_argument("--ref-audio", type=str, default=None, help="Reference audio path for voice cloning.")
+    parser.add_argument(
+        "--ref-text",
+        type=str,
+        default=None,
+        help="Transcript of --ref-audio; required for voice cloning and must match the recording.",
+    )
+    parser.add_argument("--output-dir", type=str, default="output_audio", help="Directory for output WAV files.")
+    parser.add_argument(
+        "--num-prompts",
+        type=int,
+        default=1,
+        help="Duplicate the prompt N times to smoke-test concurrent requests.",
+    )
+    parser.add_argument(
+        "--deploy-config",
+        type=str,
+        default=None,
+        help="Override the deploy config path; defaults to vllm_omni/deploy/audio8_tts.yaml via model_type.",
+    )
+    parser.add_argument("--streaming", action="store_true", help="Use AsyncOmni and report per-chunk latency.")
+    parser.add_argument("--log-stats", action="store_true", help="Enable engine stats logging.")
+    parser.add_argument("--stage-init-timeout", type=int, default=600, help="Per-stage init timeout in seconds.")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    parsed_args = parse_args()
+    if parsed_args.streaming:
+        asyncio.run(main_streaming(parsed_args))
+    else:
+        main(parsed_args)
@@ -1,9 +1,12 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 import argparse
+import functools
+import json
 import os
 import urllib.request
 from pathlib import Path
+from typing import Any

 import numpy as np
 import soundfile as sf
@@ -29,6 +32,20 @@ def _default_ref_audio() -> str:
    return str(dest)


+def parse_json_object(value: str, flag_name: str = "argument") -> dict[str, Any]:
+    """Parse a CLI value as a JSON object, attributing errors to ``flag_name``."""
+    try:
+        config = json.loads(value)
+    except json.JSONDecodeError as e:
+        raise argparse.ArgumentTypeError(f"{flag_name} must be valid JSON: {e}") from e
+    if not isinstance(config, dict):
+        raise argparse.ArgumentTypeError(f"{flag_name} must be a JSON object")
+    return config
+
+
+parse_profiler_config = functools.partial(parse_json_object, flag_name="--profiler-config")
+
+
 def run_e2e():
    parser = argparse.ArgumentParser()
    # ""FunAudioLLM/Fun-CosyVoice3-0.5B-2512
@@ -64,6 +81,12 @@ def run_e2e():
        required=True,
        help="Path to tokenizer directory (e.g., <model_path>/CosyVoice-BlankEN).",
    )
+    parser.add_argument(
+        "--profiler-config",
+        type=parse_profiler_config,
+        default=None,
+        help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
+    )
    args = parser.parse_args()
    # Ensure tokenizer directory exists
    if not os.path.exists(args.tokenizer):
@@ -79,6 +102,7 @@ def run_e2e():
        deploy_config=args.deploy_config,
        tokenizer=args.tokenizer,
        log_stats=True,
+        profiler_config=args.profiler_config,
    )

    sampling_cfg = {"top_p": 0.8, "top_k": 25, "eos_token_id": 6561 + 1}
@@ -149,8 +173,8 @@ def run_e2e():

    sampling_params_list = [gpt_sampling, s2mel_sampling]

-    # Start profiling (requires VLLM_TORCH_PROFILER_DIR env var)
-    if os.environ.get("VLLM_TORCH_PROFILER_DIR"):
+    profiler_enabled = args.profiler_config is not None
+    if profiler_enabled:
        print("Starting profiler...")
        omni.start_profile()

@@ -158,7 +182,7 @@ def run_e2e():
    outputs = list(omni.generate(prompts, sampling_params_list=sampling_params_list[:2]))

    # Stop profiling and get results
-    if os.environ.get("VLLM_TORCH_PROFILER_DIR"):
+    if profiler_enabled:
        print("Stopping profiler...")
        profile_results = omni.stop_profile()
        print(f"Profile traces saved to: {profile_results}")
@@ -264,7 +264,7 @@ python examples/online_serving/openai_chat_completion_client_for_multimodal_gene

 #### Realtime WebSocket client (`openai_realtime_client.py`)

-[`openai_realtime_client.py`](./openai_realtime_client.py) connects to **`ws://<host>:<port>/v1/realtime`**, streams a local WAV as **PCM16 mono @ 16 kHz** in fixed-size chunks (OpenAI-style `input_audio_buffer.append` / `commit`), and receives **`response.audio.delta`** (incremental PCM for the reply) plus **`transcription.*`** events. By default it concatenates audio deltas and writes **`--output-wav`** (model output is typically **24 kHz**). Optional **`--delta-dump-dir`** saves each delta as `delta_000001.wav`, … for debugging.
+[`openai_realtime_client.py`](./openai_realtime_client.py) connects to **`ws://<host>:<port>/v1/realtime`**, streams a local WAV as **PCM16 mono @ 16 kHz** in fixed-size chunks, and receives **`response.audio.*`**, **`response.output_text.*`**, and transcript events. By default it uses explicit `input_audio_buffer.commit`; pass **`--server-vad`** to let the server detect speech endpoints and commit the turn automatically. The client concatenates audio deltas and writes **`--output-wav`** (model output is typically **24 kHz**). Optional **`--delta-dump-dir`** saves each delta as `delta_000001.wav`, … for debugging.

 Streaming input works well for translation-style use cases; if the Thinker runs while input is still incomplete, consider limiting **`max_tokens`** in your session / server defaults to avoid over-generation.

@@ -282,6 +282,7 @@ python openai_realtime_client.py \
  --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
  --input-wav /path/to/input_16k_mono.wav \
  --output-wav realtime_output.wav \
+  --server-vad \
  --delta-dump-dir ./rt_delta_wavs
 ```

@@ -297,15 +298,32 @@ python openai_realtime_client.py \
 | `--chunk-ms` | `200` | Size of each uploaded audio chunk (milliseconds of audio) |
 | `--send-delay-ms` | `0` | Delay between chunk sends (simulate realtime upload) |
 | `--delta-dump-dir` | *(optional)* | Directory to write per-`response.audio.delta` WAV files |
+| `--server-vad` | disabled | Enable server-side Silero VAD instead of sending explicit commits |
 | `--num-requests` | `1` | Number of sequential sessions (see `--concurrency`) |
 | `--concurrency` | `1` | Max concurrent WebSocket sessions when `--num-requests` > 1 |

-Ensure the server is running, for example:
+Server VAD uses the pinned Silero v6.2 ONNX artifact from `istupakov/silero-vad-onnx` and never downloads model files while processing a session. Make the artifact available in the Hugging Face cache before startup, or configure a local artifact in the deployment YAML:
+
+```yaml
+duplex_session:
+  server_vad_model_path: /models/silero_vad.onnx
+```
+
+Start from the bundled Qwen3-Omni deployment YAML, add the fields above at the top level, and serve that complete configuration:

 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
+cp vllm_omni/deploy/qwen3_omni_moe.yaml /path/to/qwen3_omni_server_vad.yaml
+# Edit /path/to/qwen3_omni_server_vad.yaml and add duplex_session.
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \
+  --omni \
+  --port 8091 \
+  --deploy-config /path/to/qwen3_omni_server_vad.yaml
 ```

+Keep Qwen's default `session_mode: turn`; `duplex_session` enables the Realtime handler without changing scheduler
+semantics. Bare `/v1/realtime` selects that handler. The client uses `?duplex=0` only for the existing non-Server-VAD
+wire flow; `?duplex=1` remains a supported compatibility alias.
+
 The Python client supports the following command-line arguments:

 - `--query-type` (or `-q`): Query type (default: `use_video`). Options: `text`, `use_audio`, `use_image`, `use_video`
@@ -1,9 +1,12 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 """Realtime client for vLLM-Omni /v1/realtime (audio + text events).

 This client:
 1) Reads a local WAV file (must be mono, 16-bit PCM, 16kHz),
 2) Streams PCM16 chunks to /v1/realtime with OpenAI-style events,
-3) Receives response.audio.* and transcription.* events,
+3) Receives response.audio.*, response.output_text.*, and transcript events,
 4) Saves synthesized audio to an output WAV file and optional text file.

 By default each ``response.audio.delta`` is treated as an **incremental PCM**
@@ -18,6 +21,7 @@ Usage:
      --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
      --input-wav input_16k_mono.wav \
      --output-wav realtime_output.wav \
+      --server-vad \
      --delta-dump-dir ./rt_delta_wavs

 Dependencies:
@@ -32,6 +36,7 @@ import base64
 import json
 import wave
 from pathlib import Path
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit

 try:
    import websockets
@@ -70,6 +75,14 @@ def _write_wav_pcm16(path: Path, pcm16_bytes: bytes, sample_rate_hz: int) -> Non
        wf.writeframes(pcm16_bytes)


+def _with_realtime_route(url: str, *, server_vad: bool) -> str:
+    parts = urlsplit(url)
+    query = [(key, value) for key, value in parse_qsl(parts.query, keep_blank_values=True) if key != "duplex"]
+    if not server_vad:
+        query.append(("duplex", "0"))
+    return urlunsplit(parts._replace(query=urlencode(query)))
+
+
 async def run_client(
    url: str,
    model: str,
@@ -79,6 +92,7 @@ async def run_client(
    chunk_ms: int,
    send_delay_ms: int,
    delta_dump_dir: Path | None,
+    server_vad: bool,
    request_idx: int = 1,
    total_requests: int = 1,
 ) -> None:
@@ -92,23 +106,44 @@ async def run_client(
    delta_index = 0
    text_chunks: list[str] = []
    final_text: str = ""
+    final_audio_transcript: str = ""

    if delta_dump_dir is not None:
        delta_dump_dir.mkdir(parents=True, exist_ok=True)

+    # Keep the CLI mode authoritative even if the input URL already contains
+    # an explicit duplex query parameter.
+    url = _with_realtime_route(url, server_vad=server_vad)
    async with websockets.connect(url, max_size=64 * 1024 * 1024) as ws:
        # 1) Validate model.
-        await ws.send(
-            json.dumps(
-                {
-                    "type": "session.update",
+        if server_vad:
+            session_update = {
+                "type": "session.update",
+                "session": {
                    "model": model,
-                }
-            )
-        )
+                    "audio": {
+                        "input": {
+                            "format": {"type": "audio/pcm", "rate": 16_000},
+                            "turn_detection": {
+                                "type": "server_vad",
+                                "threshold": 0.5,
+                                "prefix_padding_ms": 300,
+                                "silence_duration_ms": 500,
+                                "create_response": True,
+                                "interrupt_response": False,
+                            },
+                        }
+                    },
+                },
+            }
+        else:
+            # Preserve the existing non-duplex wire format.
+            session_update = {"type": "session.update", "model": model}
+        await ws.send(json.dumps(session_update))

-        # 2) Start generation once (non-final commit).
-        await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": False}))
+        # 2) Preserve the legacy explicit-commit flow unless Server VAD owns endpointing.
+        if not server_vad:
+            await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": False}))

        # 3) Stream audio chunks.
        for i in range(0, len(pcm16), chunk_bytes):
@@ -124,8 +159,20 @@ async def run_client(
            if send_delay_ms > 0:
                await asyncio.sleep(send_delay_ms / 1000.0)

-        # 4) Final commit closes input stream.
-        await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
+        # 4) Finalize the turn explicitly, or append silence for Server VAD endpointing.
+        if server_vad:
+            trailing_silence = bytes(16_000 * 2)
+            for i in range(0, len(trailing_silence), chunk_bytes):
+                await ws.send(
+                    json.dumps(
+                        {
+                            "type": "input_audio_buffer.append",
+                            "audio": base64.b64encode(trailing_silence[i : i + chunk_bytes]).decode("utf-8"),
+                        }
+                    )
+                )
+        else:
+            await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))

        # 5) Receive server events until audio done.
        while True:
@@ -144,7 +191,7 @@ async def run_client(
                sr = event.get("sample_rate_hz")
                if isinstance(sr, int) and sr > 0:
                    output_sample_rate = sr
-                audio_b64 = event.get("audio", "")
+                audio_b64 = event.get("delta" if server_vad else "audio", "")
                if audio_b64:
                    pcm_delta = base64.b64decode(audio_b64)
                    incremental_pcm_parts.append(pcm_delta)
@@ -158,14 +205,14 @@ async def run_client(
                        )
                continue

-            if event_type == "transcription.delta":
+            if event_type in {"response.output_text.delta", "transcription.delta"}:
                delta = event.get("delta", "")
                if delta:
                    text_chunks.append(delta)
                    print(delta, end="", flush=True)
                continue

-            if event_type == "transcription.done":
+            if event_type in {"response.output_text.done", "transcription.done"}:
                final_text = event.get("text", "") or "".join(text_chunks)
                usage = event.get("usage")
                final_text_with_tag = f"Final transcription: {final_text}"
@@ -176,7 +223,16 @@ async def run_client(
                    print(f"{log_prefix}text usage: {usage}")
                continue

+            if event_type == "response.audio_transcript.done":
+                final_audio_transcript = event.get("transcript", "")
+                continue
+
            if event_type == "response.audio.done":
+                if not server_vad:
+                    break
+                continue
+
+            if event_type == "response.done" and server_vad:
                break

            if event_type == "error":
@@ -191,7 +247,7 @@ async def run_client(
        print(f"{log_prefix}Saved realtime audio to: {output_wav} (incremental chunks joined)")

        if output_text is not None:
-            text_to_save = final_text if final_text else "".join(text_chunks)
+            text_to_save = final_text or "".join(text_chunks) or final_audio_transcript
            output_text.parent.mkdir(parents=True, exist_ok=True)
            output_text.write_text(text_to_save, encoding="utf-8")
            print(f"{log_prefix}Saved realtime text to: {output_text}")
@@ -213,6 +269,7 @@ async def run_clients_concurrent(
    chunk_ms: int,
    send_delay_ms: int,
    delta_dump_dir: Path | None,
+    server_vad: bool,
    num_requests: int,
    concurrency: int,
 ) -> None:
@@ -235,6 +292,7 @@ async def run_clients_concurrent(
                    chunk_ms=chunk_ms,
                    send_delay_ms=send_delay_ms,
                    delta_dump_dir=per_delta_dir,
+                    server_vad=server_vad,
                    request_idx=index,
                    total_requests=num_requests,
                )
@@ -283,6 +341,11 @@ def main() -> None:
        default=None,
        help="If set, each response.audio.delta is saved as delta_NNNNNN.wav under this directory",
    )
+    parser.add_argument(
+        "--server-vad",
+        action="store_true",
+        help="Let the server detect speech endpoints instead of sending input_audio_buffer.commit",
+    )
    parser.add_argument("--num-requests", type=int, default=1, help="Total number of requests to send")
    parser.add_argument(
        "--concurrency",
@@ -309,6 +372,7 @@ def main() -> None:
                chunk_ms=args.chunk_ms,
                send_delay_ms=args.send_delay_ms,
                delta_dump_dir=args.delta_dump_dir,
+                server_vad=args.server_vad,
            )
        )
    else:
@@ -322,6 +386,7 @@ def main() -> None:
                chunk_ms=args.chunk_ms,
                send_delay_ms=args.send_delay_ms,
                delta_dump_dir=args.delta_dump_dir,
+                server_vad=args.server_vad,
                num_requests=args.num_requests,
                concurrency=concurrency,
            )
@@ -1,7 +1,7 @@
 # Streaming Video Generation

 This example uses the custom WebSocket endpoint `WS /v1/realtime/video` to receive a video byte stream as chunks are produced.
-It covers text-only video generation. Image/reference input is intentionally not included for now.
+It covers text-to-video, and image-conditioned models through `--image-reference`.

 ## Start The Server

@@ -20,7 +20,7 @@ The `--diffusion-streaming-output` CLI flag is forwarded as `streaming_output=Tr

 | Direction | Message | Format | Description |
 | --- | --- | --- | --- |
-| Client to server | `session.start` | JSON text: `{"type":"session.start","model":"...","prompt":"...","format":"m4s"}` | Starts generation. `format` is optional and accepts `m4s` (default). Sampling fields such as `width`, `height`, `fps`, `num_frames`, and `extra_params` may be included. |
+| Client to server | `session.start` | JSON text: `{"type":"session.start","model":"...","prompt":"...","format":"m4s"}` | Starts generation. `format` is optional and accepts `m4s` (default). Sampling fields such as `width`, `height`, `fps`, `num_frames`, and `extra_params` may be included. Image-conditioned models take a first frame as `image_reference`: `{"image_url":"https://..."}` or a `data:` URL. |
 | Client to server | `session.interaction` | JSON text: `{"type":"session.interaction","interaction":{"event_id":"xxx","event":{"prompt":"..."},"transition_chunks":3}}` | Updates the active prompt midway through generation. `event_id` is optional and `transition_chunks` defaults to the model setting. |
 | Server to client | `video.start` | JSON text: `{"type":"video.start","request_id":"...","format":"m4s","config":{...}}` | Confirms the session and mirrors the accepted `format`. |
 | Server to client | `video.chunk_metadata` | JSON text: `{"type":"video.chunk_metadata","request_id":"...","kind":"media","transport_chunk_index":0,"generation_chunk_index":0,"num_frames":9,"byte_length":1234,"started_event_ids":[],"active_event_ids":[],"completed_event_ids":[]}` | Precedes each binary frame and describes the immediately following payload. |
@@ -62,7 +62,7 @@ python streaming_video_client.py \
 The client sends one `session.start` message, prints each received binary video chunk with its byte size and elapsed time, and saves the received bytes to `--output` after `session.done`.
 The client remuxes the gathered stream to a regular progressive MP4 file so that local playback knows the video duration.

-Schedule midway prompt updates with `--prompt-updates`. Each entry uses `"at"` as seconds on the client clock after the server sends `video.start`:
+Schedule midway prompt updates with `--prompt-updates`. None are sent unless you ask for them, and only pipelines that implement midway prompt updates accept them (LingBot-World, for one, does not, and rejects the whole session). Each entry uses `"at"` as seconds on the client clock after the server sends `video.start`:

 ```bash
 python streaming_video_client.py \
@@ -73,6 +73,19 @@ python streaming_video_client.py \
  ]'
 ```

+The Helios demo continues the default prompt into an underwater storm:
+
+```bash
+python streaming_video_client.py \
+  --model BestWishYsh/Helios-Distilled \
+  --prompt-updates '[
+    {"at": 4,
+     "prompt": "An underwater tornado appears and affects the ocean floor in a dramatic and chaotic scene. The water is murky, swirling violently, carrying debris and marine life into the vortex. The tropical fish on the scene all swim in panic, trying to avoid the powerful currents. The camera remains stationary, capturing the intensity of the underwater tornado as it disrupts the serene ocean floor. Close-up shot emphasizing the turbulent motion and destruction."},
+    {"at": 11,
+     "prompt": "The swirling underwater vortex now seizes a heavy, encrusted treasure chest, its lid flapping open as it is smashed onto the ocean floor. Gold coins and silver trinkets spill out, glittering briefly in the murky water before being swept instantly into the violent funnel. The heavy wooden box tumbles end over end, colliding with floating rocks and adding to the debris field. Swirling sediment and bubbles surround the spilling fortune, highlighting the chaotic power of the storm as it ravages the seabed. Close-up shot emphasizing the turbulent motion and destruction."}
+  ]'
+```
+
 ## Run The Gradio Demo

 ```bash
@@ -106,3 +119,31 @@ Disable it in the CLI example with `--no-helios-distilled-preset`, or override/e
 python streaming_video_client.py \
  --extra-params '{"pyramid_num_inference_steps_list":[2, 2, 2]}'
 ```
+
+### LingBot-World 2.0
+
+`robbyant/lingbot-world-v2-14b-causal-fast-diffusers` is image-conditioned and runs on the
+AR-Diffusion engine, so it needs the stepwise deploy config and a first frame. See
+[the recipe](../../../recipes/Robbyant/LingBot-World-2.0.md) for the constraints on
+`width`/`height` and `num_frames`.
+
+```bash
+vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+  --omni \
+  --deploy-config vllm_omni/deploy/lingbot_world_v2_stepwise.yaml \
+  --port 8000
+```
+
+```bash
+python streaming_video_client.py \
+  --model robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+  --prompt "The camera moves slowly forward through the scene." \
+  --image-reference /path/to/first_frame.png \
+  --width 832 --height 480 --num-frames 33 --fps 16 --seed 42 \
+  --extra-params '{"camera_action_script":[[["w"],["w"],["w"]],[["a"],[],[]],[[],[],[]]]}' \
+  --output lingbot_world_v2_stream.mp4
+```
+
+`camera_action_script` carries one three-latent-frame WASD action list per generated
+chunk. Mid-session `session.interaction` only updates the prompt, so a LingBot rollout
+follows the camera script it started with.
@@ -1,11 +1,12 @@
 #!/usr/bin/env python3
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Text-only WebSocket client for `/v1/realtime/video`.
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""WebSocket client for `/v1/realtime/video`.

 The client prints a line for every binary video chunk it receives and writes
 the received bytes to disk when the session finishes or is interrupted.
-Image/reference input is intentionally not supported in this example yet.
+``--image-reference`` sends a first frame for image-conditioned models such as
+LingBot-World 2.0; text-only models need no image.

 Requirements:
    pip install av websockets
@@ -13,9 +14,11 @@ Requirements:

 import argparse
 import asyncio
+import base64
 import contextlib
 import io
 import json
+import mimetypes
 import os
 import time
 from dataclasses import dataclass
@@ -53,6 +56,18 @@ class ScheduledPromptUpdate:
    transition_chunks: int


+def _image_reference(value: str) -> dict[str, Any]:
+    """Accept an http(s) URL as-is, or inline a local image as a data URL."""
+    if value.startswith(("http://", "https://", "data:")):
+        return {"image_url": value}
+    path = Path(value)
+    if not path.is_file():
+        raise argparse.ArgumentTypeError(f"--image-reference is neither a URL nor a readable file: {value}")
+    media_type = mimetypes.guess_type(path.name)[0] or "image/png"
+    encoded = base64.b64encode(path.read_bytes()).decode()
+    return {"image_url": f"data:{media_type};base64,{encoded}"}
+
+
 def _parse_extra_params(value: str) -> dict[str, Any]:
    try:
        parsed = json.loads(value)
@@ -116,6 +131,7 @@ def build_session_start(args: argparse.Namespace) -> dict[str, Any]:
        "prompt": args.prompt,
    }

+    _maybe_set(payload, "image_reference", args.image_reference)
    _maybe_set(payload, "negative_prompt", args.negative_prompt)
    _maybe_set(payload, "width", args.width)
    _maybe_set(payload, "height", args.height)
@@ -362,6 +378,12 @@ def parse_args() -> argparse.Namespace:
    parser.add_argument("--fps", type=int, default=16, help="Frames per second")
    parser.add_argument("--num-frames", type=int, default=429, help="Number of generated frames")

+    parser.add_argument(
+        "--image-reference",
+        type=_image_reference,
+        default=None,
+        help="First frame for image-conditioned models: a local image path or an http(s) URL.",
+    )
    parser.add_argument("--negative-prompt", default=None, help="Negative prompt")
    parser.add_argument("--num-inference-steps", type=int, default=None, help="Number of diffusion steps")
    parser.add_argument("--guidance-scale", type=float, default=1.0, help="Classifier-free guidance scale")
@@ -383,36 +405,16 @@ def parse_args() -> argparse.Namespace:
        default=None,
        help="JSON object merged into extra_params; overrides preset keys on conflict.",
    )
+    # Midway prompt updates are pipeline-specific: a model that does not implement
+    # them (LingBot-World, for one) rejects the whole session on the first one, so
+    # nothing is scheduled unless the caller asks for it.
    parser.add_argument(
        "--prompt-updates",
        type=_parse_prompt_updates,
-        default=json.dumps(
-            [
-                {
-                    "at": 4,
-                    "prompt": (
-                        "An underwater tornado appears and affects the ocean floor in a dramatic and chaotic scene. "
-                        "The water is murky, swirling violently, carrying debris and marine life into the vortex. "
-                        "The tropical fish on the scene all swim in panic, trying to avoid the powerful currents. "
-                        "The camera remains stationary, capturing the intensity of the underwater tornado as it disrupts the serene ocean floor. "
-                        "Close-up shot emphasizing the turbulent motion and destruction."
-                    ),
-                },
-                {
-                    "at": 11,
-                    "prompt": (
-                        "The swirling underwater vortex now seizes a heavy, encrusted treasure chest, its lid flapping open as it is smashed onto the ocean floor. "
-                        "Gold coins and silver trinkets spill out, glittering briefly in the murky water before being swept instantly into the violent funnel. "
-                        "The heavy wooden box tumbles end over end, colliding with floating rocks and adding to the debris field. "
-                        "Swirling sediment and bubbles surround the spilling fortune, highlighting the chaotic power of the storm as it ravages the seabed. "
-                        "Close-up shot emphasizing the turbulent motion and destruction."
-                    ),
-                    # "transition_chunks": 3,
-                },
-            ]
-        ),
+        default=[],
        help=(
-            "JSON array of scheduled prompt updates. Each object requires "
+            "JSON array of scheduled prompt updates; none are sent by default. "
+            "Each object requires "
            "'at' (seconds after video.start) and 'prompt'. Optional "
            f"'transition_chunks' defaults to {DEFAULT_TRANSITION_CHUNKS}."
        ),
@@ -14,6 +14,7 @@ For the full list of supported architectures across all modalities, see

 | Model | HuggingFace repo | Voice cloning | Streaming | Voice presets / upload | Gradio demo |
 | --- | --- | --- | --- | --- | --- |
+| Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | uploaded audio voice only; no presets | ✓ |
 | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
 | GLM-TTS | `zai-org/GLM-TTS` | ✓ (`ref_audio`+`ref_text`, required) | ✓ (PCM stream) | — | ✓ |
 | IndexTTS-2 | `IndexTeam/IndexTTS-2` | ✓ (`ref_audio` or uploaded `voice`) | `stream=true` response, non-chunk | uploaded audio voice only; no presets | — |
@@ -253,16 +254,152 @@ python examples/online_serving/text_to_speech/indextts2/speech_client.py \

 ---

-## Fish Speech S2 Pro
+## Audio8 TTS Preview

-4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec.
+0.6B DualAR TTS at 44.1 kHz, 11 languages, zero-shot voice cloning.

 ### Prerequisites

+None beyond the base install: the neural audio codec is implemented in tree and
+its weights (`codec.pth`) ship with the checkpoint.
+
+### Launch
+
 ```bash
-pip install fish-speech
+vllm serve Audio8/Audio8-TTS-Preview-0.6b --omni --port 8092
+# or:
+./audio8_tts/run_server.sh
 ```

+The deploy config auto-loads from `vllm_omni/deploy/audio8_tts.yaml` (HF
+`model_type` is `arktts`). Do **not** pass `--trust-remote-code`: vllm-omni
+registers its own `arktts` config, and transformers would otherwise prefer the
+checkpoint's remote code and bypass it.
+
+### Text-only synthesis
+
+```bash
+curl -X POST http://localhost:8092/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Welcome to Audio8 TTS.",
+        "response_format": "wav"
+    }' --output output.wav
+```
+
+### Voice cloning
+
+```bash
+curl -X POST http://localhost:8092/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Welcome to Audio8 TTS.",
+        "ref_audio": "https://example.com/reference.wav",
+        "ref_text": "The exact transcript of the reference recording."
+    }' --output cloned.wav
+```
+
+`ref_audio` also accepts a `data:audio/wav;base64,...` URL. `ref_text` is
+mandatory whenever `ref_audio` is present and must match the recording.
+Uploading a voice via `POST /v1/audio/voices` lets the server reuse the encoded
+reference codes across requests (`voice: "<name>"`).
+
+### Streaming
+
+```bash
+python audio8_tts/speech_client.py --text "Welcome to Audio8 TTS." --stream --output out.pcm
+ffplay -f s16le -ar 44100 -ac 1 out.pcm
+```
+
+### Gradio demo
+
+```bash
+./audio8_tts/run_gradio_demo.sh          # server + demo
+python audio8_tts/gradio_demo.py --api-base http://localhost:8092   # demo only
+```
+
+### Benchmark
+
+```bash
+export VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100   # required: harness defaults to 24 kHz
+vllm bench serve --omni --host 127.0.0.1 --port 8092 \
+    --model Audio8/Audio8-TTS-Preview-0.6b \
+    --backend openai-audio-speech --endpoint /v1/audio/speech \
+    --dataset-name seed-tts-text --dataset-path benchmarks/build_dataset/seed_tts_smoke \
+    --seed-tts-locale en --num-prompts 20 --num-warmups 2 \
+    --max-concurrency 1 --request-rate inf \
+    --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration,audio_underrun
+```
+
+Measured on 1x H20 **shared with an unrelated job holding ~80% SM**, so treat
+these as a lower bound. 20 prompts from `seed_tts_smoke/en`, mean audio 4.5 s,
+deploy defaults:
+
+| | HF reference (batch=1) | vllm-omni c=1 | c=4 | c=8 |
+| --- | --- | --- | --- | --- |
+| RTF (mean) | 1.008 | **0.19** | 0.30 | 0.54 |
+| Throughput (req/s) | 0.23 | 1.18 | 2.84 | **2.93** |
+| E2E latency (mean, ms) | 4375 | 850 | 1349 | 2361 |
+| Time to first packet (mean, ms) | n/a (no streaming) | 63 | 105 | 1124 |
+| Underrun p99 / continuity OK | n/a | 0 s / 100% | 0 s / 100% | 0 s / 100% |
+
+~5.3x lower per-request RTF than the upstream reference implementation at
+concurrency 1, ~12.8x aggregate throughput at concurrency 8.
+
+#### Stage-0 `max_num_seqs`: first-packet vs throughput
+
+This subsection is operator tuning guidance, not a claim this PR optimizes
+anything: the shipped deploy default is unchanged (`max_num_seqs: 4`). The
+numbers below are a single run on the same contended H20 as above (one unrelated
+job holding ~80% SM, no repeats), so read them as directional, not as measured
+speedups. Same setup, varying stage 0's `max_num_seqs` (stage 1 stays at 1):
+
+| stage-0 `max_num_seqs` | client concurrency | req/s | RTF | TTFP mean / p99 (ms) | underrun p99 (s) |
+| --- | --- | --- | --- | --- | --- |
+| 4 (default) | 1 | 1.18 | 0.19 | 63 / 67 | 0.00 |
+| 4 | 4 | 2.84 | 0.30 | 105 / 148 | 0.00 |
+| 4 | 8 | 2.93 | 0.54 | 1124 / 1642 | 0.00 |
+| 8 | 1 | 1.16 | 0.19 | 64 / 70 | 0.00 |
+| 8 | 4 | 2.88 | 0.30 | 107 / 148 | 0.00 |
+| 8 | 8 | **3.72** | 0.40 | **247** / 733 | 0.22 |
+| 8 | 16 | 3.91 | 0.65 | 1366 / 2530 | 0.29 |
+
+Reading of this:
+
+- Below the limit (`concurrency <= max_num_seqs`) the setting is irrelevant, as
+  expected — c=1 and c=4 are identical for both.
+- At c=8, raising the limit to 8 admits every request instead of queueing half
+  of them: in this single run, roughly +27% throughput and ~4.5x lower
+  first-packet latency.
+- Throughput saturates around 3.7-3.9 req/s (~17 s of audio per second). Past
+  c=8 you only buy queueing delay.
+- The cost of admitting more streams is a 0.22-0.29 s worst-case buffer deficit
+  in 1 of 20 requests (`Streaming continuity OK rate` 95%), because stage 1 runs
+  `max_num_seqs: 1` and the codec window round-robins across streams. Client-side
+  prebuffering (the bundled Gradio demo holds 2 chunks) absorbs that; raising
+  stage 1's `max_num_seqs` is the untested knob if you need both.
+
+The shipped default stays at 4 because every measured point there had zero
+underrun. Raise it to 8 if first-packet latency under load matters more than a
+rare sub-300 ms gap.
+
+### Notes
+
+- Output: 44.1 kHz mono; ~21.5 codec frames per second.
+- No built-in speaker presets. Omit `voice` for a random timbre, or clone one.
+- `max_new_tokens` caps generated codec frames (1 frame ~= 46 ms).
+- Context is 2048 packed text+audio positions (`max_model_len: 2048`).
+- Do not send `temperature: 0`: greedy decoding is degenerate for this
+  checkpoint and truncates the utterance. The deploy defaults
+  (0.7 / top_k 50 / top_p 0.9) mirror `generation_config.json`.
+
+---
+
+## Fish Speech S2 Pro
+
+4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec, which is vendored in
+vLLM-Omni (no extra packages required).
+
 ### Kvcache attention fast path

 Fish Speech S2 Pro uses a Triton decode-only kvcache attention fast path by
@@ -425,6 +562,46 @@ python examples/online_serving/text_to_speech/ming_flash_omni_tts/speech_client.

 ---

+## MOSS-TTS Local Transformer v1.5
+
+For a single H200, the optional
+[`moss_tts_local_h200.yaml`](../../../vllm_omni/deploy/moss_tts_local_h200.yaml)
+deployment places both the talker and codec on logical GPU 0:
+
+```bash
+CUDA_VISIBLE_DEVICES=0 VLLM_OMNI_EVENT_DRIVEN_ORCH=1 \
+    vllm serve OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 \
+    --omni --trust-remote-code \
+    --deploy-config vllm_omni/deploy/moss_tts_local_h200.yaml \
+    --disable-log-stats
+```
+
+Run from the repository root and select an available physical GPU with
+`CUDA_VISIBLE_DEVICES`. The command enables event-driven orchestration and
+disables detailed per-request statistics logging to reduce CPU overhead at
+high concurrency. Remove `--disable-log-stats` when those statistics are needed;
+keep these settings identical when comparing performance.
+This preset configures 256 request slots per stage,
+a 32 GiB talker KV cache, talker CUDA Graph buckets through 512 scheduled tokens,
+and codec batch buckets through 256. It requires more than 80 GiB of GPU memory;
+request capacity also depends on input and generated lengths. The default
+[`moss_tts_local.yaml`](../../../vllm_omni/deploy/moss_tts_local.yaml) remains
+available for smaller deployments.
+
+The preset enables the optional codec backend with
+`hf_overrides.codec_attention_backend: triton` on stage 1. It preserves the
+streaming ring-cache mask and uses BF16 attention with 64-dimensional heads;
+other attention shapes use PyTorch SDPA. Set the backend to `sdpa` to use the
+default attention implementation. Codec terminal tails share execution only
+when they already map to the same padded CUDA Graph; returned audio retains
+each request's actual length.
+
+Voice cloning requests use `ref_audio` and `ref_text`. For streaming output,
+set `stream: true`, `stream_format: "audio"`, and `response_format: "pcm"`;
+the native PCM format is 48 kHz, stereo, signed 16-bit little-endian. Compute
+audio throughput as `PCM bytes / (48000 * 2 * 2) / elapsed seconds`, and compare
+the same requests, concurrency, and reference-cache state.
+
 ## MOSS-TTS-Nano

 Single-stage 0.1B AR LM + MOSS-Audio-Tokenizer-Nano codec at 48 kHz mono. Every request must include `ref_audio`; there are no built-in speaker presets.
@@ -668,13 +845,21 @@ Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and

 ### Streaming WebSocket

-The `/v1/audio/speech/stream` endpoint accepts text incrementally and synthesizes the buffered text as one continuous request on `input.done`:
+The `/v1/audio/speech/stream` endpoint accepts text incrementally and, by default, synthesizes the buffered text as one continuous request on `input.done`:

 ```bash
 python qwen3_tts/streaming_speech_client.py --text "Hello world. How are you? I am fine."
 python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-delay 0.1
 ```

+For per-sentence audio (including Indic danda `।`) before `input.done`:
+
+```bash
+python qwen3_tts/streaming_speech_client.py \
+    --text "नमस्ते। कैसे हो?" \
+    --split-granularity sentence
+```
+
 `input.done` flushes without closing, so repeating `--text` synthesizes several utterances over one connection:

 ```bash
@@ -0,0 +1,258 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Gradio demo for Audio8 TTS Preview via the /v1/audio/speech API.
+
+Supports:
+  - text-to-speech synthesis in 11 languages
+  - zero-shot voice cloning from reference audio (upload, microphone, or URL)
+  - streaming (progressive PCM chunks) and non-streaming modes
+
+Usage:
+    # Start the server first (see run_server.sh), then:
+    python gradio_demo.py --api-base http://localhost:8092
+"""
+
+import argparse
+import base64
+import io
+
+try:
+    import gradio as gr
+except ImportError:
+    raise ImportError("gradio is required to run this demo. Install it with: pip install 'vllm-omni[demo]'") from None
+import httpx
+import numpy as np
+import soundfile as sf
+
+#: Audio8 TTS decodes at 44.1 kHz; raw PCM must be interpreted at that rate.
+PCM_SAMPLE_RATE = 44100
+DEFAULT_API_BASE = "http://localhost:8092"
+
+
+def encode_audio_to_base64(audio_data: tuple) -> str:
+    """Encode a Gradio ``(sample_rate, numpy_array)`` input as a data URL."""
+    sample_rate, audio_np = audio_data
+    if audio_np.dtype != np.int16:
+        if audio_np.dtype in (np.float32, np.float64):
+            audio_np = (np.clip(audio_np, -1.0, 1.0) * 32767).astype(np.int16)
+        else:
+            audio_np = audio_np.astype(np.int16)
+    buf = io.BytesIO()
+    sf.write(buf, audio_np, sample_rate, format="WAV")
+    return f"data:audio/wav;base64,{base64.b64encode(buf.getvalue()).decode('utf-8')}"
+
+
+def build_payload(
+    text: str,
+    ref_audio: tuple | None,
+    ref_audio_url: str,
+    ref_text: str,
+    response_format: str = "wav",
+    stream: bool = False,
+) -> dict:
+    """Build the ``/v1/audio/speech`` payload.
+
+    Audio8 TTS has no built-in speakers: omit ``voice`` for a random timbre, or
+    supply ``ref_audio`` + ``ref_text`` to clone a voice.
+    """
+    if not text or not text.strip():
+        raise gr.Error("Please enter text to synthesize.")
+
+    payload: dict = {
+        "input": text.strip(),
+        "response_format": "pcm" if stream else response_format,
+        "stream": stream,
+    }
+    if stream:
+        payload["stream_format"] = "audio"
+
+    ref_url = ref_audio_url.strip() if ref_audio_url else ""
+    if ref_audio is not None:
+        payload["ref_audio"] = encode_audio_to_base64(ref_audio)
+    elif ref_url:
+        payload["ref_audio"] = ref_url
+
+    if "ref_audio" in payload:
+        if not ref_text or not ref_text.strip():
+            raise gr.Error("Voice cloning requires the transcript of the reference audio.")
+        payload["ref_text"] = ref_text.strip()
+
+    return payload
+
+
+def generate_speech(api_base: str, text: str, ref_audio, ref_audio_url, ref_text, response_format):
+    """Non-streaming request; returns the whole waveform."""
+    payload = build_payload(text, ref_audio, ref_audio_url, ref_text, response_format, stream=False)
+    try:
+        with httpx.Client(timeout=300.0) as client:
+            resp = client.post(
+                f"{api_base}/v1/audio/speech",
+                json=payload,
+                headers={"Content-Type": "application/json", "Authorization": "Bearer EMPTY"},
+            )
+    except httpx.TimeoutException:
+        raise gr.Error("Request timed out. The server may be busy.") from None
+    except httpx.ConnectError:
+        raise gr.Error(f"Cannot connect to server at {api_base}. Is the server running?") from None
+
+    if resp.status_code != 200:
+        raise gr.Error(f"Server error ({resp.status_code}): {resp.text}")
+    if "application/json" in resp.headers.get("content-type", ""):
+        raise gr.Error(f"Server error: {resp.text}")
+
+    try:
+        if response_format == "pcm":
+            audio_np = np.frombuffer(resp.content, dtype=np.int16).astype(np.float32) / 32767.0
+            return (PCM_SAMPLE_RATE, audio_np)
+        audio_np, sample_rate = sf.read(io.BytesIO(resp.content))
+        if audio_np.ndim > 1:
+            audio_np = audio_np[:, 0]
+        return (sample_rate, audio_np.astype(np.float32))
+    except Exception as e:
+        raise gr.Error(f"Failed to decode audio: {e}") from None
+
+
+def generate_speech_stream(api_base, text, ref_audio, ref_audio_url, ref_text, response_format, chunk_seconds=0.5):
+    """Streaming request; yields PCM chunks for progressive playback."""
+    payload = build_payload(text, ref_audio, ref_audio_url, ref_text, response_format, stream=True)
+    min_chunk_samples = int(PCM_SAMPLE_RATE * chunk_seconds)
+    prebuffer_chunks = 2
+    pending: list[np.ndarray] = []
+    pending_len = 0
+    chunks_yielded = 0
+    prebuffer: list[np.ndarray] = []
+    leftover = b""
+
+    try:
+        with (
+            httpx.Client(timeout=300.0) as client,
+            client.stream(
+                "POST",
+                f"{api_base}/v1/audio/speech",
+                json=payload,
+                headers={"Content-Type": "application/json", "Authorization": "Bearer EMPTY"},
+            ) as resp,
+        ):
+            if resp.status_code != 200:
+                resp.read()
+                raise gr.Error(f"Server error ({resp.status_code}): {resp.text}")
+            for chunk in resp.iter_bytes():
+                if not chunk:
+                    continue
+                raw = leftover + chunk
+                # s16le: never split a sample across chunk boundaries.
+                usable = len(raw) - (len(raw) % 2)
+                leftover = raw[usable:]
+                if usable == 0:
+                    continue
+                samples = np.frombuffer(raw[:usable], dtype=np.int16).copy().astype(np.float32) / 32767.0
+                pending.append(samples)
+                pending_len += len(samples)
+                if pending_len < min_chunk_samples:
+                    continue
+                audio_chunk = np.concatenate(pending)
+                pending.clear()
+                pending_len = 0
+                if chunks_yielded < prebuffer_chunks:
+                    prebuffer.append(audio_chunk)
+                    chunks_yielded += 1
+                    if chunks_yielded == prebuffer_chunks:
+                        yield (PCM_SAMPLE_RATE, np.concatenate(prebuffer))
+                        prebuffer.clear()
+                else:
+                    yield (PCM_SAMPLE_RATE, audio_chunk)
+        remaining = prebuffer + pending
+        if remaining:
+            yield (PCM_SAMPLE_RATE, np.concatenate(remaining))
+    except httpx.TimeoutException:
+        raise gr.Error("Request timed out. The server may be busy.") from None
+    except httpx.ConnectError:
+        raise gr.Error(f"Cannot connect to server at {api_base}. Is the server running?") from None
+
+
+def on_stream_change(stream: bool):
+    """Streaming only makes sense with raw PCM."""
+    if stream:
+        return gr.update(value="pcm", interactive=False)
+    return gr.update(value="wav", interactive=True)
+
+
+def build_interface(api_base: str, stream_chunk_seconds: float = 0.5):
+    with gr.Blocks(title="Audio8 TTS Preview Demo") as demo:
+        gr.Markdown("# Audio8 TTS Preview 0.6B - Text to Speech")
+        gr.Markdown(f"**Server:** `{api_base}` | **Model:** Audio8/Audio8-TTS-Preview-0.6b | **Output:** 44.1 kHz")
+
+        with gr.Row():
+            with gr.Column(scale=3):
+                text_input = gr.Textbox(label="Text to Synthesize", placeholder="Enter text here...", lines=4)
+
+                with gr.Accordion("Voice Cloning (optional)", open=False):
+                    gr.Markdown(
+                        "Upload or link a short reference recording (5-30 s) and paste its **exact** transcript. "
+                        "A transcript that does not match the audio degrades both stability and speaker similarity."
+                    )
+                    ref_audio = gr.Audio(label="Reference Audio", type="numpy", sources=["upload", "microphone"])
+                    ref_audio_url = gr.Textbox(
+                        label="Reference Audio URL (alternative to upload)",
+                        placeholder="https://example.com/reference.wav",
+                        lines=1,
+                    )
+                    ref_text = gr.Textbox(
+                        label="Reference Audio Transcript (required for cloning)",
+                        placeholder="Exact transcript of the reference audio...",
+                        lines=2,
+                    )
+
+                with gr.Row():
+                    response_format = gr.Dropdown(
+                        choices=["wav", "mp3", "flac", "pcm"], value="wav", label="Audio Format", scale=1
+                    )
+                    stream_checkbox = gr.Checkbox(
+                        label="Stream output", value=False, info="Progressive PCM streaming", scale=1
+                    )
+
+                generate_btn = gr.Button("Generate Speech", variant="primary", size="lg")
+
+            with gr.Column(scale=2):
+                audio_output = gr.Audio(label="Generated Audio", interactive=False, streaming=True, autoplay=True)
+                gr.Markdown(
+                    "### About\n"
+                    "- **Audio8 TTS Preview** — 0.6B DualAR model (slow AR + fast AR codebook decoder)\n"
+                    "- **Languages**: Cantonese, Chinese, Dutch, English, French, German, "
+                    "Italian, Japanese, Korean, Polish, Spanish\n"
+                    "- **Voice cloning**: zero-shot from a reference clip + transcript\n"
+                    "- **44.1 kHz** output, ~21.5 codec frames per second"
+                )
+
+        stream_checkbox.change(fn=on_stream_change, inputs=[stream_checkbox], outputs=[response_format])
+        all_inputs = [text_input, ref_audio, ref_audio_url, ref_text, response_format]
+
+        def dispatch(stream_enabled, *args):
+            if stream_enabled:
+                yield from generate_speech_stream(api_base, *args, chunk_seconds=stream_chunk_seconds)
+            else:
+                yield generate_speech(api_base, *args)
+
+        generate_btn.click(fn=dispatch, inputs=[stream_checkbox] + all_inputs, outputs=[audio_output])
+        demo.queue()
+    return demo
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Gradio demo for Audio8 TTS Preview")
+    parser.add_argument("--api-base", default=DEFAULT_API_BASE, help=f"API base URL (default: {DEFAULT_API_BASE})")
+    parser.add_argument("--host", default="0.0.0.0", help="Gradio host")
+    parser.add_argument("--port", type=int, default=7861, help="Gradio port")
+    parser.add_argument("--share", action="store_true", help="Create a public share link")
+    parser.add_argument("--stream-chunk-seconds", type=float, default=0.5, help="Seconds of audio per yielded chunk")
+    args = parser.parse_args()
+
+    print(f"Connecting to vLLM server at: {args.api_base}")
+    build_interface(args.api_base, stream_chunk_seconds=args.stream_chunk_seconds).launch(
+        server_name=args.host, server_port=args.port, share=args.share
+    )
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,44 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+# Launch the Audio8 TTS Preview server + Gradio demo together.
+#
+# Usage:
+#   ./run_gradio_demo.sh
+#   CUDA_VISIBLE_DEVICES=0 PORT=8092 GRADIO_PORT=7861 ./run_gradio_demo.sh
+
+set -e
+
+MODEL="${MODEL:-Audio8/Audio8-TTS-Preview-0.6b}"
+PORT="${PORT:-8092}"
+GRADIO_PORT="${GRADIO_PORT:-7861}"
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+
+echo "Starting Audio8 TTS Preview server (port $PORT)..."
+vllm serve "$MODEL" \
+    --omni \
+    --host 0.0.0.0 \
+    --port "$PORT" &
+SERVER_PID=$!
+
+cleanup() {
+    echo "Stopping server (PID $SERVER_PID)..."
+    kill $SERVER_PID 2>/dev/null
+    wait $SERVER_PID 2>/dev/null
+}
+trap cleanup EXIT
+
+echo "Waiting for server to start..."
+for _ in $(seq 1 120); do
+    if curl -s "http://localhost:$PORT/health" > /dev/null 2>&1; then
+        echo "Server ready."
+        break
+    fi
+    sleep 2
+done
+
+echo "Starting Gradio demo (port $GRADIO_PORT)..."
+python "$SCRIPT_DIR/gradio_demo.py" \
+    --api-base "http://localhost:$PORT" \
+    --port "$GRADIO_PORT"
@@ -0,0 +1,24 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+# Launch a vLLM-Omni server for Audio8 TTS Preview 0.6B.
+#
+# Usage:
+#   ./run_server.sh
+#   CUDA_VISIBLE_DEVICES=0 PORT=8092 ./run_server.sh
+
+set -e
+
+MODEL="${MODEL:-Audio8/Audio8-TTS-Preview-0.6b}"
+PORT="${PORT:-8092}"
+
+echo "Starting Audio8 TTS Preview server with model: $MODEL"
+
+# --trust-remote-code is deliberately NOT passed: vllm-omni registers its own
+# `arktts` config, and transformers would otherwise prefer the checkpoint's
+# remote code and bypass it.
+vllm serve "$MODEL" \
+    --omni \
+    --host 0.0.0.0 \
+    --port "$PORT"
@@ -0,0 +1,131 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Client for Audio8 TTS Preview via the /v1/audio/speech endpoint.
+
+Examples:
+    # Basic TTS
+    python speech_client.py --text "Welcome to Audio8 TTS."
+
+    # Zero-shot voice cloning (the transcript must match the reference audio)
+    python speech_client.py --text "Welcome to Audio8 TTS." \
+        --ref-audio ref.wav --ref-text "The exact transcript of the reference recording."
+
+    # Streaming PCM output
+    python speech_client.py --text "Welcome to Audio8 TTS." --stream --output output.pcm
+"""
+
+import argparse
+import base64
+import os
+
+import httpx
+
+DEFAULT_API_BASE = "http://localhost:8092"
+DEFAULT_API_KEY = "EMPTY"
+DEFAULT_MODEL = "Audio8/Audio8-TTS-Preview-0.6b"
+#: The codec runs at 44.1 kHz, so raw PCM must be played back at that rate.
+PCM_SAMPLE_RATE = 44100
+
+
+def encode_audio_to_base64(audio_path: str) -> str:
+    """Encode a local audio file as a base64 data URL."""
+    if not os.path.exists(audio_path):
+        raise FileNotFoundError(f"Audio file not found: {audio_path}")
+    ext = audio_path.lower().rsplit(".", 1)[-1]
+    mime_type = {"wav": "audio/wav", "mp3": "audio/mpeg", "flac": "audio/flac", "ogg": "audio/ogg"}.get(
+        ext, "audio/wav"
+    )
+    with open(audio_path, "rb") as handle:
+        audio_b64 = base64.b64encode(handle.read()).decode("utf-8")
+    return f"data:{mime_type};base64,{audio_b64}"
+
+
+def build_payload(args) -> dict:
+    payload = {
+        "model": args.model,
+        "input": args.text,
+        "response_format": args.response_format,
+    }
+    if args.ref_audio:
+        if args.ref_audio.startswith(("http://", "https://")):
+            payload["ref_audio"] = args.ref_audio
+        else:
+            payload["ref_audio"] = encode_audio_to_base64(args.ref_audio)
+    if args.ref_text:
+        payload["ref_text"] = args.ref_text
+    if args.max_new_tokens is not None:
+        payload["max_new_tokens"] = args.max_new_tokens
+    if args.stream:
+        payload["stream"] = True
+        payload["stream_format"] = "audio"
+        payload["response_format"] = "pcm"
+    return payload
+
+
+def run_tts(args) -> None:
+    payload = build_payload(args)
+    api_url = f"{args.api_base}/v1/audio/speech"
+    headers = {"Content-Type": "application/json", "Authorization": f"Bearer {args.api_key}"}
+
+    print(f"Model: {args.model}")
+    print(f"Text: {args.text}")
+    if args.ref_audio:
+        print(f"Voice cloning: ref_audio={args.ref_audio!r} ref_text={args.ref_text!r}")
+
+    if args.stream:
+        output_path = args.output or "output.pcm"
+        with (
+            httpx.Client(timeout=300.0) as client,
+            client.stream("POST", api_url, json=payload, headers=headers) as resp,
+        ):
+            if resp.status_code != 200:
+                print(f"Error {resp.status_code}: {resp.read().decode()}")
+                return
+            total_bytes = 0
+            with open(output_path, "wb") as handle:
+                for chunk in resp.iter_bytes():
+                    handle.write(chunk)
+                    total_bytes += len(chunk)
+        seconds = total_bytes / 2 / PCM_SAMPLE_RATE
+        print(f"Streamed {total_bytes} bytes (~{seconds:.2f}s of s16le @ {PCM_SAMPLE_RATE} Hz) to {output_path}")
+        print(f"Play with: ffplay -f s16le -ar {PCM_SAMPLE_RATE} -ac 1 {output_path}")
+        return
+
+    with httpx.Client(timeout=300.0) as client:
+        response = client.post(api_url, json=payload, headers=headers)
+    if response.status_code != 200:
+        print(f"Error {response.status_code}: {response.text}")
+        return
+    if response.headers.get("content-type", "").startswith("application/json"):
+        print(f"Error: {response.text}")
+        return
+
+    output_path = args.output or f"output.{args.response_format}"
+    with open(output_path, "wb") as handle:
+        handle.write(response.content)
+    print(f"Audio saved to: {output_path} ({len(response.content)} bytes)")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Audio8 TTS Preview client")
+    parser.add_argument("--api-base", default=DEFAULT_API_BASE, help="API base URL")
+    parser.add_argument("--api-key", default=DEFAULT_API_KEY, help="API key")
+    parser.add_argument("--model", "-m", default=DEFAULT_MODEL, help="Model name")
+    parser.add_argument("--text", required=True, help="Text to synthesize")
+    parser.add_argument("--ref-audio", default=None, help="Reference audio for voice cloning (path or URL)")
+    parser.add_argument("--ref-text", default=None, help="Transcript of the reference audio")
+    parser.add_argument("--max-new-tokens", type=int, default=None, help="Cap the number of generated codec frames")
+    parser.add_argument("--stream", action="store_true", help="Stream PCM instead of returning a whole file")
+    parser.add_argument(
+        "--response-format",
+        default="wav",
+        choices=["wav", "mp3", "flac", "pcm", "aac", "opus"],
+        help="Audio format (default: wav)",
+    )
+    parser.add_argument("--output", "-o", default=None, help="Output file path")
+    run_tts(parser.parse_args())
+
+
+if __name__ == "__main__":
+    main()
@@ -24,6 +24,11 @@ Usage:
        --text "Hello world. How are you? I am fine." \
        --simulate-stt --stt-delay 0.1

+    # Opt into per-sentence synthesis (Indic danda, CJK, Latin)
+    python streaming_speech_client.py \
+        --text "नमस्ते। कैसे हो?" \
+        --split-granularity sentence
+
    # Receive JSON sidecar chunks with word-level timestamps
    python streaming_speech_client.py \
        --text "Hello world. How are you?" \
@@ -73,8 +78,9 @@ except ImportError:
 def frame_basename(msg: dict) -> str:
    """Name a file after the utterance and sentence a frame belongs to.

-    Every utterance is sentence 0, so the utterance index is what keeps the
-    files of one connection from overwriting each other.
+    Every utterance uses `sentence_index` for units inside that flush.
+    With default `split_granularity=none` that index stays 0; sentence mode
+    uses 0, 1, ... so files from one connection do not overwrite each other.
    """
    return f"utterance_{msg['utterance_index']:03d}_sentence_{msg['sentence_index']:03d}"

@@ -368,6 +374,13 @@ def main():
    )
    parser.add_argument("--speed", type=float, default=1.0, help="Playback speed (0.25-4.0)")
    parser.add_argument("--max-new-tokens", type=int, default=None, help="Max tokens")
+    parser.add_argument("--seed", type=int, default=None, help="Sampling seed forwarded to the server")
+    parser.add_argument(
+        "--split-granularity",
+        default=None,
+        choices=["none", "sentence", "clause"],
+        help="Text split mode (default: server none = one request per input.done)",
+    )

    # Base task options
    parser.add_argument("--ref-audio", default=None, help="Reference audio")
@@ -409,6 +422,8 @@ def main():
        "max_new_tokens",
        "ref_audio",
        "ref_text",
+        "seed",
+        "split_granularity",
    ]:
        val = getattr(args, key.replace("-", "_"), None)
        if val is not None:
@@ -53,7 +53,9 @@ dev = [
    "soundfile==0.14.0",
    "imageio[ffmpeg]==2.37.4",
    "opencv-python==5.0.0.93",
-    "mooncake-transfer-engine-cuda13==0.3.11.post1",
+    # 0.3.12 includes acknowledged TCP writes (Mooncake #2850): completion
+    # must mean the receive buffer is ready before zero-copy consumers read it.
+    "mooncake-transfer-engine-cuda13==0.3.12.post1",
    "av==18.0.0", # for ComfyUI tests
    "openpyxl==3.1.5", # for nightly CI
    "pyttsx3==2.99",
@@ -75,7 +77,6 @@ dev = [
    # Provides top-level cosyvoice2; do not co-install stepaudio2-minicpmo,
    # whose top-level stepaudio2 package conflicts with this distribution.
    "step-audio2==1.0.0", # Step-Audio2 / MiniCPM-o 4.5 Token2Wav (flashcosyvoice / hyperpyyaml)
-    "silero-vad==6.1", # Realtime server-VAD duplex E2E
 ]

 demo = [
@@ -85,9 +86,6 @@ demo = [
    "cn2an>=0.5.22",  # Chinese number normalization for TTS (e.g. "1次" -> "一次")
 ]

-server-vad = [
-    "silero-vad==6.1",
-]

 forced-aligner = [
    "qwen-asr>=0.0.6",  # faithful Japanese/Korean word segmentation for the TTS forced aligner
@@ -285,6 +283,8 @@ markers = [
    "S5000: [hardware-resource] [musa] Tests that require S5000 GPU (Moore Threads/MUSA)",
    "A2: [hardware-resource] [npu] Tests that require A2 NPU",
    "A3: [hardware-resource] [npu] Tests that require A3 NPU",
+    "A5: [hardware-resource] [npu] Tests that require A5 NPU",
+    "310P: [hardware-resource] [npu] Tests that require 310P NPU",
    # Tag with [hardware-cards] for tests.helpers.mark.get_supported_card_counts().
    "cards_1: [hardware-cards] Tests that require 1 accelerator card",
    "cards_2: [hardware-cards] Tests that require 2 accelerator cards",
@@ -376,3 +376,7 @@ ba = "ba"
 SER = "SER"
 # NumPy ndarray.flags.writeable (canonical flag name, not "writable").
 writeable = "writeable"
+# "convnet" abbreviates convolutional network. It appears in the vendored Fish
+# Speech DAC codec identifiers (convnet_type, transconvnet_type) and is not a
+# typo of "convent".
+convnet = "convnet"
@@ -1,12 +1,13 @@
 # Boogu-Image

 > Text-to-image and image-editing online serving
-(Boogu-Image-0.1-Base / -Edit / -Edit-Turbo)
+(Boogu-Image-0.1-Base / -Turbo / -Edit / -Edit-Turbo)

 ## Summary

 - Vendor: Boogu
 - Model: `Boogu/Boogu-Image-0.1-Base` (text-to-image),
+  `Boogu/Boogu-Image-0.1-Turbo` (four-step text-to-image),
  `Boogu/Boogu-Image-0.1-Edit` (image editing), and
  `Boogu/Boogu-Image-0.1-Edit-Turbo` (four-step image editing)
 - Task: Text-to-image generation and text-guided image editing (TI2I)
@@ -16,8 +17,9 @@
 ## When to use this recipe

 Use this recipe when you want a known-good starting point for serving
-`Boogu/Boogu-Image-0.1-Base`, `Boogu/Boogu-Image-0.1-Edit`, or
-`Boogu/Boogu-Image-0.1-Edit-Turbo` with vLLM-Omni's native pipeline (no
+`Boogu/Boogu-Image-0.1-Base`, `Boogu/Boogu-Image-0.1-Turbo`,
+`Boogu/Boogu-Image-0.1-Edit`, or `Boogu/Boogu-Image-0.1-Edit-Turbo` with
+vLLM-Omni's native pipeline (no
 `--diffusion-load-format diffusers`, and the upstream `boogu` package is not
 required).

@@ -25,12 +27,14 @@ Boogu-Image-0.1 is an Apache-2.0 unified image generation and editing model
 family. The Base text-to-image checkpoint pairs a Qwen3-VL multimodal encoder
 with a Diffusion Transformer (DiT) and a flow-match Euler scheduler with
 time-shift. It handles photorealistic generation and Chinese/English text
-rendering. The Edit and Edit-Turbo checkpoints use the same native
-`BooguImagePipeline` for image editing.
+rendering. The Edit checkpoint uses the native `BooguImagePipeline`, while the
+distilled Turbo and Edit-Turbo checkpoints run the four-step DMD path through
+`BooguImageTurboPipeline`.

 ## References

 - Upstream model card: <https://huggingface.co/Boogu/Boogu-Image-0.1-Base>
+- Turbo model card: <https://huggingface.co/Boogu/Boogu-Image-0.1-Turbo>
 - Edit model card: <https://huggingface.co/Boogu/Boogu-Image-0.1-Edit>
 - Edit-Turbo 1K hotfix: <https://huggingface.co/Boogu/Boogu-Image-0.1-Edit-Turbo/tree/hotfix-1k-20260708>
 - Project page: <https://boogu.org>
@@ -148,7 +152,7 @@ The following configuration was validated:
 - vLLM-Omni version or commit: Use a commit that contains TorchAO FP8
  checkpoint loading for diffusion models
 - TorchAO version: 0.17.0
- `kernels` / `kernels-data`: 0.15.2 / 0.16.1
+- `kernels` / `kernels-data`: 0.16.1 / matching kernels-data

 #### Command

@@ -186,6 +190,19 @@ ID passed to `vllm serve`. For example, use
 corresponding Edit model.

 Base text-to-image:
+## Fast text-to-image (Boogu-Image-0.1-Turbo)
+
+Turbo is the distilled text-to-image checkpoint. Its `model_index.json`
+declares `BooguImageTurboPipeline`, which runs the four-step DMD path instead
+of the scheduler-driven path used by Base.
+
+### Command
+
+```bash
+vllm serve Boogu/Boogu-Image-0.1-Turbo --omni --port 8091
+```
+
+### Verification

 ```bash
 curl -X POST http://localhost:8091/v1/images/generations \
@@ -230,6 +247,23 @@ curl -X POST http://localhost:8091/v1/images/edits \
  checkpoint-declared FP8 configuration. With the regular checkpoints used for
  online FP8, the MLLM remains in BF16. The VAE and scheduler are outside this
  quantization routing.
+    "model": "Boogu/Boogu-Image-0.1-Turbo",
+    "prompt": "A mountain lake at sunset, photorealistic, cinematic lighting",
+    "size": "1024x1024",
+    "seed": 42
+  }' | jq -r '.data[0].b64_json' | base64 -d > output-turbo.png
+```
+
+### Notes
+
+- **Required settings:** `guidance_scale=1.0` and `guidance_scale_2=1.0`; any
+  other value is rejected. `num_inference_steps` defaults to 4. All three are
+  pipeline defaults, so the request above omits them.
+- **Resolution:** upstream constrains Turbo to 1K, so `1024x1024` is the
+  supported working resolution. The shared pipeline still clamps at 2K; larger
+  sizes are not validated for this checkpoint.
+- **Known limitations:** the same single-GPU limitations as Base apply; CPU
+  offload, Cache-DiT, and multi-GPU parallelism are not yet validated.

 ## Image editing (Boogu-Image-0.1-Edit)

@@ -354,16 +388,18 @@ curl -s http://localhost:8091/v1/chat/completions \
 ## Fast image editing (Boogu-Image-0.1-Edit-Turbo)

 Edit-Turbo is the distilled image-editing checkpoint. Its `model_index.json`
-declares `BooguImagePipeline`, so it uses the same native TI2I path as Edit.
-The upstream 1K hotfix is recommended for stable results and is pinned below
-to avoid silently loading the older checkpoint from the repository's default
-revision.
+declares `BooguImagePipeline`, so `--model-class-name BooguImageTurboPipeline`
+is required to reach the four-step DMD path; without it the checkpoint is
+served on the dense Edit path. The upstream 1K hotfix is recommended for
+stable results and is pinned below to avoid silently loading the older
+checkpoint from the repository's default revision.

 ### Command

 ```bash
 vllm serve Boogu/Boogu-Image-0.1-Edit-Turbo \
  --omni \
+  --model-class-name BooguImageTurboPipeline \
  --revision hotfix-1k-20260708 \
  --port 8091
 ```
@@ -407,11 +443,10 @@ curl -s http://localhost:8091/v1/chat/completions \

 - **Pinned revision:** use `hotfix-1k-20260708`; upstream recommends the 1K
  hotfix over the 1.5K variant for more stable results.
- **Recommended settings:** `num_inference_steps=4` and
-  `guidance_scale=1.0`. Edit-Turbo is guidance-distilled, meaning the guidance
-  behavior is already baked into the distilled checkpoint. A scale of `1.0`
-  avoids applying additional classifier-free guidance; unlike the regular Edit
-  checkpoint, it should not be increased to `5.0`.
+- **Required settings:** `guidance_scale=1.0` and `guidance_scale_2=1.0`; the
+  Turbo pipeline rejects any other value, so the `guidance_scale_2=2.0` shown
+  for the regular Edit checkpoint does not apply here. `num_inference_steps`
+  defaults to 4.
 - **Reference images:** the same single-reference and `align_res` behavior as
  the regular Edit checkpoint applies.
 - **Known limitations:** the same single-GPU limitations as Base and Edit apply;
@@ -219,3 +219,51 @@ the audio-track caveat.
  5 chunks, `key_frames=0` = summarize all chunk frames) are aligned to the JoyVL
  reference adapter so per-tick behavior matches the released model; the framework only
  supplies the serving structure.
+
+## Native multi-stage pipeline (experimental)
+
+An experimental native speech path is available alongside the existing Day-0 serving path:
+
+```text
+one request with image/video and text
+  -> JoyAI complete action text
+  -> Qwen3-TTS Talker
+  -> Code2Wav
+  -> 24 kHz audio
+```
+
+It runs behind one `vllm serve --omni` endpoint:
+
+```bash
+vllm serve jdopensource/JoyAI-VL-Interaction-Preview --omni \
+  --deploy-config vllm_omni/deploy/joyai_vl_interaction.yaml \
+  --port 8091
+```
+
+With `modalities: ["text", "audio"]`, its behavior is:
+
+| JoyAI action | Native output |
+|---|---|
+| `</response> <text>` | action text + speech for `<text>` |
+| `</silence>` | action text + empty audio output (zero samples) |
+| `</response> <note> </delegation> <question>` | action text + speech for `<note>` only |
+
+Native speech supports the Qwen3-TTS `CustomVoice` task. The request-level `voice` and
+`language` fields select the speaker and language; when omitted, they default to
+`Vivian` and `Auto`. Values supplied as `tts_speaker` and `tts_language` inside
+`additional_information` take priority.
+
+This experimental native multi-stage pipeline is request-scoped, stateless, and
+all-sync. It currently does not include:
+
+- continuous frame sessions, persistent standing instructions, or session memory;
+- audio input / ASR, full-duplex interaction, or barge-in;
+- background Agent execution for delegated questions;
+- async-chunk support from Talker to Code2Wav;
+- integration with the existing WebUI.
+
+For actions with spoken text, the Talker starts only after the complete JoyAI action is
+available, and Code2Wav starts only after the Talker completes. Silence actions skip both
+TTS stages and return an empty audio output with zero samples. All three stages are placed
+on GPU 0 by the provided
+[`deploy config`](../../vllm_omni/deploy/joyai_vl_interaction.yaml).
@@ -110,6 +110,29 @@ vllm serve Lightricks/LTX-2.5-Diffusers \

 Use the T2V or I2V request below after the server is ready.

+### Multi-GPU low-latency path
+
+For low-latency 1920x1088 distilled two-stage serving, match the Ulysses and
+VAE patch-parallel degrees to the GPU count. For example, on four GPUs:
+
+```bash
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+vllm serve Lightricks/LTX-2.5-Diffusers \
+  --omni \
+  --model-class-name LTX2DistilledTwoStagePipeline \
+  --usp 4 \
+  --vae-patch-parallel-size 4 \
+  --vae-use-tiling \
+  --host 0.0.0.0 \
+  --port 8000 \
+  --stage-init-timeout 900
+```
+
+Strict Ulysses only: `((F - 1) / 8 + 1) * (H / 32) * (W / 32)` and TP-local
+SP attention head counts must be divisible by `ulysses_degree`. Use each
+phase's `H, W` (half/full resolution for two-stage); non-divisible shapes fail.
+Each GPU must still fit the resident weights. Warm up with the production shape.
+
 ## Offline inference

 Choose values from the pipeline table. For example, the distilled two-stage
@@ -101,6 +101,12 @@ sigmas and input latents.
 Ordinary two-stage uses layer-fused LoRA for an unquantized BF16 Transformer
 and automatically switches to dynamic LoRA when quantization is enabled.

+## Sequence Parallelism
+
+Strict Ulysses only: `((F - 1) / 8 + 1) * (H / 32) * (W / 32)` and TP-local
+SP attention head counts must be divisible by `ulysses_degree`. Use each
+phase's `H, W` (half/full resolution for two-stage); non-divisible shapes fail.
+
 ## Serving

 Start either one-stage checkpoint:
@@ -335,9 +341,8 @@ bundled offline CLI do not currently expose `sigmas`.
  `--max-num-seqs 1`.
 - `--cfg-parallel-size` shards the complete LTX guidance plan, including STG,
  modality guidance, and rescale-compatible prediction gathering.
- Sequence parallelism may pad audio latents. Pure Ulysses masks the padding;
-  Ring cannot, so audio length must be SP-divisible. Use `ring_degree=1` or a
-  divisible request shape.
+- Ulysses masks padded audio latents; video sequences must satisfy the
+  [sequence-parallel divisibility constraints](#sequence-parallelism).
 - Cache-DiT is one-stage only; multi-stage configurations are rejected.

 ## Operational Notes
@@ -68,16 +68,24 @@ The `/v1/videos` request schema and `extra_params.task` values (`t2va`,

 ## Turbo LoRA

-MiniMax-H3 Turbo uses five sigma points, `flow_shift=6`, and
-`audio_flow_shift=3`. Start Turbo deployments with the dedicated defaults so
-requests that omit sampling controls do not inherit the 50-step base schedule:
+The deploy config below carries the four-step 768p contract -- five sigma
+points, `flow_shift=6`, `audio_flow_shift=3` -- so requests that omit sampling
+controls do not inherit the 50-step base schedule:

 ```bash
 vllm-omni serve MiniMaxAI/MiniMax-H3 \
  --omni \
-  --lora-path /path/to/MiniMax-H3-Turbo \
+  --lora-path /path/to/minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors \
  --deploy-config vllm_omni/deploy/minimax_h3_disaggregated_turbo.yaml
 ```

+`--lora-path` must name one artifact, or a directory holding exactly one; the
+Turbo repository holds several and a directory of them is rejected as
+ambiguous. Serving a different artifact means overriding those defaults with
+that artifact's own contract -- see the
+[Turbo LoRA table](MiniMax-H3.md#turbo-lora) for every published row.
+
 The base deployment intentionally retains 50 inference steps for non-LoRA
-quality. Turbo LoRA supports T2VA and FL2VA requests, not Ref2VA.
+quality. This disaggregated topology is validated on the `fl2v` artifacts,
+which serve T2VA and FL2VA; the `ref2v` artifacts need a `--task-type ref2va`
+server.
@@ -7,13 +7,14 @@ recipes it uses **no offload of any kind** — see the capacity note below.
 Validated on:

 - Host: DGX Spark (GB10), aarch64
- GPUs: 1 (unified memory)
- Driver: <FILL_DRIVER_VERSION>
+- GPUs: 1, or 2 hosts with one unified-memory GPU each
+- Driver: 580.173.02 for the two-host qualification
 - vLLM: 0.26.0
 - vLLM-Omni: `main` at `e1aa6eae75c460cd1893bc320546e81e66973831`
- Workloads: T2VA and Ref2VA, 960x576, `duration=8.0`, 50 steps, `flow_shift=12`,
-  `seed=1101`
-
+- One-host workloads: T2VA and Ref2VA, 960x576, `duration=8.0`, 50 steps
+- Two-host workload: T2VA, 1344x768, `duration=5.0` and `duration=10.0`,
+  50 steps, text-encoder TP2, and DiT USP2
+- Common sampling settings: `flow_shift=12`, `seed=1101`

 ## Capacity requirements

@@ -24,6 +25,7 @@ Validated on:
 | Checkpoint storage | 135 GiB per partition |
 | Measured peak allocator high-water, T2VA (FP8) | 97.7 GiB |
 | Measured peak allocator high-water, Ref2VA (FP8) | 102.8 GiB |
+| Measured peak per rank, two-host T2VA 5 s / 10 s (FP8) | 75.45 / 87.17 GiB |

 `FL2VA` and `Ref2VA` are separate 135 GiB checkpoint partitions. Start one server
 at a time.
@@ -164,6 +166,41 @@ curl --fail-with-body -sS -X POST http://127.0.0.1:8000/v1/videos/sync \
 The response is `200 OK` with an MP4 body: H.264 at the requested geometry plus a
 32 kHz stereo AAC track. `ffprobe` reports 8.032 s for `duration=8.0`.

+## Two GB10 hosts: T2VA at 1344x768
+
+The two-host qualification uses one GB10 GPU per DGX Spark, connected through a
+single ConnectX-7 RoCE interface. The FL2VA checkpoint and software revision are
+identical on both hosts. The distributed configuration is:
+
+- world size 2;
+- text-encoder tensor parallel size 2;
+- DiT Ulysses sequence parallel size 2 and ring degree 1;
+- VAE patch parallel size 1;
+- online FP8, eager execution, cuDNN attention, and tiled VAE decode.
+
+The current diffusion multiprocess executor starts all ranks on one host and sets
+`MASTER_ADDR=localhost`. These measurements therefore used an external `env://`
+launcher with one rank on each host and a benchmark-only adaptation that maps
+both hosts' local worker to `cuda:0`. This qualifies the two-rank MiniMax-H3
+pipeline and its output, but it is not yet a supported cross-host `vllm serve`
+command. Do not infer that the single-host launch command above works unchanged
+across two machines.
+
+One 5-second, 2-step request warmed both workers before the measured runs. The
+reported pipeline elapsed time covers distributed model execution through GPU
+completion; MP4 encoding happened afterward and is not included.
+
+| Duration | Steps | Text encode | Denoise | Per step | VAE decode | Pipeline elapsed | Max peak per rank |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
+| 5 s | 50 | 0.21 s | 1570.19 s | 31.40 s | 79.45 s | 1649.87 s (27 min 30 s) | 75.45 GiB |
+| 10 s | 50 | 0.15 s | 4448.12 s | 88.96 s | 160.71 s | 4609.00 s (76 min 49 s) | 87.17 GiB |
+
+Both ranks reported stage times and allocator peaks within measurement noise.
+The generated outputs were H.264 at 1344x768 and 24 FPS with 32 kHz stereo AAC:
+124 frames (5.167 s) and 243 frames (10.125 s). Both files decoded successfully,
+and neither run encountered an OOM or NCCL error. These are single qualification
+runs (`n=1`), not throughput guarantees.
+
 ## Measured latency

 Single GB10, FP8, eager, cuDNN attention, tiled VAE, one request at a time:
@@ -238,9 +275,11 @@ shorter than a 50-step Ref2VA run.

 ## Known limitations

- One GPU means no Ulysses, ring, TP, or VAE patch parallelism. Pass
+- With one GPU there is no Ulysses, ring, TP, or VAE patch parallelism. Pass
  `--usp 1 --ring 1 --vae-patch-parallel-size 1` explicitly if a profile or
  script would otherwise supply higher degrees.
+- The two-host results require the external launch adaptation described above;
+  cross-host diffusion worker launch is not exposed by the documented server CLI.
 - `--enforce-eager` is used because regional compile adds startup time without a
  parallelism win on a single rank.
 - `t2va` rejects requests without an explicit `aspect_ratio`. When `width` and
@@ -255,8 +294,7 @@ shorter than a 50-step Ref2VA run.
  adding references or raising the output shape.
 - Ref2VA requires a vLLM-Omni build newer than `v0.26.0`. See the version note
  under **Validated on**: FP8 weight loading and image-only Ref2VA are both
-  broken on `release/v0.26.0`.suggestion is  `v0.26.1`.
+  broken on `release/v0.26.0`. The suggested version is `v0.26.1`.
 - Online FP8 is incompatible with layerwise offload — the offload path produces a
  weight stride the Cutlass FP8 kernel rejects. This is not a practical
  restriction here since offload is unusable on GB10 anyway.
-```
@@ -869,13 +869,51 @@ balanced switch order.

 ### Turbo LoRA

-Only the native Diffusers 4-step FL2VA/T2VA v1.0 artifact is supported:
+The eight Diffusers-layout LightX2V Turbo artifacts are supported. The
+filename records the contract, so the server reads the step count, task family
+and flow shift from it and validates each request against the artifact that is
+loaded. It does not rewrite request or deploy-config sampling values: the
+request must carry that artifact's own settings, listed here, or it is
+rejected.

-```text
-lightx2v/Minimax-h3-Turbo/minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors
-```
+| Artifact | Task | Forwards | `num_inference_steps` | `flow_shift` | declared `alpha` |
+| --- | --- | ---: | ---: | ---: | ---: |
+| `minimax_h3_fl2v_turbo_4step_v0.1.safetensors` | T2VA / FL2VA | 4 | 5 | 12 | none -> 8 |
+| `minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors` | T2VA / FL2VA | 4 | 5 | 6 | 128 |
+| `minimax_h3_fl2v_turbo_4step_v1.1_768p_bf16.safetensors` | T2VA / FL2VA | 4 | 5 | 6 | 128 |
+| `minimax_h3_fl2v_turbo_4step_v1.2_768p_bf16.safetensors` | T2VA / FL2VA | 4 | 5 | 6 | 8 |
+| `minimax_h3_fl2v_turbo_8step_v1.0_bf16.safetensors` | T2VA / FL2VA | 8 | 9 | 12 | 8 |
+| `minimax_h3_fl2v_turbo_8step_v1.0_768p_bf16.safetensors` | T2VA / FL2VA | 8 | 9 | 6 | 8 |
+| `minimax_h3_ref2v_turbo_4step_v0.1_bf16.safetensors` | Ref2VA | 4 | 5 | 12 | 8 |
+| `minimax_h3_ref2v_turbo_8step_v1.0_768p_bf16.safetensors` | Ref2VA | 8 | 9 | 6 | 8 |

-Download only that file:
+`audio_flow_shift` is `3.0` across the family. Each row is the complete
+published filename; use it verbatim as `TURBO_FILE` below.
+
+Alpha needs no manual compensation: the server reads it from the artifact's
+metadata, falling back to 8 with a warning for
+`minimax_h3_fl2v_turbo_4step_v0.1`, the one artifact that declares none. The
+request-level `scale` is a further multiplier on top of it.
+
+> [!NOTE]
+> Every artifact is rank 128 and the delta is applied at `scale * alpha /
+> rank`, so the `alpha=8` rows drive at 1/16 the strength of the `alpha=128`
+> rows. 8 is the default of LightX2V's reference script, which never reads the
+> metadata; its documented `v0.1` command does not override that default.
+
+Every artifact except `minimax_h3_fl2v_turbo_4step_v0.1` also ships a
+`_comfyui_` export of the same weights. Those fuse Q/K/V into one projection
+and are **not** supported; downloading one is refused by name. Take the
+Diffusers file. The filename is the contract, so do not rename an artifact
+either -- a renamed file is rejected rather than served on a guess.
+
+FL2VA artifacts serve `t2va` and `fl2va` on any FL2VA or combined server.
+Ref2VA artifacts require
+`--task-type ref2va`: a combined server serves `ref2va` from a second DiT that
+the adapter cannot bind to, so loading one there is refused rather than silently
+running an undistilled model on the few-step schedule.
+
+Download the artifact you want:

 ```bash
 export TURBO_DIR=/path/to/minimax-h3-turbo
@@ -884,10 +922,19 @@ hf download lightx2v/Minimax-h3-Turbo "${TURBO_FILE}" --local-dir "${TURBO_DIR}"
 export TURBO_LORA="${TURBO_DIR}/${TURBO_FILE}"
 ```

+`--lora-path` accepts one artifact, or a directory holding exactly one.
+
+> [!IMPORTANT]
+> This changes earlier behaviour. `--lora-path /path/to/minimax-h3-turbo`
+> pointing at a full clone of the Turbo repository used to select the v1.0 768p
+> file implicitly; a directory holding several recognized artifacts is now
+> rejected as ambiguous. Name the artifact instead:
+> `--lora-path /path/to/minimax-h3-turbo/minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors`.
+
 Start from a non-offloaded or DLO FL2VA server command and add
 `--task-type fl2va --lora-backend peft --lora-path "${TURBO_LORA}"`.
-`--lora-path` preloads the adapter; each request still activates it and uses
-the published sampling settings:
+`--lora-path` preloads the adapter; each request still activates it and must
+carry that artifact's sampling settings:

 ```bash
 -F 'num_inference_steps=5' \
@@ -896,14 +943,25 @@ the published sampling settings:
 -F "lora={\"name\":\"h3-turbo-v1.0\",\"path\":\"${TURBO_LORA}\",\"scale\":1.0}"
 ```

-For FL2VA, change `task` and add `input_reference` as shown above. The 8-step,
-ComfyUI, Ref2VA, and v1.1 artifacts are not supported. This integration is
-dynamic-only and does not support prefusion or LoRA composition. DLO is
+Switching to another FL2VA artifact means repointing `TURBO_FILE`, which moves
+both `--lora-path` and the request's `lora.path`, and carrying that row's
+`num_inference_steps` and `flow_shift`: `9` and `6` for `8step_v1.0_768p`, `9`
+and `12` for the 544p `8step_v1.0`. A request that does not match the loaded
+artifact is rejected, so a mismatch cannot silently degrade output.
+
+The two `ref2v` rows are not served by this FL2VA command. Start a
+`--task-type ref2va` server, take a request from
+[Ref2VA](#3-ref2va-image-only-imageaudio-or-mixed-references) and override
+`num_inference_steps` and `flow_shift` with that row's values; those examples
+already send `audio_flow_shift=3.0`.
+
+For FL2VA, change `task` and add `input_reference` as shown above. This
+integration is dynamic-only and does not support prefusion or LoRA composition. DLO is
 supported by keeping the request-switchable LoRA A/B buffers resident on the
 accelerator while DLO streams only the base blocks; budget for this additional
 fixed HBM usage. Model-level and standard layerwise offload remain unsupported.
-The five requested sigma points produce the four denoiser evaluations expected
-by the Turbo artifact.
+The requested sigma points always number one more than the artifact's denoiser
+evaluations.

 ### FlashGen native LoRA

@@ -16,7 +16,7 @@ Use this recipe for 24 kHz multilingual TTS with voice cloning (20 languages inc
 Chinese and English). Choose a variant based on your latency and quality requirements:

 | Model | Params | Use case |
-|---|---|---|
+| --- | --- | --- |
 | MOSS-TTS | 8B | General TTS, highest quality |
 | MOSS-TTS-v1.5 | 8B | General TTS upgrade of 1.0: 31 languages, steadier cloning, `[pause Xs]` markers (set `language` for best results); same `MossTTSDelay` API |
 | MOSS-TTS-Realtime | 1.7B | Lowest latency (TTFB ~180 ms), streaming-first |
@@ -24,9 +24,17 @@ Chinese and English). Choose a variant based on your latency and quality require
 | MOSS-SoundEffect | 8B | Sound effect synthesis from text description |
 | MOSS-VoiceGenerator | 1.7B | Zero-shot voice design |

-All variants share the same codec (`OpenMOSS-Team/MOSS-Audio-Tokenizer`, ~7 GB) and
+The variants above share the same codec (`OpenMOSS-Team/MOSS-Audio-Tokenizer`, ~7 GB) and
 output 24 kHz mono audio.

+MOSS-TTS-Local-Transformer-v1.5 uses MOSS-Audio-Tokenizer-v2 and outputs 48 kHz
+stereo audio. For Local voice cloning through `/v1/audio/speech`, provide an
+accurate `ref_text` transcript alongside `ref_audio`. The adapter uses the
+reference transcript and audio as a continuation prefix, then generates only
+the requested `input` speech. Without a nonblank `ref_text`, Local uses
+audio-reference generation. Reference transcripts must match the reference
+audio; they are not style instructions.
+
 ## References

 - Offline inference example: [`examples/offline_inference/text_to_speech/moss_tts/`](../../examples/offline_inference/text_to_speech/moss_tts/)
@@ -263,6 +263,94 @@ curl -X POST http://localhost:8091/v1/audio/speech \
      gpu_memory_utilization: 0.15
  ```

+### 1x A100 40GB (0.6B CustomVoice)
+
+#### Environment
+
+- OS: Linux 4.18.0-553 (RHEL 8.10), x86_64
+- Python: 3.12.14
+- PyTorch: 2.13.0+cu129
+- Driver / runtime: NVIDIA 570.124.06 / CUDA 12.8 (`nvidia-smi`); PyTorch CUDA 12.9
+- GPU: NVIDIA A100-SXM4-40GB, 40960 MiB
+- vLLM version: 0.28.0+cu129
+- vLLM-Omni version or commit: `3204a0b2ade0f05424b7f11e3bcc03cb3542e0c6`
+
+#### Command
+
+```bash
+vllm serve Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice \
+    --deploy-config vllm_omni/deploy/qwen3_tts.yaml \
+    --omni --port 8091
+```
+
+The default deploy config (`qwen3_tts.yaml`) works without modification on this
+A100 40GB. Both stages (talker + code2wav) share GPU 0 with
+`gpu_memory_utilization: 0.3` each.
+
+#### Verification
+
+**English synthesis:**
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Hello, this is Qwen3-TTS running on A100 40GB.",
+        "voice": "vivian",
+        "language": "English"
+    }' --output test_english.wav
+```
+
+**Chinese synthesis:**
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "你好，这是在A100 40GB上运行的语音合成测试。",
+        "voice": "vivian",
+        "language": "Chinese"
+    }' --output test_chinese.wav
+```
+
+**With emotion instruction:**
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "I am so excited about this!",
+        "voice": "vivian",
+        "language": "English",
+        "instructions": "Speak with great enthusiasm"
+    }' --output test_emotion.wav
+```
+
+#### Notes
+
+- Memory usage: **16283 MiB / 40960 MiB** peak (`nvidia-smi`) during the three
+  verification requests with the default deploy config
+  (`gpu_memory_utilization: 0.3` per stage). Stage 0 used 12404 MiB and stage 1
+  used 3862 MiB. The 0.6B weights occupy only ~2.4 GiB (Stage 0: 1.91 GiB,
+  Stage 1: 0.45 GiB); the rest is KV cache reserved as a fraction of device
+  memory, so the same `0.3` setting reads higher in GiB on 40GB than the
+  ~13.5 GiB idle figure on the 24GB 4090 profile above. To shrink the
+  reservation, use the `gpu_memory_utilization: 0.15` override documented in
+  that 4090 section.
+- vLLM 0.28.0's default wheel targets CUDA 13.0. Driver 570 cannot load that
+  variant. This qualification used the official CUDA 12.9 wheel (same pin as
+  the A100 Wan2.2 recipe):
+
+  ```bash
+  uv pip install \
+    'https://github.com/vllm-project/vllm/releases/download/v0.28.0/vllm-0.28.0%2Bcu129-cp38-abi3-manylinux_2_28_x86_64.whl' \
+    --torch-backend=cu129
+  ```
+
+  Confirm `torch.__version__` contains `cu129`. If a CPU build is already
+  installed, `uv pip install --torch-backend=cu129` is a no-op; use
+  `--reinstall`.
+
 ### 1x AMD MI300X, 1.7B checkpoints

 #### Environment
@@ -42,6 +42,7 @@ recipes/
 | -------- | ------ | ---------- |
 | [`Baidu/ERNIE-Image.md`](./Baidu/ERNIE-Image.md) | Text-to-image online serving (ERNIE-Image 8B) | 1x or 2x RTX 4090 24GB |
 | [`Bagel/BAGEL-7B-MoT.md`](./Bagel/BAGEL-7B-MoT.md) | Text-to-image with shared online/offline examples | 1x A100 80GB / 2x CUDA GPUs |
+| [`black-forest-labs/FLUX.1-schnell.md`](./black-forest-labs/FLUX.1-schnell.md) | Offline text-to-image generation with CPU offload | 1x NVIDIA GeForce RTX 5090 32GB |
 | [`Boogu/Boogu-Image.md`](./Boogu/Boogu-Image.md) | Text-to-image online serving (Boogu-Image-0.1-Base) | 1x A100/H100 40GB+ |
 | [`BosonAI/Higgs-Audio-V3-TTS.md`](./BosonAI/Higgs-Audio-V3-TTS.md) | Online + offline multilingual TTS with voice cloning | 1x H100 80GB |
 | [`ByteDance/Lance.md`](./ByteDance/Lance.md) | Unified AR+diffusion: text/img/video gen + understanding (Lance 3B) | 1x B300 / A100 80GB |
@@ -54,6 +55,7 @@ recipes/
 | [`IndexTeam/IndexTTS-2.md`](./IndexTeam/IndexTTS-2.md) | Online serving for voice-cloned TTS with optional emotion control | 1x L4 24GB or larger CUDA GPU |
 | [`IndexTeam/IndexTTS-2_5.md`](./IndexTeam/IndexTTS-2_5.md) | Online + offline multilingual voice-cloned TTS with native speed and emotion control | 1x NVIDIA H20 96GB |
 | [`MiniMaxAI/MiniMax-H3.md`](./MiniMaxAI/MiniMax-H3.md) | T2VA, FL2VA, image+audio Ref2VA, and multi-video Ref2VA serving | 1x GPU with CPU offload / 4x B300 |
+| [`MiniMaxAI/MiniMax-H3-Spark-GB10.md`](./MiniMaxAI/MiniMax-H3-Spark-GB10.md) | MiniMax H3 T2VA and Ref2VA on DGX Spark | 1x or 2x DGX Spark (GB10) |
 | [`MiniMaxAI/MiniMax-H3-MUSA.md`](./MiniMaxAI/MiniMax-H3-MUSA.md) | MiniMax H3 T2VA, FL2VA, and Ref2VA on MUSA | 4x MTT S5000 (TP4 Ref2VA profile) |
 | [`krea/Krea-2.md`](./krea/Krea-2.md) | Text-to-image (Turbo + Raw), offline + online, with LoRA | 1x H100 80GB |
 | [`LTX/LTX-2.md`](./LTX/LTX-2.md) | LTX-2/LTX-2.3 text-to-video and image-to-video with synchronized audio | H200 141GB / 96GB-class GPU |
@@ -85,6 +87,7 @@ recipes/
 | [`Tencent/Covo-Audio-Chat.md`](./Tencent/Covo-Audio-Chat.md) | Online serving for audio chat | 1x A100 80GB |
 | [`Tencent/HunyuanImage-3.0-Instruct.md`](./Tencent/HunyuanImage-3.0-Instruct.md) | DiT-only text-to-image serving and benchmark, including ModelOpt mixed FP8/NVFP4 | 4x H100/H800 80GB / 2x B200 |
 | [`Wan-AI/Wan2.2-T2V.md`](./Wan-AI/Wan2.2-T2V.md) | Text-to-video serving (Wan2.2 14B) with Skip-Softmax sparse attention | 1x datacenter Blackwell (B200/B300) |
+| [`Wan-AI/Wan2.2-TI2V-5B-A100.md`](./Wan-AI/Wan2.2-TI2V-5B-A100.md) | Unified 720p text-to-video and image-to-video serving (Wan2.2 5B) | 1x A100-SXM4-80GB |
 | [`Wan-AI/Wan2.2-I2V.md`](./Wan-AI/Wan2.2-I2V.md) | Image-to-video serving (Wan2.2 14B) | 8x Ascend NPU (A2/A3) |
 | [`Wan-AI/Wan2.2-S2V.md`](./Wan-AI/Wan2.2-S2V.md) | Speech-to-video serving (Wan2.2 14B) | 2x A100/H100 80GB |
 | [`Wan-AI/Wan2.1-VACE.md`](./Wan-AI/Wan2.1-VACE.md) | Unified T2V, I2V, V2LF, FLF2V, inpaint, and R2V | 1x RTX 5090 (1.3B) / 1x L40S 48GB with layerwise offload (14B) |
@@ -7,13 +7,31 @@
 - Vendor: Robbyant
 - Model: `robbyant/lingbot-world-v2-14b-causal-fast-diffusers`
 - Task: image-conditioned interactive world generation
- Modes: offline trajectory replay and in-process realtime AR-Diffusion ticks
+- Modes: offline trajectory replay, stepwise one-request streaming, and in-process realtime AR-Diffusion ticks (deprecated)
 - Hardware validated: NVIDIA H200 and B200
 - Maintainer: Community

 The checkpoint is separately licensed under CC BY-NC-SA and restricted to
 non-commercial use. The vLLM-Omni integration code remains Apache-2.0.

+## Which path to use
+
+There are three ways to drive this model, each documented in its own section
+below.
+
+- **Realtime stepwise** — see [Streaming video serving](#streaming-video-serving).
+  The suggested path: one `WS /v1/realtime/video` session produces the whole
+  rollout, one video chunk per AR block. It cannot take realtime camera
+  interaction yet; that is being added, and this path is the one that will
+  remain.
+- **Realtime tick** *(deprecated)* — see
+  [Realtime in-process generation](#realtime-in-process-generation-deprecated).
+  The only path that takes realtime camera interaction today, so it stays until
+  the stepwise path can, but it takes no new work and will be removed.
+- **Offline** — see [Offline generation](#offline-generation). Replays a fixed
+  pose/intrinsics trajectory in one request and writes an MP4. Use it when the
+  camera path is known up front and streaming is not needed.
+
 ## Offline generation

 The offline path consumes one source image and an action directory containing
@@ -34,7 +52,12 @@ counts must be `9 + 12k`, up to 117 frames. Camera files may contain a
 longer bounded source trajectory (the official example contains 269 frames);
 the runtime consumes only the prefix needed by the request.

-## Realtime in-process generation
+## Realtime in-process generation (deprecated)
+
+Use [Streaming video serving](#streaming-video-serving) instead unless you need
+mid-session camera interaction, which only this path has. It still works and
+emits no runtime warning; removal is tracked as B4 of the LingBot World 2.0
+roadmap ([#6672](https://github.com/vllm-project/vllm-omni/issues/6672)).

 The realtime example keeps AR-Diffusion KV and model-owned state across
 requests. Each JSONL line describes the prompt and/or three latent-frame camera
@@ -61,12 +84,81 @@ exercises the same `ARDiffusionSessionManager -> ARDiffusionOmniTickConsumer
 -> AsyncOmni -> ARDiffusionEngine` path used by a future HTTP or WebSocket
 transport.

-This PR intentionally does not define a public realtime HTTP/WebSocket schema.
-An online serving client should be added together with the public transport
-API rather than exposing LingBot-specific event fields from the generic model
-runtime. The structured camera-interaction frontend and transport work is
-tracked separately in
-[vllm-project/vllm-omni#5527](https://github.com/vllm-project/vllm-omni/pull/5527).
+Realtime serving now goes through the generic `WS /v1/realtime/video`
+transport, which exposes no LingBot-specific event fields; the tick loop above
+stays available only for the older control plane.
+
+## Streaming video serving
+
+The stepwise path keeps AR-Diffusion paged KV but issues **one** request for
+the whole rollout: `prepare_encode` runs once, then every AR block is four DMD
+steps followed by one streamed chunk. Identity metadata uses
+`session_id = request_id` with contiguous `chunk_index` values from zero.
+Camera motion for this landing is request-scoped: one three-frame action list
+per chunk, fixed when the rollout starts.
+
+Serve it with the AR-Diffusion deploy config, which selects the AR-Diffusion
+engine and enables streamed step execution:
+
+```bash
+vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+  --omni \
+  --deploy-config vllm_omni/deploy/lingbot_world_v2_stepwise.yaml \
+  --port 8000
+```
+
+`--deploy-config` is required: the `lingbot_world` pipeline deliberately
+registers no default deploy config, so offline replay and the tick example
+keep their request-mode topology when no deploy config is given, and the
+stepwise serving topology is only ever an explicit choice.
+
+Clients then use the generic WebSocket protocol documented in
+[`docs/serving/video_stream_api.md`](../../docs/serving/video_stream_api.md):
+`session.start` begins one rollout and each AR block arrives as a binary video
+chunk. The model is image-conditioned, so the first frame is required and rides
+on `image_reference` as an `http(s)` or `data:` URL; per-chunk camera actions
+ride on `extra_params`:
+
+```json
+{"type": "session.start", "model": "robbyant/lingbot-world-v2-14b-causal-fast-diffusers",
+ "prompt": "The camera moves slowly forward through the scene.",
+ "image_reference": {"image_url": "data:image/png;base64,<first frame>"},
+ "width": 832, "height": 480, "num_frames": 33,
+ "extra_params": {"flow_shift": 5.0,
+                  "camera_action_script": [[["w"], ["w"], ["w"]], [["a"], [], []], [[], [], []]]}}
+```
+
+`camera_action_script` must hold exactly one action list per generated
+chunk, and a request generates `((num_frames - 1) // 4 + 1) // 3` chunks:
+three for `num_frames: 33`, seven for `num_frames: 81`.
+
+The bundled client speaks this protocol:
+
+```bash
+python examples/online_serving/streaming_video_generation/streaming_video_client.py \
+  --model robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+  --prompt "The camera moves slowly forward through the scene." \
+  --image-reference /path/to/first_frame.png \
+  --width 832 --height 480 --num-frames 33 --fps 16 --seed 42 \
+  --extra-params '{"camera_action_script":[[["w"],["w"],["w"]],[["a"],[],[]],[[],[],[]]]}' \
+  --output lingbot_world_v2_stream.mp4
+```
+
+A served request may instead point at a pose/intrinsics trajectory with
+`extra_params.action_path`, which is resolved inside the trusted root set by
+`model_config.lingbot_action_root` or `VLLM_OMNI_LINGBOT_ACTION_ROOT`; a server
+started without that root configured accepts only `camera_action_script`.
+
+Requested `width`/`height` must match `ar_diffusion_width`/`ar_diffusion_height`
+in the deploy config, because the AR cache geometry is fixed at load time.
+Blocks are decoded independently, so seams between chunks are possible; a
+session-owned streaming decoder is tracked separately.
+
+Mid-session `session.interaction` for camera control is not wired yet, so a
+served rollout follows the script it started with.
+
+The deprecated tick example remains available for the older
+one-block-per-`generate()` control plane.

 ## Realtime identity and controls

@@ -104,10 +196,16 @@ tested commit.
 ## Current limitations

 - Only the 14B causal-fast checkpoint is supported.
- The realtime control plane is internal; there is no public server transport yet.
+- The tick control plane is internal; the public transport is the stepwise
+  `WS /v1/realtime/video` path, which cannot take mid-session camera
+  interaction yet.
+- Stepwise serving requires an explicit
+  `--deploy-config vllm_omni/deploy/lingbot_world_v2_stepwise.yaml`; there is
+  no default deploy config for this model.
 - AR-Diffusion stages currently require one replica because session-affine
  routing across replicas is not implemented.
- One AR block is generated per request and `max_num_seqs` must be one.
+- Tick mode generates one AR block per request. Stepwise mode generates many
+  AR blocks in one request. `max_num_seqs` must be one in both cases.
 - Stateful streaming VAE decode is not implemented; the realtime example emits
  latent chunks.
 - SP/USP, pipeline/CFG parallelism, HSDP, VAE parallelism, quantization,
@@ -119,4 +217,7 @@ tested commit.
 - Checkpoint: <https://huggingface.co/robbyant/lingbot-world-v2-14b-causal-fast-diffusers>
 - Official implementation: <https://github.com/robbyant/lingbot-world-v2>
 - Offline example: [`examples/offline_inference/diffusion/lingbot_world_v2.py`](../../examples/offline_inference/diffusion/lingbot_world_v2.py)
+- Streaming serving client: [`examples/online_serving/streaming_video_generation/`](../../examples/online_serving/streaming_video_generation/README.md)
+- Streaming serving end-to-end test: [`tests/e2e/online_serving/test_lingbot_world_v2_stepwise.py`](../../tests/e2e/online_serving/test_lingbot_world_v2_stepwise.py)
+- Stepwise latent-level end-to-end test: [`tests/e2e/offline_inference/test_lingbot_world_v2_stepwise.py`](../../tests/e2e/offline_inference/test_lingbot_world_v2_stepwise.py)
 - Realtime design: [`docs/design/feature/realtime_ar_diffusion.md`](../../docs/design/feature/realtime_ar_diffusion.md)
@@ -0,0 +1,184 @@
+# Wan2.2 TI2V 5B — A100 80GB
+
+> Single-GPU, native 720p text-to-video and image-to-video serving
+
+## Summary
+
+- Vendor: Wan-AI
+- Model: `Wan-AI/Wan2.2-TI2V-5B-Diffusers`
+- Task: Unified text-to-video (T2V) and image-to-video (I2V) generation
+- Mode: Online serving with the OpenAI-compatible Videos API
+- Hardware: 1x NVIDIA A100-SXM4-80GB
+- Maintainer: Community
+
+## When to use this recipe
+
+Use this recipe to serve both T2V and I2V from one dense 5B checkpoint on one
+A100 80GB. It qualifies the model's native 720p profile without CPU offload or
+multi-GPU parallelism. The synchronous endpoint is used below because it
+returns the MP4 and request timing in one response; the asynchronous Videos API
+uses the same server and sampling parameters.
+
+## Supported model contract
+
+| Task | Required input | Qualified output | Endpoint |
+| --- | --- | --- | --- |
+| T2V | Text prompt; omit `input_reference` | 1280x704, 121 frames, 24 fps | `POST /v1/videos/sync` |
+| I2V | Text prompt plus one image uploaded as `input_reference` | 1280x704, 121 frames, 24 fps | `POST /v1/videos/sync` |
+
+The upstream checkpoint supports both tasks at 720p and 24 fps. For native
+landscape 720p, use `1280x704`; portrait generation uses `704x1280`. Wan video
+lengths follow the temporal VAE contract (`4k+1` frames). This recipe validates
+one output per request with 50 denoising steps and seed 42.
+
+## References
+
+- Upstream model card:
+  <https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B-Diffusers>
+- Online T2V example:
+  [`examples/online_serving/text_to_video`](../../examples/online_serving/text_to_video)
+- Online I2V example:
+  [`examples/online_serving/image_to_video`](../../examples/online_serving/image_to_video)
+- Offline I2V example:
+  [`examples/offline_inference/image_to_video`](../../examples/offline_inference/image_to_video)
+- Community recipe tracker:
+  [vllm-project/vllm-omni#2645](https://github.com/vllm-project/vllm-omni/issues/2645)
+
+## Hardware
+
+- Accelerator model and per-device memory: NVIDIA A100-SXM4-80GB, 81,920 MiB
+- Number of devices: 1
+- Device interconnect: not applicable to this single-GPU profile
+- Host memory: 1 TiB installed on the qualification host; CPU offload was not used
+- Qualification scope: BF16, batch size 1, native FlashAttention, eager mode,
+  VAE tiling, T2V and I2V at 1280x704 / 121 frames / 24 fps
+
+## Software environment
+
+- OS: Ubuntu 24.04.3 LTS, Linux 6.8.0
+- Python: 3.12.3
+- NVIDIA driver / host toolkit: 550.127.05 / CUDA 12.8
+- PyTorch: 2.13.0+cu129
+- Transformers / Diffusers: 5.14.1 / 0.40.0
+- vLLM: 0.28.0+cu129
+- vLLM-Omni: commit `ff6e906a25de1ca3122c0d0d5250fde5a0ddbc7b`
+
+The default vLLM 0.28.0 wheel targets CUDA 13.0. Driver 550 cannot run that
+variant, so this qualification used the official CUDA 12.9 wheel:
+
+```bash
+uv venv --python 3.12 --seed
+source .venv/bin/activate
+uv pip install \
+  'https://github.com/vllm-project/vllm/releases/download/v0.28.0/vllm-0.28.0%2Bcu129-cp38-abi3-manylinux_2_28_x86_64.whl' \
+  --torch-backend=auto
+uv pip install -e .
+```
+
+## Command
+
+Start the server from the repository root:
+
+```bash
+vllm serve Wan-AI/Wan2.2-TI2V-5B-Diffusers \
+  --omni \
+  --port 8092 \
+  --vae-use-tiling \
+  --enforce-eager
+```
+
+In another terminal, download the public reference image:
+
+```bash
+wget -O cherry_blossom.jpg \
+  https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg
+```
+
+Generate a text-only video:
+
+```bash
+curl --fail-with-body -D t2v-headers.txt \
+  -X POST http://127.0.0.1:8092/v1/videos/sync \
+  -F "prompt=Two anthropomorphic cats in comfortable boxing gear and bright gloves fight intensely on a spotlighted stage, cinematic lighting, smooth motion." \
+  -F "negative_prompt=blurry, low quality, static, distorted, artifacts, watermark, text" \
+  -F "width=1280" -F "height=704" \
+  -F "num_frames=121" -F "fps=24" \
+  -F "num_inference_steps=50" -F "guidance_scale=5.0" \
+  -F "flow_shift=5.0" -F "seed=42" \
+  --output wan22-ti2v-5b-t2v.mp4
+```
+
+Generate an image-conditioned video from the same server:
+
+```bash
+curl --fail-with-body -D i2v-headers.txt \
+  -X POST http://127.0.0.1:8092/v1/videos/sync \
+  -F "input_reference=@cherry_blossom.jpg" \
+  -F "prompt=Cherry blossoms sway gently in the breeze while petals drift across the path, cinematic, smooth natural motion." \
+  -F "negative_prompt=blurry, low quality, static, distorted, artifacts, watermark, text" \
+  -F "width=1280" -F "height=704" \
+  -F "num_frames=121" -F "fps=24" \
+  -F "num_inference_steps=50" -F "guidance_scale=5.0" \
+  -F "flow_shift=5.0" -F "seed=42" \
+  --output wan22-ti2v-5b-i2v.mp4
+```
+
+## Verification
+
+Check both MP4 files with `ffprobe`:
+
+```bash
+for video in wan22-ti2v-5b-t2v.mp4 wan22-ti2v-5b-i2v.mp4; do
+  ffprobe -v error -select_streams v:0 -count_frames \
+    -show_entries stream=codec_name,width,height,r_frame_rate,nb_read_frames,duration \
+    -of default=noprint_wrappers=1 "$video"
+done
+```
+
+Each file reported:
+
+```text
+codec_name=h264
+width=1280
+height=704
+r_frame_rate=24/1
+duration=5.041667
+nb_read_frames=121
+```
+
+The synchronous response headers and an independent 200 ms `nvidia-smi`
+sampler produced these steady-state results after server warmup:
+
+| Task | HTTP | `X-Inference-Time-S` | Server peak | Observed device peak | MP4 size |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| T2V | 200 | 288.267 s | 26,546 MB | 27,533 MiB | 7,321,179 bytes |
+| I2V | 200 | 299.439 s | 29,766 MB | 30,333 MiB | 15,667,146 bytes |
+
+Both outputs were decoded in full and visually inspected. The T2V output
+contained the prompted boxing cats. The I2V first frame preserved the uploaded
+cherry-blossom scene (5.81 mean absolute pixel error after resize and H.264
+encoding), and later frames showed motion.
+
+## Notes
+
+- Memory usage: model loading used 21.2239 GiB. The measured I2V device peak
+  was 30,333 MiB, leaving more than 50 GiB free on this A100.
+- `--vae-use-tiling` is retained in the known-good command even though this
+  profile has ample headroom; it bounds decode memory at 720p.
+- `--enforce-eager` removes graph-compilation variability from the measured
+  profile. Remove it only after separately warming and measuring compiled mode.
+- `flow_shift=5.0` is the 720p setting. The shared 480p examples use 12.0.
+- The generated MP4 contains video only; this checkpoint does not generate audio.
+- The first request includes neither model-load time nor the server's startup
+  warmup. Server initialization took approximately 26 seconds with a warm cache.
+
+## Supported features
+
+| Feature | Status for this profile | Shared guide |
+| --- | --- | --- |
+| OpenAI-compatible Videos API | Qualified for synchronous T2V and I2V | [Videos API](../../docs/serving/videos_api.md) |
+| VAE tiling | Enabled and qualified on one A100 | [VAE parallelism and tiling](../../docs/user_guide/diffusion/parallelism/vae_parallelism.md) |
+| Cache-DiT / TaylorSeer | Available but not enabled in the measurements above | [Cache-DiT](../../docs/user_guide/diffusion/cache_acceleration/cache_dit.md) |
+| TeaCache | Available but not qualified in this recipe | [TeaCache](../../docs/user_guide/diffusion/cache_acceleration/teacache.md) |
+| CPU or layerwise offload | Not needed for the qualified 80GB profile | [CPU offload](../../docs/user_guide/diffusion/cpu_offload.md) |
+| TP / SP / CFG parallelism | Not applicable to this single-GPU profile | [Parallelism overview](../../docs/user_guide/diffusion/parallelism/overview.md) |
@@ -0,0 +1,160 @@
+# FLUX.1-schnell
+
+> Offline text-to-image generation on one NVIDIA GeForce RTX 5090 32 GB
+
+## Summary
+
+- Vendor: Black Forest Labs
+- Model: `black-forest-labs/FLUX.1-schnell`
+- Task: Text-to-image generation
+- Mode: Offline inference with the shared text-to-image runner
+- Maintainer: Community
+
+## When to use this recipe
+
+Use this recipe as a personally validated starting point for generating a
+1024x1024 image with FLUX.1-schnell on one NVIDIA GeForce RTX 5090. The tested
+configuration uses BF16, four denoising steps and model-level CPU offload.
+
+This recipe covers single-image generation. It does not attempt to optimize
+throughput with quantization or cache acceleration.
+
+## References
+
+- Model card: <https://huggingface.co/black-forest-labs/FLUX.1-schnell>
+- Canonical offline text-to-image guide:
+  [`docs/user_guide/examples/offline_inference/text_to_image.md`](../../docs/user_guide/examples/offline_inference/text_to_image.md)
+- Shared runnable example:
+  [`examples/offline_inference/text_to_image`](../../examples/offline_inference/text_to_image)
+- CPU offload guide:
+  [`docs/user_guide/diffusion/cpu_offload.md`](../../docs/user_guide/diffusion/cpu_offload.md)
+- Community recipe tracker:
+  [#2645](https://github.com/vllm-project/vllm-omni/issues/2645)
+
+## Hardware Support
+
+## GPU
+
+### 1x NVIDIA GeForce RTX 5090 32 GB
+
+This configuration was personally validated end to end on September 9, 2026.
+
+#### Environment
+
+- OS: Ubuntu 22.04.3 x86_64
+- Python: 3.12.7
+- GPU: 1x NVIDIA GeForce RTX 5090, 32,607 MiB reported by `nvidia-smi`
+- Host RAM: approximately 62 GiB (64 GB class)
+- NVIDIA driver: 595.71.05; CUDA 13.2 reported by `nvidia-smi`
+- PyTorch: 2.13.0+cu130; PyTorch CUDA runtime: 13.0
+- vLLM: 0.28.0
+- vLLM-Omni: `0.28.1.dev112+ga31690041`
+- Diffusers: 0.40.0
+- Transformers: 5.14.1
+
+#### Command
+
+Run the shared example from the vLLM-Omni repository root in an environment
+with vLLM-Omni installed:
+
+```bash
+python examples/offline_inference/text_to_image/text_to_image.py \
+  --model black-forest-labs/FLUX.1-schnell \
+  --prompt 'A red fox standing in fresh snow, cinematic photography' \
+  --height 1024 --width 1024 \
+  --num-inference-steps 4 --guidance-scale 0 --seed 42 \
+  --enable-cpu-offload \
+  --output flux-schnell-rtx5090.png
+```
+
+The command uses the public model ID for portability. The measured runs used
+already downloaded FLUX.1-schnell weights; initial Hub download and
+authentication were not part of the inference validation.
+
+#### Verification
+
+Verify that the runner succeeded and inspect the saved image:
+
+```bash
+python - <<'PY'
+import hashlib
+from pathlib import Path
+
+from PIL import Image, ImageStat
+
+path = Path("flux-schnell-rtx5090.png")
+with Image.open(path) as image:
+    image.load()
+    assert image.format == "PNG"
+    assert image.size == (1024, 1024)
+    rgb = image.convert("RGB")
+    stats = ImageStat.Stat(rgb)
+    assert max(stats.stddev) > 1, "Image is nearly uniform"
+    print(f"format={image.format}")
+    print(f"size={image.width}x{image.height}")
+    print(f"mode={image.mode}")
+    print(f"bytes={path.stat().st_size}")
+    print("rgb_mean=" + ",".join(f"{value:.3f}" for value in stats.mean))
+    print(f"rgb_extrema={rgb.getextrema()}")
+
+print(f"sha256={hashlib.sha256(path.read_bytes()).hexdigest()}")
+PY
+```
+
+The repeated experiment completed 36 image requests (6 additional warmups and
+30 measured outputs) with successful process exits. Every PNG passed decoding,
+1024x1024 dimension and non-uniform pixel checks. SHA-256 hashes were recorded
+for the artifacts; matching those hashes is not required across environments.
+These checks establish file integrity, not a quantitative image-quality score.
+
+#### Performance
+
+The experiment used three independent processes. Each process initialized the
+model once, generated two warmup images, then generated ten measured images
+sequentially. All 30 measured images used BF16, CPU offload, 1024x1024, four
+steps, guidance scale 0 and TP=1, without concurrent requests.
+
+| Generation measurement | Result |
+| --- | ---: |
+| Measured images | 30 |
+| Mean latency | 25.8411 seconds |
+| P95 latency (nearest-rank) | 26.1446 seconds |
+
+| Startup / resource measurement | Maximum |
+| --- | ---: |
+| Model initialization (seconds) | 64.73 |
+| Full-lifetime RAM PSS peak (MiB) | 49306 |
+| Full-lifetime device GPU memory peak (MiB) | 28121 |
+| Measured PyTorch reserved GPU memory peak (MiB) | 27360 |
+
+Latency measures only `Omni.generate()`, excluding image saving and
+verification. The two warmup images per process are excluded. P95 uses the
+nearest-rank method over the 30 measured images. Model initialization includes
+the engine's built-in dummy warmup; weights were already downloaded and OS file
+caches were not cleared.
+
+RAM and whole-device GPU memory were sampled throughout each process lifetime
+at a target interval of 200 ms. RAM is the maximum simultaneous sum of PSS for
+the inference process and its workers, read from Linux `smaps_rollup`.
+Whole-device GPU memory is the maximum reported by `nvidia-smi`. Both are
+sampled peaks and may miss shorter spikes.
+
+The PyTorch value is the maximum worker-reported `peak_memory_mb` across the
+30 measured requests. Despite the field name, its unit is MiB; it measures the
+PyTorch allocator's reserved-memory high-water mark rather than whole-device
+usage.
+
+#### Notes
+
+- `--enable-cpu-offload` swaps the transformer and text encoders between host
+  and device memory at phase boundaries. It reduces GPU residency but adds
+  transfer overhead and requires sufficient host RAM. The minimum host RAM
+  requirement was not measured.
+- CPU offload does not imply CPU-only computation. The tested model-level
+  offload backend keeps the VAE on the GPU.
+- Only offline single-image generation on one GPU with BF16, 1024x1024,
+  four steps and CPU offload was validated. Online serving, FP8, other
+  resolutions, concurrency, execution without offload and smaller host RAM
+  configurations are outside this recipe's validated scope.
+- The upstream revision of the downloaded weights was not recorded, so the
+  measurements do not establish exact checkpoint reproducibility.
@@ -207,6 +207,18 @@ curl -sS -X POST http://localhost:8000/v1/videos/sync \
  -F 'extra_params={"depth":{"control_path":"/path/to/depth_control.mp4"},"max_frames":121,"resolution":"720","num_video_frames_per_chunk":121}' \
  -o cosmos3_transfer_depth.mp4

+# The same transfer control can be uploaded by a remote client instead of
+# being placed on the server filesystem. Uploads are limited to 512 MiB.
+curl -sS -X POST http://localhost:8000/v1/videos/sync \
+  -H "Accept: video/mp4" \
+  -F "model=nvidia/Cosmos3-Nano" \
+  -F "prompt=Generate a realistic scene following the provided world-state control." \
+  -F "input_reference=@/path/to/input.mp4;type=video/mp4" \
+  -F "control_reference=@/path/to/wsm.mp4;type=video/mp4" \
+  -F "control_type=wsm" \
+  -F 'extra_params={"wsm":{"control_weight":1.0},"max_frames":121,"resolution":"720","num_video_frames_per_chunk":121}' \
+  -o cosmos3_transfer_wsm.mp4
+
 # Text-to-video-with-sound
 curl -sS -X POST http://localhost:8000/v1/videos/sync \
  -H "Accept: video/mp4" \
@@ -36,19 +36,13 @@ Other hardware and configurations are welcome as community validation lands.

 - OS: Linux
 - Python: 3.10+
- Required package: `fish-speech` (for DAC codec)
 - CUDA 12.8
 - vLLM version: 0.19.0
 - vLLM-Omni version or commit: c93359bb354a6aa5c14d062430cb85b2c4db251e

-```bash
-# Install PortAudio dependency if missing (required by pyaudio which is a dependency of fish-speech)
-# For Ubuntu/Debian:
-if ! dpkg -l | grep -q libportaudio2; then
-    sudo apt-get update && sudo apt-get install -y libportaudio2 portaudio19-dev
-fi
-pip install fish-speech
-```
+No extra packages are required: the DAC codec architecture is vendored in
+vLLM-Omni (`vllm_omni/model_executor/models/fish_speech/dac_modules/`), so the
+model runs out of the box, including in the official Docker image.

 #### Kvcache attention fast path

@@ -129,7 +123,6 @@ curl -X POST http://localhost:8091/v1/audio/speech \

 - OS: Linux
 - Python: 3.10+
- Required package: `fish-speech` (for DAC codec)
 - CUDA 12.8
 - vLLM version: 0.19.0+

@@ -9,13 +9,15 @@
 # That breakage is now handled in code instead of by pinning:
 # MossTTSNanoForGeneration wraps weight loading in
 # transformers_keys_to_ignore_compat (vllm_omni/model_executor/models/utils.py).
-transformers >= 5.10.1, < 5.15
+# Floor is 5.13.0 so transformers.integrations.hub_kernels LayerRepository
+# entries always pass version=/revision= required by kernels>=0.16 (#6971).
+transformers >= 5.13.0, < 5.15
 av>=14.0.0
 omegaconf>=2.3.0
 diffusers==0.40.0
-# Hub kernels, including LTX-2.5 NATTEN. Keep this pin in sync with the
-# supported transformers range above.
-kernels==0.15.2
+# Hub kernels, including LTX-2.5 NATTEN. 0.16+ resolves torch-stable-abi*
+# FA3 builds on torch 2.13 / cu130; keep in sync with transformers above.
+kernels==0.16.1
 safetensors>=0.8.0
 accelerate==1.12.0
 soundfile>=0.13.1
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

 import pytest
 import torch
@@ -214,11 +214,20 @@ def test_fish_kvcache_backend_wraps_only_model_instance(monkeypatch):
    assert fish_kvcache_backend.get_fish_kvcache_attn_stats()["small_hit_count"] == 1


-def _vllm_kv_cache_shape(num_blocks, block_size, num_kv_heads, head_size):
+def _vllm_kv_cache_shape(num_blocks, block_size, num_kv_heads, head_size, dtype=torch.float16):
    # Source of truth for the paged KV cache layout the backend must unpack.
-    from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
+    # vLLM 0.29 removed AttentionBackend.get_kv_cache_shape; the per-layer shape
+    # now comes from the spec, with the content dim expressed in bytes.
+    from vllm.v1.kv_cache_interface import FullAttentionSpec, compute_layer_kv_cache_shape_bytes

-    return FlashAttentionBackend.get_kv_cache_shape(num_blocks, block_size, num_kv_heads, head_size)
+    spec = FullAttentionSpec(
+        block_size=block_size,
+        num_kv_heads=num_kv_heads,
+        head_size=head_size,
+        dtype=dtype,
+    )
+    num_pages, num_heads, num_states, content_bytes = compute_layer_kv_cache_shape_bytes(spec, num_blocks)
+    return (num_pages, num_heads, num_states, content_bytes // dtype.itemsize)


 def test_fish_kvcache_backend_unbinds_kv_on_vllm_cache_layout(monkeypatch):
@@ -457,5 +457,136 @@ def test_text_benchmark_still_reports_ttft(capsys):
    assert "Time to First Token" in out, "text benchmarks must keep TTFT"


+# ============================================================================
+# Suppress text metrics for pure image / video endpoints
+# ============================================================================
+
+
+def _make_image_output(prompt_len: int) -> MixRequestFuncOutput:
+    output = MixRequestFuncOutput()
+    output.success = True
+    output.prompt_len = prompt_len
+    output.output_tokens = 0
+    output.generated_text = ""
+    output.ttft = 0.0
+    output.text_latency = 0.0
+    output.latency = 2.5
+    output.start_time = 0.0
+    output.itl = []
+    output.image_count = 1
+    output.image_generation_time_ms = 2000.0
+    output.image_pixels = 1024 * 1024
+    output.input_audio_duration = 0.0
+    output.error = ""
+    return output
+
+
+def _make_video_output(prompt_len: int) -> MixRequestFuncOutput:
+    output = MixRequestFuncOutput()
+    output.success = True
+    output.prompt_len = prompt_len
+    output.output_tokens = 0
+    output.generated_text = ""
+    output.ttft = 0.0
+    output.text_latency = 0.0
+    output.latency = 5.0
+    output.start_time = 0.0
+    output.itl = []
+    output.video_duration = 2.0
+    output.video_frames = 48
+    output.video_generation_time_ms = 4000.0
+    output.video_rtf = 2.0
+    output.input_audio_duration = 0.0
+    output.error = ""
+    return output
+
+
+_MEDIA_PERCENTILE_METRICS = ["e2el", "ttft"]
+
+
+def test_pure_image_benchmark_omits_text_result(capsys):
+    """Pure image generation must not print Text Result or invent peak tok/s."""
+    outputs = [_make_image_output(64), _make_image_output(64)]
+
+    metrics, actual_output_lens = calculate_metrics(
+        input_requests=[],
+        outputs=outputs,
+        dur_s=5.0,
+        tokenizer=None,
+        selected_percentiles=[99.0],
+        goodput_config_dict={},
+        task_type=TaskType.GENERATION,
+        selected_percentile_metrics=_MEDIA_PERCENTILE_METRICS,
+        max_concurrency=None,
+        request_rate=float("inf"),
+        benchmark_duration=5.0,
+    )
+
+    out = capsys.readouterr().out
+    assert actual_output_lens == [0, 0]
+    assert metrics.total_output == 0
+    assert metrics.max_output_tokens_per_s == 0.0
+    assert " Text Result " not in out
+    assert "Peak output token throughput" not in out
+    assert "Time to First Token" not in out
+    assert " Image Result " in out
+
+
+def test_pure_video_benchmark_omits_text_result(capsys):
+    """Pure video generation must not print Text Result or invent peak tok/s."""
+    outputs = [_make_video_output(63), _make_video_output(63)]
+
+    metrics, actual_output_lens = calculate_metrics(
+        input_requests=[],
+        outputs=outputs,
+        dur_s=10.0,
+        tokenizer=None,
+        selected_percentiles=[99.0],
+        goodput_config_dict={},
+        task_type=TaskType.GENERATION,
+        selected_percentile_metrics=_MEDIA_PERCENTILE_METRICS,
+        max_concurrency=None,
+        request_rate=float("inf"),
+        benchmark_duration=10.0,
+    )
+
+    out = capsys.readouterr().out
+    assert actual_output_lens == [0, 0]
+    assert metrics.total_output == 0
+    assert metrics.max_output_tokens_per_s == 0.0
+    assert " Text Result " not in out
+    assert "Peak output token throughput" not in out
+    assert "Time to First Token" not in out
+    assert " Video Result " in out
+
+
+def test_image_with_generated_text_still_reports_text_result(capsys):
+    """Image edits / AR text alongside images must keep Text Result."""
+    output = _make_image_output(64)
+    output.output_tokens = 8
+    output.generated_text = "revised caption"
+    output.ttft = 0.05
+    output.itl = [0.01] * 7
+
+    calculate_metrics(
+        input_requests=[],
+        outputs=[output],
+        dur_s=3.0,
+        tokenizer=_EmptyAwareTokenizer(),
+        selected_percentiles=[99.0],
+        goodput_config_dict={},
+        task_type=TaskType.GENERATION,
+        selected_percentile_metrics=_MEDIA_PERCENTILE_METRICS,
+        max_concurrency=None,
+        request_rate=float("inf"),
+        benchmark_duration=3.0,
+    )
+
+    out = capsys.readouterr().out
+    assert " Text Result " in out
+    assert "Time to First Token" in out
+    assert " Image Result " in out
+
+
 if __name__ == "__main__":
    pytest.main([__file__, "-v", "-s"])
@@ -22,9 +22,16 @@ from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
 )
 from vllm_omni.benchmarks.patch.patch import (
    MixRequestFuncOutput,
+    _add_video_extra_body_to_form,
+    _add_video_reference_to_form,
    _apply_stage0_token_timings,
+    _apply_video_metrics_from_payload,
    _attach_seed_tts_to_request_func_input,
+    _build_benchmark_session,
+    _omni_request_timeout_s,
    async_request_openai_chat_omni_completions,
+    async_request_openai_image_edits_omni,
+    async_request_openai_image_generations_omni,
    async_request_openai_realtime_duplex,
    should_request_stage_metrics,
 )
@@ -1196,5 +1203,337 @@ async def test_prompt_len_assigned_from_usage(mocker: MockerFixture):
    )


+class TestOmniRequestTimeout:
+    """``--omni-request-timeout-s`` precedence: explicit value > 900 s default."""
+
+    _OVERRIDE = "vllm_omni.benchmarks.patch.patch._REQUEST_TIMEOUT_OVERRIDE_S"
+
+    def test_default_timeout_is_900s_when_not_set(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setattr(self._OVERRIDE, None)
+        assert _omni_request_timeout_s() == 900.0
+
+    def test_explicit_value_wins_over_default(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setattr(self._OVERRIDE, 123.5)
+        assert _omni_request_timeout_s() == 123.5
+
+    def test_non_positive_restores_legacy_6h_cap(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setattr(self._OVERRIDE, 0)
+        assert _omni_request_timeout_s() == 6 * 60 * 60.0
+
+    async def test_benchmark_session_uses_configured_timeout(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setattr(self._OVERRIDE, 42.0)
+        session = _build_benchmark_session(max_concurrency=8, ssl_setting=False)
+        try:
+            assert session.timeout.total == 42.0
+            assert session.connector.limit == 8
+            assert session.connector.limit_per_host == 8
+        finally:
+            await session.close()
+
+    async def test_hung_server_request_times_out_as_failed(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        """A server that accepts the request but never responds must surface as ``failed``."""
+
+        async def handler(reader, writer):
+            # Drain the request but never respond — the original hang failure mode.
+            try:
+                while await reader.read(4096):
+                    pass
+            except ConnectionResetError:
+                pass
+            finally:
+                # Close our side of the socket: the client aborts after the
+                # timeout and only sends FIN, so without writer.close() the
+                # half-closed connection keeps Server.wait_closed() (which on
+                # Python 3.12+ waits for every accepted connection to drop)
+                # blocked forever.
+                writer.close()
+                try:
+                    await writer.wait_closed()
+                except (ConnectionResetError, BrokenPipeError):
+                    pass
+
+        monkeypatch.setattr(self._OVERRIDE, 1.0)
+        server = await asyncio.start_server(handler, "127.0.0.1", 0)
+        port = server.sockets[0].getsockname()[1]
+        session = _build_benchmark_session(max_concurrency=1, ssl_setting=False)
+        try:
+            request_input = _seed_tts_request_func_input()
+            request_input.api_url = f"http://127.0.0.1:{port}/v1/chat/completions"
+            output = await async_request_openai_chat_omni_completions(request_input, session)
+            assert output.success is False
+            assert output.error
+        finally:
+            await session.close()
+            server.close()
+            await server.wait_closed()
+
+
+def test_video_rtf_prefers_generation_time_over_poll_latency():
+    """RTF must not grow with client poll_interval overshoot in output.latency."""
+    payload = {
+        "duration_s": 2.0,
+        "num_frames": 48,
+        "fps": 24.0,
+        "stage_durations": {"stage_0_gen_ms": 4000.0},
+    }
+    request_body: dict[str, object] = {}
+
+    short_poll = MixRequestFuncOutput()
+    short_poll.latency = 4.2  # ~generation + small poll overshoot
+    _apply_video_metrics_from_payload(short_poll, payload, request_body)
+
+    long_poll = MixRequestFuncOutput()
+    long_poll.latency = 8.0  # same job, larger poll_interval_s overshoot
+    _apply_video_metrics_from_payload(long_poll, payload, request_body)
+
+    assert short_poll.video_generation_time_ms == pytest.approx(4000.0)
+    assert long_poll.video_generation_time_ms == pytest.approx(4000.0)
+    assert short_poll.video_rtf == pytest.approx(2.0)
+    assert long_poll.video_rtf == pytest.approx(2.0)
+    assert short_poll.video_rtf == long_poll.video_rtf
+
+
+def test_video_rtf_falls_back_to_e2e_latency_without_generation_time():
+    output = MixRequestFuncOutput()
+    output.latency = 6.0
+    _apply_video_metrics_from_payload(
+        output,
+        {"duration_s": 2.0, "num_frames": 48, "fps": 24.0},
+        {},
+    )
+    assert output.video_generation_time_ms == 0.0
+    assert output.video_rtf == pytest.approx(3.0)
+
+
+# 1x1 PNG used as a valid b64_json image payload in edit-client tests.
+_MIN_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
+
+
+@pytest.mark.asyncio
+async def test_image_edits_defaults_to_non_streaming_json(mocker: MockerFixture) -> None:
+    """Single-stage servers reject stream=true; default path must send stream=false + JSON."""
+
+    class MockJsonResponse:
+        status = 200
+
+        def __init__(self):
+            self._payload = {
+                "created": 1,
+                "data": [
+                    {
+                        "b64_json": _MIN_PNG_B64,
+                        "stage_durations": {"stage_0_gen_ms": 12.5},
+                    }
+                ],
+            }
+
+        async def json(self):
+            return self._payload
+
+        async def text(self):
+            return json.dumps(self._payload)
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *_args):
+            return None
+
+    captured_stream: list[str] = []
+    real_add_field = None
+
+    def tracking_add_field(self, name, value=None, **kwargs):
+        if name == "stream":
+            captured_stream.append(str(value))
+        return real_add_field(self, name, value, **kwargs)
+
+    import aiohttp
+
+    real_add_field = aiohttp.FormData.add_field
+    mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
+
+    mock_session = mocker.AsyncMock()
+    mock_session.post = mocker.MagicMock(return_value=MockJsonResponse())
+
+    request = RequestFuncInput(
+        model="single-stage-edit",
+        model_name="single-stage-edit",
+        prompt="make it sunny",
+        api_url="http://test.com/v1/images/edits",
+        prompt_len=4,
+        output_len=1,
+        multi_modal_content=[{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_MIN_PNG_B64}"}}],
+        extra_body={"num_inference_steps": 4},
+    )
+    output = await async_request_openai_image_edits_omni(request, mock_session, pbar=None)
+
+    assert captured_stream == ["false"]
+    assert output.success is True
+    assert not output.error
+    assert output.image_count == 1
+    assert output.image_generation_time_ms == pytest.approx(12.5)
+
+
+@pytest.mark.asyncio
+async def test_image_edits_stream_true_uses_sse_path(mocker: MockerFixture) -> None:
+    """Explicit stream=true keeps the multi-stage SSE client path."""
+    sse_chunk = (
+        b'data: {"type":"image","data":[{"b64_json":"' + _MIN_PNG_B64.encode() + b'"}]}\n\n' + b"data: [DONE]\n\n"
+    )
+    mock_response = MockResponse(200, [sse_chunk])
+    captured_stream: list[str] = []
+
+    import aiohttp
+
+    real_add_field = aiohttp.FormData.add_field
+
+    def tracking_add_field(self, name, value=None, **kwargs):
+        if name == "stream":
+            captured_stream.append(str(value))
+        return real_add_field(self, name, value, **kwargs)
+
+    mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
+
+    mock_session = mocker.AsyncMock()
+    mock_session.post = mocker.MagicMock(return_value=mock_response)
+
+    request = RequestFuncInput(
+        model="multi-stage-edit",
+        model_name="multi-stage-edit",
+        prompt="edit",
+        api_url="http://test.com/v1/images/edits",
+        prompt_len=2,
+        output_len=1,
+        multi_modal_content=[{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_MIN_PNG_B64}"}}],
+        extra_body={"stream": True},
+    )
+    output = await async_request_openai_image_edits_omni(request, mock_session, pbar=None)
+
+    assert captured_stream == ["true"]
+    assert output.success is True
+    assert output.image_count == 1
+
+
+@pytest.mark.asyncio
+async def test_image_generations_e2el_includes_json_body_consume(mocker: MockerFixture) -> None:
+    """E2EL must include body transfer/decode, not stop at HTTP headers."""
+
+    class SlowJsonResponse:
+        status = 200
+
+        def __init__(self):
+            self._payload = {
+                "created": 1,
+                "data": [{"b64_json": _MIN_PNG_B64, "stage_durations": {"stage_0_gen_ms": 1.0}}],
+            }
+
+        async def json(self):
+            await asyncio.sleep(0.05)
+            return self._payload
+
+        async def text(self):
+            return json.dumps(self._payload)
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *_args):
+            return None
+
+    mock_session = mocker.AsyncMock()
+    mock_session.post = mocker.MagicMock(return_value=SlowJsonResponse())
+    request = RequestFuncInput(
+        model="img-gen",
+        model_name="img-gen",
+        prompt="a cat",
+        api_url="http://test.com/v1/images/generations",
+        prompt_len=2,
+        output_len=1,
+        extra_body={},
+    )
+    output = await async_request_openai_image_generations_omni(request, mock_session, pbar=None)
+    assert output.success is True
+    assert output.latency >= 0.05
+
+
+def test_video_local_image_reference_not_forwarded_as_raw_extra_field(tmp_path, mocker: MockerFixture) -> None:
+    """Local image_reference must only be uploaded via dedicated serializer."""
+    import aiohttp
+
+    ref_path = tmp_path / "ref.png"
+    ref_path.write_bytes(base64.b64decode(_MIN_PNG_B64))
+
+    captured: list[tuple[str, object]] = []
+    real_add_field = aiohttp.FormData.add_field
+
+    def tracking_add_field(self, name, value=None, **kwargs):
+        captured.append((str(name), value))
+        return real_add_field(self, name, value, **kwargs)
+
+    mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
+
+    form = aiohttp.FormData()
+    extra_body = {
+        "num_inference_steps": 2,
+        "image_reference": str(ref_path),
+    }
+    request_body = {"model": "vid", "prompt": "p", **extra_body}
+    _add_video_extra_body_to_form(form, extra_body, request_body)
+    assert _add_video_reference_to_form(form, extra_body["image_reference"]) is True
+
+    field_names = [name for name, _ in captured]
+    assert "image_reference" not in field_names
+    assert field_names.count("input_reference") == 1
+    # Dedicated uploader sends file bytes, not the raw local path string.
+    uploaded = next(value for name, value in captured if name == "input_reference")
+    assert uploaded == base64.b64decode(_MIN_PNG_B64)
+
+
+@pytest.mark.parametrize(
+    "reference",
+    [
+        {"image_url": "https://example.com/ref.png"},
+        {"file_id": "file-abc"},
+        [{"image_url": "https://example.com/a.png"}, {"file_id": "file-xyz"}],
+    ],
+)
+def test_video_structured_image_reference_serialized_to_form(reference: object, mocker: MockerFixture) -> None:
+    """Object-form image_reference must be JSON-serialized, not silently dropped."""
+    import aiohttp
+
+    captured: list[tuple[str, object]] = []
+    real_add_field = aiohttp.FormData.add_field
+
+    def tracking_add_field(self, name, value=None, **kwargs):
+        captured.append((str(name), value))
+        return real_add_field(self, name, value, **kwargs)
+
+    mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
+
+    form = aiohttp.FormData()
+    extra_body = {"image_reference": reference}
+    request_body = {"model": "vid", "prompt": "p", **extra_body}
+    _add_video_extra_body_to_form(form, extra_body, request_body)
+    assert _add_video_reference_to_form(form, reference) is True
+
+    field_names = [name for name, _ in captured]
+    # Reserved key must not be double-forwarded as a generic extra field;
+    # exactly one dedicated image_reference field.
+    assert field_names.count("image_reference") == 1
+    assert "input_reference" not in field_names
+    payload = next(value for name, value in captured if name == "image_reference")
+    assert json.loads(payload) == reference
+
+
+def test_video_unsupported_image_reference_raises() -> None:
+    import aiohttp
+
+    form = aiohttp.FormData()
+    with pytest.raises(ValueError, match="Unsupported image_reference"):
+        _add_video_reference_to_form(form, {"not_a_supported_key": "x"})
+    with pytest.raises(ValueError, match="Unsupported image_reference"):
+        _add_video_reference_to_form(form, "/tmp/does-not-exist-ref.png")
+
+
 if __name__ == "__main__":
    pytest.main([__file__, "-v", "-s"])
@@ -1,9 +1,11 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 # ruff: noqa: E402, I001
 import argparse
 import math
 import os
 import sys
-import types
 from pathlib import Path

 import pytest
@@ -106,11 +108,9 @@ def test_resolve_seed_tts_root_downloads_only_requested_locale(monkeypatch, tmp_
        captured["allow_patterns"] = allow_patterns
        return str(downloaded_root)

-    monkeypatch.setitem(
-        sys.modules,
-        "huggingface_hub",
-        types.SimpleNamespace(snapshot_download=fake_snapshot_download),
-    )
+    from vllm_omni.transformers_utils import repo_utils
+
+    monkeypatch.setattr(repo_utils.hf_api(), "snapshot_download", fake_snapshot_download)

    resolved = resolve_seed_tts_root(
        "zhaochenyang20/seed-tts-eval",
@@ -26,6 +26,7 @@ _MODULE_NAME = "vllm_omni.benchmarks.data_modules.seed_tts_dataset"

 if _MODULE_NAME not in sys.modules:
    _spec = importlib.util.spec_from_file_location(_MODULE_NAME, _MODULE_PATH)
+    assert _spec is not None and _spec.loader is not None
    _mod = importlib.util.module_from_spec(_spec)
    sys.modules[_MODULE_NAME] = _mod
    _spec.loader.exec_module(_mod)
@@ -237,7 +238,8 @@ def test_seed_tts_design_dataset_rejects_missing_description(seed_tts_design_roo
        assert req.seed_tts_utterance_id == "ok"


-def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch):
+@pytest.mark.parametrize("audio_seconds", [0.1, 61])
+def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch, audio_seconds):
    from vllm_omni.benchmarks.data_modules import seed_tts_eval

    calls = {}
@@ -252,8 +254,9 @@ def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch):
            return self

    class FakeProcessor:
-        def __call__(self, wav, *, sampling_rate, return_tensors, return_attention_mask=False):
+        def __call__(self, wav, *, sampling_rate, return_tensors, return_attention_mask=False, **kwargs):
            calls["return_attention_mask"] = return_attention_mask
+            calls["processor_kwargs"] = kwargs
            assert sampling_rate == 16000
            assert return_tensors == "pt"
            assert len(wav) > 0
@@ -262,9 +265,10 @@ def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch):
                attention_mask=FakeTensor("mask") if return_attention_mask else None,
            )

-        def get_decoder_prompt_ids(self, *, language, task):
+        def get_decoder_prompt_ids(self, *, language, task, no_timestamps=True):
            assert language == "english"
            assert task == "transcribe"
+            calls["no_timestamps"] = no_timestamps
            return [(1, 2)]

        def batch_decode(self, predicted_ids, *, skip_special_tokens):
@@ -283,10 +287,18 @@ def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch):
    monkeypatch.setattr(seed_tts_eval, "_en_model", FakeModel())
    monkeypatch.setattr(seed_tts_eval, "_device", "cuda:1")

-    text = seed_tts_eval._transcribe_en_f32_16k(np.ones(1600, dtype=np.float32))
+    text = seed_tts_eval._transcribe_en_f32_16k(np.ones(int(audio_seconds * 16000), dtype=np.float32))

    assert text == "hello"
    assert calls["return_attention_mask"] is True
    assert calls["input_features"].device == "cuda:1"
    assert calls["generate_kwargs"]["attention_mask"].device == "cuda:1"
    assert calls["generate_kwargs"]["forced_decoder_ids"] == [(1, 2)]
+    if audio_seconds > 30:
+        assert calls["processor_kwargs"] == {"truncation": False, "padding": "longest"}
+        assert calls["generate_kwargs"]["return_timestamps"] is True
+        assert calls["no_timestamps"] is False
+    else:
+        assert calls["processor_kwargs"] == {}
+        assert "return_timestamps" not in calls["generate_kwargs"]
+        assert calls["no_timestamps"] is True
@@ -167,6 +167,26 @@ def test_preprocess_serve_args_applies_safe_omniinteract_prompt_default(
            {"bot_task": "recaption"},
            {"backend", "print_stage", "bot_task"},
        ),
+        (
+            [
+                "--endpoint",
+                "/v1/images/edits",
+                "--image-edits-bot-task",
+                "think",
+            ],
+            {"bot_task": "think"},
+            {"endpoint", "bot_task"},
+        ),
+        (
+            [
+                "--backend",
+                "/v1/images/edits",
+                "--image-edits-bot-task",
+                "recaption",
+            ],
+            {"bot_task": "recaption"},
+            {"backend", "bot_task"},
+        ),
        (
            ["--extra-body", '{"bot_task":"vanilla"}'],
            {"bot_task": "vanilla"},
@@ -182,6 +202,7 @@ def test_omni_args_parse_and_preprocess(
    parser = TrackingArgumentParser()
    parser.add_argument("--extra-body", type=json.loads, default=None)
    parser.add_argument("--backend", default="openai-chat-omni")
+    parser.add_argument("--endpoint", default="/v1/chat/completions")
    add_omni_args(parser)

    args = parser.parse_args(argv)
@@ -358,3 +379,17 @@ def test_bench_serve_cli_mocks_http_request(tmp_path: Path):
    )
    assert bench_requests
    assert all(url == expected_url for url in bench_requests), f"Unexpected target URLs: {bench_requests}"
+
+
+def test_omni_request_timeout_s_flag_defaults_and_parses() -> None:
+    parser = TrackingArgumentParser()
+    add_omni_args(parser)
+
+    args = parser.parse_args([])
+    assert args.omni_request_timeout_s is None
+
+    args = parser.parse_args(["--omni-request-timeout-s", "0"])
+    assert args.omni_request_timeout_s == 0.0
+
+    args = parser.parse_args(["--omni-request-timeout-s", "3600"])
+    assert args.omni_request_timeout_s == 3600.0