[Bugfix][MiniMax-H3] Apply pre-commit formatting

Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
[Bugfix][MiniMax-H3] Reconcile temporal decode plans across VAE ranks
2026-09-03 08:44:26 +08:00 · 2026-09-03 08:20:00 +08:00 · 2026-09-02 07:08:40 +08:00 · 2026-09-01 23:11:29 +08:00 · 2026-09-01 23:11:29 +08:00 · 2026-09-01 23:11:28 +08:00
682 changed files with 62350 additions and 12581 deletions
@@ -1,4 +1,7 @@
 #!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 # vllm-omni customized version
 # Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/bootstrap-amd.sh
 # Last synced: 2025-12-15
@@ -6,6 +9,9 @@

 set -euo pipefail

+# The bootstrap runs from the repository root; ShellCheck does not follow
+# sourced files unless invoked with -x.
+# shellcheck disable=SC1091
 source .buildkite/common/scripts/resolve_skip_ci.sh

 if [[ -z "${RUN_ALL:-}" ]]; then
@@ -63,6 +69,8 @@ upload_pipeline() {
    # Install minijinja
    ls .buildkite || buildkite-agent annotate --style error 'Please merge upstream main branch for buildkite CI'
    curl -sSfL https://github.com/mitsuhiko/minijinja/releases/download/2.3.1/minijinja-cli-installer.sh | sh
+    # Installed by the minijinja bootstrap above and only present on the CI agent.
+    # shellcheck disable=SC1091
    source /var/lib/buildkite-agent/.cargo/env

    if [[ $BUILDKITE_PIPELINE_SLUG == "fastcheck" ]]; then
@@ -90,22 +98,82 @@ upload_pipeline() {
    cd .buildkite/amd

    # Select test definition file: merge suite for main, ready suite for PRs.
-    # For debugging, DEBUG_TEST_YAML can override the selection — accepts
-    # "merge" or "ready" (case-insensitive).
+    # For debugging, DEBUG_TEST_YAML accepts a comma-separated list containing
+    # "merge" and/or "ready" (case-insensitive). Multiple suites are combined
+    # before rendering so they share one amd-build step.
    if [[ -n "${DEBUG_TEST_YAML:-}" ]]; then
-        case "${DEBUG_TEST_YAML,,}" in
-            merge)
-                TEST_YAML="test-amd-merge.yml"
-                ;;
-            ready)
-                TEST_YAML="test-amd-ready.yml"
-                ;;
-            *)
-                echo "ERROR: DEBUG_TEST_YAML must be 'merge' or 'ready', got '$DEBUG_TEST_YAML'" >&2
+        declare -a DEBUG_TEST_SPECS=()
+        declare -A SEEN_DEBUG_TESTS=()
+        IFS=',' read -ra REQUESTED_DEBUG_TESTS <<< "${DEBUG_TEST_YAML,,}"
+
+        for requested_test in "${REQUESTED_DEBUG_TESTS[@]}"; do
+            # Permit readable values such as "ready, merge," and ignore the
+            # empty item produced by a trailing comma.
+            requested_test="${requested_test//[[:space:]]/}"
+            [[ -z "$requested_test" ]] && continue
+
+            if [[ -n "${SEEN_DEBUG_TESTS[$requested_test]:-}" ]]; then
+                echo "ERROR: duplicate DEBUG_TEST_YAML suite '$requested_test'" >&2
                exit 1
-                ;;
-        esac
-        echo "DEBUG_TEST_YAML override: using $TEST_YAML"
+            fi
+            SEEN_DEBUG_TESTS[$requested_test]=1
+
+            case "$requested_test" in
+                ready)
+                    DEBUG_TEST_SPECS+=("READY_TESTS:test-amd-ready.yml")
+                    ;;
+                merge)
+                    DEBUG_TEST_SPECS+=("MERGE_TESTS:test-amd-merge.yml")
+                    ;;
+                *)
+                    echo "ERROR: DEBUG_TEST_YAML entries must be 'merge' or 'ready', got '$requested_test'" >&2
+                    exit 1
+                    ;;
+            esac
+        done
+
+        if [[ ${#DEBUG_TEST_SPECS[@]} -eq 0 ]]; then
+            echo "ERROR: DEBUG_TEST_YAML did not contain a test suite" >&2
+            exit 1
+        elif [[ ${#DEBUG_TEST_SPECS[@]} -eq 1 ]]; then
+            TEST_YAML="${DEBUG_TEST_SPECS[0]#*:}"
+        else
+            TEST_YAML=$(mktemp "${TMPDIR:-/tmp}/amd-debug-tests.XXXXXX.yml")
+            python - "$TEST_YAML" "${DEBUG_TEST_SPECS[@]}" <<'PY'
+import sys
+
+import yaml
+
+
+output_path, *suite_specs = sys.argv[1:]
+combined = {"env": {}, "steps": []}
+
+for suite_spec in suite_specs:
+    group_name, input_path = suite_spec.split(":", 1)
+    with open(input_path, encoding="utf-8") as test_file:
+        suite = yaml.safe_load(test_file)
+
+    for name, value in (suite.get("env") or {}).items():
+        previous = combined["env"].get(name, value)
+        if previous != value:
+            raise ValueError(
+                f"Conflicting environment value for {name}: {previous!r} != {value!r}"
+            )
+        combined["env"][name] = value
+
+    suite_steps = []
+    for entry in suite.get("steps") or []:
+        if "group" in entry:
+            suite_steps.extend(entry.get("steps") or [])
+        else:
+            suite_steps.append(entry)
+    combined["steps"].append({"group": group_name, "steps": suite_steps})
+
+with open(output_path, "w", encoding="utf-8") as output_file:
+    yaml.safe_dump(combined, output_file, sort_keys=False)
+PY
+        fi
+        echo "DEBUG_TEST_YAML override: using ${DEBUG_TEST_SPECS[*]}"
    elif [[ $BUILDKITE_BRANCH == "main" ]]; then
        TEST_YAML="test-amd-merge.yml"
    else
@@ -130,19 +198,20 @@ upload_pipeline() {
            > pipeline.yaml
    )
    cat pipeline.yaml
+    if [[ "$TEST_YAML" == "${TMPDIR:-/tmp}/amd-debug-tests."*.yml ]]; then
+        rm -f -- "$TEST_YAML"
+    fi
    buildkite-agent artifact upload pipeline.yaml
    buildkite-agent pipeline upload pipeline.yaml
    exit 0
 }

 get_diff() {
-    $(git add .)
-    echo $(git diff --name-only --diff-filter=ACMDR $(git merge-base origin/main HEAD))
+    git diff --name-only --diff-filter=ACMDR "$(git merge-base origin/main HEAD)"
 }

 get_diff_main() {
-    $(git add .)
-    echo $(git diff --name-only --diff-filter=ACMDR HEAD~1)
+    git diff --name-only --diff-filter=ACMDR HEAD~1
 }

 file_diff=$(get_diff)
@@ -181,7 +250,7 @@ for file in $file_diff; do
    # First check if file matches any pattern
    matches_pattern=0
    for pattern in "${patterns[@]}"; do
-        if [[ $file == $pattern* ]] || [[ $file == $pattern ]]; then
+        if [[ $file == "$pattern"* ]] || [[ $file == "$pattern" ]]; then
            matches_pattern=1
            break
        fi
@@ -191,7 +260,7 @@ for file in $file_diff; do
    if [[ $matches_pattern -eq 1 ]]; then
        matches_ignore=0
        for ignore in "${ignore_patterns[@]}"; do
-            if [[ $file == $ignore* ]] || [[ $file == $ignore ]]; then
+            if [[ $file == "$ignore"* ]] || [[ $file == "$ignore" ]]; then
                matches_ignore=1
                break
            fi
@@ -21,7 +21,7 @@ steps:

    - label: "Simple · Diffusion Test · Shard %N/%t"
      agent_pool: mi300_1
-      parallelism: 2
+      parallelism: 4
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
@@ -71,26 +71,45 @@ steps:
      commands:
        - export VLLM_LOGGING_LEVEL=DEBUG
        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        - |
-          bash -c '
-            set +e
-            pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
-            EXIT1=\$?
-            pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
-            EXIT2=\$?
-            exit \$((EXIT1 | EXIT2))
-          '
+        - pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
+        - >-
+          pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
+          --deselect='tests/diffusion/offloader/test_diffusion_layerwise_offload.py::test_layerwise_offload_diffusion_model[stabilityai/stable-audio-open-1.0]'

-    - label: "Diffusion Sequence Parallelism Test (Need 4 GPUs)"
+    # TODO: Diagnose unstable Stable Audio layerwise-offload memory savings on ROCm.
+    - label: "Diffusion Stable Audio Layerwise Offload Test (quarantined)"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: NonBlocking
+      commands:
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - >-
+          pytest -s -v
+          'tests/diffusion/offloader/test_diffusion_layerwise_offload.py::test_layerwise_offload_diffusion_model[stabilityai/stable-audio-open-1.0]'
+
+    - label: "Diffusion Sequence Parallelism Test (Need 4 GPUs) · Shard 1/2"
      agent_pool: mi300_4
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
+      timeout_in_minutes: 40
      grade: Blocking
      commands:
        - export VLLM_LOGGING_LEVEL=DEBUG
        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py
-        - timeout 20m pytest -s -v tests/diffusion/distributed/test_ulysses_uaa_perf.py
+        - pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py
+
+    - label: "Diffusion Sequence Parallelism Test (Need 4 GPUs) · Shard 2/2"
+      agent_pool: mi300_4
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      timeout_in_minutes: 40
+      grade: Blocking
+      commands:
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - pytest -s -v tests/diffusion/distributed/test_ulysses_uaa_perf.py

    # merge-only tests
    - label: "Diffusion Tensor Parallelism Test"
@@ -171,21 +190,45 @@ steps:
      grade: Blocking
      commands:
        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+        - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py -m 'advanced_model and cuda' --run-level 'advanced_model'

    - label: "Qwen3-TTS CustomVoice E2E Test · Shard %N/%t"
      agent_pool: mi300_1
-      parallelism: 2
+      parallelism: 3
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
+      timeout_in_minutes: 80
      commands:
        - |
-          bash -c '
+          timeout --signal=TERM --kill-after=2m 80m bash -c '
            export VLLM_LOGGING_LEVEL=DEBUG
            export VLLM_WORKER_MULTIPROC_METHOD=spawn
            export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-            pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model" --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
+            TESTS=(
+              "tests/e2e/online_serving/test_qwen3_tts_customvoice.py::test_default_cuda_graph_startup[async_chunk]"
+              "tests/e2e/online_serving/test_qwen3_tts_customvoice.py::test_text_to_audio_002[async_chunk]"
+              "tests/e2e/offline_inference/test_qwen3_tts_customvoice.py::test_text_to_audio_001[no_cuda_graph]"
+            )
+            TEST="$${TESTS[$$BUILDKITE_PARALLEL_JOB]}"
+            pytest -s -v "$$TEST" -m "advanced_model" --run-level "advanced_model"
+          '
+
+    # TODO: Restore this case to the blocking sharded step once its long-running
+    # Whisper validation on AMD CI is resolved.
+    - label: "Qwen3-TTS CustomVoice async_chunk Test (quarantined)"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: NonBlocking
+      timeout_in_minutes: 80
+      commands:
+        - |
+          timeout --signal=TERM --kill-after=2m 75m bash -c '
+            export VLLM_LOGGING_LEVEL=DEBUG
+            export VLLM_WORKER_MULTIPROC_METHOD=spawn
+            export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+            pytest -s -v "tests/e2e/online_serving/test_qwen3_tts_customvoice.py::test_text_to_audio_001[async_chunk]" -m "advanced_model" --run-level "advanced_model"
          '

    - label: "Qwen3-TTS Base E2E Test"
@@ -195,13 +238,10 @@ steps:
      grade: Blocking
      timeout_in_minutes: 50
      commands:
-        - |
-          timeout 40m bash -c '
-            export VLLM_LOGGING_LEVEL=DEBUG
-            export VLLM_WORKER_MULTIPROC_METHOD=spawn
-            export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-            pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-          '
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+        - timeout 40m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model and cuda" --run-level "advanced_model"

 # TODO: Bagel test on ROCm is very unstable. @tjtanaa
 # Need to debug before reneable numerical changes across large PRs
@@ -191,10 +191,7 @@ steps:
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
-        - |
-          timeout 20m bash -c '
-            pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"
-          '
+        - pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"

 # TODO: Bagel test on ROCm is very unstable. @tjtanaa
 # Need to debug before reneable numerical changes across large PRs
@@ -85,6 +85,9 @@ steps:
          - vllm_omni/model_executor/models/common/alias_free_activation.py
          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
          - vllm_omni/deploy/qwen3_tts.yaml
+          - vllm_omni/entrypoints/openai/serving_speech.py
+          - vllm_omni/entrypoints/openai/tts_adapters/
+          - vllm_omni/engine/stage_pool.py
        commands:
          - |
            timeout 40m bash -c "
@@ -104,6 +107,9 @@ steps:
          - vllm_omni/model_executor/models/common/alias_free_activation.py
          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
          - vllm_omni/deploy/qwen3_tts.yaml
+          - vllm_omni/entrypoints/openai/serving_speech.py
+          - vllm_omni/entrypoints/openai/tts_adapters/
+          - vllm_omni/engine/stage_pool.py
          - .buildkite/common/scripts/run_cov_split.sh
          - pyproject.toml
        # 40 -> 60: the coverage split runs the model twice.
@@ -122,6 +128,7 @@ steps:
      - label: "Omni · Qwen3-Omni Test"
        source_file_dependencies:
          - tests/e2e/offline_inference/test_qwen3_omni.py
+          - tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py
          - tests/e2e/online_serving/test_qwen3_omni.py
          - tests/helpers/runtime.py
          - vllm_omni/entrypoints/openai/serving_chat.py
@@ -132,9 +139,10 @@ steps:
          - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
          - vllm_omni/deploy/qwen3_omni_moe.yaml
+          - vllm_omni/deploy/qwen3_omni_moe_thinking.yaml
        timeout_in_minutes: 50
        commands:
-          - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+          - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py -m 'advanced_model and cuda' --run-level 'advanced_model'
        mirror_hardwares: h100_2

      - label: "Omni · MiniCPM-o 4.5 Test"
@@ -166,10 +174,10 @@ steps:
          - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
          - tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
+          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py
          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
          - vllm_omni/deploy/minicpmo_4_5.yaml
          - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
-          - vllm_omni/deploy/minicpmo_4_5_duplex.yaml
          - vllm_omni/experimental/fullduplex/
          - vllm_omni/model_executor/models/minicpmo_4_5/
          - vllm_omni/model_executor/models/step_audio2/
@@ -217,3 +225,50 @@ steps:
              pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
            "
        mirror_hardwares: h100_1
+
+      - label: "Diffusion · HunyuanVideo-1.5 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
+          - vllm_omni/diffusion/models/hunyuan_video/
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+            "
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · MiniMax-H3 L3 DLO DP2 and FastH3"
+        source_file_dependencies:
+          - tests/e2e/online_serving/minimax_h3/
+          - tests/helpers/assertions.py
+          - tests/helpers/mark.py
+          - tests/helpers/media.py
+          - tests/helpers/runtime.py
+          - vllm_omni/diffusion/models/minimax_h3/
+          - vllm_omni/diffusion/lora/
+          - vllm_omni/diffusion/offloader/
+          - vllm_omni/diffusion/distributed/
+          - vllm_omni/entrypoints/openai/
+        # Keep four server lifecycles in one Buildkite job, but never in one
+        # pytest process. DLO uses multiprocessing shared-memory queues; the
+        # parent's resource_tracker can retain a stale /psm_* entry during
+        # teardown and strand the next Ref2VA collective.
+        # A per-file process plus a hard timeout also makes a regression fail
+        # instead of leaving the merge queue stuck indefinitely.
+        timeout_in_minutes: 150
+        commands:
+          - |
+            set -eu
+            export VLLM_IMAGE_FETCH_TIMEOUT=60
+            if [ -f vllm_omni/diffusion/models/minimax_h3/lora.py ]; then
+              export VLLM_TEST_MINIMAX_H3_TURBO_LORA=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id="lightx2v/Minimax-h3-Turbo", filename="minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors"))')
+            fi
+            if [ -f vllm_omni/diffusion/models/minimax_h3/fasth3.py ]; then
+              export VLLM_TEST_MINIMAX_H3_FASTH3_LORA=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id="FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA", filename="dense-datafree/adapter_model.safetensors"))')
+            fi
+            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_dlo_dp2_fl2va.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_dlo_dp2_ref2va.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_turbo_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_fasth3_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+        mirror_hardwares: h100_2
@@ -12,16 +12,22 @@ steps:
    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
    steps:

-      - label: ":full_moon: Omni · Function Test with H100"
-        timeout_in_minutes: 120
+      - label: ":full_moon: Omni · Function Test with H100 · Single-GPU"
+        timeout_in_minutes: 90
        commands:
-          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: Omni · Function Test with H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni and cards_2" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
        mirror_hardwares: h100_2

      - label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
        timeout_in_minutes: 50
        commands:
-          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni" --run-level "full_model"
+          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni and cards_1" --run-level "full_model"
        mirror_hardwares: h100_1

      - label: ":full_moon: Omni · Doc Test with L4"
@@ -31,10 +37,10 @@ steps:
          - pytest -s -v tests/examples/ -m "full_model and omni and L4" --run-level "full_model"
        mirror_hardwares: l4_4

-      - label: ":full_moon: Omni · Doc Test with H100"
+      - label: ":full_moon: Omni · Doc Test with H100 · 2-GPU"
        timeout_in_minutes: 90
        commands:
-          - pytest -s -v tests/examples/ -m "full_model and omni and H100" --run-level "full_model"
+          - pytest -s -v tests/examples/ -m "full_model and omni and H100 and cards_2" --run-level "full_model"
        mirror_hardwares: h100_2

      - label: ":full_moon: Omni · Accuracy Test"
@@ -83,7 +89,7 @@ steps:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - |
            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and full_model"
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
            exit $$EXIT
@@ -136,15 +142,29 @@ steps:
          - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
        mirror_hardwares: l4_1

-      - label: ":full_moon: TTS · Perf Test"
-        key: nightly-tts-performance
+      - label: ":full_moon: TTS · Perf Test · Single-GPU"
+        key: nightly-tts-performance-single
        timeout_in_minutes: 180
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - |
            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and cards_1"
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: TTS · Perf Test · 2-GPU"
+        key: nightly-tts-performance-2gpu
+        timeout_in_minutes: 180
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and cards_2"
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
            exit $$EXIT
@@ -179,7 +199,6 @@ steps:
      build.pull_request.labels includes "nightly-test" ||
      build.pull_request.labels includes "diffusion-x2iat-test"
    steps:
-      # Use explicit file shards so single-card cases do not reserve 2+ GPUs.

      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU"
        timeout_in_minutes: 120
@@ -187,24 +206,24 @@ steps:
          - >-
            pytest -sv
            tests/e2e/online_serving/test_qwen_image_expansion.py
-            -m "full_model and diffusion and H100 and not distributed_cuda"
+            -m "full_model and diffusion and H100 and cards_1"
            --run-level "full_model"
        mirror_hardwares: h100_1

-      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Qwen-Image"
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU"
        timeout_in_minutes: 120
        commands:
          - >-
            pytest -sv
            tests/e2e/online_serving/test_qwen_image_expansion.py
-            -m "full_model and diffusion and H100 and distributed_cuda"
+            -m "full_model and diffusion and H100 and cards_2"
            --run-level "full_model"
        mirror_hardwares: h100_2

-      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · Multi-GPU"
        timeout_in_minutes: 120
        commands:
-          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy
+          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4 and not cards_1" --run-level "full_model" --ignore=tests/e2e/accuracy
        mirror_hardwares: l4_4

      - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
@@ -235,8 +254,8 @@ steps:
          - pytest -s -v tests/e2e/accuracy/test_hidream_o1_image.py -m full_model --run-level full_model
        mirror_hardwares: h100_2

-      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image"
-        key: nightly-diffusion-x2iat-performance-qwen-image
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · Single-GPU"
+        key: nightly-diffusion-x2iat-performance-qwen-image-single
        timeout_in_minutes: 180
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
@@ -244,7 +263,23 @@ steps:
          - export CACHE_DIT_VERSION=1.5.0
          - |
            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and cards_1"
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · 4-GPU"
+        key: nightly-diffusion-x2iat-performance-qwen-image-4gpu
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - export CACHE_DIT_VERSION=1.5.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and cards_4"
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
@@ -260,16 +295,34 @@ steps:
      build.pull_request.labels includes "diffusion-x2v-test"
    steps:

-      - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test"
+      - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · Single-GPU"
        timeout_in_minutes: 90
        commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda" --run-level "full_model"
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and cards_1" --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and cards_2" --run-level "full_model"
        mirror_hardwares: h100_2

-      - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test"
+      - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · Single-GPU"
        timeout_in_minutes: 90
        commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda" --run-level "full_model"
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and cards_1" --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and cards_2" --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test"
+        timeout_in_minutes: 90
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and cuda" --run-level "full_model"
        mirror_hardwares: h100_2

      - label: ":full_moon: Diffusion X2V · Doc Test"
@@ -304,17 +357,49 @@ steps:
          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model
        mirror_hardwares: h100_2

-      - label: ":full_moon: Diffusion X2V · Perf Test"
-        key: nightly-diffusion-x2v-performance
+      - label: ":full_moon: Diffusion X2V · Perf Test · Single-GPU"
+        key: nightly-diffusion-x2v-performance-single
        timeout_in_minutes: 180
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
          - |
            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_1"
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: Diffusion X2V · Perf Test · 2-GPU"
+        key: nightly-diffusion-x2v-performance-2gpu
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_2"
            EXIT1=$$?
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json -m "H100 and cards_2"
+            EXIT2=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi
+            exit $$EXIT2
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2V · Perf Test · HunyuanVideo-1.5"
+        key: nightly-diffusion-x2v-performance-hunyuanvideo15
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
+            EXIT1=$$?
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json
            EXIT2=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
@@ -348,10 +433,16 @@ steps:
          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4' --run-level "full_model"
        mirror_hardwares: l4_4

-      - label: ":full_moon: Diffusion · Distributed Test with H100"
+      - label: ":full_moon: Diffusion · Distributed Test with H100 · 2-GPU"
        timeout_in_minutes: 60
        commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100' --run-level "full_model"
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and cards_2' --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion · Distributed Test with H100 · 4-GPU"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and cards_4' --run-level "full_model"
        mirror_hardwares: h100_4

      - label: ":full_moon: Diffusion · Offloader Test"
@@ -375,7 +466,7 @@ steps:
      - label: ":full_moon: Diffusion · Quantization Test with L4"
        timeout_in_minutes: 60
        commands:
-          - pip install "vllm-gguf-plugin>=0.0.3"
+          - pip install "vllm-gguf-plugin==0.0.4"
          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model"
        mirror_hardwares: l4_1

@@ -386,19 +477,25 @@ steps:
    depends_on:
      - nightly-omni-performance-no-async-chunk
      - nightly-omni-performance-async-chunk
-      - nightly-tts-performance
-      - nightly-diffusion-x2iat-performance-qwen-image
-      - nightly-diffusion-x2v-performance
+      - nightly-tts-performance-single
+      - nightly-tts-performance-2gpu
+      - nightly-diffusion-x2iat-performance-qwen-image-single
+      - nightly-diffusion-x2iat-performance-qwen-image-4gpu
+      - nightly-diffusion-x2v-performance-single
+      - nightly-diffusion-x2v-performance-2gpu
    if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1"
    commands:
      - pip install openpyxl
      - export DEFAULT_INPUT_DIR=tests/dfx/perf/results
      - export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results
-      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-single
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-2gpu
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
-      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image
-      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-single
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-4gpu
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance-single
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance-2gpu
      - python tools/nightly/generate_nightly_perf_excel.py
      - python tools/nightly/generate_nightly_perf_html.py
      - python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html"
@@ -205,10 +205,10 @@ steps:
          - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
          - tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
+          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py
          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
          - vllm_omni/deploy/minicpmo_4_5.yaml
          - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
-          - vllm_omni/deploy/minicpmo_4_5_duplex.yaml
          - vllm_omni/experimental/fullduplex/
          - vllm_omni/model_executor/models/minicpmo_4_5/
          - vllm_omni/model_executor/models/step_audio2/
@@ -252,6 +252,9 @@ steps:
          - vllm_omni/model_executor/models/common/alias_free_activation.py
          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
          - vllm_omni/deploy/qwen3_tts.yaml
+          - vllm_omni/entrypoints/openai/serving_speech.py
+          - vllm_omni/entrypoints/openai/tts_adapters/
+          - vllm_omni/engine/stage_pool.py
        commands:
          - |
            timeout 20m bash -c "
@@ -260,25 +263,6 @@ steps:
            "
        mirror_hardwares: l4_1

-      - label: "TTS · Qwen3-TTS Base Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen3_tts_base.py
-          - vllm_omni/model_executor/models/qwen3_tts/
-          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-          - vllm_omni/model_executor/models/common/snake_activation.py
-          - vllm_omni/model_executor/models/common/alias_free_activation.py
-          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
-          - vllm_omni/deploy/qwen3_tts.yaml
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_WORKER_MULTIPROC_METHOD=spawn
-              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
-              export VLLM_OMNI_USE_V2_RUNNER=1
-              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m 'core_model' --run-level 'core_model'
-            "
-        mirror_hardwares: l4_1
-
      - label: "Diffusion · Wan22 Test"
        source_file_dependencies:
          - tests/e2e/online_serving/test_wan22_t2v.py
@@ -307,6 +291,24 @@ steps:
            "
        mirror_hardwares: h100_1

+      - label: "Diffusion · MiniMax-H3 DLO DP2 T2VA Test"
+        timeout_in_minutes: 90
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py
+          - tests/helpers/assertions.py
+          - tests/helpers/runtime.py
+          - vllm_omni/diffusion/models/minimax_h3/
+          - vllm_omni/diffusion/offloader/
+          - vllm_omni/diffusion/distributed/
+          - vllm_omni/entrypoints/openai/
+        commands:
+          - |
+            timeout 80m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py -m 'core_model and cuda and diffusion' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_2
+
      - label: "Diffusion · Qwen Image Test"
        source_file_dependencies:
          - tests/e2e/online_serving/test_qwen_image.py
@@ -320,3 +322,15 @@ steps:
              pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model'
            "
        mirror_hardwares: h100_1
+
+      - label: "Diffusion · HunyuanVideo-1.5 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
+          - vllm_omni/diffusion/models/hunyuan_video/
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'core_model and diffusion' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_1
@@ -44,10 +44,16 @@ steps:
          - pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow"
        mirror_hardwares: h100_4

-      - label: "Reliability Test · Invalid parameters · H100"
-        timeout_in_minutes: 120
+      - label: "Reliability Test · Invalid parameters · H100 · Single-GPU"
+        timeout_in_minutes: 90
        commands:
-          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"
+          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100 and cards_1"
+        mirror_hardwares: h100_1
+
+      - label: "Reliability Test · Invalid parameters · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100 and cards_2"
        mirror_hardwares: h100_2

      - label: "Reliability Test · Invalid parameters · L4"
@@ -74,6 +80,19 @@ steps:
            exit $$EXIT
        mirror_hardwares: h100_2

+      - label: ":full_moon: Omni · Perf Test · Async Chunk · Random"
+        key: weekly-omni-performance-async-chunk-random
+        timeout_in_minutes: 180
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and slow"
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_2
+
      - label: ":full_moon: Omni · Perf Test · Multi-Replica"
        key: weekly-omni-performance-multi-replicas
        timeout_in_minutes: 180
@@ -92,27 +111,37 @@ steps:
    if: build.env("NON_CRITICAL") == "1"
    steps:

-      - label: "Omni · L4"
+      # Qwen2.5 expansion (CUDA 4-card / online 2-card) stays on this l4_4 job.
+      - label: "Omni · L4 · Multi-GPU"
        timeout_in_minutes: 90
        commands:
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
+          - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and not cards_1" --run-level "core_model"
          - >-
            pytest -s -v tests/e2e/
            --ignore=tests/e2e/online_serving/test_qwen2_5_omni_expansion.py
            --ignore=tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
-            -m "slow and L4 and omni"
+            -m "slow and L4 and omni and not cards_1"
            --run-level "full_model"
        mirror_hardwares: l4_4

-      - label: "Omni · H100"
+      - label: "Omni · H100 · Single-GPU"
        timeout_in_minutes: 90
        commands:
          - >-
            pytest -sv
-            tests/e2e/ -m "slow and H100 and omni"
+            tests/e2e/ -m "slow and H100 and omni and cards_1"
            --run-level "full_model"
-        mirror_hardwares: h100_2
+        mirror_hardwares: h100_1
+
+      - label: "Omni · H100 · 4-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/ -m "slow and H100 and omni and cards_4"
+            --run-level "full_model"
+        mirror_hardwares: h100_4

      - label: "TTS · L4"
        timeout_in_minutes: 120
@@ -156,12 +185,21 @@ steps:
            --run-level "full_model"
        mirror_hardwares: l4_1

-      - label: "Diffusion · L4"
+      - label: "Diffusion · L4 · Single-GPU"
        timeout_in_minutes: 60
        commands:
          - >-
            pytest -sv
-            tests/e2e/ -m "slow and diffusion and L4"
+            tests/e2e/ -m "slow and diffusion and L4 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: l4_1
+
+      - label: "Diffusion · L4 · Multi-GPU"
+        timeout_in_minutes: 60
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/ -m "slow and diffusion and L4 and not cards_1"
            --run-level "full_model"
        mirror_hardwares: l4_4

@@ -170,15 +208,24 @@ steps:
        commands:
          - >-
            pytest -sv
-            tests/e2e/ -m "slow and diffusion and H100 and not distributed_cuda"
+            tests/e2e/ -m "slow and diffusion and H100 and cards_1"
            --run-level "full_model"
        mirror_hardwares: h100_1

-      - label: "Diffusion · H100 · Multi-GPU"
+      - label: "Diffusion · H100 · 2-GPU"
        timeout_in_minutes: 90
        commands:
          - >-
            pytest -sv
-            tests/e2e/ -m "slow and diffusion and H100 and distributed_cuda"
+            tests/e2e/ -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · H100 · 4-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/ -m "slow and diffusion and H100 and cards_4"
            --run-level "full_model"
        mirror_hardwares: h100_4
@@ -10,7 +10,7 @@ steps:
          DOCKER_BUILDKIT: "1"
          # Buildkite will automatically replace this with the actual commit hash
          VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.27.0"
+          VLLM_VERSION: "v0.28.0"
        priority: 100
        timeout_in_minutes: 60
        soft_fail: false
@@ -5,7 +5,12 @@ set -ex

 omni_source_dir=$(git rev-parse --show-toplevel)

-base_image_name="xpu/vllm-omni-ci-base:${VLLM_VERSION:?VLLM_VERSION must be set}"
+: "${VLLM_VERSION:?VLLM_VERSION must be set}"
+# Official upstream XPU image. It is published from vllm's own
+# docker/Dockerfile.xpu, which is exactly what build_vllm_base() clones and
+# builds below, so pulling it is equivalent to building the fallback base.
+upstream_base_image="vllm/vllm-openai-xpu:${VLLM_VERSION}"
+local_base_image="xpu/vllm-omni-ci-base:${VLLM_VERSION}"
 image_name="xpu/vllm-omni-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT must be set}"
 container_name="xpu_${BUILDKITE_COMMIT}_$(
    tr -dc A-Za-z0-9 </dev/urandom | head -c 10
@@ -49,9 +54,36 @@ build_vllm_base() (
        "${vllm_source_dir}"
 )

-if [ -z "$(docker images -q "${base_image_name}")" ]; then
-    build_vllm_base
-fi
+# Resolve the vLLM base image, in priority order:
+#   1. the published upstream image for VLLM_VERSION, pulled fresh (so moving
+#      tags like nightly are refreshed rather than served stale from disk);
+#   2. a local copy of that same image, when the registry is unreachable;
+#   3. VLLM_BASE, if that image is already on disk;
+#   4. otherwise build VLLM_BASE from the matching vLLM source tag (~30min).
+# VLLM_BASE names the fallback base and defaults to ${local_base_image}; set it
+# to reuse or produce a base under a different tag.
+resolve_vllm_base() {
+    if docker pull "${upstream_base_image}"; then
+        base_image_name="${upstream_base_image}"
+        return
+    fi
+
+    # Registry unreachable, but a previous run may have left the image behind.
+    if [ -n "$(docker images -q "${upstream_base_image}")" ]; then
+        echo "WARNING: could not pull ${upstream_base_image}; using local copy" >&2
+        base_image_name="${upstream_base_image}"
+        return
+    fi
+
+    echo "WARNING: ${upstream_base_image} unavailable remotely and locally" >&2
+    base_image_name="${VLLM_BASE:-${local_base_image}}"
+    if [ -z "$(docker images -q "${base_image_name}")" ]; then
+        echo "WARNING: building ${base_image_name} from vLLM ${VLLM_VERSION} source" >&2
+        build_vllm_base
+    fi
+}
+
+resolve_vllm_base

 # Try building the docker image
 docker_build "${image_name}" --build-arg "VLLM_BASE=${base_image_name}" --build-arg "VLLM_VERSION=${VLLM_VERSION}"
@@ -10,6 +10,18 @@ steps:
        -m 'core_model and npu and A3 and omni'
        --run-level 'core_model'

+  - label: "Diffusion NPU layer tests"
+    key: ready-diffusion-npu-layer-tests
+    depends_on: upload-ready-pipeline
+    mirror_hardwares: a3_npu_2
+    commands:
+      - >-
+        pytest -s -v
+        tests/diffusion/layers/test_activation.py
+        tests/diffusion/layers/test_fused_qk_norm_rope_npu.py
+        -m 'core_model and npu and A3'
+        --run-level 'core_model'
+
  - group: ":card_index_dividers: Omni Model Test"
    key: ready-omni-group
    depends_on: upload-ready-pipeline
@@ -105,7 +105,7 @@ For custom loop structures, follow `vllm_omni/diffusion/models/progress_bar.py`

 ## Path B: Custom/Private Repo Model

-For models without a diffusers pipeline — weights in custom format, model code in a private repo. Real examples: DreamID-Omni, BAGEL, HunyuanImage3. See [references/custom-model-patterns.md](references/custom-model-patterns.md) for concrete custom-model integration patterns.
+For models without a diffusers pipeline — weights in custom format, model code in a private repo. Real examples: BAGEL, HunyuanImage3. See [references/custom-model-patterns.md](references/custom-model-patterns.md) for concrete custom-model integration patterns.

 ### B1. Understand the reference repo

@@ -126,9 +126,9 @@ This is the key design decision for custom models. Follow these placement rules:
 | **Pipeline orchestration** (init, forward, denoise loop) | `vllm_omni/diffusion/models/<name>/pipeline_<name>.py` | Always required |
 | **Custom transformer/backbone** (ported and adapted to vllm-omni) | `vllm_omni/diffusion/models/<name>/<name>_transformer.py` or similar | `wan2_2.py`, `fusion.py`, `bagel_transformer.py` |
 | **Custom sub-models** (VAE, fusion, autoencoder) | `vllm_omni/diffusion/models/<name>/` as separate files | `autoencoder.py`, `fusion.py` |
-| **External dependency code** (original repo utilities) | **External repo**, installed via download script or pip | `dreamid_omni` package via git clone |
+| **External dependency code** (original repo utilities) | **External repo**, installed via download script or pip | original-repo package via git clone |
 | **Hardcoded model configs** | Module-level dicts in pipeline file | `VIDEO_CONFIG`, `AUDIO_CONFIG` dicts |
-| **Download/setup script** | `examples/offline_inference/<name>/download_<name>.py` | `download_dreamid_omni.py` |
+| **Download/setup script** | `examples/offline_inference/<name>/download_<name>.py` | `download_<name>.py` |
 | **Custom `model_index.json`** | Generated by download script, placed at model root | Minimal: `{"_class_name": "YourPipeline", ...}` |

 ### B3. Handle external dependencies
@@ -157,7 +157,7 @@ Copy the essential model files into `vllm_omni/diffusion/models/<name>/` and ada

 Custom models have two common patterns for weight loading:

-**Pattern 1: Bypass standard loader** (DreamID-Omni style)
+**Pattern 1: Bypass standard loader** (eager custom init)

 When the original model has complex custom init functions that load weights in `__init__`:

@@ -338,7 +338,7 @@ Classify the model's **CI priority** first:

 **L2 & L3 online — same file, dual marks on the baseline smoke:** The **first / simplest** case in `test_{slug}.py` (default deploy, minimal steps, single prompt) should carry **both** `@pytest.mark.core_model` and `@pytest.mark.advanced_model` on the **same** function so L2 (`test-ready.yml`) and L3 (`test-merge.yml`) share one smoke test. Heavier deploy variants or API paths in the same file use **`advanced_model` only**. When L3 moves to nightly, migrate those heavier cases into `test_{slug}_expansion.py` with `full_model` and remove the dedicated `test-merge.yml` job (see `test_longcat_image_expansion.py`, `test_qwen_image_expansion.py`).

-**L4 design (high priority):** Combine multiple supported features (Cache-DiT, TP, USP, CFG, HSDP, CPU offload, quantization) into **few parametrized** `OmniServerParams` rows so each feature appears in at least one case without exploding GPU jobs. Shard single-GPU vs multi-GPU cases across the nightly X2I/X2V function steps (1-GPU vs `distributed_cuda`).
+**L4 design (high priority):** Combine multiple supported features (Cache-DiT, TP, USP, CFG, HSDP, CPU offload, quantization) into **few parametrized** `OmniServerParams` rows so each feature appears in at least one case without exploding GPU jobs. Shard single-GPU vs multi-GPU cases across the nightly X2I/X2V function steps (`cards_1` vs `not cards_1`).

 **L4 design (medium / low):** One or two parametrized rows covering the best quality/perf trade-off; skip perf JSON unless the model is high priority.

@@ -16,20 +16,19 @@ vllm_omni/diffusion/models/wan2_2/

 The transformer is loaded separately via `weights_sources` + `load_weights()`. Non-transformer components (VAE, text encoder) are loaded in `__init__` via `from_pretrained()`.

-### Custom model with external deps (e.g., DreamID-Omni)
+### Custom model with external deps

 ```
-vllm_omni/diffusion/models/dreamid_omni/
+vllm_omni/diffusion/models/<name>/
 ├── __init__.py                    # Exports pipeline only
-├── pipeline_dreamid_omni.py       # Pipeline: loads ALL weights in __init__ via custom helpers
-├── fusion.py                      # Custom fusion architecture (video + audio cross-attention)
-└── wan2_2.py                      # Re-implemented Wan backbone with split API
+├── pipeline_<name>.py             # Pipeline: loads ALL weights in __init__ via custom helpers
+└── <backbone>.py                  # Custom transformer / fusion architecture

-examples/offline_inference/x_to_video_audio/
-└── download_dreamid_omni.py       # Downloads weights from 3 HF repos + clones code repo
+examples/offline_inference/<task>/
+└── download_<name>.py             # Downloads weights and/or clones code repo
 ```

-All weights loaded eagerly in `__init__`. `load_weights()` is a no-op. External dependency (`dreamid_omni` package) imported with try/except.
+All weights loaded eagerly in `__init__`. `load_weights()` is a no-op. External dependency imported with try/except.

 ### Custom model with ported code (e.g., BAGEL)

@@ -76,7 +75,7 @@ self.weights_sources = [
 ]
 ```

-### Pattern 3: Fully custom loading (DreamID-Omni)
+### Pattern 3: Fully custom loading

 ```
 init → load ALL weights eagerly via custom helpers → load_weights() = no-op
@@ -108,8 +107,8 @@ Standard diffusers `model_index.json`:
 Custom model `model_index.json` (minimal):
 ```json
 {
-    "_class_name": "DreamIDOmniPipeline",
-    "fusion": "DreamID-Omni/dreamid_omni.safetensors"
+    "_class_name": "YourCustomPipeline",
+    "fusion": "weights/model.safetensors"
 }
 ```

@@ -117,7 +116,7 @@ The only **required** field is `_class_name` — it must match a key in `_DIFFUS

 ## External Dependency Management

-### Git clone + .pth injection (DreamID-Omni pattern)
+### Git clone + .pth injection

 ```python
 def download_dependency():
@@ -163,7 +162,7 @@ The engine checks `isinstance(pipeline, SupportImageInput)` at startup to config

 Diffusers models use `config.json` in each subfolder. Custom models often use:

-**Module-level config dicts** (DreamID-Omni):
+**Module-level config dicts**:
 ```python
 VIDEO_CONFIG = {
    "patch_size": [1, 2, 2], "model_type": "ti2v",
@@ -181,7 +180,7 @@ vae_cfg = bagel_cfg.get("vae_config", {})

 ## Custom Architecture Patterns

-### Split forward API (DreamID-Omni)
+### Split forward API

 When a fusion model needs to interleave blocks from two backbones:

@@ -34,7 +34,7 @@ but the changes are correct — stage the modified files and re-commit.
 | `check-forbidden-imports` | Stdlib `re`/`base64`, pickle, Hugging Face Hub API, or direct Triton/TileLang | `import regex as re` and `pybase64`; use `vllm.transformers_utils.repo_utils`. Do **not** add the file to `allowed_files` without review |
 | `check-torch-cuda-call` | New `torch.cuda.*` call site | Use `current_omni_platform` / `OmniPlatform`; do not grow `ALLOWED_FILES` without review |
 | `check-tts-adapter-migration` | New `self._tts_model_type` branch in `serving_speech.py` | Put per-model logic in `tts_adapters/`. Lower `MAX_MODEL_TYPE_BRANCHES` when removing branches; do not raise it without review |
-| `check-test-ci-coverage` | New `tests/**/test_*.py` missing level or hardware mark | Add `core_model`/`advanced_model`/… plus `cpu`/`cuda`/`hardware_test(` |
+| `check-mark` | New `tests/**/test_*.py` missing level/hardware mark, or direct `pytest.mark.H100` | Add `core_model`/`advanced_model`/… plus `cpu`/`cuda`/`hardware_test(`; SKU marks only via helpers |
 | `mypy-3.10` | Type error on changed `vllm_omni/` files | Fix the types; model trees are excluded. Extra versions: `pre-commit run --hook-stage manual mypy-3.12` |
 | `markdownlint-cli2` | Docs/README markdown lint | Accept auto-fix under `docs/` / `recipes/` / root README |
 | `check-buildkite` | Invalid `.buildkite/*.yml` | Fix the YAML; do not grow `SKIP_FILES` without review |
@@ -52,8 +52,8 @@ Always attach markers deliberately:
  - `tts` — speech synthesis / TTS-only models (`/v1/audio/speech`, voice clone, etc.)
  - `diffusion` — generative diffusion models (image / audio / text / video from noise)
 - **Cross-cutting area** (when relevant): `parallel`, `cache`, `example`, `benchmark`
- **Hardware**: `cpu`, `gpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `distributed_cuda`, …
- Optional: `slow`, distributed markers when multi-card is required
+- **Hardware**: `cpu`, `gpu`, `cuda`, `rocm`, `npu`, …; SKU (`H100`, `L4`, …) and `cards_{n}` via `@hardware_test(...)` / `hardware_marks(...)` only — never `pytest.mark.H100` (`check-mark` rejects it)
+- Optional: `slow`. Multi-card: `num_cards=N` on the helper (filter with `cards_N` / `not cards_1`)

 **Baseline smoke (L2 + L3):** The simplest e2e case per model — default deploy, minimal request — should usually carry **both** `@pytest.mark.core_model` and `@pytest.mark.advanced_model` on the **same** test function so `test-ready.yml` and `test-merge.yml` share one test. `send_*_request` picks validation depth from `--run-level`. References: `test_voxcpm2_tts.py::test_text_to_audio_001`, `test_qwen3_tts_customvoice.py::test_text_to_audio_001`. Heavier scenarios use **`advanced_model` only**; L4 expansion uses **`full_model`**.

@@ -274,7 +274,7 @@ def test_text_to_video_001(omni_server, online_client) -> None:

 **3. Cross-cutting rules**

- New `.py` files need the Omni SPDX header (`Copyright contributors to the vLLM-Omni project`, not `vLLM project`). `tests/` may keep stdlib `re`/`base64`; pickle is still banned unless the file is already on the pickle allowlist. Every collected `tests/**/test_*.py` needs a CI **level** mark and a **hardware** mark/helper (`check-test-ci-coverage` runs locally; GHA skips it). Do not grow `allowed_files` to land a test. Policy: [docs/contributing/README.md](../../../docs/contributing/README.md#linting).
+- New `.py` files need the Omni SPDX header (`Copyright contributors to the vLLM-Omni project`, not `vLLM project`). `tests/` may keep stdlib `re`/`base64`; pickle is still banned unless the file is already on the pickle allowlist. Every collected `tests/**/test_*.py` needs a CI **level** mark and a **hardware** platform mark/helper (`check-mark` runs locally; GHA skips it). Do not write `pytest.mark.H100` / `pytest.mark.L4`; use the helpers so `cards_{n}` is attached. Do not grow `allowed_files` to land a test. Policy: [docs/contributing/README.md](../../../docs/contributing/README.md#linting).
 - Reuse existing fixtures for the chosen scenario; do not mix “online client” assumptions into offline `OmniRunner` tests without a clear reason.
 - Avoid external network dependency in assertions unless the scenario is explicitly “online serving” or doc examples that require a model hub (then align with CI secrets/cache).
 - Keep **one test function = one intent** (one modality combo, one endpoint contract, or one acceleration combo).
@@ -79,7 +79,7 @@ E2E (L2+) should not rely on mocks unless documenting a rare exception; prefer r
   - `full_model` — L4 nightly (`test-nightly.yml`); some expansion tests still carry both `advanced_model` and `full_model` during migration
 2. **Model type** (pick one): `omni`, `tts`, or `diffusion`
 3. **Cross-cutting** when relevant: `parallel`, `cache`, `example`, `benchmark`
-4. **Hardware**: `cpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `distributed_cuda`, …
+4. **Hardware**: `cpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `cards_1`, …
 5. Multi-card: `@hardware_test(...)` in `tests/helpers/mark.py`

 ## Command Templates
@@ -244,7 +244,7 @@ pytest -s -v dfx/reliability/invalid_param_test/test_invalid_image_generation.py

 ```bash
 cd tests
-pytest -s -v -m "core_model and distributed_cuda and L4" --run-level=core_model
+pytest -s -v -m "core_model and L4 and not cards_1" --run-level=core_model
 ```

 ### Concrete e2e paths (common in-tree)
@@ -27,4 +27,4 @@ jobs:
        # Don't run historical-debt hooks against the whole tree in CI yet.
        # They still run locally on changed files.
        env:
-          SKIP: check-test-ci-coverage,markdownlint-cli2,shellcheck,check-spdx-header,mypy-3.10
+          SKIP: check-mark,markdownlint-cli2,shellcheck,check-spdx-header,mypy-3.10
@@ -105,8 +105,8 @@ repos:
      <<: *mypy_common
      stages: [manual]

-    - id: check-test-ci-coverage
-      name: Ensure test files have pytestmarks
+    - id: check-mark
+      name: Ensure test files have CI marks (no direct SKU)
      entry: python tools/pre_commit/check_test_marks.py
      language: python
      types: [python]
@@ -16,6 +16,7 @@ Easy, fast, and cheap omni-modality model serving for everyone

 *Latest News* 🔥

+- [2026/08] We released [0.28.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.28.0), featuring production-ready [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) serving on GPU and NPU, a unified AR/DiT paged KV cache runtime, and enhanced realtime full-duplex serving for the [MiniCPM-o series](recipes/OpenBMB/MiniCPM-o-4_5.md).
 - [2026/08] [VeRL-Omni](https://github.com/verl-project/verl-omni) `v0.2.0` is released: faster diffusion RL powered by vLLM-Omni (request-level/step-wise batching with FA3), rebuilt Qwen3-Omni multimodal training (DPO & GSPO), plus LTX-2.3, Qwen-Image-Edit support and more. See the [release notes](https://github.com/verl-project/verl-omni/releases/tag/v0.2.0).
 - [2026/08] We released [0.26.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.26.0) - aligned with the vLLM 0.26 release line, featuring [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) joint video/audio generation, an experimental full-duplex realtime runtime for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md), distributed layerwise diffusion offload, and broader model, hardware, streaming, TTS, and quantization support.
 - [2026/07] We released [0.24.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.24.0) - aligned with the vLLM 0.24 release line, expanding production-ready coverage across TTS, speech, diffusion, image/video generation, and robot-policy serving, with major Omni stage runtime refactoring, diffusion request-level batching, async output materialization, quantization/cache/memory improvements, and broad CUDA/ROCm/XPU/NPU support.
@@ -57,9 +58,9 @@ vLLM-Omni is flexible and easy to use with:
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:

 - **Omni-modality models** (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
- **TTS models** (e.g. Qwen3-TTS, VoxCPM2, Ming-Omni-TTS, CosyVoice3)
- **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, Qwen-Image, Wan2.2, FLUX)
- **Robot-policy and action models** (e.g. GR00T-N1.7, DreamZero-DROID, InternVLA-A1, Cosmos3 action policy)
+- **TTS models** (e.g. Qwen3-TTS, IndexTTS 2.5, dots.tts, CosyVoice3)
+- **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
+- **Robot-policy and action models** (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)

 ## Getting Started

@@ -67,6 +68,7 @@ Visit our [documentation](https://vllm-omni.readthedocs.io/en/latest/) to learn

 - [Installation](https://vllm-omni.readthedocs.io/en/latest/getting_started/installation/)
 - [Quickstart](https://vllm-omni.readthedocs.io/en/latest/getting_started/quickstart/)
+- [Nightly CUDA Docker images](https://hub.docker.com/r/vllm/vllm-omni/tags?name=nightly) built from the latest `main`
 - [List of Supported Models](https://vllm-omni.readthedocs.io/en/latest/models/supported_models/)
 - [Deployment Recipes](https://recipes.vllm.ai) for vLLM-Omni model serving

@@ -0,0 +1,151 @@
+"""Benchmark CosyVoice3-style CUDA stream handoff strategies."""
+
+import argparse
+import json
+import statistics
+import time
+
+import torch
+
+from vllm_omni.platforms import current_omni_platform
+
+
+def run_case(
+    *,
+    device: torch.device,
+    steps: int,
+    warmup_steps: int,
+    producer_cycles: int,
+    estimator_cycles: int,
+    use_stream_dependencies: bool,
+) -> dict[str, float | int | str]:
+    caller_stream = torch.cuda.Stream(device=device)
+    estimator_stream = torch.cuda.Stream(device=device)
+    input_tensor = torch.zeros(1024, device=device)
+    output_tensor = torch.empty_like(input_tensor)
+    submission_ms = []
+
+    def step(measure: bool) -> None:
+        with torch.cuda.stream(caller_stream):
+            torch.cuda._sleep(producer_cycles)
+            input_tensor.add_(1)
+
+        start = time.perf_counter_ns()
+        if use_stream_dependencies:
+            estimator_stream.wait_stream(caller_stream)
+        else:
+            caller_stream.synchronize()
+
+        with torch.cuda.stream(estimator_stream):
+            torch.cuda._sleep(estimator_cycles)
+            output_tensor.copy_(input_tensor)
+
+        if use_stream_dependencies:
+            caller_stream.wait_stream(estimator_stream)
+        else:
+            estimator_stream.synchronize()
+
+        if measure:
+            submission_ms.append((time.perf_counter_ns() - start) / 1e6)
+
+    for _ in range(warmup_steps):
+        step(measure=False)
+    caller_stream.synchronize()
+    estimator_stream.synchronize()
+    input_tensor.zero_()
+    torch.accelerator.reset_peak_memory_stats(device)
+    initial_allocated = torch.accelerator.memory_allocated(device)
+
+    start_event = torch.cuda.Event(enable_timing=True)
+    end_event = torch.cuda.Event(enable_timing=True)
+    wall_start = time.perf_counter()
+    with torch.cuda.stream(caller_stream):
+        start_event.record()
+    for _ in range(steps):
+        step(measure=True)
+    with torch.cuda.stream(caller_stream):
+        end_event.record()
+    end_event.synchronize()
+    wall_ms = (time.perf_counter() - wall_start) * 1e3
+    peak_delta_bytes = torch.accelerator.max_memory_allocated(device) - initial_allocated
+
+    expected = torch.full_like(output_tensor, steps)
+    torch.testing.assert_close(output_tensor, expected, rtol=0, atol=0)
+    return {
+        "mode": "wait_stream" if use_stream_dependencies else "synchronize",
+        "steps": steps,
+        "submission_median_ms": statistics.median(submission_ms),
+        "submission_p95_ms": statistics.quantiles(submission_ms, n=20)[18],
+        "submission_mean_ms": statistics.mean(submission_ms),
+        "gpu_elapsed_ms": start_event.elapsed_time(end_event),
+        "wall_elapsed_ms": wall_ms,
+        "peak_allocation_delta_bytes": peak_delta_bytes,
+        "final_value": int(output_tensor[0].item()),
+    }
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--device", default="cuda:0")
+    parser.add_argument("--steps", type=int, default=100)
+    parser.add_argument("--warmup-steps", type=int, default=10)
+    parser.add_argument("--repeats", type=int, default=7)
+    parser.add_argument("--producer-cycles", type=int, default=1_000_000)
+    parser.add_argument("--estimator-cycles", type=int, default=2_000_000)
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    device = torch.device(args.device)
+    current_omni_platform.set_device(device)
+    torch.manual_seed(0)
+
+    old_results = []
+    new_results = []
+    for repeat in range(args.repeats):
+        common_args = {
+            "device": device,
+            "steps": args.steps,
+            "warmup_steps": args.warmup_steps,
+            "producer_cycles": args.producer_cycles,
+            "estimator_cycles": args.estimator_cycles,
+        }
+        if repeat % 2 == 0:
+            old_results.append(run_case(**common_args, use_stream_dependencies=False))
+            new_results.append(run_case(**common_args, use_stream_dependencies=True))
+        else:
+            new_results.append(run_case(**common_args, use_stream_dependencies=True))
+            old_results.append(run_case(**common_args, use_stream_dependencies=False))
+
+    old_submission_ms = statistics.median(result["submission_median_ms"] for result in old_results)
+    new_submission_ms = statistics.median(result["submission_median_ms"] for result in new_results)
+    old_gpu_ms = statistics.median(result["gpu_elapsed_ms"] for result in old_results)
+    new_gpu_ms = statistics.median(result["gpu_elapsed_ms"] for result in new_results)
+    print(
+        json.dumps(
+            {
+                "device": torch.cuda.get_device_name(device),
+                "torch": torch.__version__,
+                "cuda": torch.version.cuda,
+                "repeats": args.repeats,
+                "warmup_steps": args.warmup_steps,
+                "producer_cycles": args.producer_cycles,
+                "estimator_cycles": args.estimator_cycles,
+                "old_submission_median_ms": old_submission_ms,
+                "new_submission_median_ms": new_submission_ms,
+                "submission_median_speedup": old_submission_ms / new_submission_ms,
+                "old_gpu_elapsed_median_ms": old_gpu_ms,
+                "new_gpu_elapsed_median_ms": new_gpu_ms,
+                "gpu_elapsed_change_percent": (new_gpu_ms - old_gpu_ms) / old_gpu_ms * 100,
+                "parity": all(result["final_value"] == args.steps for result in old_results + new_results),
+                "old_runs": old_results,
+                "new_runs": new_results,
+            },
+            indent=2,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

 # ruff: noqa
 # code borrowed from https://github.com/pytorch/pytorch/blob/main/torch/utils/collect_env.py
@@ -494,6 +494,16 @@ def is_xnnpack_available():


 def get_env_vars():
+    try:
+        from vllm_omni.config.environment_variable_inventory import ENVIRONMENT_VARIABLE_INVENTORY
+    except Exception:
+        # Keep environment collection usable when an incomplete/broken Omni
+        # installation is the reason the user is running this diagnostic.
+        # Importing an Omni submodule executes package initialization, which
+        # may fail with RuntimeError, AssertionError, or OSError as well as
+        # ImportError when vLLM or the accelerator stack is broken.
+        ENVIRONMENT_VARIABLE_INVENTORY = {}
+
    env_vars = ""
    secret_terms = ("secret", "token", "api", "access", "password")
    report_prefix = (
@@ -507,12 +517,17 @@ def get_env_vars():
        "MKL_",
        "NVIDIA",
    )
-    for k, v in os.environ.items():
-        if any(term in k.lower() for term in secret_terms):
+    for k, v in sorted(os.environ.items()):
+        omni_classification = ENVIRONMENT_VARIABLE_INVENTORY.get(k)
+        if (omni_classification is not None and omni_classification.redact_value) or any(
+            term in k.lower() for term in secret_terms
+        ):
            continue
-        if k in environment_variables:
-            env_vars = env_vars + "{}={}".format(k, v) + "\n"
-        if k.startswith(report_prefix):
+        if (
+            k in environment_variables
+            or (omni_classification is not None and omni_classification.is_public_omni)
+            or k.startswith(report_prefix)
+        ):
            env_vars = env_vars + "{}={}".format(k, v) + "\n"

    return env_vars
@@ -1,4 +1,4 @@
-# Target vllm version: v0.27.0 (releases/v0.27.0 branch).
+# Target vllm version: v0.28.0 (tag v0.28.0 @ 2cf0a6915ce5).
 # Keep VLLM_BASE_TAG aligned with the target vLLM release on every rebase.
 #
 # Release-boundary form: there is no precompiled-wheel pin here, because the
@@ -14,11 +14,14 @@
 # core init), registering libnvJitLink.so.13 with ldconfig, and re-pinning numpy
 # under numba's ceiling. All of it was compensation for a base image and a vLLM
 # that disagreed. At a real release boundary they agree, so none of it is
-# needed. If you pin an untagged commit again, restore the reinstall block —
-# agent/tasks/20_update_wheel_url.sh fails the run rather than letting the
-# Dockerfile silently ship the wrong vLLM.
+# needed. The 0.28 rebase repeated the cycle: the v0.28.0rc / final-tag wheels
+# rode on the v0.27.0 base image with the full reinstall block until the
+# official v0.28.0 image shipped, at which point this file dropped back to the
+# form you are reading. If you pin an untagged commit again, restore the
+# reinstall block — agent/tasks/20_update_wheel_url.sh fails the run rather
+# than letting the Dockerfile silently ship the wrong vLLM.
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai
-ARG VLLM_BASE_TAG=v0.27.0
+ARG VLLM_BASE_TAG=v0.28.0

 # We use registry-based layer caching for the dependencies,
 # where the tag is determined based on the hash of the contents
@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai:v0.26.0
+ARG BASE_IMAGE=vllm/vllm-openai:v0.28.0
 FROM ${BASE_IMAGE}

 ARG COMMON_WORKDIR=/app
@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.27.0
+ARG VLLM_ASCEND_TAG=v0.28.0
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}

 # WORKDIR /vllm-workspace/vllm
@@ -30,6 +30,7 @@ RUN python3 -m pip install -v --no-deps \
      "stepaudio2-minicpmo>=0.1.1" \
      "step-audio2>=1.0.0"

+ENV VLLM_TARGET_DEVICE=npu
 ENV VLLM_WORKER_MULTIPROC_METHOD=spawn

 ENTRYPOINT []
@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.27.0-a3
+ARG VLLM_ASCEND_TAG=v0.28.0-a3
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}

 # WORKDIR /vllm-workspace/vllm
@@ -30,6 +30,7 @@ RUN python3 -m pip install -v --no-deps \
      "stepaudio2-minicpmo>=0.1.1" \
      "step-audio2>=1.0.0"

+ENV VLLM_TARGET_DEVICE=npu
 ENV VLLM_WORKER_MULTIPROC_METHOD=spawn

 ENTRYPOINT []
@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.27.0
+ARG VLLM_ASCEND_TAG=v0.28.0
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}

 ARG APP_DIR=/vllm-workspace/vllm-omni
@@ -65,6 +65,7 @@ RUN python3 -m pip install -v --no-deps \
      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
      --default-timeout=60

+ENV VLLM_TARGET_DEVICE=npu
 ENV VLLM_WORKER_MULTIPROC_METHOD=spawn

 ENTRYPOINT []
@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.27.0-a3
+ARG VLLM_ASCEND_TAG=v0.28.0-a3
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}

 ARG APP_DIR=/vllm-workspace/vllm-omni
@@ -66,6 +66,7 @@ RUN python3 -m pip install -v --no-deps \
      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
      --default-timeout=60

+ENV VLLM_TARGET_DEVICE=npu
 ENV VLLM_WORKER_MULTIPROC_METHOD=spawn

 ENTRYPOINT []
@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.27.0
+ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.28.0
 FROM ${BASE_IMAGE} AS base

 # Declare a variable to know if we want to use the nightly build or the stable build.
@@ -10,7 +10,7 @@ FROM ${BASE_IMAGE} AS base
 #    we should swap over to use stable release ASAP.
 #    We should avoid relying on custom commits.
 ARG USE_NIGHTLY_BUILD=0
-ARG VLLM_VERSION_OR_COMMIT_HASH=v0.27.0
+ARG VLLM_VERSION_OR_COMMIT_HASH=v0.28.0
 ARG ARG_PYTORCH_ROCM_ARCH
 ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}}

@@ -1,140 +1,33 @@
-# Argument to configure vllm base image if pre-built
-ARG VLLM_BASE=vllm-base
-
-FROM intel/deep-learning-essentials:2025.3.2-0-devel-ubuntu24.04 AS vllm-base
-
-WORKDIR /workspace/
-
-ARG PYTHON_VERSION=3.12
-ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/xpu"
-
-RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \
-    echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list
-
-RUN apt clean && apt-get update -y && \
-    apt-get install -y --no-install-recommends --fix-missing \
-    curl \
-    ffmpeg \
-    git \
-    libsndfile1 \
-    libsm6 \
-    libxext6 \
-    libgl1 \
-    lsb-release \
-    libaio-dev \
-    numactl \
-    wget \
-    vim \
-    python3.12 \
-    python3.12-dev \
-    python3-pip
-
-# Add oneAPI repo, pin oneAPI to 2025.3, then install pinned packages in one layer.
-RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \
-    echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \
-    printf '%s\n' \
-    'Package: intel-oneapi-* intel-deep-learning-essentials* intel-pti*' \
-    'Pin: version 2025.3*' \
-    'Pin-Priority: 1001' \
-    > /etc/apt/preferences.d/oneapi-2025.3.pref && \
-    apt-get update -y && \
-    apt-get install -y --no-install-recommends \
-    intel-oneapi-compiler-dpcpp-cpp-2025.3 \
-    intel-oneapi-mkl-devel-2025.3 \
-    intel-oneapi-dnnl-devel-2025.3 && \
-    rm -rf /var/lib/apt/lists/*
-
-# Install UMD with fixed version
-RUN mkdir neo && \
-    cd neo && \
-    wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.24.8/intel-igc-core-2_2.24.8+20344_amd64.deb && \
-    wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.24.8/intel-igc-opencl-2_2.24.8+20344_amd64.deb && \
-    wget https://github.com/intel/compute-runtime/releases/download/25.48.36300.8/intel-ocloc_25.48.36300.8-0_amd64.deb && \
-    wget https://github.com/intel/compute-runtime/releases/download/25.48.36300.8/intel-opencl-icd_25.48.36300.8-0_amd64.deb && \
-    wget https://github.com/intel/compute-runtime/releases/download/25.48.36300.8/libigdgmm12_22.8.2_amd64.deb && \
-    wget https://github.com/intel/compute-runtime/releases/download/25.48.36300.8/libze-intel-gpu1_25.48.36300.8-0_amd64.deb && \
-    wget https://github.com/oneapi-src/level-zero/releases/download/v1.26.0/level-zero_1.26.0+u24.04_amd64.deb && \
-    dpkg -i *.deb && \
-    cd .. && \
-    rm -rf neo
-
-ENV PATH="/root/.local/bin:$PATH"
-ENV VIRTUAL_ENV="/opt/venv"
-ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python
-RUN curl -LsSf https://astral.sh/uv/install.sh | sh
-RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV}
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-
-# This oneccl contains the BMG support which is not the case for default version of oneapi 2025.3.
-ARG ONECCL_INSTALLER="intel-oneccl-2021.15.9.14_offline.sh"
-RUN wget "https://github.com/uxlfoundation/oneCCL/releases/download/2021.15.9/${ONECCL_INSTALLER}" && \
-    bash "${ONECCL_INSTALLER}" -a --silent --eula accept && \
-    rm "${ONECCL_INSTALLER}" && \
-    echo "source /opt/intel/oneapi/setvars.sh --force" >> /root/.bashrc && \
-    echo "source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force" >> /root/.bashrc
-RUN rm -f /opt/intel/oneapi/ccl/latest && \
-    ln -s /opt/intel/oneapi/ccl/2021.15 /opt/intel/oneapi/ccl/latest
-
-SHELL ["bash", "-c"]
-CMD ["bash", "-c", "source /root/.bashrc && exec bash"]
-
-WORKDIR /workspace/
-
-ENV UV_HTTP_TIMEOUT=500
-
-# Configure package index for XPU
-ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
-ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
-ENV UV_INDEX_STRATEGY="unsafe-best-match"
-ENV UV_LINK_MODE="copy"
-
-ARG VLLM_VERSION=v0.27.0
-RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
-WORKDIR /workspace/vllm
-
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip install --upgrade pip && \
-    uv pip install -r requirements/xpu.txt && \
-    uv pip install setuptools_scm &&\
-    uv pip install grpcio-tools protobuf nanobind && \
-    source /opt/intel/oneapi/setvars.sh --force && \
-    source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force && \
-    export CMAKE_PREFIX_PATH="$(python3 -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH}"
-
-ENV LD_LIBRARY_PATH="/opt/intel/oneapi/ccl/2021.15/lib:$LD_LIBRARY_PATH:/usr/local/lib/"
-
-ENV VLLM_TARGET_DEVICE=xpu
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip uninstall triton triton-xpu && \
-    uv pip install triton-xpu==3.7.2 && \
-    uv pip install --no-build-isolation --no-deps .
-
-CMD ["/bin/bash"]
-
-FROM vllm-base AS vllm-openai
-
-# install nixl from source code
-RUN uv pip install nixl
-
-# install additional dependencies for openai api server
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip install accelerate hf_transfer pytest pytest_asyncio lm_eval[api] modelscope
-
-# install development dependencies (for testing)
-RUN uv pip install -e tests/vllm_test_utils
-
-# ensure vllm is properly installed
-RUN python -c "import vllm, inspect; print(vllm.__file__)"
-RUN uv pip show vllm
-
-CMD ["/bin/bash"]
-
-ENTRYPOINT []
+# Target vLLM version: v0.28.0. Keep VLLM_VERSION aligned with the target vLLM
+# release on every rebase.
+#
+# This file layers vLLM-Omni on top of the published upstream XPU image, which
+# already ships a self-consistent XPU runtime (torch-xpu, oneCCL/UCX+NIXL, the
+# pinned UMD/compute-runtime stack, and vLLM itself) built from
+# https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.xpu. Docker
+# pulls that base automatically, so there is nothing to build here beyond the
+# Omni layers. Mirrors how docker/Dockerfile.ci consumes vllm/vllm-openai.
+#
+# If the tag does not exist yet (targeting an unreleased vLLM commit), build the
+# base from upstream's own Dockerfile and point VLLM_BASE at the result:
+#
+#   git clone --depth 1 --branch $VLLM_VERSION https://github.com/vllm-project/vllm /tmp/vllm
+#   docker build -t vllm-openai-xpu:local --target vllm-openai \
+#       -f /tmp/vllm/docker/Dockerfile.xpu /tmp/vllm
+#   docker build -f docker/Dockerfile.xpu --build-arg VLLM_BASE=vllm-openai-xpu:local .
+#
+# .buildkite/intel/scripts/run-xpu-test.sh automates that pull-then-build fallback.
+ARG VLLM_BASE_IMAGE=vllm/vllm-openai-xpu
+ARG VLLM_VERSION=v0.28.0
+ARG VLLM_BASE=${VLLM_BASE_IMAGE}:${VLLM_VERSION}

 FROM ${VLLM_BASE} AS vllm-omni

+# The base image ships ENTRYPOINT ["vllm", "serve"]; clear it so this stage stays
+# usable as a plain shell/test container regardless of which VLLM_BASE was used.
+ENTRYPOINT []
+CMD ["/bin/bash"]
+
 RUN apt-get update && \
    apt-get install -y espeak-ng ffmpeg git jq && \
    apt-get clean && \
@@ -5,14 +5,22 @@ nav:
    - getting_started/quickstart.md
    - getting_started/installation/*
  - Serving:
-    - OpenAI-Compatible API:
-      - Chat Completions API: serving/chat_completions_api.md
-      - Image Generation: serving/image_generation_api.md
-      - Image Edit: serving/image_edit_api.md
-      - Text to Speech: serving/speech_api.md
-      - Videos: serving/videos_api.md
-      - Audio Generation: serving/audio_generate_api.md
-      - Streaming Video Input: serving/video_stream_api.md
+    - API Server:
+      - Overview: serving/README.md
+      - Core APIs:
+        - Chat Completions: serving/chat_completions_api.md
+        - Text to Speech: serving/speech_api.md
+        - Audio Generation: serving/audio_generate_api.md
+        - Image Generation: serving/image_generation_api.md
+        - Image Edit: serving/image_edit_api.md
+        - Videos: serving/videos_api.md
+      - Streaming and Realtime:
+        - Realtime Audio: serving/realtime_api.md
+        - Full Duplex: serving/full_duplex_api.md
+        - Streaming Video Input: serving/video_stream_api.md
+        - Streaming Video Output: serving/streaming_video_output_api.md
+        - OpenPI Robot Policy: serving/openpi_api.md
+      - Standalone Experimental Servers: serving/standalone_servers.md
  - Examples:
    - examples/README.md
    - Offline Inference:
@@ -24,13 +32,12 @@ nav:
      - Text-To-Speech: user_guide/examples/offline_inference/text_to_speech.md
      - Text-To-Video: user_guide/examples/offline_inference/text_to_video.md
      - X-To-Text: user_guide/examples/offline_inference/x_to_text.md
-      - X-To-Video-Audio: user_guide/examples/offline_inference/x_to_video_audio.md
    - Online Serving:
      - Image-To-Image: user_guide/examples/online_serving/image_to_image.md
      - Image-To-Video: user_guide/examples/online_serving/image_to_video.md
      - Speech-To-Video: user_guide/examples/online_serving/speech_to_video.md
      - Text-To-Image: user_guide/examples/online_serving/text_to_image.md
-      - Text-To-Speech: user_guide/examples/online_serving/text_to_speech.md
+      - Text-To-Speech (Online Serving): user_guide/examples/online_serving/text_to_speech.md
      - Text-To-Video: user_guide/examples/online_serving/text_to_video.md
  - General:
    - usage/*
@@ -54,6 +61,7 @@ nav:
      - BitsAndBytes W4: user_guide/quantization/bitsandbytes.md
      - MXFP8 W8A8: user_guide/quantization/mxfp8.md
      - MXFP4 W4A4: user_guide/quantization/mxfp4.md
+      - SVDQuant W4A4: user_guide/quantization/svdquant.md
      - ModelOpt: user_guide/quantization/modelopt.md
      - AutoRound: user_guide/quantization/autoround.md
      - msModelSlim: user_guide/quantization/msmodelslim.md
@@ -119,6 +127,8 @@ nav:
        - Async Omni Output Materialization: design/feature/omni_async_output_materialization.md
        - Automatic Prefix Caching: design/feature/prefix_caching.md
        - Realtime AR-Diffusion: design/feature/realtime_ar_diffusion.md
+      - Model-Specific Capabilities:
+        - MiniMax H3 Temporal VAE Chunks: design/feature/minimax_h3_temporal_vae_chunks.md
      - Communication:
        - OmniConnector Implementations:
          - Mooncake Store Connector: design/feature/omni_connectors/mooncake_store_connector.md
@@ -57,9 +57,9 @@ vLLM-Omni is flexible and easy to use with:
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:

 - Omni-modality models (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
- TTS models (e.g. Qwen3-TTS, VoxCPM2, Ming-Omni-TTS, CosyVoice3)
- Diffusion models — image, video, and audio generation (e.g. MiniMax H3, Qwen-Image, Wan2.2, FLUX)
- Robot-policy and action models (e.g. GR00T-N1.7, DreamZero-DROID, InternVLA-A1, Cosmos3 action policy)
+- TTS models (e.g. Qwen3-TTS, IndexTTS 2.5, dots.tts, CosyVoice3)
+- Diffusion models — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
+- Robot-policy and action models (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)

 For more information, checkout the following:

@@ -114,6 +114,10 @@ Configuration classes.
 - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderDiTConfig][]
 - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1EncoderConfig][]
 - [vllm_omni.transformers_utils.configs.cosyvoice3.CosyVoice3Config][]
+- [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Config][]
+- [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLConfig][]
+- [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLTextConfig][]
+- [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLVisionConfig][]
 - [vllm_omni.transformers_utils.configs.omnivoice.OmniVoiceConfig][]
 - [vllm_omni.transformers_utils.configs.voxcpm2.VoxCPM2Config][]

@@ -2,6 +2,11 @@

 The CLI for vLLM-Omni inherits from vllm with some additional arguments.

+Environment variables are process configuration rather than CLI-only options.
+See the [Environment Variables](../configuration/environment_variables.md)
+reference for ownership, precedence, evaluation time, and per-stage `env`
+scoping.
+
 ## serve

 Starts the vLLM-Omni OpenAI Compatible API server.
@@ -39,4 +44,5 @@ vllm bench serve --omni \
    --num-prompts  5
 ```

-See [vllm bench serve](./bench/serve.md) for the full reference of all available arguments.
+See [vllm bench serve](./bench/serve.md) for serving benchmark arguments and
+dataset-specific examples, including OmniInteract native-duplex sessions.
@@ -2,7 +2,7 @@
 The vllm bench command launches the vLLM-Omni benchmark to evaluate the performance of multimodal models.

 ## Notes
-vLLM-Omni registers the `openai-chat-omni`, `openai-audio-speech`, `openai-image-edits-omni`, and `daily-omni` serving benchmark backends.
+vLLM-Omni registers the `openai-chat-omni`, `openai-audio-speech`, `openai-image-edits-omni`, `daily-omni`, and `openai-realtime-duplex` serving benchmark backends. It also adds the `omniinteract` dataset.

 ## Basic Parameter Description
 You can use `vllm bench serve --omni --help=all` to get descriptions of all parameters. The commonly used parameters are described below:
@@ -10,7 +10,7 @@ You can use `vllm bench serve --omni --help=all` to get descriptions of all para
  Enable Omni (multimodal) mode, supporting multimodal inputs and outputs such as images, videos, and audio.

 - `--backend`
-  Specify the backend adapter. vLLM-Omni adds `openai-chat-omni`, `openai-audio-speech`, `openai-image-edits-omni`, and `daily-omni` to the upstream vLLM backend choices.
+  Specify the backend adapter. vLLM-Omni adds `openai-chat-omni`, `openai-audio-speech`, `openai-image-edits-omni`, `daily-omni`, and `openai-realtime-duplex` to the upstream vLLM backend choices.

 - `--model`
  The model identifier to load, filled according to the models supported by vLLM-Omni.
@@ -19,7 +19,7 @@ You can use `vllm bench serve --omni --help=all` to get descriptions of all para
  The API endpoint exposed externally, to which clients send their requests.

 - `--dataset-name`
-  The name of the dataset used; random-mm indicates generating random multimodal inputs (images, videos, audio).
+  The name of the dataset used; random-mm indicates generating random multimodal inputs (images, videos, audio), while `omniinteract` replays official OmniInteract videos.

 - `--num-prompts`
  The total number of requests to send, an integer.
@@ -269,6 +269,44 @@ We use audio generation time / audio duration to calculate RTF.

 </details>

+### OmniInteract Realtime Benchmark
+
+OmniInteract runs each video as one native-duplex WebSocket sample in the standard serving-benchmark lifecycle:
+
+```bash
+vllm bench serve --omni \
+  --backend openai-realtime-duplex \
+  --dataset-name omniinteract \
+  --dataset-path /path/to/OmniInteract \
+  --model openbmb/MiniCPM-o-4_5 \
+  --base-url http://127.0.0.1:8000 \
+  --endpoint /v1/realtime \
+  --omniinteract-ref-audio /path/to/reference.wav \
+  --omniinteract-output-dir ./omniinteract-artifacts \
+  --num-prompts 3 \
+  --num-warmups 0
+```
+
+`--dataset-path` accepts an extracted directory, `data.tar[.gz]`, or a Hugging Face dataset ID; omitting it uses
+`lucky-lance/OmniInteract`. `--num-prompts` is the total across subsets and defaults to 3 for OmniInteract; explicit `0`
+selects all and oversize values use all available cases. Reference audio is required, and OmniInteract uses the
+`/v1/realtime` endpoint.
+
+Audio is replayed as 16 kHz PCM16 in 200 ms chunks and video at 1 FPS with real-time pacing. All selected media is decoded
+before timing and remains in client memory for the run, so `--max-concurrency` does not limit media preparation memory; use
+explicit `--num-prompts 0` only when the client has enough RAM for the full dataset. Media commands are bounded by
+`--omniinteract-media-timeout-s`, and concurrency defaults to 1. Standard request-rate, warmup, result-saving, and summary
+options apply. Use `--omniinteract-require-response` only for functional E2E cases; LISTEN is a valid benchmark result.
+
+Each completed case writes `output.wav`, `wav_transcript.json`, `events.json`, `result.json`, and a final `.done` marker under
+`--omniinteract-output-dir`. The root also contains `batch_summary.json` and `official_eval_manifest.jsonl`; failed cases write
+`.failed.json`. Runs sharing one output root are serialized. Completion validates transport, response lifecycle, and artifacts,
+not answer accuracy. Transcript timestamps are serialized playback-queue times. Clipped or cancelled outputs are ineligible and
+omitted from the official manifest; `audio_clipped_bytes` records output beyond the rounded video horizon.
+
+TTFT, TTFP, and RTF start at client receipt of `response.created`. TPOT/ITL use engine stage-0 timing; ITL is emitted only when
+every token interval is present.
+
 ### Multi-Modal Benchmark

 <details class="admonition abstract" markdown="1">
@@ -2,11 +2,14 @@

 This section lists the most common options for running vLLM-Omni.

-For options within a vLLM Engine, please refer to the [vLLM 0.26 configuration guide](https://docs.vllm.ai/en/v0.26.0/configuration/index.html).
+For options within a vLLM Engine, please refer to the [vLLM 0.28 configuration guide](https://docs.vllm.ai/en/v0.28.0/configuration/index.html).

 Each model defines fixed topology in a registered `PipelineConfig` and runtime
 overrides in a deploy YAML.

+For process-level settings shared by the CLI, deploy YAML, workers, and model
+integrations, see [Environment Variables](./environment_variables.md).
+
 For a specific example, see the [Qwen2.5-Omni deploy config](gh-file:vllm_omni/deploy/qwen2_5_omni.yaml). The matching frozen pipeline topology lives at [vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py](gh-file:vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py).

 For an introduction, see [Pipeline and deploy configurations](./stage_configs.md).
@@ -0,0 +1,238 @@
+# Environment Variables
+
+Environment variables are process-level configuration shared by the CLI,
+deploy YAML, Python APIs, worker processes, and model integrations. They are
+therefore documented under **User Guide → Configuration**, rather than under
+CLI Reference.
+
+This page describes the public vLLM-Omni environment-variable contract on the
+`latest` documentation branch. Use the documentation for your installed
+vLLM-Omni release when behavior differs.
+
+!!! important "Public contract"
+    Only names in [Public vLLM-Omni variables](#public-vllm-omni-variables) are
+    public Omni configuration. A name found in source code is not automatically
+    supported: model-specific, benchmark, platform, and internal variables have
+    separate ownership and lifecycle rules.
+
+## Naming and implementation boundary
+
+New public variables owned by vLLM-Omni must use the `VLLM_OMNI_` prefix.
+Existing public names with older prefixes remain supported for compatibility,
+but they do not establish naming precedent for new settings. Inherited vLLM
+and third-party names retain the prefix chosen by their owning project.
+
+The source [environment-variable inventory](gh-file:vllm_omni/config/environment_variable_inventory.py)
+records ownership, migration disposition, and redaction metadata. Unlike
+`vllm.envs`, it is not an executable value resolver: existing consumers still
+own parsing, defaults, precedence, and evaluation time. The public tables below
+describe those live consumer contracts. Stable model or stage behavior should
+move to typed configuration, and request-varying behavior should move to a
+request schema instead of adding another environment variable.
+
+## Precedence and evaluation time
+
+There is no repository-wide rule that makes environment variables override (or
+lose to) every CLI, YAML, or request option. The tables below state precedence
+for each public setting. In general:
+
+- request fields should control behavior that can vary per request;
+- typed deploy or stage configuration should control stable model and stage
+  behavior;
+- environment variables are appropriate for process launch, platform
+  integration, emergency diagnostics, and compatibility fallbacks.
+
+An environment variable can be read while a module is imported, when server or
+model configuration is constructed, when a worker is spawned, or whenever a
+property is accessed. Set import- and startup-time variables before invoking
+`vllm serve` or importing vLLM-Omni. Changing the parent shell after workers
+start does not update existing worker environments.
+
+The lifecycle labels used below are:
+
+- **Stable** — reviewed public configuration; changes require compatibility
+  consideration.
+- **Experimental** — public for evaluation, but its contract may still evolve.
+- **Diagnostic** — supported for investigation or temporary mitigation, not as
+  a normal tuning interface.
+- **Deprecated** — compatibility alias; migrate to the named replacement.
+
+## Public vLLM-Omni variables
+
+### Diffusion backend and cache selection
+
+| Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
+| --- | --- | --- | --- | --- |
+| `DIFFUSION_ATTENTION_BACKEND` | Backend name or `auto`; default `auto` (platform selection) | Diffusion stages; read when `OmniDiffusionConfig` is constructed | `diffusion_attention_config.default` wins. An unknown backend fails during backend resolution. | Stable fallback |
+| `DIFFUSION_CACHE_BACKEND` | `none`, `cache_dit`, `tea_cache`, `mag_cache`, `step_cache`, `stepcache`, or `step_cache_dit`; default `none` | Diffusion runner startup | Explicit `cache_backend` in config wins. Otherwise this name wins over the deprecated alias. Unsupported values raise `ValueError` during runner setup. | Stable fallback |
+| `DIFFUSION_CACHE_ADAPTER` | Same values as `DIFFUSION_CACHE_BACKEND`; default `none` | Diffusion runner startup | Used only when neither explicit `cache_backend` nor `DIFFUSION_CACHE_BACKEND` is set. Unsupported values raise `ValueError`. | Deprecated; use `DIFFUSION_CACHE_BACKEND` |
+| `OMNI_DIFFUSION_PROMPT_EMBED_CACHE` | Boolean: `1`, `true`, `yes`, `on`, `0`, `false`, `no`, or `off`; default disabled | Each diffusion runner; resolved during model setup | A recognized environment value overrides the explicit enable setting. An unrecognized value is ignored. | Experimental |
+| `OMNI_DIFFUSION_PROMPT_EMBED_CACHE_SIZE` | Positive integer; default `32` entries | Each diffusion runner; resolved during model setup | A valid environment value overrides the explicit cache size. A non-integer logs a warning; a non-positive value is ignored. | Experimental |
+| `OMNI_DIFFUSION_SESSION_STATE_MANAGER` | Boolean with the same accepted spellings as the prompt cache; default disabled | Experimental diffusion session manager; model setup | A recognized environment value overrides the explicit enable setting. An unrecognized value is ignored. | Experimental |
+| `OMNI_DIFFUSION_SESSION_STATE_MANAGER_MAX_SESSIONS` | Positive integer; default `64` | Experimental diffusion session manager; model setup | A valid environment value overrides the explicit maximum. A non-integer logs a warning; a non-positive value is ignored. | Experimental |
+
+Backend names for `DIFFUSION_ATTENTION_BACKEND` are the members of
+`DiffusionAttentionBackendEnum`, such as `FLASH_ATTN`, `TORCH_SDPA`,
+`SAGE_ATTN`, `FLASHINFER_ATTN`, and `TRTLLM_ATTN`. Platform support still
+depends on the installed kernels and model path.
+
+### Serving and runtime
+
+| Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
+| --- | --- | --- | --- | --- |
+| `SPEAKER_SAMPLES_DIR` | Filesystem path; default `~/.cache/vllm-omni/speakers` | Speech server; read when speaker storage initializes | Environment-only setting. The directory is created; filesystem errors propagate. | Stable |
+| `SPEAKER_MAX_UPLOADED` | Integer; default `1000` | Speech server; read when speaker storage initializes | Environment-only setting. A non-integer logs a warning and uses `1000`; range is not otherwise validated. | Stable |
+| `VLLM_OMNI_ASYNC_OUTPUT_TIMEOUT` | Float seconds; default `600` | Diffusion engine async-output wait in `step_streaming`; resolved per call on the request path, not at import | Environment-only setting. A non-float or `<=0` value warns once and uses the default. | Experimental |
+| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `1`, `true`, `yes` or `on` enables; default `0` (off) | Orchestration loop and the serving-side final-output drain; read once when the `Orchestrator` is constructed | Environment-only setting. Values are stripped and case-normalized; any unrecognized value leaves the legacy poll loop selected. | Experimental |
+| `VLLM_OMNI_INPUT_WAIT_TIMEOUT_S` | Float seconds; default `600`; `<=0` disables | Full-payload input coordinator, not async-chunk transfer; read when the scheduler module imports in each worker | Environment-only setting. A non-float logs a warning and uses `600`. | Stable operational control |
+| `VLLM_OMNI_ORCH_MONITOR_PATH` | Filesystem path; default `<current-working-directory>/vllm_omni_orch_monitor_<timestamp>.json` | Orchestrator monitor enabled by `--enable-orch-monitor`; read when the monitor is created | Environment-only path override. Parent directories are created; write errors are logged. | Diagnostic |
+| `VLLM_OMNI_VIDEO_SYNC_TIMEOUT` | Float seconds; default `600` | Synchronous Videos API; read when the API server module imports | Environment-only setting. A non-float raises `ValueError` during import. | Experimental |
+| `VLLM_VIDEO_ASYNC_CHUNK` | `on` or `off`; default `on` | Streaming video output; read on attribute access | Environment-only setting. Values are trimmed and case-normalized; an invalid value warns once and uses `on`. | Experimental |
+| `VLLM_VIDEO_AUDIO_DELTA_MODE` | `fast` or `slow`; default `fast` | Streaming video audio deltas; read on attribute access | Environment-only setting. Values are trimmed and case-normalized; an invalid value warns once and uses `fast`. | Experimental |
+
+### Server storage
+
+Storage names use Pydantic's nested-settings delimiter (`__`). They are read
+when `ServerSettings` is initialized, normally during server import/startup.
+Invalid integer values raise a Pydantic validation error.
+
+| Name | Type and default | Meaning | Lifecycle |
+| --- | --- | --- | --- |
+| `VLLM_OMNI_SERVER_STORAGE__PATH` | Path; default `/tmp/storage` | Directory for completed server files. | Stable |
+| `VLLM_OMNI_SERVER_STORAGE__FILE_CONCURRENCY` | Integer; default `4` | Maximum concurrent file operations. No additional range validation is currently applied. | Stable |
+| `VLLM_OMNI_SERVER_STORAGE__FILE_TTL` | Integer seconds or unset; default unset | Optional lifetime for locally stored files. | Stable |
+| `VLLM_OMNI_SERVER_STORAGE__TTL_SWEEP_INTERVAL` | Integer seconds or unset | Sweep frequency. If file TTL is set and this is unset, the effective default is `300`. | Stable |
+| `VLLM_OMNI_STORAGE_PATH` | Path | Deprecated alias for `VLLM_OMNI_SERVER_STORAGE__PATH`. The new name wins when both are set. | Deprecated |
+| `VLLM_OMNI_STORAGE_MAX_CONCURRENCY` | Integer | Deprecated alias for `VLLM_OMNI_SERVER_STORAGE__FILE_CONCURRENCY`. The new name wins when both are set. | Deprecated |
+
+The deprecated aliases emit `DeprecationWarning`. They do not affect file TTL
+settings.
+
+### Quantization diagnostics and performance
+
+| Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
+| --- | --- | --- | --- | --- |
+| `VLLM_OMNI_SKIP_NVFP4_NAN_CLAMP` | Boolean truthy spellings: `1`, `true`, `yes`, `on`; default false | ModelOpt NVFP4 compatibility patch; read when `vllm_omni.patch` imports | Environment-only escape hatch. Any other value means false. Set only to diagnose the upstream NaN-scale issue. | Diagnostic and temporary |
+| `VLLM_OMNI_USE_QUACK_FP8` | Boolean truthy spellings: `1`, `true`, `yes`, `on`; unset means hardware auto-detection | FP8 scaled matrix multiplication; evaluated when quack capability is selected | A set value overrides auto-detection. Any non-truthy value forces quack off. If quack cannot load, vLLM-Omni warns and falls back to FlashInfer. | Experimental performance control |
+
+`QUACK_CACHE_DIR` is owned by the external quack library and is not an
+Omni-owned variable, even though vLLM-Omni supplies a persistent default when
+it is unset.
+
+## Per-stage environment
+
+Deploy configurations can set arbitrary environment keys for one stage:
+
+```yaml
+stages:
+  - stage_id: 0
+    env:
+      HCCL_IF_BASE_PORT: 23000
+      MY_MODEL_CACHE: /mnt/model-cache
+```
+
+The legacy nested form under `runtime.env` is also accepted. vLLM-Omni converts
+keys and values to strings, applies them while the stage engine or child process
+is launched, logs keys but not values, and restores the parent process's prior
+environment afterward. A child process inherits the scoped values. This keeps
+sibling stages from permanently changing one another's parent environment.
+
+This scoping has two consequences:
+
+1. A setting read inside stage/model initialization can see the stage value.
+2. A setting already cached by an import that occurred before stage launch
+   cannot be changed retroactively with stage `env`.
+
+Stage `env` is an escape hatch for process and third-party settings, not a
+replacement for typed stage configuration. Do not put request-varying behavior
+there. Secret values may reach child processes; logs and diagnostics must emit
+their keys only.
+
+## Inherited vLLM variables
+
+vLLM-Omni also reads variables through its aligned vLLM dependency. Refer to
+the [vLLM 0.28 environment-variable reference](https://docs.vllm.ai/en/v0.28.0/configuration/env_vars.html)
+for their definitions. This includes vLLM launch, cache, logging, plugin, ROCm,
+XPU, ModelScope, and FlashInfer workspace settings.
+
+An inherited name does not become Omni-owned because Omni reads or forwards it.
+Where behavior differs, the Omni-specific page should document only that
+difference. For example, device-visibility values can be further narrowed by a
+stage's `devices` setting.
+
+## Platform, external, and secret variables
+
+Hardware runtimes, distributed launchers, and third-party libraries retain
+ownership of their variables. Common groups include:
+
+- device visibility (`ASCEND_RT_VISIBLE_DEVICES`, `HIP_VISIBLE_DEVICES`, and
+  `MUSA_VISIBLE_DEVICES`);
+- distributed launch (`MASTER_ADDR`, `MASTER_PORT`, `RANK`, and `WORLD_SIZE`);
+- Hugging Face cache and authentication (`HF_HOME`, `HF_TOKEN`, and related
+  names);
+- accelerator libraries such as NCCL, FlashInfer, MindIE, MORI, and quack.
+
+Consult the owning project's documentation for accepted values and support.
+`HF_TOKEN`, `HUGGINGFACE_HUB_TOKEN`, and `OPENAI_API_KEY` are explicitly marked
+for redaction in the reviewed inventory. Never print their values in environment
+collection, examples, bug reports, or logs.
+
+## Model-specific variables
+
+The audit found 56 variables read by a single model or pipeline family. They are
+not listed as public usage options here because doing so would turn implementation
+escape hatches into an accidental compatibility contract.
+
+Every audited model-specific name has a migration disposition in the
+[environment-variable inventory](gh-file:vllm_omni/config/environment_variable_inventory.py):
+
+| Disposition | Count | Required outcome |
+| --- | ---: | --- |
+| Promote | 33 | Move a stable setting into typed stage or model configuration. |
+| Request scope | 6 | Move request-varying behavior into a declared request-option schema. |
+| External | 0 | Retain only when a supported third-party library owns the contract. |
+| Internalize | 11 | Keep a debug or diagnostic switch out of public documentation and configuration. |
+| Deprecate/remove | 6 | Remove a compatibility escape hatch that has no continuing contract. |
+
+The disposition is a migration target, not a statement that the existing
+environment switch is stable. Promote or request-scope work should land in
+model-owner-reviewed follow-up changes with validation and model-page updates.
+The family-sized migration packages and their ownership gate are tracked in
+[#6232](https://github.com/vllm-project/vllm-omni/issues/6232).
+
+## Environment collection and troubleshooting
+
+Run the repository collector when preparing a bug report:
+
+```bash
+python collect_env.py
+```
+
+The environment section includes the safe public Omni allowlist, registered
+vLLM variables, and selected platform prefixes. It omits names marked for
+redaction and names containing common secret terms. Review the output before
+sharing it. A key supplied through stage `env` is not collected merely because
+it appears in deploy configuration; the same allowlist and prefix rules apply.
+
+If a setting appears ineffective, first check its read time in the tables
+above. Restart processes after changing import- or startup-time values, verify
+that the variable reaches the intended worker, and check whether an explicit
+typed configuration field takes precedence.
+
+## Benchmark and internal variables
+
+The reviewed inventory marks 20 benchmark variables as transitional while
+benchmark configuration moves to CLI/request schemas in
+[issue #5376](https://github.com/vllm-project/vllm-omni/issues/5376). They are not
+part of the serving/runtime contract.
+
+Internal process-coordination and debug variables are deliberately excluded
+from this page. The CI scanner resolves string literals and module-level string
+constants used by direct `os` access, local wrapper helpers, aliased `os`
+imports, and environment membership checks. Server-storage names generated by
+Pydantic settings are checked separately against their model fields.
+
+Names assembled from cross-module metadata and arbitrary keys supplied through
+stage `env` cannot be inferred reliably. Contributors must classify those names
+explicitly. CI fails when a statically resolved name has no classification or
+when a reviewed public name is absent from this page.
@@ -14,7 +14,7 @@ describe fixed model topology and are not accepted in deploy YAMLs.
 Common `PipelineConfig` fields include:

 | Field | Description |
-|-------|-------------|
+| ------- | ------------- |
 | `model_type` | Pipeline identifier used during model and config resolution. |
 | `default_deploy_config_name` | Bundled deploy YAML loaded when the user does not pass `deploy_config`. |
 | `model_arch` | Default Hugging Face architecture for the pipeline. |
@@ -26,7 +26,7 @@ Common `PipelineConfig` fields include:
 Common `StagePipelineConfig` fields include:

 | Field | Description |
-|-------|-------------|
+| ------- | ------------- |
 | `stage_id` | Stable stage identifier. |
 | `model_stage` | Logical stage name used by runtime and strategy resolution. |
 | `execution_type` | `LLM_AR`, `LLM_GENERATION`, or `DIFFUSION`. |
@@ -49,17 +49,17 @@ deployment overrides.
 The new deploy schema lives under `vllm_omni/deploy/` and is paired with a frozen `PipelineConfig` registered by the model's `pipeline.py`. Each deploy YAML has these top-level fields:

 | Field | Type | Required | Default | Description |
-|-------|------|----------|---------|-------------|
+| ------- | ------ | ---------- | --------- | ------------- |
 | `base_config` | str (path) | optional | — | Overlay parent (relative or absolute). `stages:` / `platforms:` deep-merged by stage_id; other scalars overlay-wins. Intended for user-authored overlays; prod yamls stay flat. |
 | `async_chunk` | bool | optional | `true` | Enable chunked streaming between stages. Pin to `false` if the pipeline runs end-to-end. |
-| `session_mode` | str | optional | `"turn"` | Session behavior. Use `"duplex"` only with a pipeline that enables duplex control. |
+| `session_mode` | str | optional | `"turn"` | Session behavior. MiniCPM-o 4.5 deploy YAMLs set `"duplex"` so `/v1/realtime` and chat share the same profile. |
 | `active_stream_window` | int | optional | `0` | Number of active downstream stream slots; `0` preserves all-stream cycling. |
 | `duplex_session` | dict | optional | runtime defaults | Full-duplex session lifecycle, buffering, replay, and capacity limits. |
 | `connectors` | dict | optional | `null` | Named connector specs (`{name, extra}`). Referenced by each stage's `input_connectors` / `output_connectors`. See [Connector schema](#connector-schema). |
 | `edges` | list | optional | `null` | Explicit edge list for the KV transfer graph. Auto-derived from stage inputs if omitted. |
 | `stages` | list | optional | `[]` | Per-stage runtime overrides matched by `stage_id`. Pipeline stages are still created from `PipelineConfig` when this list is empty. |
 | `platforms` | dict | optional | `null` | Keyed by `npu` / `rocm` / `xpu`, each contains a `stages:` list with per-platform overrides applied on top of the CUDA defaults. |
-| `pipeline` | str | optional | `null` | Override the auto-detected pipeline registry key (used for structural variants like `qwen2_5_omni_thinker_only`). |
+| `pipeline` | str | optional | `null` | Override the auto-detected pipeline registry key (used for structural variants like `qwen2_5_omni_thinker_only` / `qwen3_omni_moe_thinker_only`). |
 | `trust_remote_code` | bool \| null | optional | `null` | **Pipeline-wide.** Trust HF remote code on model load; applies to every stage when specified. |
 | `distributed_executor_backend` | str \| null | optional | `null` | **Pipeline-wide.** Distributed executor backend forwarded to vLLM (`"mp"`, `"ray"`, `"external_launcher"`). If omitted, vLLM auto-selects backend from runtime topology. |
 | `dtype` | str \| null | optional | `null` | **Pipeline-wide.** Model dtype for every stage. |
@@ -84,7 +84,7 @@ subprocess on one GPU. `ray` / `external_launcher` are not fully supported yet.
 Each entry under `stages:` accepts any `StageDeployConfig` field directly (no nested `engine_args:`). Only fields whose value legitimately varies across stages live here; pipeline-wide settings (trust_remote_code, distributed_executor_backend, dtype, quantization, prefix/chunked prefill, DP/PP sizes) are declared at the top level and applied to every stage. Unknown keys fall through to `engine_extras:` and are forwarded to the engine. Frequently used fields are listed below; the source-of-truth schema is `StageDeployConfig` in `vllm_omni/config/stage_config.py`.

 | Field | Type | Required | Default | Description |
-|-------|------|----------|---------|-------------|
+| ------- | ------ | ---------- | --------- | ------------- |
 | `stage_id` | int | required | — | Stage identity; matched against `PipelineConfig.stages[*].stage_id`. |
 | `max_num_seqs` | int \| null | optional | `null` | Max concurrent sequences per stage. |
 | `gpu_memory_utilization` | float \| null | optional | `null` | Per-stage total memory target; used for automatic KV-cache sizing. |
@@ -94,12 +94,14 @@ Each entry under `stages:` accepts any `StageDeployConfig` field directly (no ne
 | `max_num_batched_tokens` | int \| null | optional | `null` | Per-stage prefill/token budget; also contributes to the native maximum in-flight token limit. |
 | `max_model_len` | int \| null | optional | `null` | Per-sequence context or KV length; `-1` enables native cache-capacity auto-fitting, while values above the HF default auto-set `VLLM_ALLOW_LONG_MAX_MODEL_LEN=1`. |
 | `async_scheduling` | bool \| null | optional | `null` | Per-stage async scheduling toggle. |
-| `devices` | str \| null | optional | `null` | Device list assigned to this stage. |
+| `devices` | str \| null | optional | `null` | Device list assigned to this stage. The number of device ids must equal this stage's local world size (`tensor_parallel_size` × local data-parallel size × `pipeline_parallel_size`, or `num_replicas` × that product for a replica pool); a mismatch fails early — see the note below. |
 | `output_connectors` | dict \| null | optional | `null` | Keyed by `to_stage_<n>`; values are names registered under top-level `connectors:`. |
 | `input_connectors` | dict \| null | optional | `null` | Keyed by `from_stage_<n>`; values are names registered under top-level `connectors:`. |
 | `default_sampling_params` | dict \| null | optional | `null` | Baseline sampling params. Deep-merged with pipeline `sampling_constraints` (pipeline wins). |
 | `engine_extras` | dict | optional | `{}` | Catch-all for engine fields not listed above; deep-merged across overlays and forwarded to the stage engine. |

+**Note:** a stage's `devices` count must equal its local world size (`tensor_parallel_size` × `data_parallel_size_local` × `pipeline_parallel_size`, falling back to global `data_parallel_size` when the local size is unset), or `num_replicas` × that product for a replica pool. A mismatch fails early and names the offending stage. A top-level `--tensor-parallel-size` is broadcast to every stage, so it can make a single-GPU stage violate this contract ([issue #5003](gh-issue:5003)); fix that case with `--stage-overrides` (set `tensor_parallel_size` and `devices` together per stage) or set TP only on the multi-GPU stage.
+
 ### Connector schema

 Each entry under top-level `connectors:` follows this shape:
@@ -113,9 +115,9 @@ connectors:
      # Additional connector-specific options
 ```

-| Connector class | Use case | `extra` keys |
-|-----------------|----------|--------------|
-| `SharedMemoryConnector` | Same-host KV transfer between stages (default for bundled YAMLs). | None. All payloads use shared memory. |
+| Connector class          | Use case                                                              | `extra` keys                                                                                                      |
+|--------------------------|-----------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------|
+| `SharedMemoryConnector`  | Same-host KV transfer between stages (default for bundled YAMLs).     | None. All payloads use shared memory.                                                                             |
 | `MooncakeStoreConnector` | Cross-host KV transfer over TCP. Required for multi-node deployments. | `host`, `metadata_server`, `master`, `segment` (int bytes), `localbuf` (int bytes), `proto` (`"tcp"` / `"rdma"`). |

 A stage references a connector by name in its `input_connectors` / `output_connectors`:
@@ -135,7 +137,7 @@ stages:
 ### CLI flags

 | Flag | Description |
-|------|-------------|
+| ------ | ------------- |
 | `--deploy-config PATH` | Load a deploy YAML. **Optional** — when omitted, the bundled `vllm_omni/deploy/<model_type>.yaml` is auto-loaded by the model registry. |
 | `--stage-overrides JSON` | Per-stage JSON overrides, e.g. `'{"0":{"gpu_memory_utilization":0.5}}'`. Per-stage values always win over global flags. |
 | `--async-chunk` / `--no-async-chunk` | Flip the deploy YAML's `async_chunk:` bool. Unset (default) leaves the YAML value in force. |
@@ -249,7 +251,7 @@ CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
 Effective config per stage after the merge:

 | Stage | Field | Final value | Source |
-|-------|-------|-------------|--------|
+| ------- | ------- | ------------- | -------- |
 | 0 | `gpu_memory_utilization` | `0.9` | base YAML (overlay didn't touch stage 0) |
 | 0 | `max_num_seqs` | `8` | per-stage CLI (`--stage-overrides`) — wins over base `32` |
 | 0 | `max_model_len` | `16384` | global CLI |
@@ -273,12 +275,14 @@ when loading a deploy configuration.
 Examples:

 For offline inference (assuming the necessary dependencies have been imported):
+
 ```python
 model_name = "Qwen/Qwen2.5-Omni-7B"
 omni = Omni(model=model_name, deploy_config="/path/to/deploy_config.yaml")
 ```

 For online serving:
+
 ```bash
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/deploy_config.yaml
 ```
@@ -49,7 +49,7 @@ vLLM-Omni's `pre-commit` hooks will now run automatically every time you commit.
    `--all-files` run stays green while historical debt is cleaned up. The
    current `SKIP` list in
    [`.github/workflows/pre-commit.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.github/workflows/pre-commit.yml)
-    is `check-test-ci-coverage`, `markdownlint-cli2`, `shellcheck`,
+    is `check-mark`, `markdownlint-cli2`, `shellcheck`,
    `check-spdx-header`, and `mypy-3.10`. Those hooks still run on **your
    commit** for changed files. A passing GitHub pre-commit check does not mean
    they passed locally. Hooks **not** on that list (forbidden imports,
@@ -66,7 +66,7 @@ already there). `check-pickle-imports` is gone: pickle is now one rule inside
 | `markdownlint-cli2` | Markdown in `docs/`, `recipes/`, `README.md`, `CONTRIBUTING.md` (not `.claude/` / `.cursor/` / `CLAUDE.md`) | skipped |
 | `mypy-3.10` | Type-check changed `vllm_omni/` files (model trees excluded). `tests/` uses `--follow-imports skip` | skipped |
 | `mypy-3.11` / `3.12` / `3.13` | Same checker, extra Python versions | not installed; `pre-commit run --hook-stage manual mypy-3.12` |
-| `check-test-ci-coverage` | Every `tests/**/test_*.py` has a CI level mark and a hardware mark/helper | skipped |
+| `check-mark` | Every `tests/**/test_*.py` has a CI level mark and a hardware platform mark or helper; no direct `pytest.mark.H100` / SKU | skipped |
 | `check-tts-adapter-migration` | `self._tts_model_type` branches in `serving_speech.py` must not increase | runs |
 | `shellcheck` | `*.sh` quoting / undefined vars | skipped |
 | `check-spdx-header` | Omni SPDX header on `.py` / `.pyi` / `.sh` / `.rs` / `.proto` | skipped |
@@ -144,11 +144,13 @@ pre-commit run --hook-stage manual mypy-3.12

 #### Test CI marks

-`check-test-ci-coverage` requires each collected test module under `tests/` to
+`check-mark` requires each collected test module under `tests/` to
 have at least one CI **level** mark (`core_model`, `advanced_model`,
-`full_model`, `local_model`, or `slow`) and a **hardware** mark (`cpu`, `cuda`,
-`H100`, …) or helper (`hardware_test(` / `hardware_marks(`). GitHub Actions
-skips this hook; local commit does not. See the
+`full_model`, `local_model`, or `slow`) and a **hardware** platform mark
+(`cpu`, `cuda`, …) or helper (`hardware_test(` / `hardware_marks(`). SKU marks
+(`H100`, `L4`, …) must come from those helpers so `cards_{n}` is attached;
+direct `pytest.mark.H100` is rejected. GitHub Actions skips this hook; local
+commit does not. See the
 [test writing guide](./ci/test_writing_guide.md).

 #### TTS adapter ratchet
@@ -402,7 +402,7 @@ loads the model once per mode.
 Before splitting, confirm each half actually collects at least one test under the job's forwarded filters — pytest exits with code 5 ("no tests collected") if a half is empty, which the script reports as a failure, red-ing a job that used to pass as one combined invocation.

 Also check the tests' `num_cards` against the job's `mirror_hardwares` preset.
-`cuda_marks` turns `num_cards > 1` into `skipif(device_count() < num_cards)`, so a
+`_cuda_marks` turns `num_cards > 1` into `skipif(device_count() < num_cards)`, so a
 test asking for more GPUs than the preset provides is silently skipped and its
 coverage artifact comes back empty. Splitting such a job produces a file that
 looks fine and measures nothing — "Diffusion · Z Image Test" is the current
@@ -4,8 +4,8 @@ When you want to add L4-level ***performance test*** cases, add entries to JSON

 | Model type | Runner | Example JSON files |
 | ---------- | ------ | ------------------ |
-| Omni (nightly) | `run_benchmark.py` | `test_qwen3_omni_no_async_chunk.json`, `test_qwen3_omni_async_chunk.json` (`full_model` in `mark`) |
-| Omni (weekly) | `run_benchmark.py` | `test_qwen3_omni_vllm_text.json`, `test_qwen3_omni_multi_replicas.json` (`slow` in `mark`; **Perf Test** in `test-weekly.yml`) |
+| Omni (nightly) | `run_benchmark.py` | `test_qwen3_omni_no_async_chunk.json`, `test_qwen3_omni_async_chunk.json` (`full_model` without `slow` in `mark`) |
+| Omni (weekly) | `run_benchmark.py` | `test_qwen3_omni_async_chunk.json` (CUDA only), `test_qwen3_omni_vllm_text.json`, `test_qwen3_omni_multi_replicas.json` (`slow` in `mark`; **Perf Test** in `test-weekly.yml`) |
 | TTS | `run_benchmark.py` | `test_tts.json`, `test_voxcpm2.json`, `test_higgs_audio_v3.json` |
 | Diffusion | `run_diffusion_benchmark.py` | `test_qwen_image_vllm_omni.json`, `test_bagel_vllm_omni.json`, `test_wan22_i2v_vllm_omni.json`, `test_cosmos3_vllm_omni.json`, … |

@@ -20,7 +20,7 @@ Diffusion cases are detected when the JSON has `server_type` (typically `"vllm-o

 #### Running perf cases

-Pass **`--test-config-file`** to run one JSON file as-is, or omit it for the bulk scan above and filter with pytest **`-m`** on each case's JSON `mark` (for example `pytest … run_benchmark.py -m "full_model and tts and H100"` or `-m "slow and omni and H100"` for weekly Omni configs).
+Pass **`--test-config-file`** to load one JSON file, or omit it for the bulk scan above. In either mode, pytest **`-m`** filters each case using its JSON `mark`. A single file can contain nightly, weekly, and platform-specific case objects. For example, the nightly object in `test_qwen3_omni_async_chunk.json` declares both H100 and A3, while its `slow` weekly object is CUDA-only; there is no NPU weekly counterpart. Use `-m "H100 and full_model and not slow"`, `-m "H100 and full_model and slow"`, or `-m "npu"` to select the intended schedule and platform.

 ```JSON
 {
@@ -75,14 +75,14 @@ Omit `mark` only for configs not meant to be filtered by `-m`. `server_type` is

 Optional top-level field on each perf JSON **case object** (one per `test_name`). `run_benchmark.py` and `run_diffusion_benchmark.py` read it via `tests.dfx.conftest.resolve_pytest_marks` and attach the marks to the corresponding `pytest.param`, so you can filter locally with `-m`.

-When `mark` is present, it must be an **array** with exactly one ``hardware_marks`` object (same shape as `@hardware_test` / `hardware_marks()` in `tests/helpers/mark.py`), followed by registered pytest marker name strings such as `full_model`, `omni`, `tts`, `diffusion`, or `local_model`.
+When `mark` is present, it must be an **array** with exactly one ``hardware_marks`` object (same shape as `@hardware_test` / `hardware_marks()` in `tests/helpers/mark.py`), followed by registered pytest marker name strings such as `full_model`, `slow`, `omni`, `tts`, `diffusion`, or `local_model`.

 Supported form:

 | Form | Example | Effect |
 | ---- | ------- | ------ |
-| Array (single platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "diffusion"]` | Hardware marks from `hardware_marks(...)`; `num_cards` > 1 adds `distributed_*` (+ CUDA `skipif_cuda` when applicable). String entries become extra pytest marks. |
-| Array (multi-platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100", "rocm": "MI325", "npu": "A2"}, "num_cards": {"cuda": 2, "rocm": 2, "npu": 1}}}, "full_model", "omni"]` | Same as above, but declares **multiple platforms** in one case. Filter examples: `-m "full_model and H100 and cuda"`, `-m "full_model and MI325 and rocm"`. |
+| Array (single platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "diffusion"]` | Hardware marks from `hardware_marks(...)`; `num_cards` adds `cards_{n}` (including `cards_1` when omitted). String entries become extra pytest marks. |
+| Array (multi-platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100", "rocm": "MI325"}, "num_cards": 2}}, "full_model", "omni"]` | Same as above, but declares **multiple platforms** on one item. All platforms must share the same `num_cards` (int or identical dict values). Different counts need `@hardware_test` or one `hardware_marks()` per `pytest.param`. Filter examples: `-m "full_model and H100 and cards_2 and cuda"`, `-m "full_model and MI325 and cards_2 and rocm"`. |

 Recommended for L4 perf cases:

@@ -142,15 +142,22 @@ Examples:
 pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and H100 and diffusion"
 pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "full_model and omni and H100"

-# Single file (same as nightly CI Perf steps)
+# Single file (same selectors as the CI Perf steps)
 pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py \
  --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json
 pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \
-  --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
+  --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json \
+  -m "H100 and full_model and not slow"
+pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \
+  --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json \
+  -m "H100 and full_model and slow"
+pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \
+  --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json \
+  -m "npu"
 ```

 <!-- markdownlint-disable-next-line MD051 -->
-See also [Markers for Tests](#markers-for-tests) for registered hardware markers (`H100`, `L4`, `cuda`, `distributed_cuda`, …).
+See also [Markers for Tests](#markers-for-tests) for registered hardware markers (`H100`, `L4`, `cuda`, `cards_1`, …).

 #### `server_params` Configuration

@@ -88,7 +88,7 @@ Failed jobs: 1/2
    ```bash
    cd tests
    pytest -s -v test_xxxx.py --run-level=core_model
-    pytest -s -v -m "core_model and distributed_cuda and L4" --run-level=core_model
+    pytest -s -v -m "core_model and L4 and not cards_1" --run-level=core_model
    ```

 === "L3 level"
@@ -119,7 +119,7 @@ Failed jobs: 1/2

    ```bash
    pytest -s -v test_xxxx.py --run-level=advanced_model
-    pytest -s -v -m "advanced_model and distributed_cuda and L4" --run-level=advanced_model
+    pytest -s -v -m "advanced_model and L4 and not cards_1" --run-level=advanced_model
    ```

 === "L4 level"
@@ -153,12 +153,12 @@ Failed jobs: 1/2
    ```bash
    cd tests
    pytest -s -v test_xxxx.py --run-level=full_model
-    pytest -s -v -m "full_model and distributed_cuda and L4" --run-level=full_model
+    pytest -s -v -m "full_model and L4 and not cards_1" --run-level=full_model
    pytest -s -v -m "full_model and (omni or tts) and H100" --run-level=full_model
    ```
    If you only want to run specific test cases on a particular platform, you can use:
    ```bash
-    pytest -s -v -m "full_model and distributed_cuda and L4"  --run-level=full_model
+    pytest -s -v -m "full_model and L4 and not cards_1"  --run-level=full_model
    ```
    Note: ``run_benchmark.py`` and ``run_diffusion_benchmark.py`` accept an optional ``--test-config-file``. If omitted, each loads every ``*.json`` under ``tests/dfx/perf/tests/`` (omni/tts vs diffusion split by ``is_diffusion_perf_config``) and pytest ``-m`` filters by each case's JSON ``mark``:
    ```bash
@@ -48,17 +48,12 @@ Defined in `pyproject.toml`:
 | `MI325`            | Tests that require MI325 GPU (AMD/ROCm) *                                                               |
 | `A2`               | Tests that require A2 NPU *                                                                             |
 | `A3`               | Tests that require A3 NPU *                                                                             |
-| `distributed_cuda` | Tests that require multi cards on CUDA platform *                                                       |
-| `distributed_rocm` | Tests that require multi cards on ROCm platform  *                                                      |
-| `distributed_npu`  | Tests that require multi cards on NPU platform  *                                                       |
-| `skipif_cuda`      | Skip if the num of CUDA cards is less than the required *                                               |
-| `skipif_rocm`      | Skip if the num of ROCm cards is less than the required *                                               |
-| `skipif_npu`       | Skip if the num of NPU cards is less than the required *                                                |
+| `cards_{n}`        | Tests that require *n* accelerator cards (`cards_1` … `cards_8`); auto-added from `num_cards` *         |
 | `slow`             | Slow tests (may skip in quick CI)                                                                       |
 | `benchmark`        | Benchmark tests (decorator on runner test functions; perf JSON uses `full_model` + type marker instead) |
 | `local_model`      | Tests requiring local / non-HF-hub model weights                                                        |

-\* Means those markers are auto-added by `@hardware_test` (parametrization decorator) or `hardware_marks` (only returning the list of marks for flexibility).
+\* Means those markers are auto-added by `@hardware_test` (parametrization decorator) or `hardware_marks` (only returning the list of marks for flexibility). Do **not** write `pytest.mark.H100` / `pytest.mark.L4` (or other SKU names) by hand: `check-mark` rejects that so `cards_{n}` cannot be skipped.

 #### Example usage for markers

@@ -81,25 +76,23 @@ def test_video_to_audio()
 This decorator is intended to make hardware-aware, cross-platform test authoring easier and more robust for CI/CD environments. The `hardware_test` decorator in `vllm-omni/tests/helpers/mark.py` performs the following actions:

 1. **Applies platform and resource markers**  
-   Adds the appropriate pytest markers for each specified hardware platform (e.g., `cuda`, `rocm`, `xpu`, `npu`) and resource type (e.g., `L4`, `H100`, `MI325`, `B60`, `A2`, `A3`).
+   Adds the appropriate pytest markers for each specified hardware platform (e.g., `cuda`, `rocm`, `xpu`, `npu`) and resource type (e.g., `L4`, `H100`, `H200`, `B200`, `MI325`, `B60`, `A2`, `A3`).
   ```python
   @pytest.mark.cuda
   @pytest.mark.L4
   ```
-2. **Handles multi-card (distributed) scenarios**  
-   For tests requiring multiple cards, it automatically adds distributed markers such as `distributed_cuda`, `distributed_rocm`, or `distributed_npu`.
-   ```python
-   @pytest.mark.distributed_cuda(num_cards=num_cards)
-   ```
+2. **Handles card-count filtering**  
+   Adds a filterable marker `cards_{n}` (including `cards_1` when `num_cards` is omitted / 1). Use this with the SKU and platform, e.g. `-m "H100 and cards_4 and cuda"` vs `-m "H100 and cards_2 and cuda"`. Any multi-card CUDA test is `-m "cuda and not cards_1"`.
+
+   When `res` lists more than one platform **and** `num_cards` differs by platform, `@hardware_test` collects **one variant per platform** so those `cards_{n}` marks do not stack. `-m "H100 and cards_4 and cuda"` then selects only the CUDA/H100 4-card variant, not a ROCm 2-card sibling. The same count on every platform stays a single item.
 3. **Supports flexible card requirements**  
   Accepts `num_cards` as either a single integer for all platforms or as a dictionary with per-platform values. If not specified, defaults to 1 card per platform.

 4. **Integrates resource validation**  
-   On CUDA, adds a skip marker (`skipif_cuda`) if the system does not have the required number of devices.
-   Support for `skipif_rocm` and `skipif_npu` will be implemented later.
+   On CUDA and XPU, when `num_cards > 1`, attaches pytest's built-in `skipif` if the machine has fewer devices than requested. This is not a named `skipif_*` marker and is not used for `-m` filtering.

 5. **Works with pytest filtering**  
-   Allows tests to be filtered and selected at runtime using standard pytest marker expressions (e.g., `-m "distributed_cuda and L4"`).
+   Allows tests to be filtered and selected at runtime using standard pytest marker expressions (e.g., `-m "H100 and cards_2"`, `-m "L4 and not cards_1"`).

 ##### Example usage for decorator

@@ -117,13 +110,14 @@ This decorator is intended to make hardware-aware, cross-platform test authoring
        num_cards=2,
    )
    ```
- `res` must be a dict; supported resources: CUDA (L4/H100), ROCm (MI325), XPU (B60), MUSA (S5000), NPU (A2/A3)
+- `res` must be a dict. SKU values come from `pyproject.toml` markers tagged `[hardware-resource]` plus the platform key (`[cuda]`, `[rocm]`, `[xpu]`, `[npu]`, `[musa]`).
 - `num_cards` can be int (all platforms) or dict (per platform); defaults to 1 when missing
- Distributed markers (`distributed_cuda`, `distributed_rocm`, `distributed_npu`) are auto-added for multi-card cases
 - Filtering examples:
-    - CUDA only: `pytest -m "distributed_cuda and L4"`
-    - ROCm only: `pytest -m "distributed_rocm and MI325"`
-    - NPU only: `pytest -m "distributed_npu"`
+    - CUDA H100 4-card: `pytest -m "H100 and cards_4 and cuda"`
+    - CUDA H100 2-card: `pytest -m "H100 and cards_2 and cuda"`
+    - CUDA multi-card (any count): `pytest -m "L4 and not cards_1"`
+    - ROCm multi-card: `pytest -m "MI325 and not cards_1"`
+    - NPU multi-card: `pytest -m "npu and not cards_1"`

 #### Function: `hardware_marks`

@@ -179,37 +173,30 @@ If you want to add support for a new platform (e.g., "tpu" for a new accelerator
   [tool.pytest.ini_options]
   markers = [
       # ... existing markers ...
-       "tpu: Tests that require TPU device",
-       "TPU_V3: Tests that require TPU v3 hardware",
-       "distributed_tpu: Tests that require multiple TPU devices",
+       "tpu: [hardware-platform] Tests that require TPU device",
+       # Add [gpu] next to [hardware-platform] if this platform should also get pytest.mark.gpu.
+       "TPU_V3: [hardware-resource] [tpu] Tests that require TPU v3 hardware",
+       "cards_1: [hardware-cards] Tests that require 1 accelerator card",
+       "cards_2: [hardware-cards] Tests that require 2 accelerator cards",
   ]
   ```
+   `check-mark` treats `[hardware-platform]` names (including `cpu`/`gpu`) as allowed `pytest.mark.<name>` values. `hardware_test(res=...)` still rejects `cpu`/`gpu` as `res` keys.
 2. **Implement a marker construction function for your platform** in `vllm-omni/tests/helpers/mark.py`:
   ```python
   # In vllm-omni/tests/helpers/mark.py

-   def tpu_marks(*, res: str, num_cards: int):
-       test_platform = pytest.mark.tpu
-       if res == "TPU_V3":
-           test_resource = pytest.mark.TPU_V3
-       else:
-           raise ValueError(
-               f"Invalid TPU resource type: {res}. Supported: TPU_V3")
-
-       if num_cards == 1:
-           return [test_platform, test_resource]
-       else:
-           test_distributed = pytest.mark.distributed_tpu(num_cards=num_cards)
-           # Optionally: add skipif_tpu when implemented
-           return [test_platform, test_resource, test_distributed]
+   def _tpu_marks(*, res: str, num_cards: int):
+       _require_sku("tpu", res)
+       return [pytest.mark.tpu, getattr(pytest.mark, res), _cards_mark(num_cards)]
   ```
-3. **Update `hardware_marks` to recognize your new platform**:
-    In the relevant place (see the `hardware_marks` implementation), add:
+3. **Register the builder** in `_marks_for_platform`:
    ```python
-    if platform == "tpu":
-        marks = tpu_marks(res=resource, num_cards=cards)
+    builders = {
+        # ... existing platforms ...
+        "tpu": _tpu_marks,
+    }
    ```
-    (`hardware_test` calls `hardware_marks` internally, so both will pick up the change.)
+    (`hardware_test` / `hardware_marks` both go through `_marks_for_platform`.)
 4. **(Recommended) Add a test using your new markers**:
   ```python
   @hardware_test(
@@ -223,11 +210,11 @@ If you want to add support for a new platform (e.g., "tpu" for a new accelerator
 **Summary**:  

 - Add pytest markers for your new platform/resources  
- Implement a marker function (`xxx_marks`)  
- Plug into `hardware_marks`  
- You're done: tests using `@hardware_test` or `hardware_marks` with your platform now automatically get the correct markers, distribution, and isolation!
+- Implement a private marker function (`_xxx_marks`)
+- Register it in `_marks_for_platform`
+- You're done: tests using `@hardware_test` or `hardware_marks` with your platform now automatically get the correct markers and `cards_{n}` filtering.

-See code in `vllm-omni/tests/helpers/mark.py` for existing examples (`cuda_marks`, `rocm_marks`, `npu_marks`).
+See code in `vllm-omni/tests/helpers/mark.py` for existing examples (`_cuda_marks`, `_rocm_marks`, `_npu_marks`).

 ## Test case style

@@ -438,7 +438,7 @@ selects the scheduler and worker family:
 Key topology fields belong to `StagePipelineConfig`:

 | Field | Description |
-|-------|-------------|
+| ------- | ------------- |
 | `model_stage` | Logical stage name (`ar_stage`, `decoder`, etc.). |
 | `execution_type` | Runtime family for this stage. |
 | `input_sources` | Upstream stage IDs that provide input. |
@@ -960,7 +960,7 @@ Common failures and fixes:
 | `check-forbidden-imports` | Stdlib `re`/`base64`, pickle, Hugging Face Hub API, or direct Triton/TileLang | `import regex as re` and `pybase64`; do not grow the allowlist without review |
 | `check-torch-cuda-call` | New `torch.cuda.*` call site | Use `current_omni_platform`; do not grow `ALLOWED_FILES` without review |
 | `check-tts-adapter-migration` | New `_tts_model_type` branch in `serving_speech.py` | Put logic in `tts_adapters/`; lower `MAX_MODEL_TYPE_BRANCHES` when removing branches |
-| `check-test-ci-coverage` | Test file missing level or hardware mark | Add `core_model`/… plus `cpu`/`cuda`/`hardware_test(` |
+| `check-mark` | Test file missing level/hardware mark, or direct `pytest.mark.H100` | Add `core_model`/… plus `cpu`/`cuda`/`hardware_test(`; SKU marks only via helpers |
 | `shellcheck` | Native `shellcheck` missing, or script warning | Install via apt/dnf/brew (or `shellcheck.exe` on Windows). See [Linting](../README.md#linting) |
 | `mypy-3.10` / `markdownlint-cli2` / `check-buildkite` | Types, docs markdown, or Buildkite YAML | See [Linting](../README.md#linting) |

@@ -322,6 +322,15 @@ Each 1-second window records:

 On shutdown the server also logs a short summary (`loop_active_pct`, per-replica queue averages/maxima).

+`loop_idle` and `loop_active` count loop iterations, so their absolute scale is
+tied to which orchestration loop is running. Under the default poll loop an idle
+orchestrator records roughly one iteration per millisecond. Under the
+event-driven loop (`VLLM_OMNI_EVENT_DRIVEN_ORCH=1`, see
+[Speech API](../serving/speech_api.md#orchestration-loop-experimental)) an idle
+orchestrator wakes only once per 0.5 s reconcile timeout, while a busy one still
+records one iteration per routed output. Window counts and the `loop_active_pct`
+summary are therefore not comparable across the two modes.
+
 ### Relationship to other diagnostics

 This monitor is intentionally separate from the existing profiling tools:
@@ -170,6 +170,10 @@ because the user-visible output is diffusion media rather than a token stream.
 The
 [MiniMax-H3 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/MiniMaxAI/MiniMax-H3.md)
 contains the hardware-specific profiles and warmup requirements.
+The internal, opt-in
+[temporal VAE chunk callback](feature/minimax_h3_temporal_vae_chunks.md)
+documents the narrower released-source contract for publishing committed video
+frames during decode. It does not change the complete-output serving topology.

 ### Cosmos3: unified MoT reasoner and generator

@@ -63,7 +63,7 @@ vLLM-omni provides `CFGParallelMixin` that encapsulates all CFG parallel logic.

 ### N-Branch CFG (3+ branches)

-Some models require more than 2 CFG branches. For example, Bagel and OmniGen2 use 3 branches, DreamID Omni uses 4 branches.
+Some models require more than 2 CFG branches. For example, Bagel and OmniGen2 use 3 branches.

 `predict_noise_with_multi_branch_cfg()` handles these by automatically dispatching N branches across M GPUs using round-robin (rule: branch `i` → rank `i % M`):

@@ -14,12 +14,12 @@ specified in the [Host Weight Runtime module design](../module/host_weight_runti

 ## Status

-The first implementation provides contracts and a CPU local-filesystem store.
-The initial diffusion consumer contract additionally defines typed,
+The implementation provides contracts and a CPU local-filesystem store. The
+diffusion consumer additionally defines typed,
 representation-independent final-layout identity/restoration mechanics plus a
-concrete BF16-with-preserved-FP32 policy for MiniMax H3. It remains
-library-only: no loader or DLO path selects, publishes, restores, or transports
-that artifact yet.
+concrete BF16-with-preserved-FP32 policy for MiniMax H3 and
+`black-forest-labs/FLUX.2-klein-4B`. The opt-in no-AllGather DLO integration
+selects, publishes, restores, and transfers these artifacts.

 V1 includes:

@@ -33,13 +33,12 @@ V1 includes:

 V1 does not include:

- a public CLI, loader activation, or default-on model integration;
- lease handoff to DLO or another transport consumer;
- FP8, quantized, merged-adaptation, or additional model producers;
- CUDA registration, pinned staging, H2D scheduling, or GPU kernels;
+- default-on activation or consumers outside no-AllGather DLO;
+- online FP8, quantized, merged-adaptation, or additional model producers;
+- HWR interaction with DLO AllGather;
 - a remote artifact provider or cross-node coordination;
 - automatic eviction; or
- a change to DLO AllGather or no-AllGather behavior.
+- a change to DLO collective or execution behavior.

 ## Motivation and use cases

@@ -255,7 +254,8 @@ The contract is intentionally separate from loader activation:
 - `FinalLayoutBF16Producer` accepts only the matching identity context and a
  finalized CPU model. It is `POST_LOAD_ONLY` and `SINGLE_PROCESS` per exact TP
  coordinate. Its BF16 policy preserves model-declared FP32 parameters and
-  buffers and revalidates MiniMax H3 mixed-precision invariants.
+  buffers, revalidates MiniMax H3 mixed-precision invariants, and revalidates
+  FLUX.2-klein's two block stacks, packed QKV mapping, and BF16 base layout.

 Other representations reuse source identity, typed parallel identity, tensor
 ownership, and exact restoration only when their policy proves those semantics.
@@ -0,0 +1,260 @@
+# MiniMax H3 temporal VAE chunk callback
+
+!!! warning "Experimental internal capability"
+
+    This feature is an opt-in model integration seam. The current MiniMax H3
+    pipeline still uses complete-output VAE decode, and no CLI option, serving
+    path, transport, or MP4 sink enables temporal chunks by default.
+
+## Status and scope
+
+MiniMax H3 decodes video latents in temporal windows, but the released VAE
+normally exposes only the complete reconstructed video. The temporal callback
+publishes frames after the released decoder has completed overlap blending and
+padding removal. This gives a future runtime consumer a stable point at which
+it can schedule representation conversion, D2H transfer, and encoding while
+the VAE continues decoding later windows.
+
+The implementation remains model-local:
+
+- `vae.py` owns VAE topology and the thin
+  `decode_latent_with_chunks()` facade;
+- `chunked_decode.py` owns the normalized callback contract, metadata
+  validation, and fixed VAE-group rendezvous points; and
+- `temporal_chunks.py` owns the source-gated compatibility planner and the
+  released temporal assembly control flow.
+
+Crop policy, typed media, transport, backpressure, encoding, and public output
+remain runtime responsibilities. The model code does not schedule requests or
+select a presentation format.
+
+## Execution flow
+
+The default and opt-in paths are intentionally separate:
+
+```text
+Default:
+latent -> MiniMaxH3VideoVAE.decode_latent() -> complete RGB video
+
+Opt-in producer seam:
+latent
+  -> MiniMaxH3VideoVAE.decode_latent_with_chunks()
+  -> group-local preflight and callback-owner validation
+  -> released temporal decode / overlap blending / padding removal
+  -> borrowed decoder-domain chunk
+  -> normalize + clone into owned RGB float32 BCTHW [0, 1]
+  -> synchronous callback
+  -> complete RGB video is still returned
+```
+
+Calling `decode_latent_with_chunks()` is the opt-in. Supplying no callback on a
+VAE peer does not select the default decoder; every rank in a VAE patch group
+must enter the chunked method, and only the output rank receives the callback.
+
+## Producer contract
+
+The normalized callback has this structural signature:
+
+```python
+def callback(
+    frames: torch.Tensor,
+    /,
+    *,
+    chunk_index: int,
+    total_chunks: int,
+    frame_start: int,
+    is_final: bool,
+) -> None:
+    ...
+```
+
+For a successful decode:
+
+- `frames` is owned, contiguous RGB float32 in `BCTHW` layout and `[0, 1]`;
+- callback storage does not alias the returned complete video;
+- the consumer may retain or mutate a chunk;
+- `chunk_index` is zero-based, contiguous, and strictly increasing;
+- `total_chunks` is positive and stable for the decode;
+- `frame_start` equals the end of the previous published range;
+- chunks are nonempty and contain only temporally committed frames;
+- exactly one chunk has `is_final=True`; and
+- concatenating chunks along time exactly reconstructs the returned video.
+
+The callback runs synchronously on the producer's current device stream. A
+consumer that reads the tensor on another stream must establish ordering, for
+example with an event, and retain the allocation for that stream's lifetime.
+A slow callback also blocks VAE production. An overlap-oriented runtime
+consumer should therefore acquire bounded capacity, enqueue side-stream work,
+and return promptly instead of encoding frames inside the callback.
+
+Chunks retain the decoded VAE canvas dimensions. A future runtime integration
+must apply the same requested-height and requested-width crop as the complete
+MiniMax H3 pipeline before declaring the media transport-ready.
+
+The raw compatibility callback is deliberately narrower. It receives borrowed
+decoder-domain tensors and must not retain or mutate them. The coordinator is
+the ownership boundary that normalizes and clones raw chunks before invoking
+the public callback.
+
+## Temporal planning and terminal publication
+
+The released configuration advances five latent tokens per temporal window and
+includes a two-token overlap, so each `_adaptive_decode` input normally contains
+seven tokens. After temporal scaling, pre-padding removal, and five-frame
+overlap blending, the normal committed unit is 17 output frames. The final
+remainder depends on the requested frame plan.
+
+Representative released frame plans include:
+
+| Output frames | Published chunks |
+| ---: | --- |
+| 124 | `7 x 17 + 5` |
+| 209 | `12 x 17 + 5` |
+| 243 | `14 x 17 + 5` |
+| 362 | `21 x 17 + 5` |
+
+The terminal callback is held until the complete frame plan passes its logical
+frame, padding, written-frame, chunk-count, and finality checks. Earlier chunks
+are semantically stable, but they do not imply that the overall request has
+succeeded; a consumer must discard or abort partial output if a later decoder
+failure terminates the request.
+
+## Source and configuration gate
+
+The released VAE has no public temporal-callback API. vLLM-Omni therefore owns
+a compatibility implementation for one exact released code contract. The
+reference revision identifies its provenance, but runtime acceptance is based
+on source content, configuration, and structure rather than repository or
+weight metadata:
+
+- reference Hugging Face revision, not independently checked at runtime:
+  `42ed227ee7df40d41602854ae760620d6eb651fe`;
+- sorted 15-file top-level `video_vae/*.py` manifest (excluding
+  `__init__.py`) SHA-256:
+  `30e64afbaa940696bf8bfe2c86003a4b1666c84b22fa3aa12403a3e6a03f705d`;
+- clip length `17`;
+- token chunk, overlap, and drop `5`, `2`, and `3`;
+- temporal ratio `4`;
+- frame pre-padding and overlap `3` and `5`; and
+- no isolated first or last frame.
+
+The adapter does not verify Hugging Face revision metadata, repository identity,
+or model weights. The manifest pin detects Python compatibility drift; it is
+not a security boundary for remote-code execution. The source manifest, listed
+temporal attributes, structural contract, plan, and dtype are validated before
+the first `_adaptive_decode` call. A mismatch disables only the opt-in chunk path.
+`decode_latent()` remains available and performs no source fingerprint or chunk
+rendezvous.
+
+## Supported models
+
+Support in this table means the internal producer contract above, not public
+serving or end-to-end transport support.
+
+| Model or component | Source contract | Status | Validation |
+| --- | --- | --- | --- |
+| MiniMax H3 FL2VA video VAE | Released manifest/listed temporal attributes above (reference `42ed227e`) | Supported, opt-in | Same-environment full-output parity and L20X/B300 feasibility measured on the implementation through PR commit `60e9bbad` |
+| MiniMax H3 Ref2VA video VAE | Same released manifest/listed temporal attributes | Source-compatible; not separately benchmarked | Ref2VA and FL2VA use byte-identical video-VAE source and configuration in the reference snapshot |
+| MiniMax H3 with a different Python manifest or listed temporal attribute | Different enforced contract | Unsupported for chunk mode | Fails closed before decoder collectives; complete decode remains available |
+| Same manifest/temporal attributes with different weights or other component metadata | Not checked by this gate | Not independently qualified | The adapter does not inspect weights, repository identity, revision metadata, or arbitrary component configuration |
+| Other video models | Any | Unsupported | No generic temporal VAE producer ABI is declared by this feature |
+
+## Feature compatibility
+
+| Dimension or combination | Status | Constraint |
+| --- | --- | --- |
+| Existing complete-output decode | Supported and default | No source check, callback object, or callback rendezvous |
+| Explicit chunk decode | Supported internally | Caller invokes `decode_latent_with_chunks()` directly |
+| VAE patch parallel size 1 | Supported by the model seam | Caller supplies the callback only on the request output owner; no WORLD rendezvous is added |
+| VAE patch parallel size equal to the full DiT group | Supported when data parallel size is 1 | Every VAE-group rank calls; exactly group rank 0 owns the callback |
+| Partial VAE patch subgroup | Unsupported | MiniMax H3 currently accepts size 1 or the full DiT group only |
+| Data parallel size greater than 1 with VAE patch parallel size 1 | Model seam only | Per-replica callback ownership is possible, but no serving/runtime consumer is wired |
+| Data parallel size greater than 1 with VAE patch parallel size greater than 1 | Unsupported | Fails before decoder collectives because no replica-local VAE patch group is defined |
+| Too few spatial tiles for the VAE group | Correctness fallback | Decodes rank-locally and restores native tiling state; slower than patch parallel decode |
+| Device memory and full-output assembly | Not reduced or bounded | The compatibility path still assembles and returns the full decode while allocating owned callback chunks; no peak-HBM reduction is claimed |
+| DiT cache, attention, TP, SP, HSDP, LoRA, or quantization features | No direct callback dependency | This seam runs after denoising and does not independently qualify combinations beyond the MiniMax H3 model's own matrix |
+| Layerwise or distributed component offload | Not separately validated | Component residency must place the video VAE before decode |
+| Step execution or request batching | Not integrated | No pipeline caller binds a runtime chunk consumer |
+| Requested-size crop and device uint8 preparation | Not implemented here | Runtime-owned follow-up |
+| Pinned D2H, SHM/ZMQ, or cross-process transport | Not implemented here | Runtime-owned follow-up |
+| Complete MP4 or fragmented MP4 sink | Not implemented here | Existing serving behavior still waits for complete output |
+| Audio VAE chunks and incremental A/V composition | Not supported | Video chunks do not imply audio or request finality |
+| Cancellation and multi-output lease ownership | Not implemented here | Requires a bounded runtime lifecycle |
+| NVIDIA CUDA | Feasibility evidence | L20X and B300 measurements apply to the implementation through `60e9bbad`; the current coordination refactor was not reprofiled |
+| AMD ROCm | Not validated for this feature | Base MiniMax H3 support is documented separately and does not imply temporal-chunk validation |
+| Ascend, Intel, or other accelerators | No feature support claim | Consult the model support table independently; this design adds no platform qualification |
+
+## Distributed and failure semantics
+
+When VAE patch parallelism is active, all VAE-group ranks run the same
+preflight before entering checkpoint collectives. Source/planner errors and the
+resolved temporal concat dtype are reconciled at fixed group rendezvous points.
+
+Callback ownership is also a group decision. Rank `r` contributes `r + 1` when
+it owns a callback and zero otherwise. A group SUM equals one only when rank 0
+is the sole owner; every invalid owner configuration therefore fails
+identically on all ranks before decode.
+
+Callback exceptions are retained on the output rank. Further publication stops,
+but every rank completes the same remaining native decoder collective schedule.
+The original exception is then raised on the owner and
+`MiniMaxH3ChunkCallbackPeerError` on peers.
+
+Exceptions raised inside the checkpoint's native distributed decode cannot in
+general be reconciled by the adapter: a peer may already be blocked inside a
+checkpoint-owned collective. Resolving such failures requires native collective
+handling or supervisor-level timeout and worker-group teardown. The process
+group must not be assumed reusable after such a failure.
+
+## Validation evidence
+
+The contract suite covers ordinary and padded frame plans, blend boundaries,
+owned storage and mutation isolation, source-manifest/listed-temporal-attribute
+drift, callback failure, and healthy subsequent requests. Real Gloo processes cover
+rank-asymmetric preflight failures, dtype disagreement, two-rank callback
+failure propagation, three-rank wrong-owner rejection, insufficient-tile
+fallback, and group recovery for failures that occur at adapter-owned
+rendezvous points.
+
+Four-device L20X and independent four- and eight-device B300 experiments on the
+implementation through PR commit `60e9bbad` showed exact serial-versus-chunked
+frame/MP4 parity within each environment and lower VAE-to-complete-MP4 latency
+when an experimental consumer was attached. Hashes are not claimed to match
+across different hardware/software environments. The later coordination and
+module-boundary refactors were revalidated with CPU and real Gloo tests but were
+not reprofiled on GPUs. These are feasibility measurements, not a promise for
+default serving, because this change does not contain the transport or encoder
+consumer. See [RFC #6872](https://github.com/vllm-project/vllm-omni/issues/6872)
+and [PR #6885](https://github.com/vllm-project/vllm-omni/pull/6885) for the
+measurement provenance and limitations.
+
+## Generalization boundary and follow-ups
+
+This feature does not define a repository-wide `SupportsVideoChunkDecode`
+interface. H3 temporal blending, Wan causal caches, LTX temporal/spatial halos,
+and pipeline-level generation chunks have different commit and failure
+boundaries. A generic producer contract should be extracted only after a second
+model proves the same semantic event without `Any` arguments or model-name
+dispatch.
+
+The reusable future boundary is the emitted, committed video-media chunk, not a
+universal latent-to-VAE method. It should extend the typed media work in
+[RFC #6541](https://github.com/vllm-project/vllm-omni/issues/6541) and
+[PR #6615](https://github.com/vllm-project/vllm-omni/pull/6615), while keeping
+model-specific halo, cache, padding, blending, distributed, and source-gating
+logic inside each model adapter.
+
+Runtime follow-ups own:
+
+- typed video-plus-audio media and request/output identity;
+- requested-size crop and device preparation;
+- bounded leases, backpressure, cancellation, and cleanup;
+- D2H/IPC transport and observability; and
+- complete-MP4 and fragmented-MP4 consumers.
+
+## Related documentation
+
+- [Diffusion model integration](../module/diffusion/diffusion_model_integration.md)
+- [VAE patch parallelism](vae_parallel.md)
+- [Async diffusion output](async_diffusion_output.md)
+- [MiniMax H3 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/MiniMaxAI/MiniMax-H3.md)
@@ -26,6 +26,8 @@ Legend: ✅ supported, ⚠️ compatibility path or limited validation, ❌ unsu
 | **TP > 1** | ⚠️ Ordinary TP-aware loader only; no direct checkpoint mmap. | ⚠️ Ordinary TP-aware loader only; no direct checkpoint mmap. |
 | **HSDP** | ❌ Rejected to avoid double-sharding parameters. | ⚠️ Limited end-to-end coverage. |
 | **Per-tensor online FP8 linears** | ✅ Ordinary loader finalizes weights and scales before DLO sharding. | ✅ Ordinary loader retains complete rank-local tensors. |
+| **Online INT8 linears** | ✅ Ordinary loader finalizes the int8 weight (contiguous or transposed view) and fp32 scale before DLO sharding. | ✅ Ordinary loader retains complete rank-local tensors. |
+| **Online MXFP8 linears** | ✅ Ordinary loader finalizes the fp8 weight (contiguous) and e8m0 block scale (contiguous or transposed view) before DLO sharding. | ✅ Ordinary loader retains complete rank-local tensors. |
 | **Other online quantization methods** | ❌ Rejected until runtime packing and scale layouts are validated. | ⚠️ Allowed through the ordinary loader; validation is method-specific. |
 | **Model-level or standard layerwise CPU offload** | ❌ Disabled because DLO takes priority. | ❌ Disabled because DLO takes priority. |
 | **Resident leading layers** | ❌ Rejected. | ✅ Requires eligible resident paths in the model's `OffloadPlan`. |
@@ -169,18 +171,42 @@ This mode means:
  copy per rank.
 - The scheduler does not require a synchronized DP request wave for DLO.

+### Component allocator-cache retention
+
+MiniMax-H3 DLO keeps one bounded PyTorch allocator cache across its staged
+encoder, DiT, and VAE component boundaries so allocations released by one
+component can be reused by the next. The DiT prefetch path and its two shared
+device buffers are unchanged.
+
+After a component is offloaded, cached-but-unallocated memory is retained only
+while it is at most 25% of device capacity and at least 5% of device capacity
+remains physically free. Crossing either bound releases the allocator cache.
+Missing allocator telemetry also releases it conservatively. Component or
+staging failure forces a release, and an out-of-memory allocation gets one
+retry after release. This is a component-local policy rather than a global
+`empty_cache` override, so the executor's unconditional shutdown cleanup is
+preserved.
+
+Retaining allocator blocks trades higher reserved and peak device memory for
+lower lifecycle latency. The benefit is device- and workload-dependent, and a
+smaller or externally loaded device may cross a bound and return to the
+ordinary per-component cache-release behavior. Current performance validation
+covers MiniMax-H3 no-AllGather DLO on CUDA. Other MiniMax-H3 DLO transfer
+topologies and platforms receive the same bounded policy but are not claimed
+performance targets; other DLO models remain unchanged unless they explicitly
+adopt it.
+
 ### Final-layout Host Weight Runtime consumer

-> **Status:** PR2 implementation. The representation-independent identity,
-> producer, and restoration contracts landed in
-> [PR #6445](https://github.com/vllm-project/vllm-omni/pull/6445); this consumer
-> wires the final-layout BF16 path into eligible no-AllGather DLO.
+> **Status:** PR2 landed in
+> [PR #6486](https://github.com/vllm-project/vllm-omni/pull/6486). PR3 adds
+> registered direct H2D plus generic HWR publication/source-digest
+> optimizations without changing AllGather behavior.

 Final-layout Host Weight Runtime (HWR) backing is opt-in and currently applies
 only to no-AllGather DLO. Enable it with
 `--host-weight-runtime-mode preferred` (or `required`) and
-`--host-weight-runtime-root <node-local-root>`. Registration of shared mappings
-and direct asynchronous H2D remain a separate PR3 transport change.
+`--host-weight-runtime-root <node-local-root>`.

 The modes express operator fallback policy, not different artifact formats:

@@ -196,9 +222,15 @@ model revision and parallel layout with `required`. TP coordinates have distinct
 identities, while equivalent DP replicas share them.

 The current producer/consumer boundary is the model-declared final-layout BF16
-contract (MiniMax H3 today). It supports ordinary-loader final layouts for TP1
-and TP2 rank identities plus SP layout identities; online quantization, HSDP,
-LoRA/adapted weights, and non-default load formats remain ineligible. HWR mode
+contract (MiniMax H3 and `black-forest-labs/FLUX.2-klein-4B`). The FLUX.2-klein
+contract covers both transformer block stacks, constructor-stable packed QKV
+mapping state, BF16 parameters, and any persistent loader-owned `beta`/`eps`
+buffers. The shared HWR machinery supports ordinary-loader final layouts for
+TP1 and TP2 rank identities plus SP layout identities. FLUX.2-klein evidence
+in this change is TP1-only; its TP2/SP layouts rely on the existing
+per-coordinate identity mechanics and remain unmeasured. Online quantization,
+HSDP, LoRA/adapted weights, and non-default load formats remain ineligible.
+HWR mode
 `disabled`, DLO-disabled, and DLO AllGather configurations stop before source
 identity or store construction and retain the existing checkpoint-mmap or
 ordinary-loader path.
@@ -229,6 +261,44 @@ outcomes are authoritative:
 Checkpoint mmap remains an unchanged control path whenever final-layout HWR is
 not selected.

+Local source identity and publication avoid repeated full-payload passes while
+preserving the same semantic contract:
+
+- Canonical files without a trusted immutable Hub blob identity are hashed in
+  parallel. Their content IDs are cached per storage domain under the HWR root
+  and reused only when the path, inode, size, timestamps, symlink target, and
+  cache-record checksum still match. Corrupt entries are rebuilt; cache-lock or
+  cache-I/O failure falls back to hashing the canonical file directly.
+- Producers that emit payload bytes in canonical storage-key order may declare
+  `ordered=True`. The filesystem writer then computes file and tensor SHA256
+  values during the write and overlaps payload `fsync` with later producer
+  work. Producers without that guarantee retain unordered writes and use a
+  parallel readback checksum fallback.
+- Manifest and `READY` publication still wait for every payload `fsync` and
+  checksum to complete. These optimizations do not weaken artifact validation
+  or change lease ownership.
+
+#### Registered mmap transport
+
+After DLO takes an HWR lease, the no-AllGather backend may register the lease's
+complete immutable mapped ranges under the existing `pin_cpu_memory` policy.
+Registration is transport state: it does not change artifact identity, the
+store, tensor ownership, or H2D payload. On success, each tensor view copies
+directly into the existing rotating HBM block buffers and the two private host
+staging slots are not allocated.
+
+`--dlo-host-registration-limit-gib` is an optional per-worker preflight ceiling
+over page-aligned registered bytes. Zero adds no ceiling. A disabled pinned
+memory policy, unsupported platform/capability, over-budget mapping, or fully
+rolled-back registration error selects the existing two-slot staging path. A
+partial registration that cannot be rolled back aborts startup because closing
+the lease would unmap memory still owned by the platform.
+
+Direct checkpoint mmap remains unchanged and continues to use staging. It may
+require loader-owned per-block transforms, while the HWR artifact already
+contains final runtime bytes. DLO AllGather never receives an HWR final-layout
+lease and therefore never enters this registration path.
+
 #### Pre-service transaction boundary

 The startup transaction does not end at restore commit. A lease-backed model is
@@ -292,6 +362,7 @@ loader owns
  -> backend takes once before asynchronous work
  -> backend drains pending H2D work
  -> backend releases hooks, staging, and model references
+  -> backend unregisters every mapped range
  -> backend closes lease
 ```

@@ -301,13 +372,15 @@ it. Once the backend takes ownership, only backend abort or teardown may drain
 work and close it.

 The implementation keeps the final-layout lease through backend setup and
-initial prefetch. The backend uses the existing two bounded rank-local staging
-slots for the immutable HWR tensors, and its transactional `enable()` cleanup
-removes partial hooks and closes the lease before reporting a setup failure.
-Preferred mode then uses the runner's fresh canonical retry; required mode
-propagates the failure.
+initial prefetch. Successful registration copies directly from immutable HWR
+views; otherwise the backend uses the existing two bounded rank-local staging
+slots. Transactional cleanup drains device work, removes source references,
+unregisters mappings, and only then closes the lease. An unregistration failure
+retains both registration and lease for retry/process teardown. Preferred mode
+then uses the runner's fresh canonical retry; required mode propagates the
+failure.

-#### PR2 promotion gates
+#### PR2 and PR3 promotion gates

 - Warm hit performs zero ordinary DiT materialization and zero producer calls.
 - Shared warm finalization changes neither restored bytes nor backing pointers.
@@ -319,6 +392,10 @@ propagates the failure.
 - Disabled mode and AllGather emit zero final-layout HWR interaction.
 - The lease carrier rejects duplicate take and serialization.
 - Backend setup or prefetch failure drains asynchronous work before lease close.
+- Registration is all-or-nothing; rollback/unregistration failure never closes
+  a mapping still owned by the platform.
+- Registered HWR transport bypasses host staging while preserving output and
+  H2D payload; unsupported registration retains the bounded staging path.
 - Compatible checkpoint mmap remains an unchanged benchmark and control path.
 - Prewarm uses one matching producer cohort per TP/SP topology and storage
  domain.
@@ -363,17 +440,22 @@ requirements.

 Direct checkpoint mmap can back either transfer path. It is currently limited
 to proven TP1, non-HSDP, non-online-quantized layouts. Other layouts use the
-ordinary loader. Per-tensor online FP8 linears can use DLO AllGather after the
-ordinary loader finalizes their runtime weights and scales; DLO then shards and
-reconstructs those tensors with their recorded layouts. Other online methods
-must use `--dlo-no-use-allgather` or disable online quantization until their
-runtime layouts are validated.
+ordinary loader. Per-tensor online FP8, online INT8, and online MXFP8 linears
+can use DLO AllGather after the ordinary loader finalizes their runtime
+weights and scales; DLO then shards and reconstructs those tensors with their
+recorded layouts (they all keep plain 1-byte dtypes over ordinary strided
+views: online INT8 has an int8 weight plus fp32 scale, online MXFP8 has an
+fp8 weight plus e8m0 block scale, each with a contiguous or transposed-view
+finalized tensor, all covered by the same physical-order packing as online
+FP8). Other online methods must use `--dlo-no-use-allgather` or disable
+online quantization until their runtime layouts are validated.

-The Host Weight Runtime representation and publication contracts are merged;
-see [RFC #6414](https://github.com/vllm-project/vllm-omni/issues/6414) and
-[PR #6445](https://github.com/vllm-project/vllm-omni/pull/6445). The planned
-no-AllGather consumer above is the PR2 implementation; registration and direct
-asynchronous H2D remain a separate transport follow-up.
+The Host Weight Runtime representation, publication, and no-AllGather consumer
+contracts are merged; see
+[RFC #6414](https://github.com/vllm-project/vllm-omni/issues/6414),
+[PR #6445](https://github.com/vllm-project/vllm-omni/pull/6445), and
+[PR #6486](https://github.com/vllm-project/vllm-omni/pull/6486). Registered
+direct H2D is an optional transport layer over that merged lease contract.

 ## Validation coverage

@@ -383,13 +465,19 @@ Current source-level validation includes:
 - HSDP + DLO without AllGather acceptance at configuration level;
 - loader preflight fallback for TP, HSDP, online quantization, unknown custom
  loaders, missing keys, and shape/dtype mismatches;
- ordinary-loader fallback for per-tensor online FP8 linears followed by DLO
-  sharding of finalized weights and scales;
+- ordinary-loader fallback for per-tensor online FP8, online INT8, and online
+  MXFP8 linears followed by DLO sharding of finalized weights and scales;
 - exact loader-to-backend plan transfer and ordinary-loader fallback;
+- ordered publication hashing, overlapped durability, unordered parallel
+  checksum fallback, and node-local source-digest reuse/invalidation;
 - rank-local mmap source retention, bounded two-slot staging, and adapter
  transforms without parameter-side flags;
+- read-only registration preflight, direct source-to-device copies, safe
+  fallback, partial rollback, retryable unregistration, and lease ordering;
 - resident-layer requests requiring no-AllGather;
 - DP request-wave validation for denoising-step compatibility;
+- bounded component allocator-cache retention, conservative/forced release,
+  OOM retry, and immutable encoder non-block staging;
 - sharding, double-buffer, AllGather-size, and heterogeneous-block regression
  tests.

@@ -22,59 +22,60 @@ We use **Qwen-Image** as the reference implementation for decode parallel, and *

 **VAE Patch Parallelism** is an acceleration technique for both **encoding** and **decoding**. Instead of processing the entire tensor at once, the tensor is:

-+ Split into multiple spatial tiles
+- Split into multiple spatial tiles

-+ Distributed across multiple ranks
+- Distributed across multiple ranks

-+ Encoded/Decoded in parallel
+- Encoded/Decoded in parallel

-+ Merged to reconstruct the final output
+- Merged to reconstruct the final output

 This approach:

-+ Distributes computation across multiple devices
+- Distributes computation across multiple devices

-+ Reduces peak memory usage per device
+- Reduces peak memory usage per device

-+ Accelerates encoding/decoding latency
+- Accelerates encoding/decoding latency

 ### When to Use Encode vs Decode Parallel

-| Operation | Use Case | Example |
-|-----------|----------|---------|
-| **Decode Parallel** | Text-to-Image, Text-to-Video | Latent → Image/Video |
-| **Encode Parallel** | Image-to-Video (I2V) | Image → Latent (for conditioning) |
+| Operation           | Use Case                     | Example                           |
+| ------------------- | ---------------------------- | --------------------------------- |
+| **Decode Parallel** | Text-to-Image, Text-to-Video | Latent → Image/Video              |
+| **Encode Parallel** | Image-to-Video (I2V)         | Image → Latent (for conditioning) |

 ### Architecture
+
 We introduce **DistributedVaeExecutor** as the core component responsible for distributed VAE encoding/decoding.

 The executor is model-agnostic and accepts three function parameters:

-+ split – Partition the latent into tiles
+- split – Partition the latent into tiles

-+ exec – Decode a single tile
+- exec – Decode a single tile

-+ merge – Combine decoded tiles into the final output
+- merge – Combine decoded tiles into the final output

 #### Execution Flow

-+ Call split(z) to generate a list of TileTask and a GridSpec
+- Call split(z) to generate a list of TileTask and a GridSpec

-+ Dispatch tasks across ranks using workload-based balancing
+- Dispatch tasks across ranks using workload-based balancing

-+ Each rank executes exec(task) on its assigned tiles
+- Each rank executes exec(task) on its assigned tiles

-+ Gather decoded tile results to rank 0
+- Gather decoded tile results to rank 0

-+ Rank 0 performs merge(...)
+- Rank 0 performs merge(...)

-+ (Optional) Broadcast final result to all ranks
+- (Optional) Broadcast final result to all ranks

 This design separates:

-+ Distributed execution logic
+- Distributed execution logic

-+ Model-specific tiling and merging logic
+- Model-specific tiling and merging logic

 #### Why split / exec / merge is necessary?

@@ -82,23 +83,23 @@ The latent tensor cannot be arbitrarily partitioned.

 During decoding:

-+ Each output pixel may depend on neighboring pixels
+- Each output pixel may depend on neighboring pixels

-+ The receptive field is model-dependent
+- The receptive field is model-dependent

 Therefore:

-+ Tiles must include overlap
+- Tiles must include overlap

-+ Merge must perform blending to avoid seams
+- Merge must perform blending to avoid seams

 ## Step-by-Step Implementation (Decode)

 ### Step 1: Implement DistributedAutoencoderKLQwenImage
+
 `QwenImagePipeline` use `AutoencoderKLQwenImage` for vae, so implement a distributed version:

-
-```
+```python
 class DistributedAutoencoderKLQwenImage(AutoencoderKLQwenImage, DistributedVaeMixin):
    @classmethod
    def from_pretrained(cls, *args: Any, **kwargs: Any):
@@ -106,28 +107,35 @@ class DistributedAutoencoderKLQwenImage(AutoencoderKLQwenImage, DistributedVaeMi
        model.init_distributed()
        return model
 ```
+
 **Key points**:
-+ Inherit both AutoencoderKLQwenImage and DistributedVaeMixin
-+ Call init_distributed() after loading weights
+
+- Inherit both AutoencoderKLQwenImage and DistributedVaeMixin
+- Call init_distributed() after loading weights

 ### Step 2: Implement split/exec/merge
+
 Reuse `AutoencoderKLQwenImage.tiled_decode` logic and divide it into three stages. And we need return tiles with `GridSpec` and `TileTask`:
-```
+
+```python
 class GridSpec:
    split_dims: tuple[int, ...]  # Tensor dimensions being split (e.g., (2, 3) for (B, C, H, W))
    grid_shape: tuple[int, ...]  # Tile grid layout (num_rows, num_cols)
    tile_spec: dict = field(default_factory=dict) # Metadata required for merging
    output_dtype: torch.dtype | None = None # Final output dtype
 ```
-```
+
+```python
 class TileTask:
    tile_id: int # task id
    grid_coord: tuple[int, ...]  # Tile position in grid
    tensor: torch.Tensor | list[torch.Tensor]  # The tile tensor
    workload: int | float = 1 # Used for load balancing (e.g., tile area)
 ```
+
 And tiled base split/exec/merge as follow:
-```
+
+```python
 def tile_split(self, z: torch.Tensor) -> tuple[list[TileTask], GridSpec]:
    # mostly copy from AutoencoderKL
    _, _, num_frames, height, width = z.shape
@@ -210,11 +218,14 @@ def tile_merge(self, coord_tensor_map: dict[tuple[int, ...], torch.Tensor], grid
 ```

 ### Step 3: Override tiled_decode
+
 We need to override tiled_decode, the main logic is:
-+ check distributed is enabled
-+ select split/exec/merge
-+ Invoke self.distributed_executor.execute to decode
-```
+
+- check distributed is enabled
+- select split/exec/merge
+- Invoke self.distributed_executor.execute to decode
+
+```python
 def tiled_decode(self, z: torch.Tensor, return_dict: bool = True):
    if not self.is_distributed_enabled():
        return super().tiled_decode(z, return_dict=return_dict)
@@ -230,11 +241,14 @@ def tiled_decode(self, z: torch.Tensor, return_dict: bool = True):

    return DecoderOutput(sample=result)
 ```
+
 `broadcast_result` is set to True or False depending on the model; when enabled, the result will be used even on ranks other than 0.

 ### Step 4: Modify Pipeline
+
 Change vae model from AutoencoderKLQwenImage to DistributedAutoencoderKLQwenImage
-```
+
+```diff
 class YourModelPipeline(nn.Module):
    def __init__(
        self,
@@ -259,8 +273,8 @@ For models that require VAE encoding (e.g., Image-to-Video), you can also parall

 Similar to decode, split the input tensor into tiles. Key considerations:

-+ **Patchify handling**: If the model uses `patch_size`, scale tile parameters accordingly
-+ **Temporal chunking**: Video VAEs may have temporal compression (e.g., 4x)
+- **Patchify handling**: If the model uses `patch_size`, scale tile parameters accordingly
+- **Temporal chunking**: Video VAEs may have temporal compression (e.g., 4x)

 ```python
 def encode_tile_split(self, x: torch.Tensor) -> tuple[list[TileTask], GridSpec]:
@@ -406,23 +420,28 @@ def tiled_encode(self, x: torch.Tensor) -> torch.Tensor:
 **Key differences from decode parallel:**

 | Aspect | Decode Parallel | Encode Parallel |
-|--------|-----------------|-----------------|
+| -------- | ----------------- | ----------------- |
 | `broadcast_result` | Often `False` (only rank 0 needs output) | `True` (all ranks need latents for diffusion) |
 | Patchify | Applied in merge (unpatchify) | Handled by parent `_encode()` before `tiled_encode()` |
 | Temporal chunking | Frame-by-frame | Chunk-based (e.g., 1 + 4n frames) |

 ## Testing
-Verify numerical consistency between:
-+ vae_patch_parallel_size = 1

-+ vae_patch_parallel_size = N
+Verify numerical consistency between:
+
+- vae_patch_parallel_size = 1
+- vae_patch_parallel_size = N

 Example:
+
+```python
 torch.allclose(output_1, output_n, atol=1e-5)
+```

 Testing requirements:
-+ Fix random seed
-+ Use identical tiling strategy
+
+- Fix random seed
+- Use identical tiling strategy

 ```python
 m = Omni(
@@ -434,6 +453,7 @@ m = Omni(
        ),
    )
 ```
+
 When vae_patch_parallel_size is larger than the DiT world size, it will automatically fall back to using the DiT world size instead.

 ## Reference Implementations
@@ -441,7 +461,7 @@ When vae_patch_parallel_size is larger than the DiT world size, it will automati
 Complete examples in the codebase:

 | Model | Path | Decode Parallel | Encode Parallel |
-|-------|------|-----------------|-----------------|
+| ------- | ------ | ----------------- | ----------------- |
 | **Z-Image** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl.py` | ✅ | ❌ |
 | **Wan2.2** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_wan.py` | ✅ | ✅ |
 | **Qwen-Image** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py` | ✅ | ❌ |
@@ -456,7 +476,7 @@ The tile-parallel executor above assigns independent spatial **tiles** to ranks.
 ### How it differs from tile parallel

 | Aspect | Tile parallel (`"tile"`) | Spatially-sharded (`"spatial_shard_height"`/`"spatial_shard_width"`) |
-|--------|--------------------------|-----------------------------------------------|
+| -------- | -------------------------- | ----------------------------------------------- |
 | Unit of work | Independent overlapping tiles | A single global feature map sharded along H or W |
 | Cross-rank communication | Gather tiles to rank 0, stitch + blend | Per-conv **halo exchange** of boundary rows/cols (P2P) |
 | Output assembly | Blend overlapping tiles | All-gather shards on rank 0, trim padding (matches `broadcast_result=False`) |
@@ -481,7 +501,7 @@ serve.py (--vae-parallel-mode) / OmniEngineArgs
 ### Notes

 - The decoder is patched **in place** the first time spatial-shard decode runs and is bound to a single split dimension for the lifetime of the VAE instance.
- Numerical correctness vs. single-GPU decode is covered by `tests/diffusion/distributed/test_wan_spatial_shard.py::test_spatial_shard_decode_matches_reference` (multi-GPU, nightly `full_model` + `distributed_cuda`).
+- Numerical correctness vs. single-GPU decode is covered by `tests/diffusion/distributed/test_wan_spatial_shard.py::test_spatial_shard_decode_matches_reference` (multi-GPU, nightly `full_model` + `cards_2`).

 ---

@@ -25,6 +25,10 @@ implementation contract; it is not, by itself, a general support claim.
 - [Automatic Prefix Caching in Omni Models](feature/prefix_caching.md)
 - [Realtime AR-Diffusion Sessions](feature/realtime_ar_diffusion.md)

+### Model-specific capabilities
+
+- [MiniMax H3 Temporal VAE Chunks](feature/minimax_h3_temporal_vae_chunks.md)
+
 ### Communication

 #### OmniConnector implementations
@@ -6,7 +6,7 @@ This document describes how vLLM-Omni exposes Prometheus metrics for multi-stage

 - Expose pipeline-level request and latency metrics that span the full multi-stage execution (orchestrator scope).
 - Preserve all upstream vLLM per-engine metrics (`vllm:*`) for stages backed by an AR LLM engine, and reshape their `engine` label into `stage` + `replica` so multi-replica deployments gain per-replica visibility automatically.
- Expose per-modality SLO metrics that the upstream `vllm:*` families do not capture — audio TTFP / RTF / duration / frames / streaming continuity / silent-loss.
+- Expose per-modality SLO metrics that the upstream `vllm:*` families do not capture — audio TTFP / RTF / duration / frames / streaming continuity / silent-loss, plus image and diffusion workload, queueing, execution, and memory signals.
 - Expose per-replica-edge cross-stage transfer metrics so the slack between E2E latency and the sum of per-stage `gen_time` (queueing, serialization, network) becomes attributable.
 - Keep the metrics collection overhead low enough that it does not regress TTFA or throughput.

@@ -20,7 +20,7 @@ vLLM's `unregister_vllm_metrics()` function strips every `prometheus_client` col

 ### The Problem

-vLLM-Omni runs multiple engine instances (stages × replicas) within a single process, coordinated by an Orchestrator. The pipeline needs its own metrics — aggregate request counts, end-to-end latency across all stages, per-modality SLO signals, and cross-stage transfer attribution — that do not exist in upstream vLLM. All pipeline-level metrics use the `vllm:omni_` prefix to distinguish them from upstream per-engine metrics. At import time (see `vllm_omni/patch.py`) we replace `unregister_vllm_metrics()` with a scoped version that still strips upstream `vllm:*` collectors before each new `PrometheusStatLogger` registers (so multi-engine processes don't crash on duplicate registration), but preserves anything prefixed `vllm:omni_` / `vllm_omni`.
+vLLM-Omni runs multiple engine instances (stages × replicas) within a single process, coordinated by an Orchestrator. The pipeline needs its own metrics — aggregate request counts, end-to-end latency across all stages, per-modality SLO signals, and cross-stage transfer attribution — that do not exist in upstream vLLM. All pipeline-level metrics use the `vllm_omni:` prefix to distinguish them from upstream per-engine metrics. At import time (see `vllm_omni/patch.py`) we replace `unregister_vllm_metrics()` with a scoped version that still strips upstream `vllm:*` collectors before each new `PrometheusStatLogger` registers (so multi-engine processes don't crash on duplicate registration), but preserves anything prefixed `vllm_omni:` / `vllm_omni`.

 Upstream per-engine metrics retain the `vllm:` prefix but are now registered by `OmniPrometheusStatLogger`, a thin subclass of upstream's `PrometheusStatLogger` that reshapes the single `engine` label into a `stage` + `replica` pair (see "OmniPrometheusStatLogger wrap" below).

@@ -38,7 +38,7 @@ Upstream per-engine metrics retain the `vllm:` prefix but are now registered by
                                   |
        +--------+--------+--------+--------+--------+
        |                                            |
-   vllm:omni_*                                    vllm:*
+   vllm_omni:*                                    vllm:*
   collectors                                  collectors
        |                                            |
   +----+--------+   +-----------+   +----------+   +-----------+
@@ -58,17 +58,30 @@ Upstream per-engine metrics retain the `vllm:` prefix but are now registered by

 ### Data Flow

-There are four independent paths for metric collection.
+There are five independent paths for metric collection.

-**Path 1: Pipeline-level metrics (`vllm:omni_*`)**
+**Path 1: Pipeline-level metrics (`vllm_omni:*`)**

 `OmniPrometheusMetrics` registers the Gauge / Counter / Histogram collectors at import time. It is instantiated once per entrypoint, labeled with the model name. The entrypoint calls its methods as requests progress:

 - `set_running(n)` / `set_waiting(n)` — updated after each request completes. The running count comes from `OmniRequestCounter`, a simple counter incremented/decremented by the Orchestrator as it tracks requests. Waiting is derived as `total - running`.
- `request_succeeded(e2e_seconds, finished_reason="stop")` — recorded when a request finishes at the final stage. `finished_reason` is extracted from `engine_outputs.outputs[0].finish_reason` (vLLM `CompletionOutput` convention) and increments `vllm:omni_requests_success_total{finished_reason}`.
- `request_failed()` — recorded by the cleanup path when a request exits without natural completion. Internally maps to `finished_reason="abort"` so a single Counter family covers both natural and aborted completion.
+- `request_succeeded(e2e_seconds, finished_reason="stop")` — recorded when a request finishes at the final stage. `finished_reason` is extracted from `engine_outputs.outputs[0].finish_reason` (vLLM `CompletionOutput` convention) and increments `vllm_omni:requests_success_total{finished_reason}`.
+- `request_failed()` — recorded by the cleanup path when a request exits without natural completion. Internally maps to `finished_reason="abort"` on the existing completion Counter.
+- `inc_requests_failed(reason)` — increments the dedicated failure-attribution Counter once per request. Reasons are normalized to the bounded set `client_abort`, `client_disconnect`, `stage_error`, and `unknown`.

-**Path 2: Audio modality metrics (`vllm:omni_audio_*`)**
+**Path 2: Image and diffusion metrics**
+
+Image and diffusion metrics use two cooperating collectors. `OmniPrometheusMetrics` owns pipeline/stage workload families, while `OmniModalityMetrics` owns per-replica diffusion breakdowns. Finished stage messages are consumed once in `OmniBase._process_single_result`, which prevents replayed terminal messages from incrementing Counters or Histograms twice.
+
+- `stage_gen_time_s`, `stage_in_queue_s`, image workload, inference-step, and per-stage peak-memory values are emitted from `StageRequestStats` when a stage finishes. `image_pixels` and `image_count_total` are restricted to image outputs; `num_inference_steps` is restricted to diffusion stages.
+- Diffusion engine timings are accumulated per request by `OrchestratorAggregator.accumulate_diffusion_metrics`. Engine-side millisecond values are converted to seconds before `observe_modality_at_finalize(...)` dispatches the per-replica Histogram observations.
+- `request_queue_wait_s` is emitted once at final request completion from `pipeline_timings["queue_wait_ms"]`. A present value of `0` is a valid observation; a missing key produces no sample.
+- `stage_waiting_requests` is the sum of the latest scheduler snapshots for all live replicas in a stage. AR snapshots arrive through `SchedulerStats`; diffusion snapshots ride normal diffusion output metrics. A dead replica's snapshot is removed. Because this deliberately reuses existing message paths, the gauge can retain its last value while a replica produces no output.
+- `kv_wait_s` measures the AR scheduler interval from entering the KV-transfer-free wait state until extraction acknowledgement. The sample is carried to the Orchestrator through the existing engine output and labeled by connector type. Abort/error/removal paths discard incomplete timers rather than emitting partial waits.
+
+Zero and missing values have different meanings. Valid zero-duration queue observations are recorded, while unavailable optional measurements are omitted instead of being exported as synthetic zeroes. `peak_memory_mb`, `diffusion_forward_s`, `diffusion_kv_load_s`, `vae_decode_s`, and `kv_wait_s` therefore appear only when their data source is active.
+
+**Path 3: Audio modality metrics (`vllm_omni:audio_*`)**

 `OmniModalityMetrics` registers seven audio families with `{model_name, stage, replica}` (plus an extra `threshold_ms` / `reason` label on the two extra-cardinality Counters). Three observation sites:

@@ -76,7 +89,7 @@ There are four independent paths for metric collection.
 - `observe_audio_first_packet(...)` — called from the OpenAI SSE audio branch in `serving_chat.py` on the first audio packet for a request. The once-per-request guard is held by `ClientRequestState.first_audio_ts`. The `request_arrival_ts` anchor is stored in `ClientRequestState` by `async_omni.generate()`, computed at request entry.
 - `observe_audio_streaming_finalize(...)` — called from `serving_chat.py` after the streaming chunk loop exhausts. It runs the per-chunk player simulation from `vllm_omni/benchmarks/audio_continuity.py` to compute the worst-case underrun and emits `audio_underrun_s` plus (when the request stayed below the threshold) `audio_continuity_ok_total{threshold_ms}`. Per-chunk PCM byte counts and arrival timestamps are recorded by the same audio branch that updates `first_audio_ts`.

-**Path 3: Cross-stage transfer metrics (`vllm:omni_transfer_*`)**
+**Path 4: Cross-stage transfer metrics (`vllm_omni:transfer_*`)**

 `OmniTransferMetrics` registers four Histogram families with `{model_name, from_stage, from_replica, to_stage, to_replica}` labels. Each observation corresponds to one physical transfer hop (one chunk between adjacent stages), not the per-request accumulated total — so the histograms track per-transfer distribution.

@@ -87,7 +100,7 @@ The hook lives in `OrchestratorAggregator.record_transfer_tx` and `record_transf

 Defensive fail-safe: if `transfer_emitter` or `replica_resolver` is missing, or the resolver returns `None` for either side, the emit is skipped silently (the underlying `TransferEdgeStats` accumulation is unaffected).

-**Path 4: Per-engine metrics (`vllm:*`, stage/replica wrap)**
+**Path 5: Per-engine metrics (`vllm:*`, stage/replica wrap)**

 The Orchestrator instantiates `OmniPrometheusStatLogger` (a thin subclass of upstream `vllm.v1.metrics.loggers.PrometheusStatLogger`) and feeds it scheduler stats and iteration stats after processing each batch of engine outputs. This populates the standard ~37 vLLM metric families (TTFT, ITL, TPOT, KV cache usage, etc.) using the same upstream code path — but with the `engine` label reshaped into `stage` + `replica` so multi-replica deployments produce distinct series per replica. See the next section for the wrap mechanics.

@@ -97,9 +110,9 @@ The Orchestrator runs in a background thread. The API server (OmniBase) runs in

 ### Metric Registration and Lifecycle

-All `vllm:omni_*` collectors are registered once when their owning class (`OmniPrometheusMetrics` / `OmniModalityMetrics` / `OmniTransferMetrics`) is imported. Per-`(stage, replica)` labels are bound lazily on first observation to avoid registering label sets for combinations that never produce data.
+All `vllm_omni:*` collectors are registered once when their owning class (`OmniPrometheusMetrics` / `OmniModalityMetrics` / `OmniTransferMetrics`) is imported. Per-`(stage, replica)` labels are bound lazily on first observation to avoid registering label sets for combinations that never produce data.

-The `prometheus_client` default registry holds all collectors. FastAPI's `/metrics` endpoint serves the default registry, so `vllm:omni_*` and the wrapped `vllm:*` metrics appear in the same scrape response alongside `http_*` and `process_*` metrics from the instrumentator and the Python client runtime.
+The `prometheus_client` default registry holds all collectors. FastAPI's `/metrics` endpoint serves the default registry, so `vllm_omni:*` and the wrapped `vllm:*` metrics appear in the same scrape response alongside `http_*` and `process_*` metrics from the instrumentator and the Python client runtime.

 ## OmniPrometheusStatLogger Wrap

@@ -130,11 +143,11 @@ A reverse map `(stage_id, replica_id) -> flat_idx` is maintained on the Orchestr

 ## Gating: `--log-stats`

-All metrics — both the 15 `vllm:omni_*` families and the ~65 upstream `vllm:*` wrap families — are gated by the user's `--log-stats` CLI flag (default off). The flag is plumbed from `OmniBase.__init__(log_stats=...)` through `AsyncOmniEngine` to the stage-spawn helpers and to the three Prometheus metric classes (`OmniPrometheusMetrics` / `OmniModalityMetrics` / `OmniTransferMetrics`), and is forwarded to `Orchestrator._init_metrics_state(...)`.
+All metrics — both the omni-specific `vllm_omni:*` families and the upstream `vllm:*` wrap families — are gated by the user's `--log-stats` CLI flag (default off). The flag is plumbed from `OmniBase.__init__(log_stats=...)` through `AsyncOmniEngine` to the stage-spawn helpers and to the three Prometheus metric classes (`OmniPrometheusMetrics` / `OmniModalityMetrics` / `OmniTransferMetrics`), and is forwarded to `Orchestrator._init_metrics_state(...)`.

 Behavior with `--log-stats=off` (default):

- The three `Omni*Metrics` classes register their module-level Gauge / Counter / Histogram families at import time (prometheus_client requires up-front registration), but each `observe / inc / set` method early-returns. The per-label child series for `vllm:omni_*` stay materialized but never have data written to them.
+- The three `Omni*Metrics` classes register their module-level Gauge / Counter / Histogram families at import time (prometheus_client requires up-front registration), but each `observe / inc / set` method early-returns. The per-label child series for `vllm_omni:*` stay materialized but never have data written to them.
 - `OmniPrometheusStatLogger` is not constructed in `_init_metrics_state`, so the ~65 upstream `vllm:*` wrap families are not registered in the default registry at all.
 - The engine core's `Scheduler.make_stats()` also short-circuits inside upstream (`if not self.log_stats: return None`), so no `SchedulerStats` is produced per step — the per-iteration cost is bounded by the existing upstream gate.

@@ -162,10 +175,43 @@ second.

 | Metric | Type | Labels | Description |
 |--------|------|--------|-------------|
-| `vllm:omni_num_requests_running` | Gauge | `model_name` | Requests currently executing across all stages |
-| `vllm:omni_num_requests_waiting` | Gauge | `model_name` | Requests queued but not yet scheduled |
-| `vllm:omni_requests_success_total` | Counter | `model_name`, `finished_reason` | Total requests by completion reason ({stop, length, abort, ...}); aborts cover client-disconnect / cancellation paths in addition to upstream `FinishReason.ABORT` |
-| `vllm:omni_e2e_request_latency_s` | Histogram | `model_name` | Pipeline-global end-to-end latency in seconds |
+| `vllm_omni:num_requests_running` | Gauge | `model_name` | Requests currently executing across all stages |
+| `vllm_omni:num_requests_waiting` | Gauge | `model_name` | Requests queued but not yet scheduled |
+| `vllm_omni:requests_success_total` | Counter | `model_name`, `finished_reason` | Total requests by completion reason ({stop, length, abort, ...}); aborts cover client-disconnect / cancellation paths in addition to upstream `FinishReason.ABORT` |
+| `vllm_omni:e2e_request_latency_s` | Histogram | `model_name` | Pipeline-global end-to-end latency in seconds |
+
+### Image and diffusion service-level metrics
+
+| Metric | Type | Labels | Description |
+|--------|------|--------|-------------|
+| `vllm_omni:stage_gen_time_s` | Histogram | `model_name`, `stage`, `stage_type` | Stage submit to finished output; includes in-stage queueing |
+| `vllm_omni:request_queue_wait_s` | Histogram | `model_name` | Orchestration-layer queue wait; a present zero is recorded |
+| `vllm_omni:stage_waiting_requests` | Gauge | `model_name`, `stage` | Sum of the latest waiting snapshots across live replicas in the stage |
+| `vllm_omni:stage_in_queue_s` | Histogram | `model_name`, `stage` | Diffusion scheduler wait from enqueue to initial execution admission |
+| `vllm_omni:num_inference_steps` | Histogram | `model_name` | Denoising-step distribution for diffusion stages |
+| `vllm_omni:image_count_total` | Counter | `model_name` | Cumulative number of generated image outputs |
+| `vllm_omni:image_pixels` | Histogram | `model_name` | Pixel-count distribution for image outputs |
+| `vllm_omni:peak_memory_mb` | Gauge | `model_name`, `stage` | Positive per-stage peak-memory samples reported by engine outputs |
+| `vllm_omni:requests_failed_total` | Counter | `model_name`, `reason` | Request failures with bounded reason values (`client_abort`, `client_disconnect`, `stage_error`, `unknown`) |
+| `vllm_omni:kv_wait_s` | Histogram | `model_name`, `connector_type` | Completed AR KV-transfer wait intervals, grouped by connector backend |
+
+### Diffusion execution breakdown
+
+Labels: `{model_name, stage, replica}`.
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| `vllm_omni:diffusion_exec_s` | Histogram | Core diffusion-step execution time per request |
+| `vllm_omni:diffusion_exec_per_step_s` | Histogram | Core diffusion execution divided by inference-step count |
+| `vllm_omni:diffusion_preprocess_s` | Histogram | Diffusion input preprocessing time |
+| `vllm_omni:diffusion_postprocess_s` | Histogram | Diffusion output postprocessing time |
+| `vllm_omni:vae_decode_s` | Histogram | VAE decode time when the engine reports it |
+| `vllm_omni:diffusion_forward_s` | Histogram | Forward-only denoise-loop time when pipeline profiling is enabled |
+| `vllm_omni:diffusion_kv_load_s` | Histogram | Diffusion-side KV receive/load time when upstream KV is used |
+| `vllm_omni:image_ttfp_s` | Histogram | Image-stage submit to first materialized image output |
+| `vllm_omni:denoise_step_latency_s` | Histogram | Mean diffusion forward time per denoise step for image output stages |
+
+The optional breakdown and memory families are sparse by design: when an engine does not produce the source measurement, no sample is emitted. This differs from queue timings, where an explicitly measured zero is meaningful and is retained.

 ### Audio (7)

@@ -173,13 +219,13 @@ Labels: `{model_name, stage, replica}` plus the listed extra label.

 | Metric | Type | Extra label | Description |
 |--------|------|-------------|-------------|
-| `vllm:omni_audio_ttfp_s` | Histogram | — | Time from request arrival to first audio packet/frame |
-| `vllm:omni_audio_duration_s` | Histogram | — | Audio content duration (`audio_frames / sample_rate`) |
-| `vllm:omni_audio_rtf` | Histogram | — | Real-time factor `stage_gen_time_s / audio_duration_s` (SLO `< 1`); uses `RTF_BUCKETS` |
-| `vllm:omni_audio_frames_total` | Counter | — | Cumulative audio frames generated |
-| `vllm:omni_audio_underrun_s` | Histogram | — | Per-request worst-case player deficit; `> 0` indicates listener heard silent gaps |
-| `vllm:omni_audio_continuity_ok_total` | Counter | `threshold_ms` | Incremented when the request's worst underrun stayed below `threshold_ms` |
-| `vllm:omni_audio_skipped_requests_total` | Counter | `reason` | Silent-loss counter — code2wav rejected malformed codec input and returned `200 OK` with empty audio |
+| `vllm_omni:audio_ttfp_s` | Histogram | — | Time from request arrival to first audio packet/frame |
+| `vllm_omni:audio_duration_s` | Histogram | — | Audio content duration (`audio_frames / sample_rate`) |
+| `vllm_omni:audio_rtf` | Histogram | — | Real-time factor `stage_gen_time_s / audio_duration_s` (SLO `< 1`); uses `RTF_BUCKETS` |
+| `vllm_omni:audio_frames_total` | Counter | — | Cumulative audio frames generated |
+| `vllm_omni:audio_underrun_s` | Histogram | — | Per-request worst-case player deficit; `> 0` indicates listener heard silent gaps |
+| `vllm_omni:audio_continuity_ok_total` | Counter | `threshold_ms` | Incremented when the request's worst underrun stayed below `threshold_ms` |
+| `vllm_omni:audio_skipped_requests_total` | Counter | `reason` | Silent-loss counter — code2wav rejected malformed codec input and returned `200 OK` with empty audio |

 ### Cross-stage transfer (4)

@@ -187,10 +233,10 @@ Labels: `{model_name, from_stage, from_replica, to_stage, to_replica}`.

 | Metric | Type | Description |
 |--------|------|-------------|
-| `vllm:omni_transfer_size_bytes` | Histogram | Per-transfer payload size in bytes |
-| `vllm:omni_transfer_tx_s` | Histogram | Sender-side time (serialize + submit to connector) |
-| `vllm:omni_transfer_rx_s` | Histogram | Receiver-side time (recv + deserialize) |
-| `vllm:omni_transfer_in_flight_s` | Histogram | Network in-flight time (TX done → RX recv start) |
+| `vllm_omni:transfer_size_bytes` | Histogram | Per-transfer payload size in bytes |
+| `vllm_omni:transfer_tx_s` | Histogram | Sender-side time (serialize + submit to connector) |
+| `vllm_omni:transfer_rx_s` | Histogram | Receiver-side time (recv + deserialize) |
+| `vllm_omni:transfer_in_flight_s` | Histogram | Network in-flight time (TX done → RX recv start) |

 ### LLM stage-level (wrapped `vllm:*`)

@@ -203,7 +249,7 @@ After the wrap, every upstream `vllm:*` family — TTFT, ITL, TPOT, e2e latency,
  - `SECONDS_FAST_BUCKETS` (0.001 s – 60 s) for fine-grained cross-stage transfer and audio-underrun values that need millisecond-level resolution.
 - Counters use the `_total` suffix (auto-appended by `prometheus_client`).
 - Sizes use the `_bytes` suffix.
- All omni-specific families are prefixed `vllm:omni_`. The upstream `unregister_vllm_metrics()` function is monkey-patched to a scoped version that still strips upstream `vllm:*` collectors (so multi-engine init within one process does not crash on duplicate registration) but preserves anything prefixed `vllm:omni_` / `vllm_omni`.
+- All omni-specific families are prefixed `vllm_omni:`. The upstream `unregister_vllm_metrics()` function is monkey-patched to a scoped version that still strips upstream `vllm:*` collectors (so multi-engine init within one process does not crash on duplicate registration) but preserves anything prefixed `vllm_omni:` / `vllm_omni`.

 ## Logging vs. Prometheus

@@ -57,3 +57,10 @@ cross-stage routing.

 Test registry selection, checkpoint loading, minimal inference, input and output
 contracts, and every declared optional capability.
+
+## Model-specific capability documents
+
+- [MiniMax H3 temporal VAE chunk callback](../../feature/minimax_h3_temporal_vae_chunks.md)
+  defines the source-gated producer seam and its boundary with future runtime
+  media transport. It is an internal opt-in capability, not a public serving
+  feature.
@@ -18,7 +18,7 @@ validation_paths:
  - tests/diffusion/offloader/**
 upstream_refs:
  - torch.nn.Module.to
-last_reviewed: 2026-07-16
+last_reviewed: 2026-08-25
 ---

 # Diffusion offloader
@@ -48,6 +48,11 @@ and correctness required by the selected execution mode.
 **Rule:** Retained host and device copies MUST respect configured limits and
 provide deterministic teardown.

+Allocator-cache retention MUST remain local to an explicit component owner,
+declare both cache and physical-free-memory bounds, release conservatively
+when telemetry is unavailable, and force release on failure or memory
+pressure. It MUST NOT replace unconditional executor shutdown cleanup.
+
 ## Safe-change guide

 Test enabled and disabled paths, repeated execution, asynchronous transfer,
@@ -37,6 +37,7 @@ validation_paths:
  - tests/engine/test_async_omni_engine_input.py
  - tests/engine/test_async_omni_engine_outputs.py
  - tests/engine/test_async_omni_engine_abort.py
+  - tests/e2e/offline_inference/test_qwen3_omni.py
  - tests/engine/test_async_omni_engine_stage_init.py
  - tests/engine/test_orchestrator.py
  - tests/engine/test_orchestrator_error_handling.py
@@ -66,6 +67,12 @@ the boundary affected by the in-flight stage client/process refactor in
 [#5441](https://github.com/vllm-project/vllm-omni/pull/5441). Names and
 responsibilities proposed only by that PR are not current contracts.

+The orchestration loop also has an opt-in event-driven mode
+(`VLLM_OMNI_EVENT_DRIVEN_ORCH=1`, default off) proposed in
+[#5221](https://github.com/vllm-project/vllm-omni/pull/5221). It changes poll
+cadence only: the routing, ordering, and terminal-state contracts below hold
+identically on both loops.
+
 ## Ownership boundary

 This document owns `AsyncOmniEngine`, `Orchestrator`, request-state creation,
@@ -107,6 +114,15 @@ Numbers become append-only after normative promotion.

 Test routing, output ordering, queue correlation, cancellation, failure
 propagation, shutdown ordering, and representative multi-stage execution.
+AR abort of a final-stage LLM request must deliver a terminal output with
+`finish_reason="abort"` and the generated prefix so collocated training can
+resume; diffusion abort remains whole-sample retry. Aborting a parallel-sampling
+child must drop `parent_requests` once no children remain. EngineCore control
+RPCs (`sleep` / `wake_up` / `pause_scheduler` / `resume_scheduler`) must
+propagate worker exceptions rather than returning `{"supported": False}`.
+When a request has multiple final output stages, only the last abort message
+is request-terminal. Output-processor abort state is committed only after the
+physical EngineCore abort succeeds.
 Changes that cross into stage runtime or public error behavior require review
 from that contract's owners.

@@ -43,6 +43,7 @@ depends_on:
 validation_paths:
  - tests/entrypoints/test_omni_entrypoints.py
  - tests/entrypoints/test_async_omni.py
+  - tests/entrypoints/test_async_omni_pause_sleep_routing.py
  - tests/entrypoints/test_async_omni_duplex.py
  - tests/entrypoints/test_serve.py
  - tests/entrypoints/test_stream_finish_reason.py
@@ -119,7 +120,13 @@ promotion.

 Test request validation, protocol conversion, model-adapter routing,
 streaming, session identity, disconnect/cancellation, and error mapping for
-each affected offline or serving entrypoint.
+each affected offline or serving entrypoint. Sleep must wait for in-flight
+`generate()` admission before EngineCore offload; `wake_up` does not resume
+admission — callers must `resume_generation()`. Sleeping tags are tracked per
+stage so `wake_up(stage_ids=[0])` does not skip a later `wake_up(stage_ids=[1])`.
+Streaming input pumps take an admission slot immediately before each EngineCore
+ADD or update, not while waiting for the next client chunk. Frontend abort
+keeps `request_states` until `generate()` consumes the terminal output.

 ## Promotion gate

@@ -16,7 +16,7 @@ validation_paths:
  - tests/host_weight_runtime/**
 upstream_refs:
  - https://github.com/vllm-project/vllm-omni/issues/6414
-last_reviewed: 2026-08-21
+last_reviewed: 2026-08-25
 ---

 # Host Weight Runtime
@@ -84,6 +84,13 @@ another. `WeightArtifactIdentity` includes:
 - producer version, implementation fingerprint, manifest schema, and restorer
  schema.

+Loader integrations may persist content digests for canonical source shards in
+the selected storage domain. Reuse requires the same path, inode, size,
+timestamps, symlink target, and cache-record checksum. Corrupt records are
+rebuilt, and coordination or cache-I/O failure falls back to hashing the
+canonical source directly. This is an identity-computation optimization, not a
+second source of truth or an artifact substitution rule.
+
 The identity excludes transfer policy such as registered mmap versus private
 pinned staging. Those paths move the same host representation and must not
 produce duplicate artifacts.
@@ -237,6 +244,12 @@ Every publication computes full file and tensor hashes. Local lookup supports:
  and
 - `full_checksum`, which additionally reads and hashes every payload file.

+An ordered producer may compute file and tensor digests while payload bytes are
+written and overlap each closed payload's `fsync` with later producer work. An
+unordered producer retains parallel readback hashing. In both cases manifest
+and `READY` publication wait for every payload digest and `fsync`; the atomic
+publication and validation contract is unchanged.
+
 Filesystem verity is modeled but unsupported. Invalid artifacts receive an
 external deny marker before their shared lookup lock is released; a build owner
 then quarantines the immutable directory under the exclusive lock.
@@ -244,6 +257,53 @@ Lookup checks the marker both before validation and after constructing the
 lease so a weaker overlapping lookup cannot return a lease after a stronger
 validation has denied the artifact.

+If publishing the external deny marker fails, lookup returns a retryable
+storage failure and marks the artifact directory with a persistent invalidation
+mode before releasing the shared lock. Cooperating readers check that mode both
+while opening and immediately before returning a lease. Lookup then makes a
+bounded attempt to take the exclusive artifact lock and quarantine the entry;
+the quarantine transition removes the internal mode marker. If a cooperating
+builder replaces the marked inode while the locks are exchanged, containment
+preserves that new publication instead of quarantining it. If neither the
+external marker nor the directory-mode marker can be persisted, cross-process
+exclusion cannot be guaranteed on an unavailable filesystem and the domain
+must remain failed rather than treating the artifact as an ordinary invalid
+cache entry.
+
+New publications carry the internal invalidation mode through the staging
+rename. Hardening clears it and is the logical authority transition for a
+process crash; the temporary- and artifact-parent syncs then make the rename
+durable, with an artifact-inode sync first making the hardened mode durable. A
+process that dies before hardening leaves a persistently denied
+entry, while one that dies after successful hardening leaves an authoritative
+ready entry. Any synchronous failure after rename is marked invalid and
+contained in quarantine before `PUBLICATION_FAILED` is returned, so that
+nonretryable result cannot later resolve as a ready hit. If the filesystem
+refuses both the inode marker and the quarantine move, the store instead returns
+a retryable storage failure with `outcome_uncertain` details; as with deny-marker
+double failure, cross-process exclusion cannot be guaranteed until storage
+recovers. Once the artifact-parent sync succeeds, a later artifact- or
+build-lock release error is logged and lookup resolves the committed entry
+instead of reporting a contradictory publication failure. The same authority
+rule applies when a competing publisher already installed byte-identical
+semantic content; staging-cleanup errors are retried without downgrading that
+validated ready entry to publication failure.
+
+Before widening a hardened artifact for a lifecycle move, the store durably
+sets the same invalidation mode. A process crash before rename therefore leaves
+the writable source denied. After rename, the store re-hardens the destination
+but retains the marker until both parent-directory syncs make the move durable;
+only then does it durably clear the marker. An interrupted move therefore leaves
+a marked transition that the next exact-key lifecycle operation re-hardens.
+Explicit cleanup uses the locked move and retries exact-key
+`.cleanup.*` tombstones left by an interrupted removal; a later build performs
+the same tombstone reconciliation before producing replacement content.
+
+An operational failure while inspecting a noncooperative competing publication
+is a retryable storage failure. The competing entry remains authoritative and
+is not mislabeled as corrupt or quarantined merely because its manifest could
+not be read transiently.
+
 `inspect_domain()` and `inspect_artifact()` return typed, read-only snapshots.
 Inspection validation does not create a deny marker. Reported lock activity is
 point-in-time evidence only; kernel locks are not a persistent owner registry.
@@ -3,4 +3,6 @@
 vLLM-Omni's examples are split into two categories:

 - If you are using vLLM-Omni from within Python code, see the *Offline Inference* section.
- If you are using vLLM-Omni from an HTTP application or client, see the *Online Serving* section.
+- If you are using vLLM-Omni from an HTTP application or client, start with
+  the [API Server guide](../serving/README.md) to choose an endpoint, then use
+  the *Online Serving* section for model-specific examples.
@@ -1,7 +1,7 @@
 # Installation

 !!! important
-    vLLM-Omni is released against the matching upstream vLLM major/minor version. vLLM-Omni 0.26.x requires vLLM 0.26.x; the stable v0.26.0 instructions pin vLLM 0.26.0.
+    vLLM-Omni is released against the matching upstream vLLM major/minor version. vLLM-Omni 0.28.x requires vLLM 0.28.x; the stable v0.28.0 instructions pin vLLM 0.28.0.

 vLLM-Omni supports the following hardware platforms:

@@ -34,7 +34,7 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr

 ### Pre-built wheels

-Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, 0.25.0, and 0.26.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
+Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, 0.25.0, 0.26.0, and 0.28.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).

 === "NVIDIA CUDA"

@@ -5,7 +5,7 @@
 # --8<-- [end:requirements]
 # --8<-- [start:set-up-using-python]

-vLLM-Omni depends on the matching major/minor release of vLLM. The vLLM-Omni 0.26.x release line uses vLLM 0.26.x.
+vLLM-Omni depends on the matching major/minor release of vLLM. The vLLM-Omni 0.28.x release line uses vLLM 0.28.x.

 !!! note
    PyTorch installed via `conda` will statically link `NCCL` library, which can cause issues when vLLM tries to use `NCCL`. See <gh-issue:8420> for more details.
@@ -20,7 +20,7 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new**

 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm==0.26.0 --torch-backend=auto
+uv pip install vllm==0.28.0 --torch-backend=auto
 ```

 #### Installation of vLLM-Omni
@@ -39,13 +39,13 @@ uv pip install 'vllm-omni[demo]'
 # --8<-- [start:build-wheel-from-source]

 #### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.26.0 release version of the library
+If you do not need to modify source code of vLLM, you can directly install the stable 0.28.0 release version of the library

 ```bash
-uv pip install vllm==0.26.0 --torch-backend=auto
+uv pip install vllm==0.28.0 --torch-backend=auto
 ```

-The 0.26.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The 0.28.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.

 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -66,12 +66,12 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.26.0
+git checkout v0.28.0
 ```
 Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
 ```bash
-#For CUDA 13.0 (the default for v0.26.0; the wheel filename has no `+cu130` suffix)
-export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.26.0/vllm-0.26.0-cp38-abi3-manylinux_2_28_x86_64.whl
+#For CUDA 13.0 (the default for v0.28.0; the wheel filename has no `+cu130` suffix)
+export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.28.0/vllm-0.28.0-cp38-abi3-manylinux_2_28_x86_64.whl
 ```
 Install vllm with command below (If you have no existing PyTorch).
 ```bash
@@ -95,6 +95,13 @@ uv pip install --no-build-isolation --editable .

 vLLM-Omni offers an official docker image for deployment. These images are built on top of vLLM docker images and available on Docker Hub as [vllm/vllm-omni](https://hub.docker.com/r/vllm/vllm-omni/tags). The version of vLLM-Omni indicates which release of vLLM it is based on.

+Nightly CUDA images built from the latest `main` are also available:
+
+- [`vllm/vllm-omni:nightly`](https://hub.docker.com/r/vllm/vllm-omni/tags?name=nightly) is the rolling multi-architecture tag.
+- [`vllm/vllm-omni:nightly-<commit>`](https://hub.docker.com/r/vllm/vllm-omni/tags?name=nightly-) provides commit-pinned multi-architecture tags. The newest 14 are retained.
+
+Use a commit-pinned tag when you need a reproducible nightly environment.
+
 Here's an example deployment command that has been verified on 2 x H100's:
 ```bash
 docker run --runtime nvidia --gpus 2 \
@@ -102,7 +109,7 @@ docker run --runtime nvidia --gpus 2 \
    --env "HF_TOKEN=$HF_TOKEN" \
    -p 8091:8091 \
    --ipc=host \
-    vllm/vllm-omni:v0.26.0 \
+    vllm/vllm-omni:v0.28.0 \
    vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```

@@ -124,7 +131,7 @@ If you want to specify the base vLLM version:
 ```bash
 DOCKER_BUILDKIT=1 docker build \
  -f docker/Dockerfile.cuda \
-  --build-arg BASE_IMAGE=vllm/vllm-openai:v0.26.0 \
+  --build-arg BASE_IMAGE=vllm/vllm-openai:v0.28.0 \
  -t vllm-omni-cuda .
 ```

@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.

 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm==0.26.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.26.0/rocm723
+uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.28.0/rocm723
 ```

 #### Installation of vLLM-Omni
@@ -34,13 +34,13 @@ uv pip install onnxruntime-rocm
 # --8<-- [start:build-wheel-from-source]

 #### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.26.0 release version of the library
+If you do not need to modify source code of vLLM, you can directly install the stable 0.28.0 release version of the library

 ```bash
-uv pip install vllm==0.26.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.26.0/rocm723
+uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.28.0/rocm723
 ```

-The pre-built 0.26.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The pre-built 0.28.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.

 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -58,7 +58,7 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.26.0
+git checkout v0.28.0
 python3 -m pip install -r requirements/rocm.txt
 python3 setup.py develop
 ```
@@ -130,7 +130,7 @@ docker run --rm \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  --env "HF_TOKEN=$HF_TOKEN" \
  -p 8091:8091 \
-  vllm/vllm-omni-rocm:v0.26.0 \
+  vllm/vllm-omni-rocm:v0.28.0 \
  --model Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```

@@ -149,7 +149,7 @@ docker run --rm -it \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  --env "HF_TOKEN=$HF_TOKEN" \
  --entrypoint bash \
-  vllm/vllm-omni-rocm:v0.26.0
+  vllm/vllm-omni-rocm:v0.28.0
 ```

 # --8<-- [end:pre-built-images]
@@ -23,6 +23,19 @@ vLLM-Omni currently recommends using the Docker image setup steps below.
 DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.xpu -t vllm-omni-xpu --shm-size=4g .
 ```

+This layers vLLM-Omni on top of the published `vllm/vllm-openai-xpu:<VLLM_VERSION>`
+base image, which Docker pulls automatically. To target a different vLLM release,
+pass `--build-arg VLLM_VERSION=<tag>`. If that tag has not been published yet, build
+the base from upstream's own Dockerfile first and point the build at it:
+
+```bash
+git clone --depth 1 --branch "$VLLM_VERSION" https://github.com/vllm-project/vllm /tmp/vllm
+DOCKER_BUILDKIT=1 docker build -t vllm-openai-xpu:local --target vllm-openai \
+  -f /tmp/vllm/docker/Dockerfile.xpu /tmp/vllm
+DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.xpu -t vllm-omni-xpu --shm-size=4g \
+  --build-arg VLLM_BASE=vllm-openai-xpu:local .
+```
+
 #### Launch the docker image

 ##### Launch with OpenAI API Server
@@ -9,8 +9,8 @@ The recommended way to use vLLM-Omni on NPU is through the vLLM-Ascend pre-built

 ```bash
 # vLLM-Ascend image
-# Atlas A2: quay.io/atlas-ci/vllm-ascend:v0.26.0
-# Atlas A3: quay.io/atlas-ci/vllm-ascend:v0.26.0-a3
+# Atlas A2: quay.io/atlas-ci/vllm-ascend:v0.28.0
+# Atlas A3: quay.io/atlas-ci/vllm-ascend:v0.28.0-a3
 docker run --rm \
    --name vllm-omni-npu \
    --shm-size=1g \
@@ -28,11 +28,11 @@ docker run --rm \
    -v /etc/ascend_install.info:/etc/ascend_install.info \
    -v /root/.cache:/root/.cache \
    -p 8000:8000 \
-    -it quay.io/atlas-ci/vllm-ascend:v0.26.0-a3 bash
+    -it quay.io/atlas-ci/vllm-ascend:v0.28.0-a3 bash

 # Inside the container, install vLLM-Omni from source
 cd /vllm-workspace
-git clone -b v0.26.0 https://github.com/vllm-project/vllm-omni.git
+git clone -b v0.28.0 https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
 pip install -v -e . --no-build-isolation
 # or VLLM_OMNI_TARGET_DEVICE=npu pip install -v -e .
@@ -50,21 +50,11 @@ We are keeping [issue #886](https://github.com/vllm-project/vllm-omni/issues/886

 # --8<-- [start:installation-main]

-You can also build vLLM-Omni from the latest main branch if you want to use the latest features or bug fixes. (But sometimes it will break for a while. You can check [issue #886](https://github.com/vllm-project/vllm-omni/issues/886) for the status of the latest commit of vLLM-Omni main branch on NPU.)
+You can also install vLLM-Omni from the latest main branch if you want to use the latest features or bug fixes. Use the aligned vLLM-Ascend v0.28.0 image described above, and check [issue #886](https://github.com/vllm-project/vllm-omni/issues/886) for the status of the latest vLLM-Omni commit on NPU.

 ```bash
-# Pin vLLM and vLLM-Ascend to the v0.26 release line
-git clone -b v0.26.0 https://github.com/vllm-project/vllm.git
-cd vllm
-VLLM_TARGET_DEVICE=empty pip install -v -e .
-cd ..
-
-git clone -b releases/v0.26.0rc https://github.com/vllm-project/vllm-ascend.git
-cd vllm-ascend
-pip install -v -e .
-cd ..
-
-# Install vLLM-Omni from the latest main branch
+# Inside the vLLM-Ascend v0.28.0 container, install vLLM-Omni from main
+cd /vllm-workspace
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
 pip install -v -e . --no-build-isolation
@@ -88,8 +78,8 @@ Supported images as following.
 Here's an example deployment command that has been verified on 4 x NPUs:

 ```bash
-# Atlas A2: quay.io/ascend/vllm-omni:v0.26.0
-# Atlas A3: quay.io/ascend/vllm-omni:v0.26.0-a3
+# Atlas A2: quay.io/ascend/vllm-omni:v0.28.0
+# Atlas A3: quay.io/ascend/vllm-omni:v0.28.0-a3
 docker run --rm \
    --name vllm-omni-a3 \
    --shm-size=4g \
@@ -114,7 +104,7 @@ docker run --rm \
    -v /etc/ascend_install.info:/etc/ascend_install.info \
    -v ~/.cache:/root/.cache \
    -p 8091:8091 \
-    -it quay.io/ascend/vllm-omni:v0.26.0-a3 bash
+    -it quay.io/ascend/vllm-omni:v0.28.0-a3 bash
 ```

 !!! tip
@@ -19,10 +19,10 @@ uv venv --python 3.12 --seed
 source .venv/bin/activate

 # On CUDA
-uv pip install vllm==0.26.0 --torch-backend=auto
+uv pip install vllm==0.28.0 --torch-backend=auto

 # On ROCm
-uv pip install vllm==0.26.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.26.0/rocm723
+uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.28.0/rocm723

 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
@@ -34,7 +34,7 @@ For additional installation methods — please see the [installation guide](inst
 !!! note
    It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.

-    If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.26.0` and vLLM Omni `0.26.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
+    If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.28.0` and vLLM Omni `0.28.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.

 ## Offline Inference

@@ -91,7 +91,7 @@ if __name__ == "__main__":

 For more usages, please refer to [offline inference](../user_guide/examples/offline_inference/qwen2_5_omni.md)

-## Online Serving with OpenAI-Completions API
+## Online Serving with the API Server

 Text-to-image generation quickstart with vLLM-Omni:

@@ -100,20 +100,16 @@ vllm serve Tongyi-MAI/Z-Image-Turbo --omni --port 8091
 ```

 ```bash
-curl -s http://localhost:8091/v1/chat/completions \
+curl -s http://localhost:8091/v1/images/generations \
  -H "Content-Type: application/json" \
  -d '{
-    "messages": [
-      {"role": "user", "content": "a cup of coffee on the table"}
-    ],
-    "extra_body": {
-      "height": 1024,
-      "width": 1024,
-      "num_inference_steps": 50,
-      "guidance_scale": 4.0,
-      "seed": 42
-    }
-  }' | jq -r '.choices[0].message.content[0].image_url.url' | cut -d',' -f2 | base64 -d > coffee.png
+    "prompt": "a cup of coffee on the table",
+    "size": "1024x1024",
+    "response_format": "b64_json",
+    "seed": 42
+  }' | jq -r '.data[0].b64_json' | base64 -d > coffee.png
 ```

-For more details, please refer to [online serving](../user_guide/examples/online_serving/text_to_image.md).
+See the [API Server guide](../serving/README.md) to choose an endpoint. For
+model-specific details, refer to the [text-to-image online serving
+example](../user_guide/examples/online_serving/text_to_image.md).
@@ -37,7 +37,6 @@ GENERAL_EXAMPLE_SLUGS = frozenset(
        "text_to_speech",
        "text_to_video",
        "x_to_text",
-        "x_to_video_audio",
    }
 )

@@ -21,20 +21,20 @@ th {
 |--------------|--------|-------------------|------------|---------|-----|-----------|--------|
 | `Qwen3OmniMoeForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen3-Omni.md) |
 | `Qwen2_5OmniForConditionalGeneration` | Qwen2.5-Omni | `Qwen/Qwen2.5-Omni-7B`, `Qwen/Qwen2.5-Omni-3B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `MingFlashOmniForConditionalGeneration` + `MingImagePipeline` | Ming-flash-omni-2.0 (omni-speech + imagegen<sup>1</sup>) | `Jonathan1909/Ming-flash-omni-2.0` | ✅︎ |   |   |   | — |
+| `MingFlashOmniForConditionalGeneration` + `MingImagePipeline` | Ming-flash-omni-2.0 (omni-speech + imagegen<sup>1</sup>) | `Jonathan1909/Ming-flash-omni-2.0` | ✅︎ | | | | — |
 | `BagelForConditionalGeneration` | BAGEL (DiT-only) | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `InternVLAA1Pipeline` | InternVLA-A1 | `InternRobotics/InternVLA-A1-3B` | ✅︎ | ✅︎ | | | — |
 | `Gr00tN1d7Pipeline` | GR00T N1.7 | `nvidia/GR00T-N1.7-3B` | ✅︎ | | | | — |
 | `HunyuanImage3ForCausalMM` | HunyuanImage3.0 (DiT-only) | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` | ✅︎ | | | | [Published](https://recipes.vllm.ai/Qwen/Qwen-Image) |
-| `QwenImagePipeline` | Qwen-Image-2512 | `Qwen/Qwen-Image-2512` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen-Image-2512.md) |
+| `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` | ✅︎ | | | ✅︎ | [Published](https://recipes.vllm.ai/Qwen/Qwen-Image) |
+| `QwenImagePipeline` | Qwen-Image-2512 | `Qwen/Qwen-Image-2512` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen-Image-2512.md) |
 | `QwenImageEditPipeline` | Qwen-Image-Edit | `Qwen/Qwen-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2509 | `Qwen/Qwen-Image-Edit-2509` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `QwenImageLayeredPipeline` | Qwen-Image-Layered | `Qwen/Qwen-Image-Layered` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2511 | `Qwen/Qwen-Image-Edit-2511` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `GlmImagePipeline` | GLM-Image | `zai-org/GLM-Image` | ✅︎ | ✅︎ | | | — |
 | `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `Krea2Pipeline` | Krea 2 (Raw + Turbo) | `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | ✅︎ | | | | — |
+| `Krea2Pipeline` | Krea 2 (Raw + Turbo) | `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | ✅︎ | | | ✅︎ | — |
 | `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `LingBotWorldCausalDMDPipeline` | LingBot-World 2.0 v1 (experimental) | `robbyant/lingbot-world-v2-14b-causal-fast-diffusers` | Experimental | | | | — |
@@ -42,7 +42,7 @@ th {
 | `SanaWmPipeline` | SANA-WM | `BBBBruce/SANA-WM_bidirectional-stage1-diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/SANA-WM.md) |
 | `Wan22S2VPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ |   |   |   | — |
+| `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ |   |   | ✅︎ | — |
 | `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | — |
 | `LTX2TwoStagePipeline` | LTX-2 / LTX-2.3 ordinary two-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` + matching Lightricks LoRA and upsampler | ✅︎ | ✅︎ | | | — |
 | `LTX2DistilledOneStagePipeline` | LTX-2 / LTX-2.3 merged-distilled one-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` | ✅︎ | ✅︎ | | | — |
@@ -58,18 +58,19 @@ th {
 | `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Pi0Pipeline` | π0 (Pi-Zero) VLA | `lerobot/pi0_base` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/lerobot/Pi0.md) |
 | `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | | — |
-| `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | | — |
 | `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `LongCatVideoAvatarPipeline` | LongCat-Video-Avatar-1.5 A2V/AI2V (native single-speaker and multi-speaker AI2V/AVC) | `meituan-longcat/LongCat-Video-Avatar-1.5` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/meituan-longcat/LongCat-Video-Avatar-1.5.md) |
-| `BooguImagePipeline` | Boogu-Image | `Boogu/Boogu-Image-0.1-Base`, `Boogu/Boogu-Image-0.1-Edit` | ✅︎ | | | | — |
+| `BooguImagePipeline` | Boogu-Image | `Boogu/Boogu-Image-0.1-Base`, `Boogu/Boogu-Image-0.1-Edit`, `Boogu/Boogu-Image-0.1-Edit-Turbo` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Boogu/Boogu-Image.md) |
 | `StableDiffusionXLPipeline` | Stable-Diffusion-XL | `stabilityai/stable-diffusion-xl-base-1.0` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ | — |
-| `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | | — |
-| `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | | — |
+| `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | ✅︎ | — |
+| `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | ✅︎ | — |
 | `DotsTTSForConditionalGeneration` | dots.tts | `rednote-hilab/dots.tts-soar` | ✅︎ | | | | — |
+| `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | | — |
+| `MammothModa2ForConditionalGeneration` | MammothModa2-Dev (AR-only image understanding) | `bytedance-research/MammothModa2-Dev` | ✅︎ | | | | — |
 | `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | | — |
 | `FluxPipeline` | FLUX.1-dev | `black-forest-labs/FLUX.1-dev` | ✅︎ | ✅︎ | | ✅︎ | — |
@@ -77,10 +78,10 @@ th {
 | `OmniGen2Pipeline` | OmniGen2 | `OmniGen2/OmniGen2` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `MiniMaxMusic3ForConditionalGeneration` | MiniMax Music 3 (text-to-music) | `MiniMaxAI/MiniMax-Music3` | ✅︎ | | | | — |
-| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | | | | [Published](https://recipes.vllm.ai/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice) |
+| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | | | ✅︎ | [Published](https://recipes.vllm.ai/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice) |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `MingTTSForConditionalGeneration` | Ming-omni-tts dense 0.5B | `inclusionAI/Ming-omni-tts-0.5B` | ✅︎ | | | | — |
+| `MingTTSForConditionalGeneration` | Ming-omni-tts dense 0.5B | `inclusionAI/Ming-omni-tts-0.5B` | ✅︎ | | | ✅︎ | — |
 | `GLMTTSForConditionalGeneration` | GLM-TTS | `zai-org/GLM-TTS` | ✅︎ | | | | — |
 | `MossTTSNanoForCausalLM` | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✅︎ | | | | — |
 | `MossTTSDelayModel` | MOSS-TTS, MOSS-TTSD, MOSS-SoundEffect, MOSS-VoiceGenerator | `OpenMOSS-Team/MOSS-VoiceGenerator` | ✅︎ | | | | — |
@@ -94,17 +95,16 @@ th {
 | `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | | — |
 | `Flux2Pipeline` | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | ✅︎ | ✅︎ | | | — |
 | `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | ✅︎ | — |
-| `DreamIDOmniPipeline` | DreamID-Omni | `XuGuo699/DreamID-Omni` | ✅︎ | ✅︎ | | | — |
-| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only) | `SenseNova/SenseNova-U1-8B-MoT` | ✅︎ | | | | — |
+| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only), SenseNova-U1.5 | `SenseNova/SenseNova-U1-8B-MoT`, `sensenova/SenseNova-U1.5-8B-MoT` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SenseNova/SenseNova-U1.5.md) |
 | `LancePipeline` | Lance | `bytedance-research/Lance` | ✅︎ | | | | — |
 | `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | | — |
 | `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | | — |
 | `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | | — |
 | `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | | — |
-|`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | | — |
+|`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | ✅︎ | — |
 | `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | | — |
 | `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `GepardTalkerForConditionalGeneration` | Gepard-1.0 | `nineninesix/gepard-1.0` | ✅︎ | | | | — |
+| `GepardTalkerForConditionalGeneration` | Gepard-1.0 | `nineninesix/gepard-1.0` | ✅︎ | | | ✅︎ | — |
 |`HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | | — |
 | `HiDreamO1ImagePipeline` | HiDream-O1-Image | `HiDream-ai/HiDream-O1-Image` | ✅︎ | | | | — |

@@ -0,0 +1,187 @@
+# API Server
+
+vLLM-Omni exposes OpenAI-compatible HTTP APIs plus vLLM-Omni extensions for
+image, audio, video, realtime, and robot-policy workloads. Use this page to
+choose an endpoint; follow the linked reference page for model-specific fields,
+wire protocols, and response details.
+
+Each server instance hosts one model. An endpoint is usable only when the
+loaded model supports that task.
+
+This guide focuses on vLLM-Omni's primary task endpoints. Compatible models
+may also expose standard routes inherited from vLLM, such as
+`POST /v1/completions`; those routes remain model- and task-dependent.
+
+## Start and Verify the Server
+
+Start a model with the shared serving command:
+
+```bash
+vllm serve <model> --omni --port 8091
+```
+
+Some models require additional flags or a deployment configuration. Use the
+model-specific guide when one is provided.
+
+After the server starts, check its health and served model name:
+
+```bash
+export VLLM_OMNI_BASE_URL=http://localhost:8091
+
+curl "$VLLM_OMNI_BASE_URL/health"
+curl "$VLLM_OMNI_BASE_URL/v1/models" | jq .
+```
+
+Use `http://localhost:8091/v1` as the `base_url` for an OpenAI SDK client. If
+the server was started with `--api-key`, include an `Authorization` header with
+`Bearer <api-key>` in requests.
+
+## Choose a Core HTTP Endpoint
+
+Prefer a task-specific endpoint when one matches your workload. Use Chat
+Completions for conversational or heterogeneous omni pipelines, rather than as
+the default wrapper for every generation task.
+
+| Task | Endpoint | Request | Response | Details |
+|------|----------|---------|----------|---------|
+| Conversation or multimodal understanding/generation | `POST /v1/chat/completions` | JSON | OpenAI-style JSON or SSE | [Chat Completions](chat_completions_api.md) |
+| Text-to-speech | `POST /v1/audio/speech` | JSON | Audio bytes or SSE | [Speech](speech_api.md) |
+| Sound, music, or ambient audio generation | `POST /v1/audio/generate` | JSON | Audio bytes | [Audio Generation](audio_generate_api.md) |
+| Text-to-image | `POST /v1/images/generations` | JSON | Base64 JSON or an image file | [Image Generation](image_generation_api.md) |
+| Image editing | `POST /v1/images/edits` | Multipart form | Base64 JSON or SSE | [Image Edit](image_edit_api.md) |
+| Video generation (recommended) | `POST /v1/videos` | Multipart form | Asynchronous job JSON | [Videos](videos_api.md) |
+| Video generation (blocking) | `POST /v1/videos/sync` | Multipart form | Video bytes | [Videos](videos_api.md#synchronous-response) |
+
+`POST /v1/videos/sync` is intended for tests and simple scripts. Use the
+asynchronous job API for production requests so clients can poll, download,
+and delete generated videos independently.
+
+## Minimal Requests
+
+The examples below assume a compatible model is already running. TTS voices,
+media inputs, generation controls, and output capabilities vary by model.
+
+=== "Chat"
+
+    ```bash
+    curl "$VLLM_OMNI_BASE_URL/v1/chat/completions" \
+      -H "Content-Type: application/json" \
+      -d '{
+        "messages": [{"role": "user", "content": "Describe vLLM-Omni briefly."}],
+        "modalities": ["text"]
+      }'
+    ```
+
+=== "Speech"
+
+    ```bash
+    curl "$VLLM_OMNI_BASE_URL/v1/audio/speech" \
+      -H "Content-Type: application/json" \
+      -d '{
+        "input": "Hello from vLLM-Omni.",
+        "voice": "vivian"
+      }' \
+      --output speech.wav
+    ```
+
+=== "Audio"
+
+    ```bash
+    curl "$VLLM_OMNI_BASE_URL/v1/audio/generate" \
+      -H "Content-Type: application/json" \
+      -d '{
+        "input": "Ocean waves on a quiet beach",
+        "audio_length": 5.0
+      }' \
+      --output audio.wav
+    ```
+
+=== "Image"
+
+    ```bash
+    curl "$VLLM_OMNI_BASE_URL/v1/images/generations" \
+      -H "Content-Type: application/json" \
+      -d '{
+        "prompt": "A small robot reading beside a window",
+        "size": "1024x1024",
+        "response_format": "file"
+      }' \
+      --output image.png
+    ```
+
+=== "Image edit"
+
+    ```bash
+    curl "$VLLM_OMNI_BASE_URL/v1/images/edits" \
+      -F "image=@input.png" \
+      -F "prompt=Turn the background into a snowy mountain" \
+      | jq -r '.data[0].b64_json' \
+      | base64 --decode > edited.png
+    ```
+
+=== "Video"
+
+    ```bash
+    curl "$VLLM_OMNI_BASE_URL/v1/videos" \
+      -F "prompt=A cinematic tracking shot of a mountain lake at sunrise" \
+      -F "size=1280x720" \
+      -F "seconds=5"
+    ```
+
+    The response contains a video job `id`. Poll and download it with:
+
+    ```bash
+    curl "$VLLM_OMNI_BASE_URL/v1/videos/<video_id>"
+    curl -L "$VLLM_OMNI_BASE_URL/v1/videos/<video_id>/content" --output video.mp4
+    ```
+
+## Choose a Streaming or Realtime Endpoint
+
+These WebSocket APIs have different event schemas and cannot be used
+interchangeably.
+
+| Workload | Endpoint | Interaction model | Details |
+|----------|----------|-------------------|---------|
+| Incremental text input for speech synthesis | `WS /v1/audio/speech/stream` | Send text events and receive audio | [Streaming Text to Speech](speech_api.md#streaming-text-input-websocket) |
+| Live video understanding | `WS /v1/video/chat/stream` | Send video frames and receive text/audio | [Streaming Video Input](video_stream_api.md) |
+| Turn-based realtime audio | `WS /v1/realtime` | Stream one audio input and receive transcript/audio events | [Realtime Audio](realtime_api.md) |
+| Continuous speech-to-speech interaction | `WS /v1/realtime?duplex=1` or `WS /v1/duplex` | Listen and speak concurrently with session control | [Full Duplex](full_duplex_api.md) |
+| Generated video chunks | `WS /v1/realtime/video` | Start a diffusion request and receive fragmented MP4 | [Streaming Video Output](streaming_video_output_api.md) |
+| Robot policy inference | `WS /v1/realtime/robot/openpi` | Send MessagePack observations and receive action arrays | [OpenPI Robot Policy](openpi_api.md) |
+
+All six routes are model- or configuration-dependent. In particular,
+`/v1/realtime` is not full duplex unless the client sets `duplex=1` and the
+deployment explicitly enables duplex sessions. Clients should also verify the
+duplex capability payload because the query-parameter form falls back to the
+ordinary realtime handler when duplex is unavailable.
+
+## Related Endpoints
+
+| Purpose | Endpoints | Reference |
+|---------|-----------|-----------|
+| Discovery and readiness | `GET /health`, `GET /v1/models` | This page |
+| Batched conversations | `POST /v1/chat/completions/batch` | [Batch requests](chat_completions_api.md#batch-requests) |
+| Batched speech | `POST /v1/audio/speech/batch` | [Batch speech generation](speech_api.md#batch-speech-generation) |
+| TTS voice management | `GET/POST /v1/audio/voices`, `DELETE /v1/audio/voices/{name}` | [Voices](speech_api.md#voices-endpoint) |
+| Video job lifecycle | `GET /v1/videos`, `GET/DELETE /v1/videos/{video_id}`, `GET /v1/videos/{video_id}/content` | [Video endpoints](videos_api.md#endpoints) |
+| Release and restore stage memory | `POST /v1/omni/sleep`, `POST /v1/omni/wakeup` | [Sleep Mode](../features/sleep_mode.md) |
+
+## Standalone Experimental Servers
+
+PersonaPlex also provides a Moshi-compatible WebSocket server, while JoyVL
+provides a stateful interaction orchestrator in front of another model server.
+These are separate processes with their own routes; they are not additional
+paths on every `vllm serve --omni` instance. See [Standalone Experimental
+Servers](standalone_servers.md) before deploying either one.
+
+## Compatibility Notes
+
+- A registered path does not mean every model supports it. Unsupported tasks
+  return an OpenAI-style error or a WebSocket `error` event.
+- Use JSON for Chat, Speech, Audio Generation, and Image Generation. Use
+  `multipart/form-data` when uploading images, videos, or other files.
+- Generation can take minutes for large diffusion models. Configure client
+  timeouts accordingly; prefer asynchronous video jobs over long blocking
+  requests.
+- Model-specific parameters are vLLM-Omni extensions. Check the endpoint
+  reference before assuming an OpenAI SDK exposes them directly.
@@ -4,14 +4,14 @@ vLLM-Omni provides an API for text-to-audio generation using diffusion-based mod

 Unlike the [Speech API](speech_api.md) which targets text-to-speech synthesis, the Audio Generate API is designed for general-purpose audio generation from text descriptions (sound effects, music, ambient soundscapes, etc.).

-Each server instance runs a single model (specified at startup via `vllm-omni serve <model> --omni`).
+Each server instance runs a single model (specified at startup via `vllm serve <model> --omni`).

 ## Quick Start

 ### Start the Server

 ```bash
-vllm-omni serve stabilityai/stable-audio-open-1.0 \
+vllm serve stabilityai/stable-audio-open-1.0 \
    --host 0.0.0.0 \
    --port 8091 \
    --gpu-memory-utilization 0.9 \
@@ -326,7 +326,7 @@ curl http://localhost:8091/health
 Enable debug logging:

 ```bash
-vllm-omni serve stabilityai/stable-audio-open-1.0 \
+vllm serve stabilityai/stable-audio-open-1.0 \
    --host 0.0.0.0 \
    --port 8091 \
    --gpu-memory-utilization 0.9 \
@@ -1,78 +1,110 @@
 # Chat Completions API

-vLLM-Omni supports generating and editing images via the `/v1/chat/completions`
-endpoint using diffusion models. This page explains how to pass generation
-parameters (such as `num_inference_steps`, `height`, `width`) to diffusion
-models through this endpoint.
+Use `/v1/chat/completions` for conversational and multimodal pipelines that
+follow the OpenAI Chat API schema. Depending on the loaded model, a request can
+contain text, image, audio, or video input and can return text, audio, images,
+or other model-specific output.

-!!! tip
-    For dedicated endpoints that accept generation parameters as top-level
-    fields, see [Image Generation API](image_generation_api.md) and
-    [Image Edit API](image_edit_api.md).
+Some diffusion pipelines also support image generation and editing through
+Chat Completions. Prefer the dedicated [Image Generation
+API](image_generation_api.md) or [Image Edit API](image_edit_api.md) for
+task-specific clients; their request fields and response contracts are more
+direct.

-## Passing Generation Parameters
-
-The `/v1/chat/completions` endpoint follows the OpenAI Chat API schema, which
-does not natively include diffusion-specific fields like `num_inference_steps`
-or `height`. How you pass these extra fields depends on your client.
-
-### curl / Python `requests`
-
-Wrap generation parameters inside an `"extra_body"` key in the JSON body:
+## Basic Request

 ```bash
-curl -s http://localhost:8091/v1/chat/completions \
+curl http://localhost:8091/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "messages": [
-      {"role": "user", "content": "A beautiful landscape painting"}
+      {"role": "user", "content": "Describe vLLM-Omni briefly."}
    ],
-    "extra_body": {
-      "num_inference_steps": 50,
-      "seed": 42
-    }
+    "modalities": ["text"]
  }'
 ```

-### OpenAI Python SDK
+Use `stream: true` for Server-Sent Events. Input media syntax, supported output
+`modalities`, and response choices depend on the model; see the
+[model-specific online serving examples](../user_guide/examples/online_serving/qwen3_omni.md).

-Use the `extra_body` **keyword argument**. The SDK automatically merges these
-fields into the top-level request body:
+!!! tip
+    Each server hosts one model. Query `GET /v1/models` for its served model
+    name when your client requires the `model` field.

-```python
-response = client.chat.completions.create(
-    model="Qwen/Qwen-Image",
-    messages=[{"role": "user", "content": "A beautiful landscape painting"}],
-    extra_body={
-        "num_inference_steps": 50,
-        "seed": 42,
-    },
-)
+## vLLM Extension Parameters
+
+Send standard and vLLM-specific Chat parameters as top-level fields in a
+direct HTTP request. When using the OpenAI Python SDK, pass vLLM-specific
+fields through the SDK's `extra_body` keyword; the SDK merges them into the
+top-level JSON sent to the server.
+
+=== "curl"
+
+    ```bash
+    curl http://localhost:8091/v1/chat/completions \
+      -H "Content-Type: application/json" \
+      -d '{
+        "messages": [{"role": "user", "content": "Write a short haiku."}],
+        "top_k": 40
+      }'
+    ```
+
+=== "OpenAI Python SDK"
+
+    ```python
+    from openai import OpenAI
+
+    client = OpenAI(base_url="http://localhost:8091/v1", api_key="none")
+
+    response = client.chat.completions.create(
+        model="your-served-model",
+        messages=[{"role": "user", "content": "Write a short haiku."}],
+        extra_body={"top_k": 40},
+    )
+    ```
+
+??? note "Diffusion compatibility"
+
+    Diffusion models exposed through Chat Completions can accept supported
+    fields such as `num_inference_steps`, `seed`, `height`, and `width`. Send
+    them at the top level with direct HTTP, or in the SDK's `extra_body`
+    keyword argument.
+
+    A literal nested `"extra_body"` JSON object is accepted for compatibility,
+    but it is not recommended for new direct HTTP clients. Do not provide the
+    same parameter in multiple locations; the server returns a `400` error for
+    duplicate diffusion parameters. Prefer the dedicated [Image Generation
+    API](image_generation_api.md) or [Image Edit API](image_edit_api.md) when
+    either endpoint matches the task.
+
+## Batch Requests
+
+`POST /v1/chat/completions/batch` accepts the same shared generation fields,
+but `messages` is a list of conversations. The response contains one choice
+per conversation in input order.
+
+```bash
+curl http://localhost:8091/v1/chat/completions/batch \
+  -H "Content-Type: application/json" \
+  -d '{
+    "messages": [
+      [{"role": "user", "content": "Summarize vLLM in one sentence."}],
+      [{"role": "user", "content": "Summarize vLLM-Omni in one sentence."}]
+    ],
+    "max_tokens": 64
+  }'
 ```

-!!! note "SDK `extra_body` vs. JSON `extra_body`"
-    These two `extra_body` usages look similar but work differently under the
-    hood. The SDK flattens the dict into the top-level request JSON, while the
-    curl/requests approach sends it as a nested `"extra_body"` key. Both are
-    handled correctly by the server.
-
-!!! note "About the `ignored fields` warning"
-    You may see a log message like:
-
-    ```
-    WARNING: The following fields were present in the request but ignored: {'height', 'width', ...}
-    ```
-
-    This is **harmless**. It is emitted by vLLM's request validation layer
-    because these fields are not part of the standard OpenAI
-    `ChatCompletionRequest` schema. The fields are still stored internally
-    and correctly forwarded to the diffusion pipeline.
+Batch Chat Completions does not support streaming, tools, beam search, or
+`n > 1`.

 ## Model-Specific Examples

-For complete examples with full request/response details, see the model-specific
-guides:
+For complete examples with model-specific inputs and outputs, see:

+- [Qwen3-Omni](../user_guide/examples/online_serving/qwen3_omni.md)
+- [Qwen2.5-Omni](../user_guide/examples/online_serving/qwen2_5_omni.md)
 - [Text-to-Image (Qwen-Image)](../user_guide/examples/online_serving/text_to_image.md)
 - [Image-to-Image (Qwen-Image-Edit, Qwen-Image-Layered)](../user_guide/examples/online_serving/image_to_image.md)
 - [GLM-Image](../user_guide/examples/online_serving/glm_image.md)
@@ -0,0 +1,116 @@
+# Full-Duplex WebSocket API
+
+vLLM-Omni provides an experimental full-duplex runtime for models that can
+continue receiving speech while producing speech. It adds persistent session
+state, model-specific turn policy, overlap handling, playback acknowledgement,
+and optional session resume.
+
+Full duplex is distinct from the turn-based [Realtime Audio API](realtime_api.md).
+
+## Choose an Endpoint
+
+| Endpoint | Protocol | Recommended use |
+|----------|----------|-----------------|
+| `WS /v1/realtime?duplex=1` | OpenAI Realtime-style event projection | Applications and browser clients |
+| `WS /v1/duplex` | Native vLLM-Omni duplex events | Runtime integration and low-level testing |
+
+Both endpoints use the same duplex engine and require the same model-side
+adapter. Prefer the Realtime projection unless the native lifecycle events are
+specifically required.
+
+## Enable Full Duplex
+
+The route becomes usable only when the deployment configuration explicitly
+sets:
+
+```yaml
+session_mode: duplex
+```
+
+The selected model pipeline must also provide a duplex serving adapter.
+Model-native deployments configure the corresponding engine runtime extension
+and control plane as part of their registered pipeline.
+
+!!! warning
+
+    `WS /v1/duplex` fails with `Duplex API is not available` when duplex is not
+    enabled. By contrast, `/v1/realtime?duplex=1` falls back to the ordinary
+    turn-based realtime handler when no duplex handler exists. Confirm that
+    `session.created.session.capabilities` is present before treating the
+    connection as full duplex.
+
+The current unified-runtime integrations are:
+
+- MiniCPM-o 4.5, using `vllm_omni/deploy/minicpmo_4_5_duplex.yaml`;
+- PersonaPlex, whose default `vllm_omni/deploy/personaplex.yaml` enables duplex.
+
+JoyVL is a separate HTTP interaction orchestrator and does not use these
+WebSocket endpoints. See [Standalone Experimental Servers](standalone_servers.md).
+
+## MiniCPM-o Quick Start
+
+Start the duplex deployment:
+
+```bash
+vllm serve openbmb/MiniCPM-o-4_5 --omni \
+  --deploy-config vllm_omni/deploy/minicpmo_4_5_duplex.yaml \
+  --trust-remote-code \
+  --port 8091
+```
+
+Stream a mono, PCM16, 16 kHz WAV file with the provided client:
+
+```bash
+python examples/online_serving/minicpmo/realtime_duplex_demo.py \
+  --url 'ws://localhost:8091/v1/realtime?duplex=1' \
+  --model openbmb/MiniCPM-o-4_5 \
+  --input-wav input_16k_mono.wav \
+  --ref-audio reference_voice.wav \
+  --output-dir /tmp/minicpmo-duplex
+```
+
+## Realtime Event Lifecycle
+
+A typical `/v1/realtime?duplex=1` session follows this lifecycle:
+
+1. Send `session.update` with the model, modalities, audio formats, and session
+   options.
+2. Wait for `session.created`; inspect `session.capabilities` instead of
+   assuming every model supports the same controls.
+3. Send `input_audio_buffer.append` events while microphone audio arrives.
+4. Send `input_audio_buffer.commit` at a user-turn boundary when required by
+   the model policy.
+5. Consume `response.created`, transcript deltas, `response.audio.delta`, and
+   `response.done` or `response.listen` events.
+6. Send `playback.ack` after audio has been played when the session advertises
+   playback acknowledgement support.
+7. Send `session.close` and wait for `session.closed`.
+
+Unlike the turn-based realtime endpoint, input may continue while a response
+is active. The server can emit `overlap.decision` to describe whether input was
+deferred, treated as a short acknowledgement, or used to interrupt output.
+
+## Capabilities and Model Differences
+
+The `session.created` payload includes capability fields such as
+`supports_barge_in`, `supports_playback_ack`, `supports_multi_session`,
+`supports_session_resume`, and `chunk_period_ms`. Treat this payload as the
+runtime contract.
+
+For example, PersonaPlex supports native overlapping speech but currently
+advertises `supports_barge_in=false`; destructive output interruption and
+model-state rewind have not been validated for that integration. Capacity and
+session-resume behavior also depend on the selected deployment configuration.
+
+## Native Protocol
+
+`WS /v1/duplex` exposes lower-level lifecycle names including
+`session.create`, `input_audio_buffer.append`, `turn.signal`, `playback.ack`,
+and `session.close`. It returns native session, input, response, overlap, and
+error events. This protocol is experimental and may evolve with the runtime;
+applications should use the provided Realtime client where possible.
+
+See the [MiniCPM-o example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/minicpmo),
+[PersonaPlex example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/personaplex),
+and [full-duplex runtime design](https://github.com/vllm-project/vllm-omni/blob/main/vllm_omni/experimental/fullduplex/DESIGN.md)
+for model-specific validation and architecture details.
@@ -0,0 +1,109 @@
+# OpenPI Robot Policy WebSocket API
+
+Use `WS /v1/realtime/robot/openpi` for low-latency robot-policy inference with
+OpenPI-compatible clients. A persistent WebSocket carries observations to the
+loaded policy model and returns action arrays.
+
+Despite the `/realtime` prefix, this endpoint does not use the OpenAI Realtime
+event schema. It uses binary MessagePack frames with NumPy extensions.
+
+## Availability
+
+The endpoint is available only when the loaded diffusion policy configuration
+contains `policy_server_config`. The server sends that model-specific
+configuration to the client as the first binary frame.
+
+For example, DreamZero declares:
+
+```yaml
+policy_server_config:
+  image_resolution: [180, 320]
+  n_external_cameras: 2
+  needs_wrist_camera: true
+  needs_stereo_camera: false
+  needs_session_id: true
+  action_space: joint_position
+```
+
+Without this configuration, the WebSocket returns `Robot policy not available`
+and closes.
+
+## DreamZero Quick Start
+
+Start the bundled DreamZero deployment:
+
+```bash
+vllm serve GEAR-Dreams/DreamZero-DROID --omni --port 8091 \
+  --served-model-name dreamzero-droid \
+  --deploy-config vllm_omni/deploy/dreamzero_tp1_cfg2.yaml \
+  --enforce-eager --disable-log-stats
+```
+
+Install the optional client dependencies, download the sample camera inputs,
+and run the client:
+
+```bash
+pip install openpi-client websockets opencv-python
+
+hf download YangshenDeng/vllm-omni-dreamzero-assets \
+  --repo-type dataset \
+  --local-dir outputs/dreamzero/assets
+
+python examples/online_serving/dreamzero/openpi_client.py \
+  --host 127.0.0.1 \
+  --port 8091 \
+  --video-dir outputs/dreamzero/assets
+```
+
+## Protocol
+
+The connection is request-response after an initial server handshake:
+
+```text
+connect
+  <- msgpack(policy_server_config)
+
+infer
+  -> msgpack({"endpoint": "infer", "session_id": "...", ...observation})
+  <- msgpack(ndarray | dict[str, ndarray])
+
+reset
+  -> msgpack({"endpoint": "reset"})
+  <- msgpack({"status": "reset successful"})
+```
+
+If `endpoint` is omitted, the server treats the message as `infer`. Each
+inference message produces one action response after the engine request
+completes; action tokens or intermediate tensors are not streamed.
+
+NumPy arrays use the marker format implemented by `openpi-client`. The server
+also accepts the legacy vLLM NumPy marker representation on input. JSON text
+frames are not observation messages.
+
+## Sessions and Reset
+
+- The API layer's current-session and first-call counters are scoped to each
+  WebSocket connection.
+- `session_id` identifies model-side state across observations on that
+  connection. If omitted, it defaults to `default`.
+- The first inference for a session is sent to the model with `reset=true`.
+- Changing `session_id`, or sending the `reset` command, causes the next
+  inference to start with `reset=true`.
+- The policy pipeline owns observation transforms and persistent model state,
+  normally keyed by `session_id`; the API layer forwards the raw observation
+  dictionary.
+
+## Limits and Errors
+
+- Maximum inbound payload size is 64 MiB.
+- The server closes an idle connection after 30 seconds.
+- Invalid binary input returns a MessagePack
+  `{"type":"error","message":"Invalid request payload"}` response.
+- Inference failures return a generic `Internal inference error` without
+  exposing an internal traceback.
+- Unsupported NumPy object, structured, and complex dtypes are rejected.
+
+Observation keys, camera layout, state tensors, and action shapes are defined
+by the loaded policy rather than by this transport. Read the handshake before
+constructing observations. See the [DreamZero example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/dreamzero)
+for a complete OpenPI client and DROID simulation loop.
@@ -0,0 +1,82 @@
+# Realtime Audio WebSocket API
+
+Use `WS /v1/realtime` to stream PCM audio into a compatible omni model and
+receive incremental transcription and synthesized response audio. This is a
+turn-based streaming API: it starts one generation over an audio stream and
+finishes after the client closes that input stream.
+
+For a model that listens while it is already speaking, use the experimental
+[Full-Duplex API](full_duplex_api.md) instead.
+
+## Quick Start
+
+Start a compatible model. The currently documented example uses Qwen3-Omni:
+
+```bash
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
+```
+
+Run the provided client with a mono, 16-bit PCM, 16 kHz WAV file:
+
+```bash
+python examples/online_serving/qwen3_omni/openai_realtime_client.py \
+  --url ws://localhost:8091/v1/realtime \
+  --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
+  --input-wav input_16k_mono.wav \
+  --output-wav response.wav \
+  --output-text response.txt
+```
+
+The client dependency is `websockets`.
+
+## Protocol
+
+Messages in both directions are JSON text frames. Audio carried inside an
+event is base64-encoded raw PCM16.
+
+| Direction | Event | Purpose |
+|-----------|-------|---------|
+| Server to client | `session.created` | Confirms that the WebSocket connection is ready |
+| Client to server | `session.update` | Selects and validates the served model |
+| Client to server | `input_audio_buffer.commit` with `final: false` | Starts generation over the incoming stream |
+| Client to server | `input_audio_buffer.append` | Appends base64 PCM16 audio |
+| Client to server | `input_audio_buffer.commit` with `final: true` | Marks the end of input |
+| Server to client | `transcription.delta` | Carries incremental response text |
+| Server to client | `transcription.done` | Carries final text and token usage |
+| Server to client | `response.audio.delta` | Carries incremental PCM16 response audio |
+| Server to client | `response.audio.done` | Marks the end of response audio |
+| Server to client | `error` | Reports an invalid event, model, or audio payload |
+
+A minimal client sends events in this order:
+
+```json
+{"type":"session.update","model":"Qwen/Qwen3-Omni-30B-A3B-Instruct"}
+{"type":"input_audio_buffer.commit","final":false}
+{"type":"input_audio_buffer.append","audio":"<base64-pcm16>"}
+{"type":"input_audio_buffer.commit","final":true}
+```
+
+The initial non-final commit intentionally starts generation before all audio
+has arrived. Continue sending `input_audio_buffer.append` events while the
+engine is consuming the stream.
+
+## Audio Handling
+
+- Input is mono PCM16 at 16 kHz for the Qwen3-Omni example.
+- `response.audio.delta.audio` contains base64-encoded PCM16 bytes.
+- Read `sample_rate_hz` from each audio event instead of assuming an output
+  rate. Qwen3-Omni output is typically 24 kHz.
+- Concatenate audio deltas in receive order to construct the output waveform.
+- Set `input_audio_buffer.commit.final` to `true` after the last input chunk so
+  the server can terminate the streaming request.
+
+## Availability and Limitations
+
+The path is registered on the unified API server, but it is usable only when
+the loaded pipeline implements realtime audio input and produces compatible
+audio output. Unsupported deployments return an `error` event. The endpoint
+does not provide duplex session resume, playback acknowledgement, overlap
+policy, or barge-in controls.
+
+See the [Qwen3-Omni online serving example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/qwen3_omni)
+for concurrency options, chunk pacing, and per-delta audio debugging.
@@ -2,7 +2,7 @@

 vLLM-Omni provides an OpenAI-compatible API for text-to-speech (TTS) generation. Supported TTS models include:

- **Qwen3-TTS** (`Qwen/Qwen3-TTS-12Hz-*`) -- Qwen3-based TTS with CustomVoice, VoiceDesign, and Base (voice cloning) task types. Output: 24 kHz.
+- **Qwen3-TTS** (`Qwen/Qwen3-TTS-12Hz-*`) -- Qwen3-based TTS with CustomVoice, VoiceDesign, and Base (voice cloning) task types. Native output: 24 kHz; API output can be resampled to 8 kHz.
 - **Fish Speech S2 Pro** (`fishaudio/s2-pro`) -- Dual-AR TTS with DAC codec. Supports text-to-speech and voice cloning via reference audio. Output: 44.1 kHz.
 - **Voxtral TTS** (`mistralai/Voxtral-4B-TTS-2603`) -- AR + FlowMatching TTS with preset voices. Output: 24 kHz.
 - **CosyVoice3** (`FunAudioLLM/Fun-CosyVoice3-0.5B-2512`) -- 2-stage talker + flow-matching code2wav. Voice cloning via `ref_audio` + `ref_text` (no presets). Output: 24 kHz.
@@ -116,6 +116,7 @@ Content-Type: application/json

 | Parameter | Type | Default | Description |
 |-----------|------|---------|-------------|
+| `sample_rate` | integer | model native | Target output sample rate. Qwen3-TTS supports 8000 or 24000 Hz; the model remains native 24 kHz internally and is resampled before encoding. |
 | `task_type` | string | "CustomVoice" | TTS task type: CustomVoice, VoiceDesign, or Base |
 | `language` | string | "Auto" | Language (see supported languages below) |
 | `instructions` | string | "" | Voice style/emotion instructions |
@@ -811,6 +812,51 @@ If you encounter OOM errors:

 Use `/v1/audio/voices` to list available voices for the loaded model.

+## Orchestration Loop (experimental)
+
+Multi-stage omni deployments route stage outputs through a single orchestrator
+loop. By default that loop polls every stage replica on a 1 ms cadence. An
+opt-in event-driven mode replaces the poll with one reader task per live stage
+replica awaiting its client directly, and switches the serving-side
+final-output drain to a condition-variable wakeup at the same time.
+
+**Configuration (environment variables):**
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `0` (off) | Switches the orchestration loop and the final-output drain from the legacy 1 ms poll to event-driven wakeups. Enabled by `1`, `true`, `yes`, or `on`, matched case-insensitively after surrounding whitespace is stripped; any other value leaves it off. |
+
+Set it on the process that runs the orchestrator (stage 0 of an omni
+deployment) before starting the server:
+
+```bash
+export VLLM_OMNI_EVENT_DRIVEN_ORCH=1
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base \
+    --omni \
+    --port 8091
+```
+
+The server logs the selected loop mode and its reader/poller counts once at
+startup, so you can confirm which loop is live.
+
+Routing, output ordering, and terminal-state behavior are identical on both
+loops; only the poll cadence changes. Leaving the variable unset keeps the
+legacy poll loop, which is the supported default.
+
+**Known limitations:**
+
+- The measured serving A/B (idle CPU 2.43% to 0.07%; TTFP p99 -32% at
+  concurrency 8) predates the rebuild on the per-replica fault-isolation work
+  in [#4285](https://github.com/vllm-project/vllm-omni/pull/4285). That work
+  changed dead-replica handling and reader/poller lifecycle rather than the
+  steady-state output path, and the parity suite covers it, but the serving
+  A/B has not been re-run on the current head.
+- The diffusion-poller branch is covered by unit tests only. Deployments whose
+  stages all run as standard engine cores never exercise it, including GLM-TTS,
+  which deploys its DiT without `stage_type: diffusion`.
+- Concurrency 1 and 32 measured at parity with the legacy loop. At 32 the
+  latency is admission-bound, which this mode does not address.
+
 ## Development

 Enable debug logging:
@@ -0,0 +1,109 @@
+# Standalone Experimental Servers
+
+Most vLLM-Omni endpoints run on the unified server started with
+`vllm serve <model> --omni`. The experimental full-duplex package also contains
+two separate user-facing server processes. Their routes are not added to the
+unified API server.
+
+| Server | Transport | Role |
+|--------|-----------|------|
+| PersonaPlex compatibility server | WebSocket | Hosts the official PersonaPlex browser protocol and a raw-PCM alternative |
+| JoyVL interaction server | HTTP | Adds state, memory, proactive decisions, and delegation in front of a separate OpenAI-compatible model backend |
+
+## PersonaPlex Compatibility Server
+
+Use this server when a client requires the official Moshi/PersonaPlex wire
+protocol or the bundled browser UI. For the engine-owned unified duplex path,
+use [`WS /v1/realtime?duplex=1`](full_duplex_api.md) instead.
+
+Start it with:
+
+```bash
+HF_TOKEN=... CUDA_VISIBLE_DEVICES=0 python -m \
+  vllm_omni.experimental.fullduplex.personaplex.serving.server \
+  --port 8091
+```
+
+| Route | Purpose |
+|-------|---------|
+| `GET /` | Official PersonaPlex browser client, when its assets are available |
+| `GET /health` | Readiness check |
+| `WS /api/chat` | Moshi-compatible tagged binary protocol with Opus audio |
+| `WS /v1/audio/duplex` | Simpler float32 raw-PCM protocol |
+
+### `/api/chat` Protocol
+
+The first byte of every binary frame is a message tag:
+
+| Direction | Tag | Payload |
+|-----------|-----|---------|
+| Server to client | `0x00` | Ready handshake |
+| Client to server | `0x01` | Opus microphone audio at 24 kHz |
+| Server to client | `0x01` | Opus response audio |
+| Server to client | `0x02` | UTF-8 inner-monologue text |
+
+The optional `text_prompt` and `voice_prompt` query parameters select a persona
+and voice. The default server admits one conversation; use `--batch-size N` to
+configure multiple lockstep slots. Connections beyond capacity close with
+WebSocket code 1013.
+
+### `/v1/audio/duplex` Protocol
+
+Send `{"type":"open","persona":"...","voice":"..."}` as a JSON text frame,
+then exchange binary float32 PCM frames. The server returns generated float32
+audio as binary frames and text as `{"type":"text","text":"..."}`. Send
+`{"type":"close"}` and wait for `{"type":"done"}` to flush and finish.
+
+See the [PersonaPlex example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/personaplex)
+for the browser and headless clients.
+
+## JoyVL Interaction Server
+
+JoyVL is an orchestration layer, not a model-serving engine. It calls a
+separate OpenAI-compatible backend and adds per-session frame history, memory,
+persona policy, and optional delegation.
+
+Start the model backend and orchestrator separately:
+
+```bash
+vllm serve jdopensource/JoyAI-VL-Interaction-Preview \
+  --served-model-name JoyAI-VL-Interaction-Preview \
+  --port 8092 \
+  --max-model-len 131072 \
+  --enable-prefix-caching \
+  --limit-mm-per-prompt '{"image":256,"video":1}'
+
+python -m vllm_omni.experimental.fullduplex.joyvl.serving.server \
+  --port 8091 \
+  --main-backend-url http://127.0.0.1:8092/v1 \
+  --main-model JoyAI-VL-Interaction-Preview
+```
+
+| Method and route | Purpose |
+|------------------|---------|
+| `GET /health` | Readiness check |
+| `GET /v1/models` | Reports the configured interaction model |
+| `POST /v1/chat/completions` | Processes one frame or interaction tick |
+| `POST /reset` | Resets a session |
+| `POST /v1/streaming/reset` | Alias for `/reset` |
+| `POST /v1/streaming/persona` | Changes the session persona |
+
+Send one multimodal Chat Completions request per video frame, normally around
+one frame per second, and identify the session with `x-session-id`. Responses
+include an `interaction` block whose action is `silence`, `response`, or
+`delegate`.
+
+```bash
+curl http://127.0.0.1:8091/v1/chat/completions \
+  -H 'content-type: application/json' \
+  -H 'x-session-id: demo' \
+  -d '{
+    "messages": [{"role": "user", "content": [
+      {"type": "text", "text": "Alert me if a fire appears"},
+      {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,..."}}
+    ]}]
+  }'
+```
+
+See the [JoyVL recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/JD/JoyAI-VL-Interaction.md)
+for memory, personas, delegation backends, and the browser UI.
@@ -0,0 +1,92 @@
+# Streaming Video Output WebSocket API
+
+Use `WS /v1/realtime/video` to receive fragmented MP4 chunks while a compatible
+diffusion video model is generating. This endpoint streams generated video
+output; it is unrelated to [`WS /v1/video/chat/stream`](video_stream_api.md),
+which streams live video input into a conversational model.
+
+## Quick Start
+
+Start a model that supports step execution and enable diffusion streaming
+output:
+
+```bash
+vllm serve BestWishYsh/Helios-Distilled --omni \
+  --diffusion-streaming-output \
+  --port 8091
+```
+
+Install the client dependency and run the provided client:
+
+```bash
+pip install av websockets
+
+python examples/online_serving/streaming_video_generation/streaming_video_client.py \
+  --host 127.0.0.1 \
+  --port 8091 \
+  --model BestWishYsh/Helios-Distilled \
+  --prompt "A serene lakeside sunrise with mist over the water" \
+  --output output.mp4
+```
+
+## Protocol
+
+Control events are JSON text frames. Each `video.chunk_metadata` event is
+immediately followed by one binary WebSocket frame containing fragmented MP4
+bytes.
+
+| Direction | Event or frame | Purpose |
+|-----------|----------------|---------|
+| Client to server | `session.start` | Starts generation with a prompt and video parameters |
+| Server to client | `video.start` | Confirms the request ID, format, and accepted configuration |
+| Server to client | `video.chunk_metadata` | Describes the next binary media frame |
+| Server to client | Binary frame | Carries one fragmented MP4 chunk |
+| Client to server | `session.interaction` | Queues an optional prompt update during generation |
+| Server to client | `session.interaction.queued` | Confirms that an interaction was queued |
+| Client to server | `session.ping` | Refreshes the stall clock |
+| Server to client | `session.pong` | Acknowledges a ping |
+| Client to server | `session.stop` | Cancels the active generation |
+| Server to client | `session.done` | Reports completion or cancellation and the chunk count |
+| Server to client | `error` | Reports validation, generation, or timeout failures |
+
+Start a minimal session with:
+
+```json
+{
+  "type": "session.start",
+  "model": "BestWishYsh/Helios-Distilled",
+  "prompt": "A serene lakeside sunrise",
+  "format": "m4s",
+  "width": 640,
+  "height": 384,
+  "fps": 16,
+  "num_frames": 99
+}
+```
+
+Only `m4s` output is currently accepted. Concatenate the binary frames in
+receive order, then remux the fragmented stream if the target player requires
+a progressive MP4 file.
+
+## Mid-Generation Interaction
+
+Models that implement interaction updates can change the active prompt:
+
+```json
+{
+  "type": "session.interaction",
+  "interaction": {
+    "event_id": "sunlight-update",
+    "event": {"prompt": "Sunlight breaks through the mist"},
+    "transition_chunks": 2
+  }
+}
+```
+
+This feature is pipeline-dependent. An enabled endpoint does not imply that
+the loaded model supports midway prompt updates.
+
+The default start timeout is 10 seconds and the server reports a stall after
+about 60 seconds without generation progress or a `session.ping`. See the
+[streaming video generation example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/streaming_video_generation)
+for prompt-update scheduling and browser playback.
@@ -17,6 +17,8 @@ globally.
 | --- | --- | --- |
 | `sparsity` | finite, `[0, 1]` | Nominal dropped-key-block fraction; default `0.8`; `0` disables sparsity |
 | `start_step` | integer, `>= 0` | Number of early denoise steps kept dense |
+| `end_step` | integer, `>= 0` | Number of final denoise steps kept dense (tail fallback) |
+| `precision` | `"bf16"`, `"fp8"`, `"mix"` | Kernel precision mode; default `"bf16"`. Requires MindIE-SD with `sparse_attention(precision=...)` |
 | `skip_layers` | selector such as `"0-3,38"` | DiT blocks kept dense |

 ```bash
@@ -52,6 +54,12 @@ platform raises. It is incompatible with ring sequence parallelism because
 the kernel needs the complete key sequence for ranking; use Ulysses sequence
 parallelism with `ring_degree=1`.

+The `precision` knob requires a MindIE-SD release whose `sparse_attention`
+accepts `precision=`; older releases accept it through `**kwargs` but
+silently ignore it and run the BF16 path. `RAINFUSION_ATTN` raises at runtime
+when a non-`bf16` precision is requested without that support, so pin a
+compatible MindIE-SD release when using `precision="mix"` or `"fp8"`.
+
 ## Geometry handling

 RainFusion handles arbitrary video grids by rearranging video spatially and,
@@ -12,15 +12,17 @@ LoRA adapters are lightweight, model-specific fine-tuning weights that can be ap
 ## LoRA Adapter Format

 ### PEFT (Parameter-Efficient Fine-Tuning) format (default)
+
 A typical PEFT-format LoRA adapter directory structure:

-```
+```text
 lora_adapter/
 ├── adapter_config.json
 └── adapter_model.safetensors
 ```

 The `adapter_config.json` file contains metadata about the LoRA adapter, including:
+
 - `r`: LoRA rank
 - `lora_alpha`: LoRA alpha scaling factor
 - `target_modules`: List of module names to apply LoRA to
@@ -67,7 +69,7 @@ For distilled few-step LoRAs, pass `lora_backend="distill"` together with one or
 For Qwen-Image and Wan pipelines, the distill backend calls `pipeline.load_lora_weights(...)` during worker initialization.

 | Pipeline | Supported distilled LoRA repo | Notes |
-|----------|--------------------------------|-------|
+| ---------- | -------------------------------- | ------- |
 | `QwenImagePipeline` | `lightx2v/Qwen-Image-2512-Lightning` | Used with Qwen-Image-2512 Lightning-style few-step inference. |
 | `Wan22Pipeline` | `lightx2v/Wan2.1-Distill-Loras`, `lightx2v/Wan2.2-Distill-Loras` | Wan2.1 uses one LoRA file. For dual-transformer Wan2.2 MoE, pass high-noise then low-noise LoRA files. |
 | `Wan22I2VPipeline` | `lightx2v/Wan2.2-Distill-Loras` | For dual-transformer Wan2.2 MoE, pass high-noise then low-noise LoRA files. |
@@ -240,6 +242,37 @@ Notes:
 - Output quality and speed depend on the replacement checkpoints and sampling params you choose.
 - If you only need to fuse distilled LoRAs into a Wan2.2 checkpoint at load time (without the full LightX2V convert + assemble pipeline), you can instead pass them directly via `--lora-backend distill --lora-path <high>.safetensors <low>.safetensors`. See the [Distill backend](#distill-backend-fuse-distilled-lora-at-init) section above.

+## MiniMax-H3 adapter-declared schedules
+
+MiniMax-H3 supports two few-step mechanisms that must not be conflated:
+
+- **Checkpoint-pinned schedule**: a merged release writes `base_schedule` into
+  `model_index.json`. Requests must pass `num_inference_steps` as the interval
+  count (for example `4` for `[1.0, 0.7, 0.4, 0.15, 0.0]`).
+- **Runtime Turbo LoRA**: LightX2V Turbo artifacts request five sigma points and
+  enforce `flow_shift=6`, `audio_flow_shift=3`.
+- **Runtime native LoRA**: FlashGen-style artifacts declare
+  `key_format=minimax-h3-native` and embed `base_schedule` in safetensors
+  metadata. When active, the adapter schedule overrides the base checkpoint and
+  requests must use the interval-count contract (`num_inference_steps=4`).
+  Request-mode generation may omit the field and take the count from the adapter
+  schedule; step execution requires it explicitly, because the step scheduler
+  derives the total step count from the request before the adapter schedule is
+  known.
+
+Native artifacts also declare `qkv_layout=grouped`. The H3 loader reorders fused
+`qkv_proj` LoRA rows with the same `_reorder_grouped_qkv_to_qkv` helper used for
+base-weight loading, then binds the packed Q/K/V slices through the legacy PEFT
+manager without modifying `DiffusionLoRAManager`.
+
+Both runtime adapters run with distributed layerwise offload in request-mode
+generation, where the manager keeps the LoRA A/B buffers resident on the compute
+device while DLO streams the base blocks. MiniMax-H3 step execution still
+rejects DLO, so `--step-execution` cannot be combined with
+`--enable-distributed-layerwise-offload`. Model-level CPU offload and standard
+layerwise offload are rejected in every mode, because the dynamic LoRA tensors
+are neither parameters nor registered buffers and therefore do not participate
+in those weight lifecycles.

 ## See Also

@@ -65,6 +65,7 @@ omni = Omni(
 | `--dlo-resident-layers N` | Keep N leading main-DiT blocks on device; requires no-AllGather and model-declared resident paths | `0` |
 | `--host-weight-runtime-mode {disabled,preferred,required}` | HWR policy: no interaction, populate on a miss, or require an exact hit | `disabled` |
 | `--host-weight-runtime-root PATH` | Writable node-local HWR store shared by workers in one storage domain; required for `preferred` and `required` | unset |
+| `--dlo-host-registration-limit-gib N` | Optional per-worker ceiling for registering an HWR mmap; zero adds no ceiling | `0` |

 ## Host-weight loading

@@ -111,6 +112,12 @@ perform a collective and uses the same rank-local transfer behavior.
 HWR is an opt-in startup optimization for models that declare the final-layout
 BF16 restore contract. Use it only with no-AllGather DLO:

+The validated BF16 model contracts currently cover MiniMax H3 and
+`black-forest-labs/FLUX.2-klein-4B`. FLUX.2-klein-9B shares the same model
+class but has not been validated against this contract. FLUX.2-dev, online
+FP8, HSDP, LoRA/adapted weights, and non-default load formats remain outside
+the validated scope.
+
 ```bash
 vllm serve /path/to/model --omni \
  --enable-distributed-layerwise-offload \
@@ -178,12 +185,48 @@ returning to `required`.

 On a cold start, the canonical loader remains authoritative and publishes a
 validated final-layout artifact for later workers. A warm start restores the
-DiT final tensors without ordinary DiT materialization, then DLO streams them
-through the same two bounded host staging slots. The artifact identity includes
-the TP rank/size and SP layout, so TP1, TP2 rank-local shards, and distinct SP
-layouts do not alias one another. Publication failure remains separate from a
-`preferred` serving startup; the next `required` startup provides the explicit
-artifact-availability check.
+DiT final tensors without ordinary DiT materialization. DLO then attempts to
+register the immutable mapping for direct H2D; if registration is unavailable,
+it streams through the same two bounded host staging slots. The artifact
+identity includes the TP rank/size and SP layout, so TP1, TP2 rank-local shards,
+and distinct SP layouts do not alias one another. Publication failure remains
+separate from a `preferred` serving startup; the next `required` startup
+provides the explicit artifact-availability check.
+
+For local canonical checkpoints, the first eligible worker may hash source
+shards to establish immutable identity. HWR caches those digests in the same
+node-local storage domain and validates file metadata before reuse, so later
+workers normally avoid repeating that read. Cold BF16 publication also hashes
+ordered payloads as they are written and overlaps payload durability work with
+later shards; this changes startup work only, not artifact contents or runtime
+H2D behavior.
+
+#### Registered direct H2D
+
+Registration is attempted automatically only for an eligible warm HWR hit when
+`pin_cpu_memory` is enabled. A successful path is:
+
+```text
+shared read-only HWR mmap -> existing rotating HBM block buffer -> GPU kernel
+```
+
+It removes the recurrent mmap-to-private-staging CPU copy and does not allocate
+the two host staging slots. It does not reduce the H2D payload or make GPU
+kernels access host memory directly.
+
+CUDA requires read-only host-registration capability for the immutable HWR
+mapping. Unsupported capability, a positive registration limit smaller than
+the complete page-aligned mapping, or a safely rolled-back registration error
+falls back to bounded staging. Programmatic configurations can set
+`pin_cpu_memory=False` to disable registration explicitly. A successful
+registration locks the complete mapped range in host memory for that worker's
+lifetime, so use
+`--dlo-host-registration-limit-gib` when an operator-enforced ceiling is
+required.
+
+Shutdown drains H2D work, releases hook/source references, unregisters every
+range, and only then closes the HWR lease. AllGather and direct checkpoint mmap
+retain their existing transfer paths.

 When HWR is disabled, DLO is disabled, or AllGather is enabled, the loader does
 not resolve HWR sources or construct its store.
@@ -129,7 +129,7 @@ The following tables show which models support each feature:
 | **Krea 2**               |     ❌     |     ✅      |           ❌           |       ❌        |         ❌         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **LongCat-Image**        |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **LongCat-Image-Edit**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
-| **MagiHuman**            |     ❌     |     ❌      |           ❌           |       ❓        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **MammothModa2(T2I)**    |     ❌     |     ❌      |           ❌           |       ❌        |         ❌         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **Nextstep_1(T2I)**      |     ❓     |     ❓      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **OmniGen2**             |     ❌     |     ✅      |           ✅           |       ❌        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **Ovis-Image**           |     ❌     |     ✅      |           ❌           |       ✅        |         ❌         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
@@ -138,7 +138,7 @@ The following tables show which models support each feature:
 | **Qwen-Image-Edit**      |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Qwen-Image-Edit-2509** |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |        ✅ (decode)         |          ✅           |       ❌        |        ❌         |
 | **Qwen-Image-Layered**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
-| **SenseNova-U1**         |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **SenseNova-U1 / U1.5**  |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **Stable-Diffusion-XL**  |     ❌     |     ❌      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Stable-Diffusion3.5**  |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Z-Image**              |     ✅     |     ✅      |           ✅           |       ❓        |   ✅ (TP=2 only)   |          ❌          |   ✅    |             ❌             |      ✅ (decode)      |       ✅        |        ❌         |
@@ -147,7 +147,7 @@ The following tables show which models support each feature:

 > Notes:
 > 1. Nextstep_1(T2I) does not support cache acceleration methods such as TeaCache or Cache-DiT.
-> 2. `Tongyi-MAI/Z-Image-Turbo` and `SII-GAIR/daVinci-MagiHuman-Base-1080p` are distilled models with minimal NFEs; CFG-Parallel is not necessary.
+> 2. `Tongyi-MAI/Z-Image-Turbo` is a distilled model with minimal NFEs; CFG-Parallel is not necessary.
 > 3. Cosmos3 T2I uses `Cosmos3OmniDiffusersPipeline` with `modalities=["image"]`. Model-level CPU offload swaps the nested UND reasoner and GEN generator pathways; layerwise offload remains available for blockwise GEN/UND offload.
 > 4. Krea 2 currently supports single-GPU inference plus LoRA, Cache-DiT, HSDP, CPU/layerwise offload, and VAE-patch-parallel (decode). TP/SP/CFG-Parallel are not yet wired. The few-step distilled (Turbo) checkpoint uses `is_distilled=true` (fixed timestep shift `mu=1.15`); generate at 2048x2048 by default with `num_inference_steps≈8` and `guidance_scale=0`. The Raw checkpoint uses 1024x1024, `num_inference_steps=28`, and `guidance_scale=4.5`.
 > 5. HunyuanImage3 supports step execution. Multi-request step execution requires `TORCH_SDPA`; see [Diffusion Execution Modes](diffusion/execution_modes.md#step-execution).
@@ -165,7 +165,6 @@ The following tables show which models support each feature:
 | **SANA-Video-2B T2V I2V**   |     ❓     |     ❓      |           ❓           |       ❓        |         ❓         |         ❓         |   ❓    |             ❓             |          ❓           |       ❓        |        ❌         |
 | **Helios**                   |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ✅*        |
 | **HunyuanVideo-1.5 T2V I2V** |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
-| **DreamID-Omni**             |     ❌     |     ❌      |           ❌           |       ✅        |         ❌         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **Cosmos3**                  |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
 | **LongCat-Video-Avatar-1.5** |     ❌     |     ❌      |           ❌           |       ❌        |         ❌         |         ❌         |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **MiniMax-H3**               | ✅ (FL2VA) |     ✅      |           ✅           |       ❌        |       ✅ (DiT/TE)  |         ❌         |   ✅    |             ✅             |       ✅ (tile)       |      ✅ (DiT)      |        ❌         |
@@ -1,115 +0,0 @@
-# X-To-Video-Audio
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/x_to_video_audio>.
-
-
-The `DreamID-Omni` pipeline generates short videos from text, image and video.
-
-## Local CLI Usage
-### Download the Model locally
-Since DreamID-Omni combine multiple models, and without any config, so we need to download them locally.
-
-```bash
-python download_dreamid_omni.py --output-dir ./dreamid_omni
-```
-After download, the model directory will look like this:
-
-```
-dreamid_omni/
-├── DreamID-Omni/
-│   ├── dreamid_omni.safetensors
-├── MMAudio/
-│   ├── ext_weights/
-│   │   ├── best_netG.pt
-│   │   ├── v1-16.pth
-├── Wan2.2-TI2V-5B/
-│   ├── google/*
-│   ├── models_t5_umt5-xxl-enc-bf16.pth
-│   ├── Wan2.2_VAE.pth
-│
-├── model_index.json # create by download_dreamid_omni.py
-```
-
-### Run the Inference
-```python
-python x_to_video_audio.py \
-  --model /path/to/dreamid_omni \
-  --prompt "Two people walking together and singing happily" \
-  --image-path ./example0.png ./example1.png \
-  --audio-path ./example0.wav ./example1.wav \
-  --video-negative-prompt "jitter, bad hands, blur, distortion" \
-  --audio-negative-prompt "robotic, muffled, echo, distorted" \
-  --cfg-parallel-size 2 \
-  --num-inference-steps 45 \
-  --height 704 \
-  --width 1280 \
-  --output out_dreamid_omni_twoip.mp4
-```
-In the current test scenario (2 images + 2 audio inputs), the VRAM requirement is 72GB, regardless of whether cfg-parallel is enabled or disabled.
-The VRAM usage can be reduced by enabling CPU offload via --enable-cpu-offload.
-For multi-GPU memory reduction on the fused DreamID-Omni transformer, you can also enable HSDP:
-
-```python
-python x_to_video_audio.py \
-  --model /path/to/dreamid_omni \
-  --prompt "Two people walking together and singing happily" \
-  --image-path ./example0.png ./example1.png \
-  --audio-path ./example0.wav ./example1.wav \
-  --video-negative-prompt "jitter, bad hands, blur, distortion" \
-  --audio-negative-prompt "robotic, muffled, echo, distorted" \
-  --cfg-parallel-size 2 \
-  --num-inference-steps 45 \
-  --height 704 \
-  --width 1280 \
-  --use-hsdp \
-  --hsdp-shard-size 2 \
-  --output out_dreamid_omni_twoip.mp4
-```
-
-
-You could take reference images/audios from the test cases in the official repo: https://github.com/Guoxu1233/DreamID-Omni
-
-For example, single IP ref resources can be found under https://github.com/Guoxu1233/DreamID-Omni/tree/main/test_case/oneip, you could download them correspondingly to your local and use them for testing.
-
-```python
-# Example usage for oneip, ref media from the official repo DreamID-Omni
-python x_to_video_audio.py \
-  --model /path/to/dreamid_omni \
-  --prompt "<img1>: In the frame, a woman with black long hair is identified as <sub1>.\n**Overall Environment/Scene**: A lively open-kitchen café at night; stove flames flare, steam rises, and warm pendant lights swing slightly as staff move behind her. The shot is an upper-body close-up.\n**Main Characters/Subjects Appearance**: <sub1> is a young woman with thick dark wavy hair and a side part. She wears a fitted black top under a light apron, a thin gold chain necklace, and small stud earrings.\n**Main Characters/Subjects Actions**: <sub1> tastes the sauce with a spoon, then turns her face toward the camera while still holding the spoon, her expression shifting from focused to conflicted.\n<sub1> maintains eye contact, swallows as if choosing her words, and says, <S>I keep telling myself I’m fine,but some nights it feels like I’m just performing calm.<E>" \
-  --image-path 9.png \
-  --audio-path 9.wav \
-  --video-negative-prompt "jitter, bad hands, blur, distortion" \
-  --audio-negative-prompt "robotic, muffled, echo, distorted" \
-  --cfg-parallel-size 2 \
-  --num-inference-steps 45 \
-  --height 704 \
-  --width 1280 \
-  --output out_dreamid_omni_oneip.mp4
-```
-
-
-Key arguments:
- `--prompt`: text description (string).
- `--model`: path to the model local directory.
- `--height/--width`: output resolution (defaults 704 * 1024).
- `--image-path`: path to the input image list.
- `--audio-path`: path to the input audio list, indicate the timbre of the output video.
- `--cfg-parallel-size`: number of parallel cfg parallel (defaults 1).
- `--use-hsdp`: enable HSDP weight sharding for DreamID-Omni fused blocks.
- `--hsdp-shard-size`: number of GPUs used for HSDP sharding.
- `--hsdp-replicate-size`: number of HSDP replica groups.
- `--num-inference-steps`: number of denoising steps (defaults 45).
- `--video-negative-prompt`: negative prompt for video generation.
- `--audio-negative-prompt`: negative prompt for audio generation.
- `--enable-cpu-offload`: enable CPU offload (defaults False).
-
-## Example materials
-
-??? abstract "download_dreamid_omni.py"
-    ``````py
-    --8<-- "examples/offline_inference/x_to_video_audio/download_dreamid_omni.py"
-    ``````
-??? abstract "x_to_video_audio.py"
-    ``````py
-    --8<-- "examples/offline_inference/x_to_video_audio/x_to_video_audio.py"
-    ``````
@@ -28,6 +28,18 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
    --deploy-config /path/to/deploy_config_file
 ```

+To serve **thinker-only** (text output, no talker / code2wav loaded) with Instruct
+weights, pass the bundled thinker-only deploy YAML. The `pipeline:` key
+`qwen3_omni_moe_thinker_only` overrides the HF `enable_audio_output` resolver:
+
+```bash
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
+    --deploy-config vllm_omni/deploy/qwen3_omni_moe_thinking.yaml
+```
+
+Captioner / Thinking checkpoints (`enable_audio_output=false`) still auto-select
+the same single-stage pipeline without `--deploy-config`.
+
 ### Launch individual stages (stage-based CLI)

 Adopt the stage-based CLI architecture to independently instantiate execution processes per functional stage.
@@ -1,193 +0,0 @@
-# Text-To-Audio
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/stable_audio>.
-
-This example demonstrates how to deploy Stable Audio models for online text-to-audio generation using vLLM-Omni.
-
-## Supported Models
-
-| Model | Description |
-|-------|-------------|
-| `stabilityai/stable-audio-open-1.0` | Open-source audio generation, up to ~47 seconds, 44.1 kHz stereo |
-
-## Start Server
-
-### Basic Start
-
-```bash
-vllm-omni serve stabilityai/stable-audio-open-1.0 \
-    --host 0.0.0.0 \
-    --port 8091 \
-    --gpu-memory-utilization 0.9 \
-    --trust-remote-code \
-    --enforce-eager \
-    --omni
-```
-
-## API Calls
-
-### Method 1: Using curl
-
-```bash
-# Run all curl examples
-bash curl_examples.sh
-
-# Or execute directly
-curl -X POST http://localhost:8091/v1/audio/generate \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "The sound of a cat purring",
-        "audio_length": 10.0
-    }' --output cat.wav
-```
-
-### Method 2: Using Python Client
-
-```bash
-cd examples/online_serving/stable_audio
-
-# Simple generation
-python stable_audio_client.py \
-    --text "The sound of a cat purring"
-
-# With custom duration
-python stable_audio_client.py \
-    --text "A dog barking" \
-    --audio_length 5.0
-
-# With all parameters
-python stable_audio_client.py \
-    --text "Thunder and rain" \
-    --audio_length 15.0 \
-    --negative_prompt "Low quality" \
-    --guidance_scale 7.0 \
-    --num_inference_steps 100 \
-    --seed 42 \
-    --output thunder.wav
-```
-
-The Python client supports the following command-line arguments:
-
- `--api_url`: API endpoint URL (default: `http://localhost:8091/v1/audio/generate`)
- `--text`: Text prompt for audio generation (default: `"The sound of a cat purring"`)
- `--audio_length`: Audio length in seconds (default: `10.0`, max ~47s for `stable-audio-open-1.0`)
- `--audio_start`: Audio start time in seconds (default: `0.0`)
- `--negative_prompt`: Negative prompt for classifier-free guidance (default: `"Low quality"`)
- `--guidance_scale`: Guidance scale for diffusion (default: `7.0`)
- `--num_inference_steps`: Number of inference steps (default: `100`)
- `--seed`: Random seed for reproducibility (default: `None`)
- `--response_format`: Audio output format (default: `wav`). Options: `wav`, `mp3`, `flac`, `pcm`
- `--output`: Output file path (default: `stable_audio_output.wav`)
-
-### Method 3: Using Python httpx
-
-```python
-import httpx
-
-response = httpx.post(
-    "http://localhost:8091/v1/audio/generate",
-    json={
-        "input": "The sound of ocean waves crashing on a beach",
-        "audio_length": 10.0,
-        "negative_prompt": "Low quality, distorted",
-        "guidance_scale": 7.0,
-        "num_inference_steps": 100,
-    },
-    timeout=300.0,
-)
-
-with open("ocean.wav", "wb") as f:
-    f.write(response.content)
-```
-
-## Request Format
-
-### Simple Generation
-
-```json
-{
-    "input": "The sound of ocean waves"
-}
-```
-
-### Generation with Parameters
-
-```json
-{
-    "input": "A piano playing a gentle melody",
-    "audio_length": 10.0,
-    "negative_prompt": "Low quality, distorted, noisy",
-    "guidance_scale": 8.0,
-    "num_inference_steps": 150,
-    "seed": 42,
-    "response_format": "wav"
-}
-```
-
-## API Reference
-
-### Endpoint
-
-```
-POST /v1/audio/generate
-Content-Type: application/json
-```
-
-### Generation Parameters
-
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `input` | string | **required** | Text prompt describing the audio to generate |
-| `model` | string | server's model | Model to use (optional, should match server if specified) |
-| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, opus |
-| `speed` | float | 1.0 | Playback speed (0.25 - 4.0) |
-| `audio_length` | float | null | Audio duration in seconds (max ~47s for `stable-audio-open-1.0`) |
-| `audio_start` | float | 0.0 | Audio start time in seconds |
-| `negative_prompt` | string | null | Text describing what to avoid in generation |
-| `guidance_scale` | float | model default | Classifier-free guidance scale (higher = more adherence to prompt) |
-| `num_inference_steps` | int | model default | Number of denoising steps (higher = better quality, slower) |
-| `seed` | int | null | Random seed for reproducible generation |
-
-### Response Format
-
-Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/wav`).
-
-## Tuning Tips
-
-1. **Audio Length**: Keep under 47 seconds for `stable-audio-open-1.0`.
-2. **Quality vs Speed**:
-   - 50 steps: Fast, decent quality (quick previews)
-   - 100 steps: Good balance (general purpose)
-   - 150+ steps: High quality, slower (final / critical audio)
-3. **Guidance Scale**:
-   - Lower (3 - 5): More creative / varied output
-   - Default (7): Good balance
-   - Higher (10+): Strict adherence to the prompt
-4. **Negative Prompts**: Use to avoid unwanted characteristics such as `"Low quality"`, `"distorted"`, `"noisy"`.
-5. **Seeds**: Set a fixed seed to get deterministic, reproducible results.
-
-## File Description
-
-| File | Description |
-|------|-------------|
-| `curl_examples.sh` | Curl examples covering common use cases |
-| `stable_audio_client.py` | Python client with full CLI argument support |
-
-## Troubleshooting
-
-1. **Audio generation model did not produce audio output**: Verify the server started successfully and the model loaded without errors.
-2. **Connection refused**: Make sure the server is running on the correct port.
-3. **Generation timeout**: Reduce `num_inference_steps` or `audio_length`, and check GPU memory with `nvidia-smi`.
-4. **Out of memory**: Lower `--gpu-memory-utilization` or reduce `audio_length`.
-5. **Audio quality issues**: Increase `num_inference_steps`, add a negative prompt, or raise `guidance_scale`.
-
-## Example materials
-
-??? abstract "stable_audio_client.py"
-    ``````py
-    --8<-- "examples/online_serving/stable_audio/stable_audio_client.py"
-    ``````
-??? abstract "curl_examples.sh"
-    ``````sh
-    --8<-- "examples/online_serving/stable_audio/curl_examples.sh"
-    ``````
@@ -0,0 +1,50 @@
+# SVDQuant W4A4
+
+## Overview
+
+[SVDQuant](https://arxiv.org/abs/2411.05007) combines four-bit weights
+and activations with a small low-rank branch that corrects part of the
+quantization error. vLLM-Omni consumes an offline-quantized checkpoint;
+it does not calibrate the model while loading.
+
+## Checkpoint contract
+
+Place the following entry in the diffusion transformer's `config.json`:
+
+```json
+{
+  "quantization_config": {
+    "quant_method": "svdquant",
+    "rank": 32,
+    "precision": "nvfp4",
+    "act_unsigned": false,
+    "modules_to_not_convert": []
+  }
+}
+```
+
+For a quantized linear with input size `K`, output size `N`, and correction
+rank `R`, the checkpoint stores:
+
+| Suffix | Shape | dtype |
+| --- | --- | --- |
+| `qweight` | `(N, K / 2)` | `int8` (two packed FP4 values per byte) |
+| `wscales` | `(K / 16, N)` | `float8_e4m3fn` |
+| `proj_down` | `(K, R)` | `bfloat16` |
+| `proj_up` | `(N, R)` | `bfloat16` |
+| `smooth_factor` | `(K,)` | `bfloat16` |
+| `wcscales` | `(N,)` | `bfloat16` |
+| `wtscale` | `(1,)` | `bfloat16` |
+
+`K` must be divisible by 16 on every tensor-parallel rank. Set `wcscales` to
+ones when no per-output correction is needed. Modules listed in
+`modules_to_not_convert` keep their checkpoint precision.
+
+## Runtime support
+
+The compatibility path accepts BF16 inputs and executes an NVFP4 GEMM followed
+by the BF16 rank correction. It supports vLLM's FlashInfer, CUTLASS, and FBGEMM
+NVFP4 tensor layouts; incompatible forced backends fail during model loading.
+SM103 is the currently validated and enabled hardware target. Native fusion of
+the NVFP4 GEMM and rank correction is separate from this checkpoint-loading
+contract.
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""vLLM-Omni example scripts, importable as a package.
+
+Keep this ``__init__.py``: with the upstream vLLM installed editable (which also
+places an ``examples`` package on ``sys.path``), a regular package at this repo
+root wins over the upstream ``examples`` namespace during ``import examples``.
+"""
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

 """
 Image-to-Video generation example using Wan2.2 I2V/TI2V models, LTX2/LTX-2.3,
@@ -111,6 +111,13 @@ def build_image_to_video_prompt(
    return result


+def _validate_video_output_type(output_type: str) -> None:
+    if output_type not in {"image", "video"}:
+        raise ValueError(
+            f"Unexpected output type '{output_type}', expected 'video' or legacy 'image' for video generation."
+        )
+
+
 def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description=(
@@ -665,10 +672,7 @@ def main():
        frames = frames[0] if frames else None

    if isinstance(frames, OmniRequestOutput):
-        if frames.final_output_type != "image":
-            raise ValueError(
-                f"Unexpected output type '{frames.final_output_type}', expected 'image' for video generation."
-            )
+        _validate_video_output_type(frames.final_output_type)
        if frames.multimodal_output and "audio" in frames.multimodal_output:
            audio = frames.multimodal_output["audio"]
            audio_sample_rate = frames.multimodal_output.get("audio_sample_rate", audio_sample_rate)
@@ -0,0 +1,117 @@
+"""
+Offline inference example: MammothModa2 image summarization (single AR stage).
+
+Example:
+  uv run python examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py \
+    --model path/to/MammothModa2-Preview \
+    --deploy-config vllm_omni/deploy/mammoth_moda2_ar.yaml \
+    --image /path/to/input.jpg \
+    --question "Please summarize the content of this image."
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+
+from PIL import Image
+from vllm import SamplingParams
+from vllm.multimodal.image import convert_image_mode
+
+from vllm_omni import Omni
+
+DEFAULT_SYSTEM = "You are a helpful assistant."
+DEFAULT_QUESTION = "Please summarize the content of this image."
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="MammothModa2 image summarization (offline, AR only).")
+    parser.add_argument("--model", type=str, required=True, help="Path to model directory or model id.")
+    parser.add_argument("--deploy-config", type=str, required=True, help="Path to deploy config yaml.")
+    parser.add_argument("--image", type=str, required=True, help="Path to input image.")
+    parser.add_argument("--question", type=str, default=DEFAULT_QUESTION, help="Question/instruction for the model.")
+    parser.add_argument("--system", type=str, default=DEFAULT_SYSTEM, help="System prompt.")
+    parser.add_argument(
+        "--max-tokens",
+        type=int,
+        default=512,
+        help="Max new tokens to generate.",
+    )
+    parser.add_argument("--temperature", type=float, default=0.2)
+    parser.add_argument("--top-p", type=float, default=0.9)
+    parser.add_argument("--seed", type=int, default=42)
+    parser.add_argument("--trust-remote-code", action="store_true")
+    parser.add_argument(
+        "--enable-diffusion-pipeline-profiler",
+        action="store_true",
+        help="Enable diffusion pipeline profiler to display stage durations.",
+    )
+    return parser.parse_args()
+
+
+def build_prompt(system: str, question: str) -> str:
+    return (
+        f"<|im_start|>system\n{system}<|im_end|>\n"
+        "<|im_start|>user\n"
+        "<|vision_start|><|image_pad|><|vision_end|>"
+        f"{question}<|im_end|>\n"
+        "<|im_start|>assistant\n"
+    )
+
+
+def main() -> None:
+    args = parse_args()
+
+    if not os.path.exists(args.image):
+        raise FileNotFoundError(f"Image file not found: {args.image}")
+
+    pil_image = Image.open(args.image)
+    image_data = convert_image_mode(pil_image, "RGB")
+    prompt = build_prompt(args.system, args.question)
+
+    omni = Omni(
+        model=args.model,
+        deploy_config=args.deploy_config,
+        trust_remote_code=args.trust_remote_code,
+        enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
+    )
+    try:
+        sp = SamplingParams(
+            temperature=float(args.temperature),
+            top_p=float(args.top_p),
+            top_k=-1,
+            max_tokens=int(args.max_tokens),
+            seed=int(args.seed),
+            detokenize=True,
+        )
+        # NOTE: omni.generate() returns a Generator[OmniRequestOutput, None, None].
+        # Consume it inside the try block so the worker isn't closed early.
+        outputs = list(
+            omni.generate(
+                [
+                    {
+                        "prompt": prompt,
+                        "multi_modal_data": {"image": image_data},
+                        "additional_information": {"omni_task": ["chat"]},
+                    }
+                ],
+                [sp],
+            )
+        )
+    finally:
+        omni.close()
+
+    lines: list[str] = []
+    for stage_outputs in outputs:
+        ro = stage_outputs
+        text = ro.outputs[0].text if getattr(ro, "outputs", None) else str(ro)
+        lines.append(f"request_id: {getattr(ro, 'request_id', 'unknown')}\n")
+        lines.append("answer:\n")
+        lines.append(text.strip() + "\n")
+        lines.append("\n")
+
+    print("\n".join(lines))
+
+
+if __name__ == "__main__":
+    main()
@@ -599,6 +599,17 @@ def main():
            if args.seed is not None and hasattr(params, "seed"):
                params.seed = args.seed

+            # MammothModa2's AR stage emits one visual token per grid cell,
+            # one EOL token per row, and one final look-ahead token whose hidden
+            # state is unavailable. Size the first stage from the prompt metadata
+            # instead of SamplingParams' default of 16 tokens.
+            prompt_info = prompt_dict.get("additional_information", {})
+            if idx == 0 and prompt_info.get("omni_task") == ["t2i"]:
+                ar_width = int(prompt_info.get("ar_width", [0])[0])
+                ar_height = int(prompt_info.get("ar_height", [0])[0])
+                if ar_width > 0 and ar_height > 0:
+                    params.max_tokens = ar_height * (ar_width + 1) + 1
+
    if not diffusion_replaced and len(sampling_params_list) == 1:
        sampling_params_list = [diffusion_params]

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

 import argparse
 import json
@@ -174,6 +174,13 @@ def build_text_to_video_prompt(prompt: str, negative_prompt: str | None) -> dict
    return result


+def _validate_video_output_type(output_type: str) -> None:
+    if output_type not in {"image", "video"}:
+        raise ValueError(
+            f"Unexpected output type '{output_type}', expected 'video' or legacy 'image' for video generation."
+        )
+
+
 def _normalize_float_tensor(tensor: torch.Tensor, source_range: str) -> torch.Tensor:
    """Normalize decoded frames according to the pipeline's output contract."""
    if not tensor.is_floating_point():
@@ -620,10 +627,7 @@ def main():
        frames = frames[0] if frames else None

    if isinstance(frames, OmniRequestOutput):
-        if frames.final_output_type != "image":
-            raise ValueError(
-                f"Unexpected output type '{frames.final_output_type}', expected 'image' for video generation."
-            )
+        _validate_video_output_type(frames.final_output_type)
        if frames.multimodal_output and "audio" in frames.multimodal_output:
            audio = frames.multimodal_output["audio"]
            audio_sample_rate = frames.multimodal_output.get("audio_sample_rate", audio_sample_rate)
@@ -10,10 +10,12 @@ Generate text from text or image inputs with vLLM-Omni's shared offline entrypoi
 | --- | --- | --- | --- |
 | `ByteDance-Seed/BAGEL-7B-MoT` | Yes | Yes | `vllm_omni/deploy/bagel.yaml` |
 | `tencent/HunyuanImage-3.0-Instruct` | Yes | Yes | `vllm_omni/deploy/hunyuan_image3_ar.yaml` |
+| `bytedance-research/MammothModa2-Preview` | Yes | Yes | `vllm_omni/deploy/mammoth_moda2_ar.yaml` |
+| `bytedance-research/MammothModa2-Dev` | Yes | Yes | `vllm_omni/deploy/mammoth_moda2_ar.yaml` |

 The script recognizes these model families from `config.json`, applies the
-model-specific prompt format, and selects an AR-only deploy for HunyuanImage-3.
-BAGEL uses its registered default deploy.
+model-specific prompt format, and selects an AR-only deploy for HunyuanImage-3
+and MammothModa2. BAGEL uses its registered default deploy.

 ## Text-To-Text

@@ -33,6 +35,22 @@ python examples/offline_inference/x_to_text/x_to_text.py \
  --prompt "Explain multimodal inference in three concise sentences."
 ```

+### MammothModa2
+
+```bash
+python examples/offline_inference/x_to_text/x_to_text.py \
+  --model bytedance-research/MammothModa2-Preview \
+  --prompt "Explain multimodal inference in three concise sentences."
+```
+
+The same command supports the Dev checkpoint:
+
+```bash
+python examples/offline_inference/x_to_text/x_to_text.py \
+  --model bytedance-research/MammothModa2-Dev \
+  --prompt "Explain multimodal inference in three concise sentences."
+```
+
 ## Image-To-Text

 Pass one image with `--image`. The default question can be replaced with any
@@ -54,6 +72,16 @@ python examples/offline_inference/x_to_text/x_to_text.py \
  --prompt "Describe this image in detail."
 ```

+```bash
+python examples/offline_inference/x_to_text/x_to_text.py \
+  --model bytedance-research/MammothModa2-Preview \
+  --image image.png \
+  --prompt "Describe this image in detail."
+```
+
+Use `bytedance-research/MammothModa2-Dev` in the same command to run I2T with
+the Dev checkpoint.
+
 ## Key Arguments

 | Argument | Default | Description |
@@ -74,4 +102,4 @@ python examples/offline_inference/x_to_text/x_to_text.py \

 The committed HunyuanImage-3 AR-only default uses four GPUs with TP=4. To use a
 different topology, pass a custom AR-only YAML through `--deploy-config`.
-BAGEL uses its repository deploy default unless overridden.
+BAGEL and MammothModa2 use their repository deploy defaults unless overridden.
@@ -15,6 +15,7 @@ from vllm_omni.model_extras import build_x_to_text_prompt, get_x_to_text_model_f

 _REPO_ROOT = Path(__file__).resolve().parents[3]
 _HUNYUAN_AR_DEPLOY_CONFIG = _REPO_ROOT / "vllm_omni" / "deploy" / "hunyuan_image3_ar.yaml"
+_MAMMOTH_AR_DEPLOY_CONFIG = _REPO_ROOT / "vllm_omni" / "deploy" / "mammoth_moda2_ar.yaml"


 def parse_args() -> argparse.Namespace:
@@ -61,7 +62,7 @@ def main() -> None:
    omni_kwargs: dict[str, Any] = {
        "model": args.model,
        "mode": "image-to-text" if image is not None else "text-to-text",
-        "trust_remote_code": args.trust_remote_code or family == "hunyuan_image3",
+        "trust_remote_code": args.trust_remote_code or family in {"hunyuan_image3", "mammoth_moda2"},
        "log_stats": args.log_stats,
    }
    if args.enforce_eager is not None:
@@ -70,6 +71,8 @@ def main() -> None:
        omni_kwargs["deploy_config"] = args.deploy_config
    elif family == "hunyuan_image3":
        omni_kwargs["deploy_config"] = str(_HUNYUAN_AR_DEPLOY_CONFIG)
+    elif family == "mammoth_moda2":
+        omni_kwargs["deploy_config"] = str(_MAMMOTH_AR_DEPLOY_CONFIG)

    omni = Omni(**omni_kwargs)
    try:
@@ -1,108 +0,0 @@
-import argparse
-import fcntl
-import os
-import site
-import subprocess
-import tempfile
-import time
-from pathlib import Path
-
-from huggingface_hub import snapshot_download
-
-DEPENDENCY_REPO = "https://github.com/bytedance/DreamID-V.git"
-DEPENDENCY_BRANCH = "omni"
-CACHE_DIR = Path(tempfile.gettempdir()) / "vllm-omni-dependency"
-LOCK_FILE = CACHE_DIR / ".install.lock"
-DEPENDENCY_DIR = CACHE_DIR / "DreamID-Omni"
-
-
-def download_dependency():
-    CACHE_DIR.mkdir(parents=True, exist_ok=True)
-
-    with open(LOCK_FILE, "w") as f:
-        fcntl.flock(f, fcntl.LOCK_EX)
-        if not DEPENDENCY_DIR.exists():
-            print(f"Downloading DreamID-Omni to {DEPENDENCY_DIR} ...")
-            subprocess.run(
-                ["git", "clone", "--depth", "1", DEPENDENCY_REPO, "--branch", DEPENDENCY_BRANCH, str(DEPENDENCY_DIR)],
-                check=True,
-            )
-            print("Download finished.")
-        fcntl.flock(f, fcntl.LOCK_UN)
-
-    # write .pth to site-packages
-    site_packages = Path(site.getsitepackages()[0])
-    pth_file = site_packages / "vllm_omni_dependency.pth"
-    pth_file.write_text(str(DEPENDENCY_DIR))
-    print(f"Added {DEPENDENCY_DIR} to site-packages via {pth_file}")
-
-
-def timed_download(repo_id: str, local_dir: str, allow_patterns: list | None = None):
-    """Download files from HF repo and log time + destination."""
-    if os.path.exists(local_dir):
-        print(f"Directory {local_dir} already exists. Skipping download.")
-        return
-    print(f"Starting download from {repo_id} into {local_dir}")
-    start_time = time.time()
-
-    snapshot_download(
-        repo_id=repo_id,
-        local_dir=local_dir,
-        local_dir_use_symlinks=False,
-        allow_patterns=allow_patterns,
-    )
-
-    elapsed = time.time() - start_time
-    print(f"✅ Finished downloading {repo_id} in {elapsed:.2f} seconds. Files saved at: {local_dir}")
-
-
-def main(output_dir: str):
-    # Wan2.2
-    wan_dir = os.path.join(output_dir, "Wan2.2-TI2V-5B")
-    timed_download(
-        repo_id="Wan-AI/Wan2.2-TI2V-5B",
-        local_dir=wan_dir,
-        allow_patterns=["google/*", "models_t5_umt5-xxl-enc-bf16.pth", "Wan2.2_VAE.pth"],
-    )
-
-    # MMAudio
-    mm_audio_dir = os.path.join(output_dir, "MMAudio")
-    timed_download(
-        repo_id="hkchengrex/MMAudio",
-        local_dir=mm_audio_dir,
-        allow_patterns=["ext_weights/best_netG.pt", "ext_weights/v1-16.pth"],
-    )
-
-    dreamid_dir = os.path.join(output_dir, "DreamID-Omni")
-
-    timed_download(repo_id="XuGuo699/DreamID-Omni", local_dir=dreamid_dir)
-
-    # Now we construct the config file
-    import json
-
-    data = {
-        "_class_name": "DreamIDOmniPipeline",
-    }
-
-    with open(os.path.join(output_dir, "model_index.json"), "w", encoding="utf-8") as f:
-        json.dump(data, f, indent=2)
-
-    print(f"model_index.json created at {os.path.join(output_dir, 'model_index.json')}")
-
-    transformer_dir = os.path.join(output_dir, "transformer")
-    os.makedirs(transformer_dir, exist_ok=True)
-    with open(os.path.join(transformer_dir, "config.json"), "w", encoding="utf-8") as f:
-        json.dump({"fusion": "DreamID-Omni/dreamid_omni.safetensors"}, f)
-    print(f"transformer/config.json created at {os.path.join(transformer_dir, 'config.json')}")
-
-    # now we download the dependency code
-    download_dependency()
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Download models from Hugging Face")
-    parser.add_argument(
-        "--output-dir", type=str, default="./dreamid_omni", help="Base directory to save downloaded models"
-    )
-    args = parser.parse_args()
-    main(args.output_dir)
@@ -1,163 +0,0 @@
-# X-To-Video-Audio
-
-The `DreamID-Omni` pipeline generates short videos from text, image and video.
-
-## Local CLI Usage
-### Download the Model locally
-Since DreamID-Omni combine multiple models, and without any config, so we need to download them locally.
-
-```bash
-python download_dreamid_omni.py --output-dir ./dreamid_omni
-```
-After download, the model directory will look like this:
-
-```
-dreamid_omni/
-├── DreamID-Omni/
-│   ├── dreamid_omni.safetensors
-├── MMAudio/
-│   ├── ext_weights/
-│   │   ├── best_netG.pt
-│   │   ├── v1-16.pth
-├── Wan2.2-TI2V-5B/
-│   ├── google/*
-│   ├── models_t5_umt5-xxl-enc-bf16.pth
-│   ├── Wan2.2_VAE.pth
-│
-├── model_index.json
-└── transformer/
-    └── config.json   # create by download_dreamid_omni.py
-```
-
-### Run the Inference
-```python
-python x_to_video_audio.py \
-  --model /path/to/dreamid_omni \
-  --prompt "Two people walking together and singing happily" \
-  --image-path ./example0.png ./example1.png \
-  --audio-path ./example0.wav ./example1.wav \
-  --video-negative-prompt "jitter, bad hands, blur, distortion" \
-  --audio-negative-prompt "robotic, muffled, echo, distorted" \
-  --cfg-parallel-size 2 \
-  --num-inference-steps 45 \
-  --height 704 \
-  --width 1280 \
-  --output out_dreamid_omni_twoip.mp4
-```
-In the current test scenario (2 images + 2 audio inputs), the VRAM requirement is 72GB, regardless of whether cfg-parallel is enabled or disabled.
-The VRAM usage can be reduced by enabling CPU offload via --enable-cpu-offload.
-For multi-GPU memory reduction on the fused DreamID-Omni transformer, you can also enable HSDP:
-
-```python
-python x_to_video_audio.py \
-  --model /path/to/dreamid_omni \
-  --prompt "Two people walking together and singing happily" \
-  --image-path ./example0.png ./example1.png \
-  --audio-path ./example0.wav ./example1.wav \
-  --video-negative-prompt "jitter, bad hands, blur, distortion" \
-  --audio-negative-prompt "robotic, muffled, echo, distorted" \
-  --cfg-parallel-size 2 \
-  --num-inference-steps 45 \
-  --height 704 \
-  --width 1280 \
-  --use-hsdp \
-  --hsdp-shard-size 2 \
-  --output out_dreamid_omni_twoip.mp4
-```
-
-
-You could take reference images/audios from the test cases in the official repo: https://github.com/Guoxu1233/DreamID-Omni
-
-For example, single IP ref resources can be found under https://github.com/Guoxu1233/DreamID-Omni/tree/main/test_case/oneip, you could download them correspondingly to your local and use them for testing.
-
-```python
-# Example usage for oneip, ref media from the official repo DreamID-Omni
-python x_to_video_audio.py \
-  --model /path/to/dreamid_omni \
-  --prompt "<img1>: In the frame, a woman with black long hair is identified as <sub1>.\n**Overall Environment/Scene**: A lively open-kitchen café at night; stove flames flare, steam rises, and warm pendant lights swing slightly as staff move behind her. The shot is an upper-body close-up.\n**Main Characters/Subjects Appearance**: <sub1> is a young woman with thick dark wavy hair and a side part. She wears a fitted black top under a light apron, a thin gold chain necklace, and small stud earrings.\n**Main Characters/Subjects Actions**: <sub1> tastes the sauce with a spoon, then turns her face toward the camera while still holding the spoon, her expression shifting from focused to conflicted.\n<sub1> maintains eye contact, swallows as if choosing her words, and says, <S>I keep telling myself I’m fine,but some nights it feels like I’m just performing calm.<E>" \
-  --image-path 9.png \
-  --audio-path 9.wav \
-  --video-negative-prompt "jitter, bad hands, blur, distortion" \
-  --audio-negative-prompt "robotic, muffled, echo, distorted" \
-  --cfg-parallel-size 2 \
-  --num-inference-steps 45 \
-  --height 704 \
-  --width 1280 \
-  --output out_dreamid_omni_oneip.mp4
-```
-
-
-## MagiHuman (`--model-type magi-human`)
-
-MagiHuman is a text → video+audio model with a DiT MoE backbone and a ~9B-param
-T5Gemma text encoder. A detailed text prompt is the only required input; an optional
-image and/or audio file may be supplied for conditioning. Natively supports Tensor
-Parallelism. For an 80GB node, `--tensor-parallel-size 4` is recommended to shard
-the MoE weights and the text encoder.
-
-> Install [MagiCompiler](https://github.com/SandAI-org/MagiCompiler) for correct
-> attention-kernel behaviour (the pipeline otherwise falls back to stubs).
-
-Text-only generation:
-
-```bash
-python x_to_video_audio.py \
-  --model-type magi-human \
-  --model /path/to/daVinci-MagiHuman \
-  --prompt "A young woman with long, wavy golden blonde hair... <dialogue and background sound>" \
-  --tensor-parallel-size 4 \
-  --height 256 --width 448 \
-  --num-inference-steps 8 \
-  --seed 52 \
-  --extra-body '{"seconds": 5, "sr_height": 1080, "sr_width": 1920, "sr_num_inference_steps": 5}' \
-  --output output_magihuman.mp4
-```
-
-With optional image and audio conditioning:
-
-```bash
-python x_to_video_audio.py \
-  --model-type magi-human \
-  --model /path/to/daVinci-MagiHuman \
-  --prompt "A young woman..." \
-  --tensor-parallel-size 4 \
-  --height 256 --width 448 \
-  --num-inference-steps 8 \
-  --seed 52 \
-  --extra-body '{"image_path": "/path/to/ref.jpg", "audio_path": "/path/to/ref.wav", "sr_height": 1080, "sr_width": 1920, "sr_num_inference_steps": 5}' \
-  --output output_magihuman.mp4
-```
-
-MagiHuman-specific arguments are passed as a JSON dict via `--extra-body` (declared
-in `vllm_omni/model_extras/magi_human.py`, routed via `extra_args`):
-
- `seconds`: output duration in seconds (default 10; ignored when `audio_path` is set,
-  because the audio length then determines the number of frames).
- `audio_path`: path to an audio file for audio-to-video conditioning. When provided,
-  the audio drives the frame count and is encoded as a latent condition into the DiT
-  (`is_a2v` mode). Omit for pure text-to-video+audio generation.
- `image_path`: path to an image file for visual conditioning. Applied at both the
-  base-resolution (BR) and super-resolution (SR) stages when SR is enabled.
- `sr_height` / `sr_width`: super-resolution output resolution. SR stage is skipped
-  when these are omitted.
- `sr_num_inference_steps`: denoising steps for the SR stage.
-
-## DreamID-Omni arguments
-
-Key arguments:
- `--prompt`: text description (string).
- `--model`: path to the model local directory.
- `--height/--width`: output resolution (defaults 704 * 1024).
- `--image-path`: path to the input image list.
- `--audio-path`: path to the input audio list, indicate the timbre of the output video.
- `--cfg-parallel-size`: number of parallel cfg parallel (defaults 1).
- `--use-hsdp`: enable HSDP weight sharding for DreamID-Omni fused blocks.
- `--hsdp-shard-size`: number of GPUs used for HSDP sharding.
- `--hsdp-replicate-size`: number of HSDP replica groups.
- `--num-inference-steps`: number of denoising steps (defaults 45).
- `--video-negative-prompt`: negative prompt for video generation.
- `--audio-negative-prompt`: negative prompt for audio generation.
- `--enable-cpu-offload`: enable CPU offload (defaults False).
- `--cache-backend`: enable `cache_dit` for acceleration.
- `--quantization`: online (dynamic) quantization method — `fp8` or `int8`.
-  (VAEs, the T5 text encoder, norms and modulation stay bf16.)
@@ -1,279 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-import argparse
-import json
-import re
-import time
-from typing import Any
-
-import numpy as np
-from PIL import Image
-from vllm.multimodal.media.audio import load_audio
-
-from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes
-from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-
-IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".gif"}
-AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".m4a", ".aac", ".ogg"}
-
-
-def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Offline inference for X -> video+audio models.")
-    parser.add_argument("--model", required=True, help="Model ckpt root directory.")
-    parser.add_argument(
-        "--model-type",
-        default="dreamid-omni",
-        choices=["dreamid-omni", "magi-human"],
-        help="Model type.",
-    )
-    parser.add_argument("--prompt", default=None, help="Text prompt.")
-
-    parser.add_argument("--image-path", type=str, nargs="+", help="list of image-path")
-    parser.add_argument("--audio-path", type=str, nargs="+", help="list of audio-path")
-    parser.add_argument("--prompt-file", type=str, default=None, help="Text prompt in json format.")
-    parser.add_argument(
-        "--extra-body",
-        type=str,
-        default=None,
-        help="[magi-human] JSON dict of model-specific extra params (declared in vllm_omni/model_extras/), "
-        'merged into sampling extra_args. Example: \'{"image_path": "/path/to/img.jpg", "seconds": 5}\'.',
-    )
-
-    parser.add_argument("--height", type=int, default=704, help="Video height.")
-    parser.add_argument("--width", type=int, default=1280, help="Video width.")
-    parser.add_argument("--num-inference-steps", type=int, default=45, help="Sampling steps.")
-    parser.add_argument("--solver-name", default="unipc", help="Solver name: unipc|dpm++|euler.")
-    parser.add_argument("--shift", type=float, default=5.0, help="Scheduler shift.")
-    parser.add_argument("--seed", type=int, default=103, help="Random seed for reproducible generation.")
-    parser.add_argument(
-        "--cfg-parallel-size",
-        type=int,
-        default=1,
-        choices=[1, 2, 3, 4],
-        help="Number of GPUs used for classifier free guidance parallel size (max 4 branches).",
-    )
-    parser.add_argument(
-        "--tensor-parallel-size",
-        type=int,
-        default=1,
-        help=("Number of GPUs used for tensor parallelism (TP) inside the DiT."),
-    )
-    parser.add_argument(
-        "--video-negative-prompt",
-        default="jitter, bad hands, blur, distortion",
-        help="Negative prompt for video.",
-    )
-    parser.add_argument(
-        "--audio-negative-prompt",
-        default="robotic, muffled, echo, distorted",
-        help="Negative prompt for audio.",
-    )
-    parser.add_argument("--output", default="dreamid_output.mp4", help="Output video path.")
-    parser.add_argument(
-        "--quantization",
-        type=str,
-        default=None,
-        choices=["fp8", "int8"],
-        help=("Online (dynamic) quantization method for the DreamID-Omni transformer. "),
-    )
-    parser.add_argument(
-        "--enable-cpu-offload",
-        action="store_true",
-        default=False,
-        help="Enable CPU offloading for diffusion models.",
-    )
-    parser.add_argument(
-        "--enable-layerwise-offload",
-        action="store_true",
-        help="Enable layerwise (blockwise) offloading on DiT modules.",
-    )
-    parser.add_argument("--cache-backend", type=str, default=None, choices=["cache_dit"], help="Cache backend.")
-    parser.add_argument(
-        "--use-hsdp",
-        action="store_true",
-        help="Enable HSDP for DreamID-Omni fused transformer blocks.",
-    )
-    parser.add_argument(
-        "--hsdp-shard-size",
-        type=int,
-        default=1,
-        help="Number of GPUs used for HSDP sharding when HSDP is enabled.",
-    )
-    parser.add_argument(
-        "--hsdp-replicate-size",
-        type=int,
-        default=1,
-        help="Number of HSDP replica groups. Default 1 means pure sharding.",
-    )
-    return parser.parse_args()
-
-
-def load_image_and_audio(image_paths, audio_paths):
-    image = []
-    audio = []
-
-    for path in image_paths:
-        with Image.open(path) as img:
-            img = img.convert("RGB")
-            image.append(img)
-
-    for path in audio_paths:
-        audio_array, sr = load_audio(path, sr=16000)
-        audio_array = audio_array[int(sr * 1) : int(sr * 3)]
-        audio.append(audio_array)
-    return image, audio
-
-
-def _extract_peak_memory_mb(result: Any) -> float:
-    """Pull worker-reported peak VRAM (MiB) generation result.
-
-    Mirrors vllm_omni/entrypoints/openai/serving_video.py:_extract_peak_memory_mb.
-    """
-    if isinstance(result, list):
-        result = result[0] if result else None
-    if result is None:
-        return 0.0
-    val = getattr(result, "peak_memory_mb", 0.0)
-    if not val:
-        inner = result
-        if isinstance(inner, list):
-            inner = inner[0] if inner else None
-        val = getattr(inner, "peak_memory_mb", 0.0)
-    try:
-        return float(val or 0.0)
-    except (TypeError, ValueError):
-        return 0.0
-
-
-def main() -> None:
-    args = parse_args()
-    if args.prompt is None and args.prompt_file is None:
-        raise ValueError("Either --prompt or --prompt-file must be provided.")
-
-    text_prompt = args.prompt
-    if args.prompt_file:
-        with open(args.prompt_file) as f:
-            text_prompt = json.load(f)
-            text_prompt = re.sub(
-                r"\[SPEAKER_TIMESTAMPS_START\].*?\[SPEAKER_TIMESTAMPS_END\]", "", text_prompt, flags=re.DOTALL
-            ).strip()
-            text_prompt = re.sub(
-                r"\[AUDIO_DESCRIPTION_START].*?\[AUDIO_DESCRIPTION_END]", "", text_prompt, flags=re.DOTALL
-            ).strip()
-            text_prompt = re.sub(r"\[[A-Z_]+\]", "", text_prompt)
-            text_prompt = re.sub(r"\n\s*\n", "\n", text_prompt).strip()
-
-    if args.model_type == "magi-human":
-        prompt = text_prompt
-        extra_args = json.loads(args.extra_body) if args.extra_body else {}
-        sampling_params = OmniDiffusionSamplingParams(
-            height=args.height,
-            width=args.width,
-            num_inference_steps=args.num_inference_steps,
-            seed=args.seed,
-            extra_args=extra_args,
-        )
-    else:
-        image, audio = load_image_and_audio(args.image_path, args.audio_path)
-
-        prompt = {
-            "prompt": text_prompt,
-            "video_negative_prompt": args.video_negative_prompt,
-            "audio_negative_prompt": args.audio_negative_prompt,
-            "multi_modal_data": {"image": image, "audio": audio},
-        }
-
-        sampling_params = OmniDiffusionSamplingParams(
-            height=args.height,
-            width=args.width,
-            num_inference_steps=args.num_inference_steps,
-            seed=args.seed,
-            extra_args={
-                "solver_name": args.solver_name,
-                "shift": args.shift,
-            },
-        )
-
-    cache_config = None
-    if args.cache_backend == "cache_dit":
-        cache_config = {
-            "Fn_compute_blocks": 1,
-            "Bn_compute_blocks": 0,
-            "max_warmup_steps": 4,
-            "max_cached_steps": 20,
-            "residual_diff_threshold": 0.24,
-            "max_continuous_cached_steps": 3,
-            "enable_taylorseer": False,
-            "taylorseer_order": 1,
-            "scm_steps_mask_policy": None,
-            "scm_steps_policy": "dynamic",
-        }
-
-    omni_kwargs = dict(
-        model=args.model,
-        cfg_parallel_size=args.cfg_parallel_size,
-        tensor_parallel_size=args.tensor_parallel_size,
-        use_hsdp=args.use_hsdp,
-        hsdp_shard_size=args.hsdp_shard_size,
-        hsdp_replicate_size=args.hsdp_replicate_size,
-        model_type=args.model_type,
-        enable_cpu_offload=args.enable_cpu_offload,
-        enable_layerwise_offload=args.enable_layerwise_offload,
-        cache_backend=args.cache_backend,
-        cache_config=cache_config,
-    )
-    if args.quantization is not None:
-        omni_kwargs["quantization"] = args.quantization
-    omni = Omni(**omni_kwargs)
-    start = time.perf_counter()
-    outputs = omni.generate(prompt, sampling_params)
-    elapsed = time.perf_counter() - start
-
-    if not outputs:
-        raise RuntimeError("No output returned from the model.")
-    result = outputs[0]
-    if not result.images:
-        raise RuntimeError("No video frames found in model output.")
-    generated_video = result.images[0]
-    mm = result.multimodal_output or {}
-    generated_audio = mm.get("audio")
-
-    if args.model_type == "magi-human":
-        # MagiHuman returns frames already as (F, H, W, C) uint8, ready to mux.
-        fps = float(mm.get("fps", 25))
-        sample_rate = int(mm.get("audio_sample_rate", 24000))
-        frames = generated_video
-    else:
-        fps = float(mm.get("fps", 24))
-        sample_rate = int(mm.get("audio_sample_rate", 16000))
-        # DreamID-Omni returns video as (C, F, H, W) float32 in [-1, 1].
-        # mux_video_audio_bytes expects (F, H, W, C) uint8.
-        if not isinstance(generated_video, np.ndarray) or generated_video.ndim != 4:
-            raise RuntimeError(f"Unexpected video shape: {getattr(generated_video, 'shape', None)}")
-        frames = generated_video.transpose(1, 2, 3, 0)
-        frames = (np.clip((frames + 1.0) / 2.0, 0.0, 1.0) * 255.0).round().astype(np.uint8)
-
-    audio_np = None
-    if generated_audio is not None:
-        audio_np = np.squeeze(np.asarray(generated_audio)).astype(np.float32)
-
-    output_path = args.output
-    video_bytes = mux_video_audio_bytes(
-        frames,
-        audio_np,
-        fps=float(fps),
-        audio_sample_rate=sample_rate,
-    )
-    with open(output_path, "wb") as f:
-        f.write(video_bytes)
-    print(f"Saved generated video to {output_path}")
-    print(f"Total time: {elapsed:.2f}s")
-    peak_mb = _extract_peak_memory_mb(outputs)
-    if peak_mb:
-        print(f"Worker peak GPU memory (reserved): {peak_mb:.2f} MiB ({peak_mb / 1024:.2f} GiB)")
-
-
-if __name__ == "__main__":
-    main()
@@ -32,12 +32,14 @@ GPU 0 (90%) and colocates the Talker (55%) and Code2Wav (35%) on GPU 1. That
 profile admits at most four concurrent sequences per stage.

 | deploy config | GPUs | Notes |
-|---|---|---|
+| --- | --- | --- |
 | `minicpmo_4_5.yaml` (default) | 1 | Memory-constrained compatibility layout. |
 | `minicpmo_4_5_2gpu.yaml` | 2 | Recommended continuous-batching layout; Talker and Code2Wav share GPU 1. |
 | `minicpmo_4_5_3gpu.yaml` | 3 | One GPU per stage. |
 | `minicpmo_4_5_8x4090.yaml` | 8 | Full 8x4090 layout. |
-| `minicpmo_4_5_duplex.yaml` | 1 | Experimental native full-duplex overlay. |
+
+Every profile sets `session_mode: duplex`, so native full-duplex is served
+from the same process as `/v1/chat/completions`.

 The split pipeline preserves native-duplex epoch/turn identity, segment text,
 turn completion, reference voice, and terminal-audio metadata through
@@ -47,7 +49,7 @@ scenario below for live barge-in validation on the target GPU.
 Default:

 ```bash
-vllm-omni serve openbmb/MiniCPM-o-4_5 \
+vllm serve openbmb/MiniCPM-o-4_5 \
    --omni \
    --deploy-config vllm_omni/deploy/minicpmo_4_5.yaml \
    --trust-remote-code \
@@ -55,8 +57,8 @@ vllm-omni serve openbmb/MiniCPM-o-4_5 \
 ```

 For local ModelScope checkpoints, replace `openbmb/MiniCPM-o-4_5` with the
-checkpoint path. To start the experimental native duplex backend, use
-`vllm_omni/deploy/minicpmo_4_5_duplex.yaml`.
+checkpoint path. For native full-duplex, connect `/v1/realtime?duplex=1` on
+this same server.

 ### Per-stage overrides

@@ -145,7 +147,7 @@ assistant template, so it is not an apples-to-apples accuracy run.

 ## Run the Realtime duplex CLI demo

-After the duplex backend is running, stream one WAV through the Realtime
+After the server is running, stream one WAV through the Realtime
 WebSocket endpoint:

 ```bash
@@ -157,6 +159,33 @@ python examples/online_serving/minicpmo/realtime_duplex_demo.py \
    --output-dir /tmp/minicpmo_realtime_duplex_demo
 ```

+Video input uses the same session. PyAV demuxes JPEG frames at `--video-fps`
+(default 1.0) and vLLM `load_audio` extracts a 16 kHz mono WAV unless
+`--input-wav` overrides the soundtrack. Frames keep their capture size
+(`--frame-max-side 0`); the server `process_image` normalizes at 448.
+
+`--stack-frames N` raises the visual refresh rate the way official duplex does:
+each 1 s unit also samples the `N-1` sub-frames captured inside it and tiles
+them into a single composite image sent next to that unit's base frame. The
+audio cadence never changes — a unit is always one second — and the wire always
+carries 2 images per unit however large `N` is, because the sub-frames share one
+composite. Official uses 5 for high refresh rate mode:
+
+```bash
+python examples/online_serving/minicpmo/realtime_duplex_demo.py \
+    --url ws://localhost:8099/v1/realtime?duplex=1 \
+    --model openbmb/MiniCPM-o-4_5 \
+    --input-video /path/to/clip.mp4 \
+    --stack-frames 5 \
+    --ref-audio /path/to/MiniCPM-o-Demo/assets/ref_audio/ref_minicpm_signature.wav \
+    --output-dir /tmp/minicpmo_realtime_duplex_video_demo
+```
+
+Detail inside a composite is capped by `scale_resolution=448` at
+`max_slice_nums=1`: official suggests HD slicing (`max_slice_nums=[2, 1]`) for
+stacked frames, which the duplex adapter does not implement yet. Reading small
+text or digits out of a wide scene is limited by that, not by frame timing.
+
 ## Open the experimental browser client

 The browser UI serves the page and proxies the same-origin Realtime WebSocket to
@@ -190,6 +219,64 @@ stronger `response-required` mode is diagnostic: it requires a purpose-built
 two-response WAV, its `--input-sha256`, and an
 `--expect-second-response-substring` value.

+## Run Omni-DuplexEval
+
+Omni-DuplexEval generation uses the vLLM-Omni native MiniCPM-o duplex
+endpoint. Evaluation uses a separate OpenAI-compatible multimodal judge
+served from the same vLLM-Omni environment. This validates the vLLM-Omni
+duplex client and local judge integration; it does not reproduce the paper's
+original MiniCPM-o inference implementation.
+
+Start the duplex generation server:
+
+```bash
+vllm serve openbmb/MiniCPM-o-4_5 --omni --trust-remote-code \
+    --deploy-config vllm_omni/deploy/minicpmo_4_5_duplex.yaml \
+    --served-model-name openbmb/MiniCPM-o-4_5 \
+    --host 0.0.0.0 --port 8099
+```
+
+Start a separate multimodal judge. The allowed path must contain any local
+videos passed with `--judge-video-mode video_url`:
+
+```bash
+vllm serve Qwen/Qwen2.5-VL-7B-Instruct \
+    --served-model-name Qwen/Qwen2.5-VL-7B-Instruct \
+    --allowed-local-media-path /data/omni-duplex-eval \
+    --host 0.0.0.0 --port 8000
+```
+
+Generate, evaluate, and summarize:
+
+```bash
+vllm bench omni-duplex-eval --omni generate \
+    --url ws://127.0.0.1:8099/v1/realtime?duplex=1 \
+    --model openbmb/MiniCPM-o-4_5 \
+    --ref-audio /data/ref.wav \
+    --dataset Hothan/Omni-DuplexEval \
+    --concurrency 2 \
+    --response-root /data/omni-duplex-eval/responses
+
+vllm bench omni-duplex-eval --omni evaluate \
+    --dataset Hothan/Omni-DuplexEval \
+    --response-root /data/omni-duplex-eval/responses \
+    --score-root /data/omni-duplex-eval/scores \
+    --judge-base-url http://127.0.0.1:8000 \
+    --judge-model Qwen/Qwen2.5-VL-7B-Instruct \
+    --judge-video-mode video_url \
+    --eval-workers 4
+
+vllm bench omni-duplex-eval --omni summarize \
+    --score-root /data/omni-duplex-eval/scores
+```
+
+The dataset names such as `RTD_OCR` and `PR_correction` are Hugging Face
+splits, not dataset configurations. Pass one with `--split`, or omit it to
+run all nine splits. `--limit 1` is useful for a smoke test. Realtime
+generation records `clock=media`; artifacts generated with
+`--pace as-fast-as-possible` record `clock=invalid` and evaluation rejects
+them unless `--allow-invalid-clock` is explicit.
+
 ## Related examples

 - [Offline MiniCPM-o inference](../../offline_inference/minicpmo/)
@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 """Minimal single-input MiniCPM-o 4.5 Realtime duplex demo.

 Run this after starting the duplex server. Strict lifecycle, overlap, and
@@ -9,7 +12,9 @@ from __future__ import annotations
 import argparse
 import asyncio
 import base64
+import io
 import json
+import math
 import sys
 import time
 from pathlib import Path
@@ -28,6 +33,17 @@ from vllm_omni.experimental.fullduplex.client import (  # noqa: E402
    wait_for,
    write_pcm16_wav,
 )
+from vllm_omni.experimental.fullduplex.client import chunk_period_ms as _chunk_period_ms  # noqa: E402
+from vllm_omni.experimental.fullduplex.client import (  # noqa: E402
+    has_residual_model_unit as _has_residual_model_unit,
+)
+from vllm_omni.experimental.fullduplex.client import (  # noqa: E402
+    reference_audio_data_url as _ref_audio_data_url,
+)
+from vllm_omni.experimental.fullduplex.video_stacking import (  # noqa: E402
+    concat_frames_b64,
+    unit_subframe_offsets,
+)


 class _StreamingOutputWriter:
@@ -133,31 +149,29 @@ def _latest_model_decision(
    return decision


-def _chunk_period_ms(events: list[dict[str, object]]) -> int:
-    for event in reversed(events):
-        session = event.get("session")
-        if not isinstance(session, dict):
-            continue
-        capabilities = session.get("capabilities")
-        if not isinstance(capabilities, dict):
-            continue
-        chunk_period_ms = capabilities.get("chunk_period_ms")
-        if isinstance(chunk_period_ms, int) and chunk_period_ms > 0:
-            return chunk_period_ms
-    return 1000
-
-
-def _has_residual_model_unit(pcm16: bytes, *, chunk_period_ms: int) -> bool:
-    unit_bytes = PCM16_SAMPLE_RATE * PCM16_BYTES_PER_SAMPLE * chunk_period_ms // 1000
-    return bool(unit_bytes > 0 and len(pcm16) % unit_bytes)
-
-
 def _response_in_progress(events: list[dict[str, object]]) -> bool:
    return sum(event.get("type") == "response.created" for event in events) > sum(
        event.get("type") == "response.done" for event in events
    )


+async def _commit_input_after_playback_checkpoint(client: RealtimeDuplexClient) -> float:
+    """Commit input only after preserving already-played assistant history.
+
+    Native duplex can finish several responses while the user is still
+    streaming.  Under the ``ack_only`` playback policy, acknowledging those
+    responses after the final input commit is intentionally rejected: it could
+    append an old assistant message after the newly committed user message.
+    Checkpoint playback first so the server commits each assistant item at its
+    original history position.  A second acknowledgement after the commit can
+    then safely advance response-local playback that drained in the meantime.
+    """
+    await client.acknowledge_playback()
+    commit_sent_at_s = time.monotonic()
+    await client.commit()
+    return commit_sent_at_s
+
+
 def _event_count_after(
    events: list[dict[str, object]],
    event_type: str,
@@ -168,21 +182,258 @@ def _event_count_after(
    return sum(event.get("type") == event_type for event in events[index + 1 :])


-def _ref_audio_data_url(path: str | None) -> str | None:
-    if path is None:
-        return None
-    ref_path = Path(path).expanduser()
-    return "data:audio/wav;base64," + base64.b64encode(ref_path.read_bytes()).decode("ascii")
+def _probe_video_frames_and_fps(video_path: Path) -> tuple[int, float]:
+    """Frame count and average FPS via PyAV (same approach as Daily-Omni)."""
+    import av
+
+    with av.open(str(video_path)) as container:
+        stream = container.streams.video[0]
+        avg_fps = float(stream.average_rate) if stream.average_rate else 0.0
+        num_frames = stream.frames or 0
+        if num_frames <= 0:
+            duration: float | None = None
+            if stream.duration is not None and stream.time_base is not None:
+                duration = float(stream.duration * stream.time_base)
+            elif container.duration is not None:
+                duration = container.duration / av.time_base
+            if duration and avg_fps:
+                num_frames = int(math.floor(duration * avg_fps))
+        if num_frames <= 0:
+            stream.thread_type = "AUTO"
+            num_frames = sum(1 for _ in container.decode(stream))
+    if num_frames <= 0:
+        raise ValueError(f"No decodable video frames in {video_path}")
+    return num_frames, avg_fps or 1.0
+
+
+def _upright(image, rotation: int):
+    """Apply a container display-matrix rotation, in degrees, to ``image``.
+
+    PyAV hands back the stored raster and leaves the display matrix for the
+    caller, unlike the ffmpeg CLI which autorotates. Phone clips routinely
+    carry +/-90, so skipping this feeds the model a sideways frame: arrows and
+    digits stop being readable and the model mirrors the motion it reports.
+    """
+    from PIL import Image
+
+    quarter_turns = {
+        90: Image.Transpose.ROTATE_90,
+        180: Image.Transpose.ROTATE_180,
+        270: Image.Transpose.ROTATE_270,
+    }
+    turn = quarter_turns.get(int(rotation) % 360)
+    return image.transpose(turn) if turn is not None else image
+
+
+def _decode_video_frames_rgb(video_path: Path, frame_idx: list[int]) -> list:
+    """Decode presentation-order ``frame_idx`` into upright RGB ``PIL.Image``s."""
+    import av
+
+    wanted = sorted({int(i) for i in frame_idx})
+    if not wanted:
+        return []
+
+    decoded: dict[int, object] = {}
+    with av.open(str(video_path)) as container:
+        stream = container.streams.video[0]
+        stream.thread_type = "AUTO"
+        cursor = 0
+        for position, frame in enumerate(container.decode(stream)):
+            if wanted[cursor] > position:
+                continue
+            image = _upright(frame.to_image(), getattr(frame, "rotation", 0) or 0)
+            while cursor < len(wanted) and wanted[cursor] <= position:
+                decoded[wanted[cursor]] = image
+                cursor += 1
+            if cursor >= len(wanted):
+                break
+
+    if not decoded:
+        raise ValueError(f"Decoded no frames from {video_path} for indices {wanted[:8]}")
+    last_image = decoded[max(decoded)]
+    return [decoded.get(int(i), last_image) for i in frame_idx]
+
+
+def _resize_max_side(image, max_side: int):
+    width, height = image.size
+    longest = max(width, height)
+    if longest <= max_side or max_side <= 0:
+        return image
+    scale = max_side / float(longest)
+    new_size = (max(1, int(round(width * scale))), max(1, int(round(height * scale))))
+    return image.resize(new_size)
+
+
+def _jpeg_bytes(image, *, quality: int = 95) -> bytes:
+    buf = io.BytesIO()
+    image.convert("RGB").save(buf, format="JPEG", quality=quality)
+    return buf.getvalue()
+
+
+def _to_pcm16_bytes(audio_np) -> bytes:
+    import numpy as np
+
+    arr = np.asarray(audio_np).reshape(-1)
+    if arr.dtype != np.int16:
+        arr = (np.clip(arr, -1.0, 1.0) * 32767.0).astype(np.int16)
+    return arr.tobytes()
+
+
+def _pcm16_wav_duration_s(path: Path) -> float:
+    pcm16 = read_pcm16_wav(path)
+    return len(pcm16) / float(PCM16_SAMPLE_RATE * PCM16_BYTES_PER_SAMPLE)
+
+
+def _extract_video_frames(
+    video_path: Path,
+    work_dir: Path,
+    *,
+    fps: float,
+    max_side: int,
+    duration_s: float,
+    stack_frames: int = 1,
+) -> tuple[list[str], list[str | None]]:
+    """Demux JPEG frames only. Frame count follows ``duration_s``, not soundtrack.
+
+    Returns the base frames (one per model unit) and the parallel stacked track.
+    With ``stack_frames=N`` each unit also samples its ``N-1`` interior
+    sub-frames and tiles them into one composite, which is how official duplex
+    raises the visual refresh rate while keeping one second of audio per unit.
+    A unit whose interior falls past the end of the clip stacks nothing.
+    """
+    num_frames, avg_fps = _probe_video_frames_and_fps(video_path)
+    sample_fps = max(float(fps), 1e-6)
+    unit_s = 1.0 / sample_fps
+    timeline_s = max(float(duration_s), unit_s)
+    num_samples = max(1, int(math.ceil(timeline_s * sample_fps - 1e-9)))
+    offsets = unit_subframe_offsets(stack_frames, unit_s=unit_s)
+
+    def frame_at(seconds: float) -> int:
+        # Official get_video_frame_audio_segments uses truncating int(), not
+        # round(), so 29.97 fps lands on the same presentation index.
+        return min(int(seconds * avg_fps), num_frames - 1)
+
+    base_idx = [frame_at(unit * unit_s) for unit in range(num_samples)]
+    sub_times = [
+        [unit * unit_s + offset for offset in offsets if unit * unit_s + offset < timeline_s]
+        for unit in range(num_samples)
+    ]
+    sub_idx = [frame_at(seconds) for times in sub_times for seconds in times]
+    images = _decode_video_frames_rgb(video_path, base_idx + sub_idx)
+    sub_images = iter(images[len(base_idx) :])
+
+    frame_dir = work_dir / "frames"
+    frame_dir.mkdir(parents=True, exist_ok=True)
+    for stale in frame_dir.glob("*.jpg"):
+        stale.unlink()
+    frames_b64: list[str] = []
+    stacked_b64: list[str | None] = []
+    for index, (image, times) in enumerate(zip(images[: len(base_idx)], sub_times), start=1):
+        jpeg = _jpeg_bytes(_resize_max_side(image, max_side))
+        (frame_dir / f"frame_{index:05d}.jpg").write_bytes(jpeg)
+        frames_b64.append(base64.b64encode(jpeg).decode("ascii"))
+        interior = [_resize_max_side(next(sub_images), max_side) for _ in times]
+        if not interior:
+            stacked_b64.append(None)
+            continue
+        composite = concat_frames_b64(interior)
+        (frame_dir / f"stack_{index:05d}.jpg").write_bytes(base64.b64decode(composite))
+        stacked_b64.append(composite)
+    return frames_b64, stacked_b64
+
+
+def _extract_video_soundtrack(video_path: Path, work_dir: Path) -> Path:
+    """Write a 16 kHz mono WAV from the video soundtrack."""
+    try:
+        from vllm.multimodal.media.audio import load_audio
+    except ImportError as exc:  # pragma: no cover
+        raise SystemExit("--input-video needs vLLM audio helpers (load_audio)") from exc
+
+    audio_np, sr = load_audio(str(video_path), sr=PCM16_SAMPLE_RATE, mono=True)
+    work_dir.mkdir(parents=True, exist_ok=True)
+    audio_path = work_dir / "input_16k.wav"
+    write_pcm16_wav(audio_path, _to_pcm16_bytes(audio_np), sample_rate_hz=int(sr))
+    return audio_path
+
+
+def _extract_video_input(
+    video_path: Path,
+    work_dir: Path,
+    *,
+    fps: float,
+    max_side: int,
+    stack_frames: int,
+) -> tuple[Path, list[str], list[str | None]]:
+    """Demux a video into duplex inputs: 16 kHz mono WAV plus JPEG frames.
+
+    Use this only when the video soundtrack is the audio source. When the
+    caller already has ``--input-wav``, extract frames against that WAV
+    duration instead so a silent clip cannot fail ``load_audio``.
+    """
+    audio_path = _extract_video_soundtrack(video_path, work_dir)
+    frames_b64, stacked_b64 = _extract_video_frames(
+        video_path,
+        work_dir,
+        fps=fps,
+        max_side=max_side,
+        duration_s=_pcm16_wav_duration_s(audio_path),
+        stack_frames=stack_frames,
+    )
+    return audio_path, frames_b64, stacked_b64
+
+
+def _resolve_duplex_av_inputs(
+    *,
+    input_wav: str | None,
+    input_video: str | None,
+    work_dir: Path,
+    fps: float,
+    max_side: int,
+    stack_frames: int = 1,
+) -> tuple[str, list[str], list[str | None]]:
+    """Return ``(wav_path, frames_b64, stacked_b64)``. External WAV wins over video audio."""
+    if input_video:
+        video_path = Path(input_video)
+        if input_wav:
+            frames, stacked = _extract_video_frames(
+                video_path,
+                work_dir,
+                fps=fps,
+                max_side=max_side,
+                duration_s=_pcm16_wav_duration_s(Path(input_wav)),
+                stack_frames=stack_frames,
+            )
+            return input_wav, frames, stacked
+        extracted_wav, frames, stacked = _extract_video_input(
+            video_path,
+            work_dir,
+            fps=fps,
+            max_side=max_side,
+            stack_frames=stack_frames,
+        )
+        return str(extracted_wav), frames, stacked
+    if not input_wav:
+        raise SystemExit("provide --input-wav, --input-video, or both")
+    return input_wav, [], []


 async def run_demo(args: argparse.Namespace) -> dict[str, object]:
-    input_pcm16 = read_pcm16_wav(Path(args.input_wav))
-    if not input_pcm16:
-        raise ValueError("input WAV has no audio")
-
    output_dir = Path(args.output_dir)
    stream_writer = _StreamingOutputWriter(output_dir)

+    input_wav, video_frames, stacked_video_frames = _resolve_duplex_av_inputs(
+        input_wav=args.input_wav,
+        input_video=args.input_video,
+        work_dir=output_dir / "video_input",
+        fps=args.video_fps,
+        max_side=args.frame_max_side,
+        stack_frames=args.stack_frames,
+    )
+
+    input_pcm16 = read_pcm16_wav(Path(input_wav))
+    if not input_pcm16:
+        raise ValueError("input WAV has no audio")
+
    url = build_realtime_url(
        args.url,
        args.model,
@@ -200,10 +451,12 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
            timeout_s=args.timeout_s,
        )
        stream_event_cursor = len(client.events.events)
-        await client.stream_pcm16(
+        frames_sent = await client.stream_pcm16(
            input_pcm16,
            chunk_ms=args.chunk_ms,
            realtime=not args.no_realtime_pacing,
+            video_frames=video_frames,
+            stacked_video_frames=stacked_video_frames,
        )
        commit_event_cursor = len(client.events.events)
        stream_decision = _latest_model_decision(client.events.events, stream_event_cursor)
@@ -212,8 +465,7 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
            chunk_period_ms=_chunk_period_ms(client.events.events),
        )
        wait_for_post_commit_decision = False
-        commit_sent_at_s = time.monotonic()
-        await client.commit()
+        commit_sent_at_s = await _commit_input_after_playback_checkpoint(client)
        wait_error: str | None = None
        committed_index: int | None = None
        post_commit_decision: str | None = None
@@ -326,6 +578,10 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
                    committed_index,
                ),
            },
+            "input_video": args.input_video,
+            "video_frames_extracted": len(video_frames),
+            "video_frames_sent": frames_sent,
+            "video_stacked_frames_extracted": sum(frame is not None for frame in stacked_video_frames),
            "audio_bytes": len(audio),
            "audio_chunk_count": len(stream_writer.audio_chunk_paths),
            "audio_chunk_files": [str(path) for path in stream_writer.audio_chunk_paths],
@@ -372,7 +628,48 @@ def parse_args() -> argparse.Namespace:
    parser.add_argument("--url", default="ws://localhost:8099/v1/realtime?duplex=1")
    parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
    parser.add_argument("--session-id")
-    parser.add_argument("--input-wav", required=True)
+    parser.add_argument(
+        "--input-wav",
+        help="Mono 16 kHz PCM16 clip to stream. Optional when --input-video carries the audio track.",
+    )
+    parser.add_argument(
+        "--input-video",
+        help=(
+            "Video to stream as omni duplex input. PyAV demuxes frames and "
+            "vLLM load_audio extracts a 16 kHz mono WAV (unless --input-wav overrides the audio)."
+        ),
+    )
+    parser.add_argument(
+        "--video-fps",
+        type=float,
+        default=1.0,
+        help="Frame extraction rate. The wire contract carries ~1 frame per second of audio.",
+    )
+    parser.add_argument(
+        "--frame-max-side",
+        type=int,
+        default=0,
+        help=(
+            "Optional client-side downscale of the longest side, in pixels. "
+            "0 (default) keeps the capture resolution so the server "
+            "process_image can normalize at scale_resolution=448 from a "
+            "higher-quality source. The web demo never resizes on the client."
+        ),
+    )
+    parser.add_argument(
+        "--stack-frames",
+        type=int,
+        default=1,
+        help=(
+            "Frames sampled per 1 s audio unit. 1 (default) sends the unit's "
+            "frame alone. N > 1 also samples the N-1 sub-frames inside the unit "
+            "and tiles them into one composite sent next to the base frame, so "
+            "motion (arrows, doors, digits changing) is visible at N fps while "
+            "the audio stays 1 s per unit. Official uses 5 for high refresh "
+            "rate mode; the wire carries 2 images per unit at any N because the "
+            "sub-frames share one composite."
+        ),
+    )
    parser.add_argument(
        "--ref-audio",
        required=True,
@@ -388,7 +685,7 @@ def parse_args() -> argparse.Namespace:
        "--temperature",
        type=float,
        default=None,
-        help="Stage0 sampling temperature; response-required soft-interrupt uses 0.0.",
+        help="Stage 0 sampling temperature; omit to preserve the model default.",
    )
    parser.add_argument("--no-realtime-pacing", action="store_true")
    parser.add_argument("--require-audio", action="store_true")
@@ -24,6 +24,18 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
    --deploy-config /path/to/your_deploy_config.yaml
 ```

+To serve **thinker-only** (text output, no talker / code2wav loaded) with Instruct
+weights, pass the bundled thinker-only deploy YAML. The `pipeline:` key
+`qwen3_omni_moe_thinker_only` overrides the HF `enable_audio_output` resolver:
+
+```bash
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
+    --deploy-config vllm_omni/deploy/qwen3_omni_moe_thinking.yaml
+```
+
+Captioner / Thinking checkpoints (`enable_audio_output=false`) still auto-select
+the same single-stage pipeline without `--deploy-config`.
+
 For a 3x-GPU multi-replica layout (talker/code2wav scale-out on cuda:1,2),
 use `--stage-overrides` on top of the default config:

@@ -1,234 +0,0 @@
-# Stable Audio Online Serving
-
-Generate audio from text prompts using Stable Audio models via an OpenAI-compatible API endpoint.
-
-## Features
-
- **OpenAI-compatible API**: Use `/v1/audio/generate` endpoint
- **Flexible control**: Adjust audio length, guidance scale, inference steps
- **Quality control**: Use negative prompts to avoid unwanted characteristics
- **Reproducible**: Set random seed for deterministic generation
-
-## Quick Start
-
-### 1. Start the Server
-
-```bash
-vllm-omni serve stabilityai/stable-audio-open-1.0 \
-    --host 0.0.0.0 \
-    --port 8091 \
-    --gpu-memory-utilization 0.9 \
-    --trust-remote-code \
-    --enforce-eager \
-    --omni
-```
-
-### 2. Generate Audio
-
-#### Using curl
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/generate \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "The sound of a cat purring",
-        "audio_length": 10.0
-    }' --output cat.wav
-```
-
-#### Using Python Client
-
-```bash
-python stable_audio_client.py \
-    --text "The sound of a cat purring" \
-    --audio_length 10.0 \
-    --output cat.wav
-```
-
-#### Using Bash Script
-
-```bash
-bash curl_examples.sh
-```
-
-## API Reference
-
-### Endpoint
-
-```
-POST /v1/audio/generate
-```
-
-### Request Body
-
-```json
-{
-    "input": "Text description of the audio",
-    "audio_length": 10.0,
-    "audio_start": 0.0,
-    "negative_prompt": "Low quality",
-    "guidance_scale": 7.0,
-    "num_inference_steps": 100,
-    "seed": 42,
-    "response_format": "wav"
-}
-```
-
-### Parameters
-
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `input` | string | **required** | Text prompt describing the audio to generate |
-| `audio_length` | float | ~47s | Audio duration in seconds (max ~47s for stable-audio-open-1.0) |
-| `audio_start` | float | 0.0 | Audio start time in seconds |
-| `negative_prompt` | string | null | Text describing what to avoid in generation |
-| `guidance_scale` | float | 7.0 | Classifier-free guidance scale (higher = more adherence to prompt) |
-| `num_inference_steps` | int | 50 | Number of denoising steps (higher = better quality, slower) |
-| `seed` | int | null | Random seed for reproducibility |
-| `response_format` | string | "wav" | Output format: wav, mp3, flac, pcm |
-
-### Response
-
-Returns audio data in the requested format (default: WAV).
-
-## Usage Examples
-
-### Basic Generation
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/generate \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "The sound of ocean waves"
-    }' --output ocean.wav
-```
-
-### Custom Duration
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/generate \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "A dog barking",
-        "audio_length": 5.0
-    }' --output dog_5s.wav
-```
-
-### High Quality with Negative Prompt
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/generate \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "A piano playing a gentle melody",
-        "audio_length": 10.0,
-        "negative_prompt": "Low quality, distorted, noisy",
-        "guidance_scale": 8.0,
-        "num_inference_steps": 150
-    }' --output piano_hq.wav
-```
-
-### Reproducible Generation
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/generate \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Thunder and rain sounds",
-        "audio_length": 15.0,
-        "seed": 42
-    }' --output thunder.wav
-```
-
-### Quick Generation (Fewer Steps)
-
-For faster generation with slightly lower quality:
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/generate \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Birds chirping in a forest",
-        "audio_length": 8.0,
-        "num_inference_steps": 50
-    }' --output birds_quick.wav
-```
-
-## Python Client Examples
-
-### Simple Generation
-
-```bash
-python stable_audio_client.py \
-    --text "The sound of a cat purring"
-```
-
-### Custom Parameters
-
-```bash
-python stable_audio_client.py \
-    --text "Thunder and rain" \
-    --audio_length 15.0 \
-    --negative_prompt "Low quality" \
-    --guidance_scale 7.0 \
-    --num_inference_steps 100 \
-    --seed 42 \
-    --output thunder.wav
-```
-
-### Different Output Format
-
-```bash
-python stable_audio_client.py \
-    --text "Guitar playing" \
-    --response_format mp3 \
-    --output guitar.mp3
-```
-
-## Tips
-
-1. **Audio Length**: Keep under 47 seconds for `stable-audio-open-1.0`
-2. **Quality vs Speed**:
-   - 50 steps: Fast, decent quality
-   - 100 steps: Good balance (default)
-   - 150+ steps: High quality, slower
-3. **Guidance Scale**:
-   - Lower (3-5): More creative/varied
-   - Default (7): Good balance
-   - Higher (10+): More literal to prompt
-4. **Negative Prompts**: Use to avoid "Low quality", "distorted", "noisy", etc.
-5. **Seeds**: Use same seed for reproducible results
-
-## Performance
-
-| Inference Steps | Quality | Speed | Use Case |
-|----------------|---------|-------|----------|
-| 50 | Good | Fast | Quick previews |
-| 100 (default) | Very Good | Medium | Production |
-| 150+ | Excellent | Slow | Final/critical audio |
-
-## Troubleshooting
-
-### Server not responding
- Check if server is running: `curl http://localhost:8091/health`
- Check server logs for errors
-
-### Audio quality issues
- Increase `num_inference_steps` (e.g., 150)
- Add negative prompts: `"Low quality, distorted, noisy"`
- Increase `guidance_scale` for more prompt adherence
-
-### Generation timeout
- Reduce `num_inference_steps`
- Reduce `audio_length`
- Check GPU memory with `nvidia-smi`
-
-### Wrong audio length
- Ensure `audio_length` is within model limits (~47s max)
- Adjust `audio_start` if trimming is needed
-
-## See Also
-
- [Offline Inference Example](../../offline_inference/text_to_audio/README.md)
- [Stable Audio Model Card](https://huggingface.co/stabilityai/stable-audio-open-1.0)
- [vLLM-Omni Documentation](https://github.com/vllm-project/vllm-omni)
@@ -1,54 +0,0 @@
-#!/bin/bash
-# Examples for using Stable Audio with curl via /v1/audio/generate endpoint
-
-# Example 1: Simple request with default parameters
-echo "Example 1: Simple request with default parameters"
-curl -X POST http://localhost:8091/v1/audio/generate \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "The sound audience clapping and cheering in a stadium"
-    }' --output stadium.wav
-
-# Example 2: Request with custom audio_length
-echo "Example 2: Custom audio length (5 seconds)"
-curl -X POST http://localhost:8091/v1/audio/generate \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "The sound of a dog barking",
-        "audio_length": 5.0
-    }' --output dog_5s.wav
-
-# Example 3: Request with negative prompt for quality control
-echo "Example 3: With negative prompt"
-curl -X POST http://localhost:8091/v1/audio/generate \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "A piano playing a gentle melody",
-        "audio_length": 10.0,
-        "negative_prompt": "Low quality, distorted, noisy"
-    }' --output piano.wav
-
-# Example 4: Full control with all parameters
-echo "Example 4: Full control (custom length, guidance, steps, seed)"
-curl -X POST http://localhost:8091/v1/audio/generate \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Thunder and rain sounds",
-        "audio_length": 15.0,
-        "negative_prompt": "Low quality",
-        "guidance_scale": 7.0,
-        "num_inference_steps": 100,
-        "seed": 42
-    }' --output thunder_rain.wav
-
-# Example 5: Quick generation with fewer steps (faster but lower quality)
-echo "Example 5: Quick generation (fewer steps)"
-curl -X POST http://localhost:8091/v1/audio/generate \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Ocean waves crashing on a beach",
-        "audio_length": 8.0,
-        "num_inference_steps": 50
-    }' --output ocean.wav
-
-echo "All examples completed!"
@@ -1,170 +0,0 @@
-#!/usr/bin/env python3
-"""
-OpenAI-compatible client for Stable Audio via /v1/audio/generate endpoint.
-
-This script demonstrates how to use the OpenAI-compatible speech API
-to generate audio from text using Stable Audio models.
-
-Examples:
-    # Simple generation
-    python stable_audio_client.py --text "The sound of a cat purring"
-
-    # With custom duration
-    python stable_audio_client.py --text "A dog barking" --audio_length 5.0
-
-    # With all parameters
-    python stable_audio_client.py --text "Thunder and rain" \
-        --audio_length 15.0 \
-        --negative_prompt "Low quality" \
-        --guidance_scale 7.0 \
-        --num_inference_steps 100 \
-        --seed 42 \
-        --output thunder.wav
-"""
-
-import argparse
-import sys
-
-import requests
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(description="Generate audio with Stable Audio via OpenAI-compatible API")
-    parser.add_argument(
-        "--api_url",
-        default="http://localhost:8091/v1/audio/generate",
-        help="API endpoint URL",
-    )
-    parser.add_argument(
-        "--text",
-        default="The sound of a cat purring",
-        help="Text prompt for audio generation",
-    )
-    parser.add_argument(
-        "--audio_length",
-        type=float,
-        default=10.0,
-        help="Audio length in seconds (max ~47s for stable-audio-open-1.0)",
-    )
-    parser.add_argument(
-        "--audio_start",
-        type=float,
-        default=0.0,
-        help="Audio start time in seconds",
-    )
-    parser.add_argument(
-        "--negative_prompt",
-        default="Low quality",
-        help="Negative prompt for classifier-free guidance",
-    )
-    parser.add_argument(
-        "--guidance_scale",
-        type=float,
-        default=7.0,
-        help="Guidance scale for diffusion (higher = more adherence to prompt)",
-    )
-    parser.add_argument(
-        "--num_inference_steps",
-        type=int,
-        default=100,
-        help="Number of inference steps (higher = better quality, slower)",
-    )
-    parser.add_argument(
-        "--seed",
-        type=int,
-        default=None,
-        help="Random seed for reproducibility",
-    )
-    parser.add_argument(
-        "--output",
-        default="stable_audio_output.wav",
-        help="Output file path",
-    )
-    parser.add_argument(
-        "--response_format",
-        default="wav",
-        choices=["wav", "mp3", "flac", "pcm"],
-        help="Audio output format",
-    )
-    return parser.parse_args()
-
-
-def generate_audio(args):
-    """Generate audio using the API."""
-
-    # Build request payload
-    payload = {
-        "input": args.text,
-        "audio_length": args.audio_length,
-        "audio_start": args.audio_start,
-        "response_format": args.response_format,
-    }
-
-    # Add optional parameters
-    if args.negative_prompt:
-        payload["negative_prompt"] = args.negative_prompt
-    if args.guidance_scale:
-        payload["guidance_scale"] = args.guidance_scale
-    if args.num_inference_steps:
-        payload["num_inference_steps"] = args.num_inference_steps
-    if args.seed is not None:
-        payload["seed"] = args.seed
-
-    print(f"\n{'=' * 60}")
-    print("Stable Audio - Text-to-Audio Generation")
-    print(f"{'=' * 60}")
-    print(f"API URL: {args.api_url}")
-    print(f"Prompt: {args.text}")
-    print(f"Audio length: {args.audio_length}s")
-    print(f"Negative prompt: {args.negative_prompt}")
-    print(f"Guidance scale: {args.guidance_scale}")
-    print(f"Inference steps: {args.num_inference_steps}")
-    if args.seed is not None:
-        print(f"Seed: {args.seed}")
-    print(f"Output: {args.output}")
-    print(f"{'=' * 60}\n")
-
-    try:
-        # Make the API request
-        print("Generating audio...")
-        response = requests.post(
-            args.api_url,
-            json=payload,
-            headers={"Content-Type": "application/json"},
-            timeout=300,  # 5 minute timeout for long generations
-        )
-
-        # Check for errors
-        if response.status_code != 200:
-            print(f"Error: API returned status code {response.status_code}")
-            print(f"Response: {response.text}")
-            return False
-
-        # Save the audio
-        with open(args.output, "wb") as f:
-            f.write(response.content)
-
-        print(f"✓ Audio saved to {args.output}")
-        print(f"  File size: {len(response.content) / 1024:.1f} KB")
-        return True
-
-    except requests.exceptions.Timeout:
-        print("Error: Request timed out. Try reducing inference steps or audio length.")
-        return False
-    except requests.exceptions.ConnectionError:
-        print(f"Error: Could not connect to {args.api_url}")
-        print("Make sure the server is running.")
-        return False
-    except Exception as e:
-        print(f"Error: {e}")
-        return False
-
-
-def main():
-    args = parse_args()
-    success = generate_audio(args)
-    sys.exit(0 if success else 1)
-
-
-if __name__ == "__main__":
-    main()