[Doc] Add Qwen-Image-2.1 recipe to recipes index (#8070)

Signed-off-by: SamitHuang <285365963@qq.com>

.buildkite/amd/scripts/build-ci-image.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+set -euo pipefail
+
+# Build and push the ROCm CI image with a registry-backed BuildKit cache. The
+# AMD builders are ephemeral, so a local Docker cache cannot survive between
+# jobs. Keeping the cache beside the runtime image also reuses the registry
+# credentials already configured on the amd-cpu agents.
+
+REGISTRY="${ROCM_CI_IMAGE_REGISTRY:-rocm/vllm-omni}"
+DOCKERFILE="docker/Dockerfile.rocm"
+BUILDER_NAME="vllm-omni-rocm-builder"
+ROCM_ARCH="${PYTORCH_ROCM_ARCH:-gfx942}"
+
+if [[ -z "${BUILDKITE_COMMIT:-}" ]]; then
+    echo "ERROR: BUILDKITE_COMMIT is not set" >&2
+    exit 1
+fi
+
+dependency_files=(
+    "$DOCKERFILE"
+    "${DOCKERFILE}.dockerignore"
+    pyproject.toml
+    setup.py
+    tools/install_torchcodec_rocm.sh
+)
+while IFS= read -r dependency_file; do
+    dependency_files+=("$dependency_file")
+done < <(find requirements -maxdepth 1 -type f -name '*.txt' -print | LC_ALL=C sort)
+
+for dependency_file in "${dependency_files[@]}"; do
+    if [[ ! -f "$dependency_file" ]]; then
+        echo "ERROR: cache-key input does not exist: $dependency_file" >&2
+        exit 1
+    fi
+done
+
+CACHE_KEY=$(
+    {
+        printf 'rocm-arch=%s\n' "$ROCM_ARCH"
+        for dependency_file in "${dependency_files[@]}"; do
+            printf 'path=%s\n' "$dependency_file"
+            sha256sum "$dependency_file"
+        done
+    } | sha256sum | cut -c1-16
+)
+CACHE_REF="${REGISTRY}:rocm-deps-cache-${CACHE_KEY}"
+IMAGE_REF="${REGISTRY}:${BUILDKITE_COMMIT}"
+
+echo "ROCm dependency cache: ${CACHE_REF}"
+echo "ROCm CI image: ${IMAGE_REF}"
+
+# The docker-container driver supports exporting the full cache to a registry;
+# the default docker driver on CI does not.
+if ! docker buildx inspect "$BUILDER_NAME" >/dev/null 2>&1; then
+    docker buildx create --name "$BUILDER_NAME" --driver docker-container
+fi
+docker buildx use "$BUILDER_NAME"
+docker buildx inspect --bootstrap
+
+docker buildx build --push --pull --progress=plain \
+    --target test \
+    --build-arg "ARG_PYTORCH_ROCM_ARCH=${ROCM_ARCH}" \
+    --cache-from "type=registry,ref=${CACHE_REF}" \
+    --cache-to "type=registry,ref=${CACHE_REF},mode=max,compression=zstd,ignore-error=true" \
+    --file "$DOCKERFILE" \
+    --tag "$IMAGE_REF" .

.buildkite/amd/test-amd-merge.yml

@@ -9,15 +9,26 @@ steps:
   steps:
     - label: "Simple · Model Executor Test · Shard %N/%t"
       agent_pool: mi300_1
-      parallelism: 2
+      parallelism: 3
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      timeout_in_minutes: 90
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+        - "pytest -sv tests/model_executor -m 'core_model and cpu and not omni' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+
+    - label: "Simple · Omni Processor Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 60
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/model_executor/models/test_omni_processing.py -m 'core_model and cpu and omni'"
 
     - label: "Simple · Diffusion Test · Shard %N/%t"
       agent_pool: mi300_1
@@ -25,6 +36,7 @@ steps:
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      timeout_in_minutes: 45
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM

.buildkite/amd/test-amd-nightly.yml

@@ -64,3 +64,21 @@ steps:
         commands:
           - export VLLM_ROCM_USE_AITER=1
           - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_expansion.py::test_text_to_text_audio_001 -m "full_model and rocm and MI325 and cards_2" --run-level "full_model"
+
+  - group: ":speaker: CosyVoice3 MI300 Nightly"
+    steps:
+      # Exercise all sync/async and Chinese/English cases with the platform's
+      # default attention backend. On ROCm this retains end-to-end AITER
+      # coverage without putting its cold JIT build on the READY critical path.
+      - label: "CosyVoice3-TTS E2E Test"
+        agent_pool: mi300_1
+        depends_on: amd-build
+        mirror_hardwares: [amdproduction]
+        grade: NonBlocking
+        timeout_in_minutes: 90
+        retry:
+          automatic:
+            - exit_status: 134
+              limit: 1
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"

.buildkite/amd/test-amd-ready.yml

@@ -9,26 +9,39 @@ steps:
   steps:
     - label: "Simple · Model Executor Test · Shard %N/%t"
       agent_pool: mi300_1
-      parallelism: 2
+      parallelism: 3
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      timeout_in_minutes: 90
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+        - "pytest -sv tests/model_executor -m 'core_model and cpu and not omni' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
 
-    - label: "Simple · Diffusion Test"
+    - label: "Simple · Omni Processor Test"
       agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      timeout_in_minutes: 60
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/model_executor/models/test_omni_processing.py -m 'core_model and cpu and omni'"
+
+    - label: "Simple · Diffusion Test · Shard %N/%t"
+      agent_pool: mi300_1
+      parallelism: 4
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 45
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "pytest -sv tests/diffusion -m 'core_model and cpu'  --ignore=tests/diffusion/cache/test_teacache_extractors.py"
+        - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
 
     - label: "Simple · Engine&Entrypoints Test"
       agent_pool: mi300_1
@@ -69,6 +82,27 @@ steps:
 
 - group: ":card_index_dividers: Diffusion Test"
   steps:
+    # Keep real FlashAttention execution out of the CPU-marked Simple lane.
+    # AITER may need to build several kernels on a cold worker, so run one
+    # pytest worker with job-local build directories and a bounded timeout.
+    - label: "ROCm · FlashAttention/AITER GPU Coverage"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: NonBlocking
+      timeout_in_minutes: 60
+      commands:
+        - export VLLM_ROCM_USE_AITER=1
+        - export AITER_JIT_DIR="/tmp/vllm-omni-aiter-$$BUILDKITE_JOB_ID"
+        - export TORCH_EXTENSIONS_DIR="/tmp/vllm-omni-torch-extensions-$$BUILDKITE_JOB_ID"
+        - mkdir -p "$$AITER_JIT_DIR" "$$TORCH_EXTENSIONS_DIR"
+        - >-
+          timeout --signal=TERM --kill-after=2m 55m
+          pytest -s -v -n 1
+          tests/diffusion/attention/
+          -m 'core_model and rocm and MI325 and cards_1'
+          --run-level "core_model"
+
     - label: "Diffusion · Batch Test"
       agent_pool: mi300_1
       depends_on: amd-build
@@ -294,13 +328,23 @@ steps:
         - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
         - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"
 
-    - label: "CosyVoice3-TTS E2E Test"
+    # Keep a fast model/server integration check in the blocking lane. The
+    # full default-backend suite remains in the nightly pipeline because a
+    # cold ROCm AITER MHA build can add more than 25 minutes to this job.
+    - label: "CosyVoice3-TTS E2E Smoke (SDPA)"
       agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      # A ROCm hardware hang aborts the interpreter with 134. Let Buildkite
+      # create one fresh pod instead of retrying on the poisoned GPU process.
+      retry:
+        automatic:
+          - exit_status: 134
+            limit: 1
       commands:
-        - pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"
+        - export DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA
+        - pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py::test_voice_clone_zh_002 -m "slow" --run-level "core_model"
 
 # TODO: Bagel test on ROCm is very unstable. @tjtanaa
 # Need to debug before reneable numerical changes across large PRs

.buildkite/amd/test-template-amd-omni.j2

@@ -3,7 +3,8 @@
    Last synced: 2026-08-04
    Modifications: Use the shared MI300 queues and Omni command-passing contract
 #}
-{% set docker_image_amd = "rocm/vllm-omni:$BUILDKITE_COMMIT" %}
+{% set docker_registry_amd = "rocm/vllm-omni" %}
+{% set docker_image_amd = docker_registry_amd ~ ":$BUILDKITE_COMMIT" %}
 {% set default_working_dir = "/app/vllm-omni" %}
 
 {# MI300 runs tests natively in Kubernetes; Docker-in-Docker is unavailable. #}
@@ -74,11 +75,12 @@ steps:
         depends_on: ~
         soft_fail: false
         commands:
-          - "docker build -f docker/Dockerfile.rocm -t {{ docker_image_amd }} --target test --build-arg ARG_PYTORCH_ROCM_ARCH=gfx942 --progress plain ."
-          - "docker push {{ docker_image_amd }}"
+          - "bash .buildkite/amd/scripts/build-ci-image.sh"
         key: "amd-build"
         env:
           DOCKER_BUILDKIT: "1"
+          PYTORCH_ROCM_ARCH: "gfx942"
+          ROCM_CI_IMAGE_REGISTRY: "{{ docker_registry_amd }}"
         retry:
           automatic:
             - exit_status: -1  # Agent was lost
@@ -156,6 +158,14 @@ steps:
 {% for path in step.artifact_paths %}
           - "{{ path }}"
 {% endfor %}
+{% endif %}
+{% if step.retry %}
+        retry:
+          automatic:
+{% for retry_rule in step.retry.automatic %}
+            - exit_status: {{ retry_rule.exit_status }}
+              limit: {{ retry_rule.limit }}
+{% endfor %}
 {% endif %}
         {% if step.timeout_in_minutes %}
         timeout_in_minutes: {{ step.timeout_in_minutes }}
@@ -224,6 +234,14 @@ steps:
 {% for path in step.artifact_paths %}
       - "{{ path }}"
 {% endfor %}
+{% endif %}
+{% if step.retry %}
+    retry:
+      automatic:
+{% for retry_rule in step.retry.automatic %}
+        - exit_status: {{ retry_rule.exit_status }}
+          limit: {{ retry_rule.limit }}
+{% endfor %}
 {% endif %}
     {% if step.timeout_in_minutes %}
     timeout_in_minutes: {{ step.timeout_in_minutes }}

.buildkite/common/ci_source_file_dependencies.yml

@@ -23,6 +23,16 @@ x-qwen3-tts: &qwen3_tts
   - vllm_omni/entrypoints/openai/serving_speech.py
   - vllm_omni/entrypoints/openai/tts_adapters/
   - vllm_omni/engine/stage_pool.py
+  - vllm_omni/worker_v2/
+  - vllm_omni/core/sched/
+  - vllm_omni/distributed/omni_connectors/
+  - vllm_omni/config/stage_config.py
+  - vllm_omni/config/omni_config.py
+  - vllm_omni/engine/orchestrator.py
+  - vllm_omni/engine/stage_engine_core_proc.py
+  - vllm_omni/deploy/qwen3_tts_mrv2.yaml
+  - vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2.yaml
+  - vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2_b4.yaml
 
 x-minicpmo-4-5: &minicpmo_4_5
   - vllm_omni/engine/async_omni_engine.py
@@ -44,6 +54,7 @@ x-minicpmo-4-5-duplex: &minicpmo_4_5_duplex
   - vllm_omni/entrypoints/duplex/
   - vllm_omni/entrypoints/duplex_omni.py
   - vllm_omni/entrypoints/async_omni_base.py
+  - vllm_omni/protocol/
   - vllm_omni/clients/
   - vllm_omni/outputs/duplex.py
   - vllm_omni/model_executor/duplex_sampling.py
@@ -99,6 +110,7 @@ x-hunyuan-image3: &hunyuan_image3
 
 x-minimax-h3: &minimax_h3
   - vllm_omni/diffusion/models/minimax_h3/
+  - vllm_omni/diffusion/layers/indexed_modulation.py
 
 x-hunyuan-video: &hunyuan_video
   - vllm_omni/diffusion/models/hunyuan_video/
@@ -243,6 +255,8 @@ source_file_dependencies:
     - *minicpmo_4_5_duplex
     - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
     - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+    - tests/e2e/online_serving/test_minicpmo_4_5_window.py
+    - tests/e2e/online_serving/helpers/minicpmo_window_e2e.py
     - tests/e2e/online_serving/test_duplex_client_live.py
     - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
     - tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
@@ -259,6 +273,7 @@ source_file_dependencies:
     - tests/entrypoints/test_duplex_omni.py
     - tests/entrypoints/openai_api/test_duplex_api_server.py
     - tests/model_executor/models/minicpmo_4_5/duplex/
+    - tests/protocol/
 
   omni_minicpmo_4_5_duplex_perf:
     - *minicpmo_4_5
@@ -440,6 +455,7 @@ source_file_dependencies:
     - *lingbot_video
     - *lingbot_world
     - tests/e2e/online_serving/test_lingbot_video.py
+    - tests/e2e/online_serving/test_lingbot_video_expansion.py
     - tests/e2e/online_serving/test_lingbot_video_moe.py
     - tests/e2e/offline_inference/test_lingbot_world_v2.py
 

.buildkite/cuda/test-merge.yml

@@ -130,7 +130,19 @@ steps:
         source_file_dependencies: omni_minicpmo_4_5_duplex_function
         timeout_in_minutes: 50
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py tests/e2e/online_serving/test_minicpmo_4_5_window.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Qwen Image 2.1 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen_image_21.py
+          - tests/helpers/
+          - vllm_omni/diffusion/models/qwen_image_21/
+          - vllm_omni/diffusion/distributed/autoencoders/
+          - vllm_omni/diffusion/sched/
+          - vllm_omni/entrypoints/openai/
+        commands:
+          - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen_image_21.py -m 'advanced_model and cuda' --run-level 'advanced_model'
         mirror_hardwares: h100_1
 
       - label: "Diffusion · Qwen Image Test"
@@ -163,6 +175,16 @@ steps:
             "
         mirror_hardwares: h100_1
 
+      - label: "Diffusion · LingBot-Video Test"
+        source_file_dependencies: diffusion_lingbot_function
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_lingbot_video.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+            "
+        mirror_hardwares: h100_1
+
       - label: "Diffusion · MiniMax-H3 L3 DLO DP2 and FastH3"
         source_file_dependencies: diffusion_minimax_h3_function
         # Keep four server lifecycles in one Buildkite job, but never in one

.buildkite/cuda/test-nightly.yml

@@ -223,6 +223,17 @@ steps:
             -m "full_model and diffusion and H100 and B200 and cards_1"
             --run-level "full_model"
 
+      - label: ":robot_face: Robot Policy OpenPI · π0.5 · H100 · Single-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - pip install --no-deps "openpi-client @ git+https://github.com/Physical-Intelligence/openpi.git@215abfb217dbac7d5f1273282331b9b1866c0479#subdirectory=packages/openpi-client"
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_pi05_expansion.py
+            -m "full_model and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
       - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU"
         source_file_dependencies: diffusion_qwen_image_function
         timeout_in_minutes: 120
@@ -338,7 +349,7 @@ steps:
         commands:
           - >-
             pytest -s -v
-            tests/e2e/online_serving/test_lingbot_video.py
+            tests/e2e/online_serving/test_lingbot_video_expansion.py
             tests/e2e/online_serving/test_lingbot_video_moe.py
             tests/e2e/offline_inference/test_lingbot_world_v2.py
             -m "full_model and diffusion and H100 and B200 and cards_1"

.buildkite/cuda/test-ready.yml

@@ -38,6 +38,12 @@ steps:
     depends_on: upload-ready-pipeline
     steps:
 
+      - label: "Diffusion · FlashAttention Test"
+        timeout_in_minutes: 20
+        commands:
+          - timeout 15m pytest -s -v tests/diffusion/attention/ -m 'core_model and cuda and L4 and cards_1' --run-level "core_model"
+        mirror_hardwares: l4_1
+
       - label: "Diffusion · Offloader Test"
         commands:
           - timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
@@ -96,7 +102,7 @@ steps:
     commands:
       - |
         timeout 15m bash -c "
-                  pytest -sv tests/engine/ tests/model_executor/ tests/worker/test_batched_omni_output.py -m 'core_model and cuda' --run-level 'core_model'
+                  pytest -sv tests/engine/ tests/model_executor/ tests/worker/test_batched_omni_output.py tests/worker_v2/ -m 'core_model and cuda' --run-level 'core_model'
         "
     mirror_hardwares: l4_1
 
@@ -223,6 +229,18 @@ steps:
             "
         mirror_hardwares: h100_2
 
+      - label: "Diffusion · Qwen Image 2.1 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen_image_21.py
+          - tests/helpers/
+          - vllm_omni/diffusion/models/qwen_image_21/
+          - vllm_omni/diffusion/distributed/autoencoders/
+          - vllm_omni/diffusion/sched/
+          - vllm_omni/entrypoints/openai/
+        commands:
+          - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen_image_21.py -m 'core_model and cuda' --run-level 'core_model'
+        mirror_hardwares: h100_1
+
       - label: "Diffusion · Qwen Image Test"
         source_file_dependencies: diffusion_qwen_image_function
         commands:
@@ -242,3 +260,13 @@ steps:
               pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'core_model and diffusion' --run-level 'core_model'
             "
         mirror_hardwares: h100_1
+
+      - label: "Diffusion · LingBot-Video Test"
+        source_file_dependencies: diffusion_lingbot_function
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_lingbot_video.py -m 'core_model and cuda' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_1

.buildkite/npu/test-npu-nightly.yml

@@ -235,6 +235,37 @@ steps:
       build.env("NIGHTLY") == "1" ||
       build.pull_request.labels includes "nightly-test"
     steps:
+      - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test"
+        source_file_dependencies: diffusion_hunyuan_video_function
+        timeout_in_minutes: 180
+        mirror_hardwares: a2b3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+          HF_HUB_OFFLINE: "1"
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "npu and A2" --run-level "full_model"
+      - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Perf Test"
+        source_file_dependencies: diffusion_hunyuan_video_perf
+        key: nightly-diffusion-x2v-performance-hunyuanvideo15
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_2
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+          HF_HUB_OFFLINE: "1"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
       - label: ":full_moon: Diffusion X2V · Wan 2.2 Function Test"
         source_file_dependencies: diffusion_wan22_function
         mirror_hardwares: a2b3_npu_4

.claude/skills/production-add-diffusion-model/references/production-validation.md

@@ -340,7 +340,7 @@ On supported GPU hosts, wrap a test or soak with the repository monitor:
 
 ```bash
 bash tests/dfx/stability/scripts/resource_monitor.sh run --backend gpu -- \
-  pytest -s -v tests/dfx/stability/scripts/test_stability_<model>.py -m slow
+  pytest -s -v tests/dfx/stability/scripts/run_stability_<model>.py -m slow
 ```
 
 Check the target revision's monitor backends. Do not claim CPU/NPU monitoring

.github/CODEOWNERS

@@ -67,6 +67,11 @@
 # File-level overrides within entrypoints/
 /vllm_omni/entrypoints/openai/errors.py @alex-jw-brooks @NickCao
 /vllm_omni/entrypoints/cli/benchmark/ @alex-jw-brooks @Bounty-hunter
+# Shared wire codec: protocol/realtime (OpenAI Realtime GA) and protocol/duplex
+# (vLLM-Omni's extension of it) — RFC #6592 P0a. Layer-neutral by design — the
+# engine depends on it — so it is owned by the serving-boundary document rather
+# than by any one consumer. — docs/design/module/entrypoints.md
+/vllm_omni/protocol/ @alex-jw-brooks @linyueqian @NickCao
 
 # Error contracts — docs/design/module/error_contracts.md
 /vllm_omni/errors.py @alex-jw-brooks @NickCao

README.md

@@ -58,7 +58,7 @@ vLLM-Omni is flexible and easy to use with:
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
 - **Omni-modality models** (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
-- **TTS models** (e.g. Qwen3-TTS, IndexTTS 2.5, dots.tts, CosyVoice3)
+- **TTS models** (e.g. Qwen3-TTS, IndexTTS 2.5, CosyVoice3)
 - **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
 - **Robot-policy and action models** (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
 

apps/ComfyUI-vLLM-Omni/README.md

@@ -49,6 +49,7 @@ This extension offers the following nodes based on the output modalities (at **C
 
 - **Generate Image** for text-to-image and image-to-image tasks
 - **Generate Video** for text-to-video, first-frame/image-to-video, and reference-conditioned video
+- **Latent Mask Editing** for MiniMax-H3 latent-mask editing (source media plus video/audio noise masks)
 - **FastH3 Deployment** for routing text-to-video requests to a MiniMax-H3 server with FastH3 fused at startup
 - **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks
 - **TTS** and **TTS Voice Clone** for TTS tasks
@@ -67,7 +68,7 @@ Every node carries the vLLM-Omni mark in its title bar and is tinted by what it
 | Amber | AR / Diffusion / Multi-Stage Sampling Params | Sampling parameters that apply to any model |
 | Purple | Qwen TTS Params, Wan Video Params, MiniMax-H3 Video Params | Parameters that only one model family accepts |
 | Red | LoRA, FastH3 Deployment | Which weights the server is expected to have loaded |
-| Teal | Video References | Reference media |
+| Teal | Video References, Latent Mask Editing | Reference media |
 
 Recolouring a node by hand (right click -> Colors) overrides its tint, and the choice is kept.
 
@@ -134,11 +135,13 @@ You can configure per-stage sampling parameters for multi-stage models.
 </p>
 
 > [!TIP]
-> Connect a **frame** image for first-frame / image-to-video (e.g., Wan I2V, MiniMax-H3 FL2VA).
+> Connect **frame** for backward-compatible first-frame / image-to-video behavior. For MiniMax-H3 FL2VA,
+> connect **first_frame**, **last_frame**, or both to condition the start frame, end frame, or both.
 >
 > For reference-conditioned generation (MiniMax-H3 Ref2VA), connect a **Video References** node instead.
 >
-> Do not use `frame` and `references` together. Task routing is automatic from which inputs you connect.
+> Do not combine `frame` with `first_frame` or `last_frame`, and do not combine any frame input with
+> `references`. Task routing is automatic from which inputs you connect.
 
 For MiniMax-H3 Ref2VA, **Video References** accepts up to 9 images (`image_1`–`image_9`),
 3 videos (`video_1`–`video_3`), and 3 audio clips (`audio_1`–`audio_3`), with at most
@@ -261,6 +264,18 @@ Open the **vLLM-Omni FastH3 Text to Video** template, then:
 
 The node records which server the workflow targets; it does not start one, nor switch adapters or attention backends on a running server.
 
+#### Latent-mask editing (MiniMax-H3)
+
+The [WF-05 template](example_workflows/vLLM-Omni%20MiniMax-H3%20Latent%20Mask%20Editing.json) contains inpainting, object removal, continuation, and extension examples in one graph. See the [workflow guide](docs/wf05-h3-latent-editing.md) for inputs, mask settings, dependencies, and preview limitations.
+
+Connect a **Latent Mask Editing** node to **Generate Video → latent_edit** to edit a source clip instead of generating from scratch. It uploads the source media and serializes the video/audio noise masks the MiniMax H3 API accepts:
+
+- `source_video` / `source_audio` — the media to edit.
+- `video_mask` — a ComfyUI mask image; `0` preserves a region, `1` regenerates it, fractional values blend. A 2D mask `[H, W]` is applied to every frame; a 3D mask `[T, H, W]` is treated as a temporal mask (one slice per frame) for continuation or extension. It is resized to the video latent grid.
+- `audio_mask` — a scalar in `[0, 1]`; `0` keeps the source audio, `1` regenerates it, fractional values blend.
+
+A non-trivial mask requires its matching source, and a source without a mask is rejected. This node only forwards inputs to the server; the served model must declare latent-mask editing support (MiniMax-H3).
+
 ### TTS (e.g., Qwen TTS series)
 
 (Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni TTS**)

apps/ComfyUI-vLLM-Omni/__init__.py

@@ -21,7 +21,9 @@ from .comfyui_vllm_omni.nodes import (
     VLLMOmniGenerateImage,
     VLLMOmniGenerateMusic,
     VLLMOmniGenerateVideo,
+    VLLMOmniLatentMaskEditing,
     VLLMOmniMiniMaxH3Params,
+    VLLMOmniMiniMaxH3TemporalMask,
     VLLMOmniQwenTTSParams,
     VLLMOmniRemoteLoRA,
     VLLMOmniSamplingParamsList,
@@ -34,6 +36,7 @@ from .comfyui_vllm_omni.nodes import (
 
 # A dictionary that contains all nodes you want to export with their names
 NODE_CLASS_MAPPINGS = {
+    "VLLMOmniMiniMaxH3TemporalMask": VLLMOmniMiniMaxH3TemporalMask,
     # === Generation ===
     "VLLMOmniGenerateImage": VLLMOmniGenerateImage,
     "VLLMOmniGenerateVideo": VLLMOmniGenerateVideo,
@@ -42,6 +45,7 @@ NODE_CLASS_MAPPINGS = {
     "VLLMOmniGenerateMusic": VLLMOmniGenerateMusic,
     "VLLMOmniVoiceClone": VLLMOmniVoiceClone,
     "VLLMOmniVideoReferences": VLLMOmniVideoReferences,
+    "VLLMOmniLatentMaskEditing": VLLMOmniLatentMaskEditing,
     # === Params ===
     "VLLMOmniARSampling": VLLMOmniARSampling,
     "VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling,
@@ -55,6 +59,7 @@ NODE_CLASS_MAPPINGS = {
 
 # A dictionary that contains the friendly/humanly readable titles for the nodes
 NODE_DISPLAY_NAME_MAPPINGS = {
+    "VLLMOmniMiniMaxH3TemporalMask": "MiniMax-H3 Temporal Mask",
     # === Generation ===
     "VLLMOmniGenerateImage": "Generate Image",
     "VLLMOmniGenerateVideo": "Generate Video",
@@ -63,6 +68,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "VLLMOmniGenerateMusic": "Generate Music",
     "VLLMOmniVoiceClone": "TTS Voice Cloning",
     "VLLMOmniVideoReferences": "Video References",
+    "VLLMOmniLatentMaskEditing": "Latent Mask Editing",
     # === Params ===
     "VLLMOmniARSampling": "AR Sampling Params",
     "VLLMOmniDiffusionSampling": "Diffusion Sampling Params",

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py

@@ -1,12 +1,14 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
+import math
 from typing import Literal
 
 import torch
 from comfy_api.input import AudioInput, VideoInput
 
 from .utils.api_client import VLLMOmniClient
+from .utils.latent_mask import _align_frame_count, _video_latent_t
 from .utils.logger import get_logger
 from .utils.models import lookup_model_spec
 from .utils.types import (
@@ -17,6 +19,7 @@ from .utils.types import (
     AutoregressionSamplingParams,
     DiffusionSamplingParams,
     FastH3Deployment,
+    LatentMaskEditing,
     MiniMaxH3ModelSpecificParams,
     QwenTTSModelSpecificParams,
     VideoReferences,
@@ -203,11 +206,14 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
             },
             "optional": {
                 "frame": ("IMAGE",),
+                "first_frame": ("IMAGE",),
+                "last_frame": ("IMAGE",),
                 "references": ("VIDEO_REFERENCES",),
                 "sampling_params": ("SAMPLING_PARAMS",),
                 "lora": ("REMOTE_LORA",),
                 "model_params": ("VIDEO_PARAMS",),
                 "fast_h3": ("FASTH3_DEPLOYMENT",),
+                "latent_edit": ("LATENT_MASK_EDITING",),
             },
         }
 
@@ -216,12 +222,30 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
     FUNCTION = "generate"
 
     @classmethod
-    def VALIDATE_INPUTS(cls, url, model, frame=None, references=None, **_kwargs) -> str | Literal[True]:
+    def VALIDATE_INPUTS(
+        cls,
+        url,
+        model,
+        frame=None,
+        first_frame=None,
+        last_frame=None,
+        references=None,
+        fast_h3=None,
+        **_kwargs,
+    ) -> str | Literal[True]:
         base = super().VALIDATE_INPUTS(url, model)
         if base is not True:
             return base
         if frame is not None and references is not None:
             return "Provide only one of frame or references, not both."
+        if frame is not None and (first_frame is not None or last_frame is not None):
+            return "Provide either frame or first_frame/last_frame, not both."
+        if references is not None and (first_frame is not None or last_frame is not None):
+            return "Provide either first_frame/last_frame or references, not both."
+        if fast_h3 is not None and any(value is not None for value in (frame, first_frame, last_frame, references)):
+            return (
+                "FastH3 Preview supports T2VA only; disconnect frame, first_frame, last_frame, and references inputs."
+            )
         return True
 
     async def generate(
@@ -235,11 +259,14 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
         duration: float,
         negative_prompt: str | None = None,
         frame: torch.Tensor | None = None,
+        first_frame: torch.Tensor | None = None,
+        last_frame: torch.Tensor | None = None,
         references: dict | None = None,
         sampling_params: dict | list[dict] | None = None,
         model_params: dict | None = None,
         lora: dict | None = None,
         fast_h3: dict | None = None,
+        latent_edit: dict | None = None,
         **kwargs,
     ):
         if kwargs:
@@ -252,8 +279,11 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
         spec_model = model
 
         if fast_h3 is not None:
-            if frame is not None or references is not None:
-                raise ValueError("FastH3 Preview supports T2VA only; disconnect frame and references inputs.")
+            if any(value is not None for value in (frame, first_frame, last_frame, references)):
+                raise ValueError(
+                    "FastH3 Preview supports T2VA only; disconnect frame, first_frame, "
+                    "last_frame, and references inputs."
+                )
             if lora is not None:
                 raise ValueError(
                     "FastH3 is already fused into the selected server; disconnect the request-level LoRA input."
@@ -314,6 +344,8 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
             spec_model=spec_model,
             prompt=prompt,
             frame=frame,  # frame present => fl2va / Wan I2V
+            first_frame=first_frame,
+            last_frame=last_frame,
             references=references,
             width=width,
             height=height,
@@ -323,6 +355,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
             sampling_params=sampling_params,
             lora=lora,
             model_params=model_params,
+            latent_edit=latent_edit,
         )
         return (output,)
 
@@ -1024,3 +1057,89 @@ class VLLMOmniVideoReferences:
                 if value is not None:
                     refs[f"{kind}_{index}"] = value
         return (refs,)
+
+
+class VLLMOmniLatentMaskEditing:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {},
+            "optional": {
+                "source_video": ("VIDEO",),
+                "source_audio": ("AUDIO",),
+                "video_mask": ("MASK",),
+                "audio_mask": ("FLOAT", {"default": -1.0, "min": -1.0, "max": 1.0, "step": 0.01}),
+            },
+        }
+
+    RETURN_TYPES = ("LATENT_MASK_EDITING",)
+    RETURN_NAMES = ("latent_edit",)
+    FUNCTION = "get_latent_edit"
+    CATEGORY = "vLLM-Omni"
+
+    def get_latent_edit(
+        self,
+        source_video: VideoInput | None = None,
+        source_audio: AudioInput | None = None,
+        video_mask: torch.Tensor | None = None,
+        audio_mask: float = -1.0,
+        **kwargs,
+    ):
+        if kwargs:
+            logger.info("Uncaught kwargs: %s", kwargs)
+        edit = LatentMaskEditing()
+        if source_video is not None:
+            edit["source_video"] = source_video
+        if source_audio is not None:
+            edit["source_audio"] = source_audio
+        if video_mask is not None:
+            edit["video_mask"] = video_mask
+        if audio_mask >= 0.0:
+            edit["audio_mask"] = audio_mask
+        return (edit,)
+
+
+class VLLMOmniMiniMaxH3TemporalMask:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "images": ("IMAGE",),
+                "source_fps": ("FLOAT", {"default": 24.0, "min": 0.01}),
+                "duration": ("FLOAT", {"default": 5.0, "min": 0.01, "max": 15.0}),
+                "mode": (["continuation", "extension"],),
+                "preserve_fraction": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0}),
+            }
+        }
+
+    RETURN_TYPES = ("MASK", "FLOAT", "IMAGE", "MASK")
+    RETURN_NAMES = ("mask", "preview_fps", "preview_images", "preview_mask")
+    FUNCTION = "build"
+    CATEGORY = "vLLM-Omni"
+
+    def build(self, images, source_fps, duration, mode, preserve_fraction=0.5):
+        if images.shape[0] <= 0 or not math.isfinite(source_fps) or source_fps <= 0:
+            raise ValueError("Source must contain frames and have a positive finite FPS.")
+        if not math.isfinite(duration) or duration <= 0 or not 0 <= preserve_fraction <= 1:
+            raise ValueError("Invalid duration or preserve_fraction.")
+        frames = _align_frame_count(max(1, round(duration * 24)))
+        source_seconds = images.shape[0] / source_fps
+        if mode == "extension":
+            if frames / 24 <= source_seconds:
+                raise ValueError("Extension output must be longer than the source; increase duration.")
+            boundary = source_seconds
+        elif mode == "continuation":
+            boundary = min(source_seconds, frames / 24) * preserve_fraction
+        else:
+            raise ValueError(f"Unknown temporal mask mode: {mode}")
+        available = min(frames, math.floor(boundary * 24 + 1e-8))
+        prefix = 0 if available < 5 else 5 + 17 * ((available - 5) // 17)
+        preserved = _video_latent_t(prefix) if prefix else 0
+        total = _video_latent_t(frames)
+        mask = torch.ones(total, 1, 1)
+        mask[:preserved] = 0
+        indices = torch.arange(frames, device=images.device)
+        source_indices = (indices * (source_fps / 24)).floor().long().clamp(max=images.shape[0] - 1)
+        preview_images = images.index_select(0, source_indices)
+        preview_mask = mask.index_select(0, torch.arange(frames) * total // frames)
+        return mask, 24.0, preview_images, preview_mask

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py

@@ -30,6 +30,7 @@ from .format import (
     video_to_base64,
     video_to_bytes,
 )
+from .latent_mask import scalar_mask_to_json, video_mask_to_grid_json
 from .logger import get_logger, pretty_printer
 from .models import lookup_model_spec
 from .types import (
@@ -277,10 +278,13 @@ class VLLMOmniClient:
         fps: int,
         negative_prompt: str | None = None,
         frame: torch.Tensor | None = None,
+        first_frame: torch.Tensor | None = None,
+        last_frame: torch.Tensor | None = None,
         references: dict | None = None,
         sampling_params: dict | None = None,
         model_params: dict | None = None,
         lora: dict | None = None,
+        latent_edit: dict | None = None,
         spec_model: str | None = None,
         **extra_params,
     ) -> VideoInput:
@@ -292,6 +296,16 @@ class VLLMOmniClient:
         """
         if frame is not None and references is not None:
             raise ValueError("Provide only one of frame or references, not both.")
+        if frame is not None and (first_frame is not None or last_frame is not None):
+            raise ValueError("Provide either frame or first_frame/last_frame, not both.")
+        if references is not None and (first_frame is not None or last_frame is not None):
+            raise ValueError("Provide either first_frame/last_frame or references, not both.")
+
+        spec, matched_pattern = lookup_model_spec(spec_model or model)
+        if (first_frame is not None or last_frame is not None) and (
+            matched_pattern is None or "MiniMax-H3" not in matched_pattern
+        ):
+            raise ValueError("first_frame and last_frame are supported only for MiniMax-H3; use frame for this model.")
 
         # === regular payload fields ===
         form = aiohttp.FormData()
@@ -311,11 +325,24 @@ class VLLMOmniClient:
 
         # === multimodal inputs (first-last-frames, references, etc.) ===
         input_reference_image: torch.Tensor | None = None
+        keyframe_images: list[tuple[str, torch.Tensor]] = []
         video_task: str | None = None
 
         if frame is not None:
             input_reference_image = frame
             video_task = "fl2va"
+        elif first_frame is not None or last_frame is not None:
+            frame_indices: list[int] = []
+            if first_frame is not None:
+                keyframe_images.append(("first_frame.png", first_frame))
+                frame_indices.append(0)
+            if last_frame is not None:
+                keyframe_images.append(("last_frame.png", last_frame))
+                frame_indices.append(-1)
+            if len(keyframe_images) == 1:
+                input_reference_image = keyframe_images[0][1]
+            extra_params["frame_indices"] = frame_indices
+            video_task = "fl2va"
         elif references is not None:
             reference_formats = (
                 ("image", MAX_REFERENCE_IMAGES, "png", "image/png", image_tensor_to_png_bytes),
@@ -353,7 +380,7 @@ class VLLMOmniClient:
             video_task = "t2va"
 
         if input_reference_image is not None:
-            image_filename = "image.png"  # Required for multipart form
+            image_filename = keyframe_images[0][0] if keyframe_images else "image.png"
             form.add_field(
                 "input_reference",
                 image_tensor_to_png_bytes(input_reference_image, image_filename),
@@ -361,12 +388,67 @@ class VLLMOmniClient:
                 content_type="image/png",
             )
 
+        # === latent-mask editing (MiniMax H3) ===
+        if latent_edit is not None:
+            source_video = latent_edit.get("source_video")
+            source_audio = latent_edit.get("source_audio")
+            video_mask = latent_edit.get("video_mask")
+            audio_mask = latent_edit.get("audio_mask")
+
+            if video_mask is None and audio_mask is None:
+                raise ValueError("Latent-mask editing requires at least one mask.")
+
+            video_mask_trivial = video_mask is None or bool((video_mask == 1.0).all().item())
+            audio_mask_trivial = audio_mask is None or audio_mask == 1.0
+            if not video_mask_trivial and source_video is None:
+                raise ValueError("A non-trivial video mask requires a source video.")
+            if not audio_mask_trivial and source_audio is None and source_video is None:
+                raise ValueError("A non-trivial audio mask requires a source audio or a source video with audio.")
+
+            if source_video is not None:
+                form.add_field(
+                    "source_video",
+                    video_to_bytes(source_video, "source.mp4"),
+                    filename="source.mp4",
+                    content_type="video/mp4",
+                )
+            if source_audio is not None:
+                form.add_field(
+                    "source_audio",
+                    audio_to_bytes(source_audio, "source_audio.mp3"),
+                    filename="source_audio.mp3",
+                    content_type="audio/mpeg",
+                )
+            if video_mask is not None:
+                mask_json = video_mask_to_grid_json(video_mask, width=width, height=height, num_frames=num_frames)
+                form.add_field(
+                    "video_noise_mask",
+                    mask_json.encode("utf-8"),
+                    filename="video-mask.json",
+                    content_type="application/json",
+                )
+            if audio_mask is not None:
+                form.add_field(
+                    "audio_noise_mask",
+                    scalar_mask_to_json(audio_mask).encode("utf-8"),
+                    filename="audio-mask.json",
+                    content_type="application/json",
+                )
+
+        if len(keyframe_images) == 2:
+            for image_filename, image in keyframe_images:
+                form.add_field(
+                    "input_references",
+                    image_tensor_to_png_bytes(image, image_filename),
+                    filename=image_filename,
+                    content_type="image/png",
+                )
+
         # === model specific params. Either use a specialized builder, or add flattened fields as-is ===
         if model_params is not None:
             model_params = dict(model_params)
             model_params.pop("type", None)
 
-        spec, _ = lookup_model_spec(spec_model or model)
         params_builder = spec.get("params_builder") if spec else None
         if params_builder is not None:
             form_fields = params_builder(

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py

@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+import json
+
+import torch
+import torch.nn.functional as F
+
+
+def scalar_mask_to_json(value: float) -> str:
+    if not 0.0 <= value <= 1.0:
+        raise ValueError(f"mask value must be in [0, 1], got {value}")
+    return str(value)
+
+
+def _align_frame_count(frame_count: int) -> int:
+    if frame_count <= 0:
+        return 1
+    current = int(frame_count)
+    while current % 17 != 5:
+        current += 1
+    return current
+
+
+def _video_latent_t(frame_count: int) -> int:
+    if frame_count <= 5:
+        return 2
+    return ((int(frame_count) - 5) // 17) * 5 + 2
+
+
+def video_mask_to_grid(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> torch.Tensor:
+    tv = _video_latent_t(_align_frame_count(num_frames))
+    height = int(height) // 32 * 32
+    width = int(width) // 32 * 32
+    gh, gw = height // 16, width // 16
+
+    if mask.ndim == 2:
+        mask = mask.unsqueeze(0)
+    elif mask.ndim != 3:
+        raise ValueError(f"expected a 2D or 3D mask tensor, got {mask.ndim}D")
+
+    grid = F.interpolate(mask.unsqueeze(1).float(), size=(gh, gw), mode="area").squeeze(1)
+    if grid.shape[0] == 1:
+        grid = grid.expand(tv, gh, gw)
+    elif grid.shape[0] != tv:
+        grid = F.interpolate(grid.unsqueeze(0).unsqueeze(0), size=(tv, gh, gw), mode="nearest").squeeze(0).squeeze(0)
+    return grid
+
+
+def video_mask_to_grid_json(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> str:
+    return json.dumps(video_mask_to_grid(mask, width=width, height=height, num_frames=num_frames).tolist())

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py

@@ -40,6 +40,10 @@ class VideoReferences(dict):
     pass
 
 
+class LatentMaskEditing(dict):
+    pass
+
+
 class FastH3Deployment(dict):
     """Descriptor for a server that fused FastH3 at startup."""
 

apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md

@@ -0,0 +1,17 @@
+# WF-05: MiniMax-H3 latent-mask editing
+
+Edit a source video with MiniMax-H3 using spatial or temporal masks. The workflow includes object removal, inpainting, continuation, and extension, with a mask preview for each case.
+
+## Requirements
+
+- ComfyUI with ComfyUI-vLLM-Omni installed.
+- An H3 service using the FL2VA checkpoint partition with latent-mask editing support. Follow the [H3 recipe](../../../recipes/MiniMaxAI/MiniMax-H3.md) for server setup.
+
+## Use
+
+Open **vLLM-Omni MiniMax-H3 Latent Mask Editing.json** from the example workflows and set the Generate Video URL/model for the case you want to run.
+
+1. Upload your source video in Load Video.
+2. Adjust the case's mask. For removal and inpainting, set Mask Canvas to the video dimensions, Mask Rectangle to the region size, and Combine Masks x/y to its position. For continuation and extension, set MiniMax-H3 Temporal Mask's mode and duration; continuation also uses `preserve_fraction`. Match the duration in Generate Video, and use a duration longer than the source for extension. Mask values are `0` to preserve and `1` to regenerate.
+3. Adjust the prompt to describe the desired content. Set the output dimensions and duration in Generate Video. If editing audio, update the sound description and audio mask (`0` preserves, `1` regenerates the whole clip).
+4. Select the case's Mask Preview Save Video node and choose **Execute to selected output nodes** to preview the mask as a red overlay without inference, or select its Result Save Video node to generate the edited video. Running the entire workflow generates all four cases.

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json

@@ -0,0 +1,499 @@
+{
+  "id": "latent-mask-editing-7380",
+  "revision": 0,
+  "last_node_id": 7,
+  "last_link_id": 5,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "MarkdownNote",
+      "pos": [
+        40,
+        20
+      ],
+      "size": [
+        420,
+        210
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Latent-mask editing server prerequisite",
+      "properties": {},
+      "widgets_values": [
+        "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends."
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 2,
+      "type": "LoadVideo",
+      "pos": [
+        40,
+        280
+      ],
+      "size": [
+        420,
+        120
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "file",
+          "name": "file",
+          "type": "COMBO",
+          "widget": {
+            "name": "file"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        ""
+      ]
+    },
+    {
+      "id": 3,
+      "type": "LoadImageMask",
+      "pos": [
+        40,
+        460
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image",
+          "type": "STRING",
+          "widget": {
+            "name": "image"
+          },
+          "link": null
+        },
+        {
+          "name": "channel",
+          "type": "STRING",
+          "widget": {
+            "name": "channel"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "mask",
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadImageMask"
+      },
+      "widgets_values": [
+        "",
+        "red"
+      ]
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniLatentMaskEditing",
+      "pos": [
+        560,
+        260
+      ],
+      "size": [
+        420,
+        300
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "source_video",
+          "name": "source_video",
+          "type": "VIDEO",
+          "link": 1,
+          "shape": 7
+        },
+        {
+          "localized_name": "source_audio",
+          "name": "source_audio",
+          "type": "AUDIO",
+          "link": null
+        },
+        {
+          "localized_name": "video_mask",
+          "name": "video_mask",
+          "type": "MASK",
+          "link": 2,
+          "shape": 7
+        },
+        {
+          "localized_name": "audio_mask",
+          "name": "audio_mask",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_mask"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniLatentMaskEditing"
+      },
+      "widgets_values": [
+        0.5
+      ]
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        40,
+        680
+      ],
+      "size": [
+        420,
+        140
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "audio_flow_shift",
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "flow_shift",
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video params",
+          "name": "video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ]
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        1080,
+        200
+      ],
+      "size": [
+        540,
+        680
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "frame",
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "references",
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "sampling_params",
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": null
+        },
+        {
+          "localized_name": "lora",
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "localized_name": "model_params",
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 4,
+          "shape": 7
+        },
+        {
+          "localized_name": "fast_h3",
+          "name": "fast_h3",
+          "type": "FASTH3_DEPLOYMENT",
+          "link": null
+        },
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "link": 3,
+          "shape": 7
+        },
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "prompt",
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "negative_prompt",
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "fps",
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "duration",
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "Partially restyle the clip and soundtrack",
+        "",
+        160,
+        120,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 7,
+      "type": "SaveVideo",
+      "pos": [
+        1700,
+        300
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "link": 5,
+          "shape": 7
+        },
+        {
+          "localized_name": "filename_prefix",
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "format",
+          "name": "format",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "vllm-omni/latent-edit",
+        "mp4"
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      2,
+      0,
+      4,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      3,
+      0,
+      4,
+      2,
+      "MASK"
+    ],
+    [
+      3,
+      4,
+      0,
+      6,
+      6,
+      "LATENT_MASK_EDITING"
+    ],
+    [
+      4,
+      5,
+      0,
+      6,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      5,
+      6,
+      0,
+      7,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json

@@ -0,0 +1,560 @@
+{
+  "last_node_id": 9,
+  "last_link_id": 7,
+  "nodes": [
+    {
+      "id": 2,
+      "type": "LoadVideo",
+      "pos": [
+        40,
+        100
+      ],
+      "size": [
+        420,
+        120
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "file",
+          "type": "STRING",
+          "widget": {
+            "name": "file"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        ""
+      ]
+    },
+    {
+      "id": 3,
+      "type": "SolidMask",
+      "pos": [
+        40,
+        300
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "value",
+          "type": "FLOAT",
+          "widget": {
+            "name": "value"
+          },
+          "link": null
+        },
+        {
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SolidMask"
+      },
+      "widgets_values": [
+        0.0,
+        160,
+        120
+      ]
+    },
+    {
+      "id": 4,
+      "type": "SolidMask",
+      "pos": [
+        40,
+        500
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "value",
+          "type": "FLOAT",
+          "widget": {
+            "name": "value"
+          },
+          "link": null
+        },
+        {
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SolidMask"
+      },
+      "widgets_values": [
+        1.0,
+        160,
+        120
+      ]
+    },
+    {
+      "id": 5,
+      "type": "BatchMasksNode",
+      "pos": [
+        250,
+        380
+      ],
+      "size": [
+        200,
+        100
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "masks.mask0",
+          "type": "MASK",
+          "link": 2,
+          "slot_index": 0
+        },
+        {
+          "name": "masks.mask1",
+          "type": "MASK",
+          "link": 3,
+          "slot_index": 1
+        }
+      ],
+      "outputs": [
+        {
+          "name": "MASK",
+          "type": "MASK",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "BatchMasksNode"
+      },
+      "widgets_values": []
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniLatentMaskEditing",
+      "pos": [
+        450,
+        180
+      ],
+      "size": [
+        400,
+        120
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "source_video",
+          "type": "VIDEO",
+          "link": 1
+        },
+        {
+          "name": "source_audio",
+          "type": "AUDIO",
+          "link": null
+        },
+        {
+          "name": "video_mask",
+          "type": "MASK",
+          "link": 4
+        },
+        {
+          "name": "audio_mask",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_mask"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniLatentMaskEditing"
+      },
+      "widgets_values": [
+        -1.0
+      ]
+    },
+    {
+      "id": 7,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        40,
+        700
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            6
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ]
+    },
+    {
+      "id": 8,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        700,
+        180
+      ],
+      "size": [
+        450,
+        380
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null
+        },
+        {
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null
+        },
+        {
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": null
+        },
+        {
+          "name": "lora",
+          "type": "LORA",
+          "link": null
+        },
+        {
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 6
+        },
+        {
+          "name": "fast_h3",
+          "type": "FAST_H3",
+          "link": null
+        },
+        {
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "link": 5
+        },
+        {
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            7
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "temporal mask: regenerate the ending",
+        "",
+        160,
+        120,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 9,
+      "type": "SaveVideo",
+      "pos": [
+        900,
+        180
+      ],
+      "size": [
+        420,
+        200
+      ],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "link": 7
+        },
+        {
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "name": "format",
+          "type": "STRING",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "temporal_mask",
+        "mp4"
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      2,
+      0,
+      6,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      3,
+      0,
+      5,
+      0,
+      "MASK"
+    ],
+    [
+      3,
+      4,
+      0,
+      5,
+      1,
+      "MASK"
+    ],
+    [
+      4,
+      5,
+      0,
+      6,
+      2,
+      "MASK"
+    ],
+    [
+      5,
+      6,
+      0,
+      8,
+      6,
+      "LATENT_MASK_EDITING"
+    ],
+    [
+      6,
+      7,
+      0,
+      8,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      7,
+      8,
+      0,
+      9,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json

@@ -0,0 +1,511 @@
+{
+  "id": "latent-mask-editing-7380",
+  "revision": 0,
+  "last_node_id": 7,
+  "last_link_id": 5,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "MarkdownNote",
+      "pos": [
+        40,
+        20
+      ],
+      "size": [
+        420,
+        210
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Latent-mask editing server prerequisite",
+      "properties": {},
+      "widgets_values": [
+        "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends."
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 2,
+      "type": "LoadVideo",
+      "pos": [
+        40,
+        280
+      ],
+      "size": [
+        420,
+        120
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "file",
+          "name": "file",
+          "type": "COMBO",
+          "widget": {
+            "name": "file"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        ""
+      ]
+    },
+    {
+      "id": 3,
+      "type": "SolidMask",
+      "pos": [
+        40,
+        460
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "value",
+          "name": "value",
+          "type": "FLOAT",
+          "widget": {
+            "name": "value"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "mask",
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SolidMask"
+      },
+      "widgets_values": [
+        1.0,
+        160,
+        120
+      ]
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniLatentMaskEditing",
+      "pos": [
+        560,
+        260
+      ],
+      "size": [
+        420,
+        300
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "source_video",
+          "name": "source_video",
+          "type": "VIDEO",
+          "link": 1,
+          "shape": 7
+        },
+        {
+          "localized_name": "source_audio",
+          "name": "source_audio",
+          "type": "AUDIO",
+          "link": null
+        },
+        {
+          "localized_name": "video_mask",
+          "name": "video_mask",
+          "type": "MASK",
+          "link": 2,
+          "shape": 7
+        },
+        {
+          "localized_name": "audio_mask",
+          "name": "audio_mask",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_mask"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniLatentMaskEditing"
+      },
+      "widgets_values": [
+        0.5
+      ]
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        40,
+        680
+      ],
+      "size": [
+        420,
+        140
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "audio_flow_shift",
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "flow_shift",
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video params",
+          "name": "video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ]
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        1080,
+        200
+      ],
+      "size": [
+        540,
+        680
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "frame",
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "references",
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "sampling_params",
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": null
+        },
+        {
+          "localized_name": "lora",
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "localized_name": "model_params",
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 4,
+          "shape": 7
+        },
+        {
+          "localized_name": "fast_h3",
+          "name": "fast_h3",
+          "type": "FASTH3_DEPLOYMENT",
+          "link": null
+        },
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "link": 3,
+          "shape": 7
+        },
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "prompt",
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "negative_prompt",
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "fps",
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "duration",
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "Partially restyle the clip and soundtrack",
+        "",
+        160,
+        120,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 7,
+      "type": "SaveVideo",
+      "pos": [
+        1700,
+        300
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "link": 5,
+          "shape": 7
+        },
+        {
+          "localized_name": "filename_prefix",
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "format",
+          "name": "format",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "vllm-omni/latent-edit",
+        "mp4"
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      2,
+      0,
+      4,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      3,
+      0,
+      4,
+      2,
+      "MASK"
+    ],
+    [
+      3,
+      4,
+      0,
+      6,
+      6,
+      "LATENT_MASK_EDITING"
+    ],
+    [
+      4,
+      5,
+      0,
+      6,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      5,
+      6,
+      0,
+      7,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax H3 Image to Video.json

@@ -0,0 +1,219 @@
+{
+  "id": "6676f19a-7f41-4c80-8cb7-f80adcb4ac44",
+  "revision": 0,
+  "last_node_id": 9,
+  "last_link_id": 4,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "LoadImage",
+      "pos": [40, 20],
+      "size": [270, 314],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "image", "name": "image", "type": "COMBO", "widget": {"name": "image"}, "link": null},
+        {"localized_name": "choose file to upload", "name": "upload", "type": "IMAGEUPLOAD", "widget": {"name": "upload"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "IMAGE", "name": "IMAGE", "type": "IMAGE", "links": [1]},
+        {"localized_name": "MASK", "name": "MASK", "type": "MASK", "links": null}
+      ],
+      "title": "First Frame",
+      "properties": {"Node name for S&R": "LoadImage"},
+      "widgets_values": ["first_frame.png", "image"]
+    },
+    {
+      "id": 2,
+      "type": "LoadImage",
+      "pos": [40, 390],
+      "size": [270, 314],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "image", "name": "image", "type": "COMBO", "widget": {"name": "image"}, "link": null},
+        {"localized_name": "choose file to upload", "name": "upload", "type": "IMAGEUPLOAD", "widget": {"name": "upload"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "IMAGE", "name": "IMAGE", "type": "IMAGE", "links": null},
+        {"localized_name": "MASK", "name": "MASK", "type": "MASK", "links": null}
+      ],
+      "title": "Last Frame (optional)",
+      "properties": {"Node name for S&R": "LoadImage"},
+      "widgets_values": ["last_frame.png", "image"]
+    },
+    {
+      "id": 3,
+      "type": "MarkdownNote",
+      "pos": [350, 20],
+      "size": [390, 240],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "MiniMax H3 I2V modes",
+      "properties": {},
+      "widgets_values": ["This workflow uses the remote vLLM-Omni Generate Video node.\n\n- First-frame only (default): keep First Frame connected.\n- Last-frame only: disconnect First Frame and connect Last Frame to last_frame.\n- First+last-frame: connect both image loaders.\n\nLeave the legacy frame and references inputs disconnected. Select images from ComfyUI's input folder. H3 runs at 24 FPS; the 5.167-second duration resolves to 124 frames, satisfying 17k+5."],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniDiffusionSampling",
+      "pos": [350, 300],
+      "size": [300, 226],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "n", "name": "n", "type": "INT", "widget": {"name": "n"}, "link": null},
+        {"localized_name": "num_inference_steps", "name": "num_inference_steps", "type": "INT", "widget": {"name": "num_inference_steps"}, "link": null},
+        {"localized_name": "guidance_scale", "name": "guidance_scale", "type": "FLOAT", "widget": {"name": "guidance_scale"}, "link": null},
+        {"localized_name": "true_cfg_scale", "name": "true_cfg_scale", "type": "FLOAT", "widget": {"name": "true_cfg_scale"}, "link": null},
+        {"localized_name": "vae_use_slicing", "name": "vae_use_slicing", "type": "BOOLEAN", "widget": {"name": "vae_use_slicing"}, "link": null},
+        {"localized_name": "vae_use_tiling", "name": "vae_use_tiling", "type": "BOOLEAN", "widget": {"name": "vae_use_tiling"}, "link": null},
+        {"localized_name": "seed", "name": "seed", "type": "INT", "widget": {"name": "seed"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "diffusion sampling params", "name": "diffusion sampling params", "type": "SAMPLING_PARAMS", "links": [2]}
+      ],
+      "title": "Base H3 Sampling (50 steps)",
+      "properties": {"Node name for S&R": "VLLMOmniDiffusionSampling"},
+      "widgets_values": [1, 50, 1, 1, false, false, 757358688076805, "fixed"]
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [350, 570],
+      "size": [300, 82],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "audio_flow_shift", "name": "audio_flow_shift", "type": "FLOAT", "widget": {"name": "audio_flow_shift"}, "link": null},
+        {"localized_name": "flow_shift", "name": "flow_shift", "type": "FLOAT", "widget": {"name": "flow_shift"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "MiniMax-H3 video params", "name": "MiniMax-H3 video params", "type": "VIDEO_PARAMS", "links": [3]}
+      ],
+      "title": "Base H3 Flow Shifts",
+      "properties": {"Node name for S&R": "VLLMOmniMiniMaxH3Params"},
+      "widgets_values": [3, 12]
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniRemoteLoRA",
+      "pos": [700, 550],
+      "size": [360, 150],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "local_path", "name": "local_path", "type": "STRING", "widget": {"name": "local_path"}, "link": null},
+        {"localized_name": "name", "name": "name", "type": "STRING", "widget": {"name": "name"}, "link": null},
+        {"localized_name": "scale", "name": "scale", "type": "FLOAT", "widget": {"name": "scale"}, "link": null},
+        {"localized_name": "int_id", "name": "int_id", "type": "INT", "widget": {"name": "int_id"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "lora", "name": "lora", "type": "REMOTE_LORA", "links": null}
+      ],
+      "title": "Optional H3 Turbo Remote LoRA",
+      "properties": {"Node name for S&R": "VLLMOmniRemoteLoRA"},
+      "widgets_values": ["minimax_h3_fl2v_turbo_8step_v1.0_768p_bf16.safetensors", "h3-turbo-8step", 1, 0]
+    },
+    {
+      "id": 7,
+      "type": "MarkdownNote",
+      "pos": [700, 300],
+      "size": [420, 210],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Optional Turbo settings",
+      "properties": {},
+      "widgets_values": ["Base H3 is configured by default: 50 steps, flow_shift 12, audio_flow_shift 3.\n\nFor the shown 8-step Diffusers Turbo artifact, set its full local_path on the server, start the server with the matching preloaded LoRA, connect this Remote LoRA to Generate Video, set num_inference_steps to 9, and set flow_shift to 6. Keep audio_flow_shift at 3 and scale at 1.\n\nDo not use the _comfyui_ repack or FastH3 here."],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 8,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [1160, 80],
+      "size": [440, 610],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "frame", "name": "frame", "shape": 7, "type": "IMAGE", "link": null},
+        {"localized_name": "first_frame", "name": "first_frame", "shape": 7, "type": "IMAGE", "link": 1},
+        {"localized_name": "last_frame", "name": "last_frame", "shape": 7, "type": "IMAGE", "link": null},
+        {"localized_name": "references", "name": "references", "shape": 7, "type": "VIDEO_REFERENCES", "link": null},
+        {"localized_name": "sampling_params", "name": "sampling_params", "shape": 7, "type": "SAMPLING_PARAMS", "link": 2},
+        {"localized_name": "lora", "name": "lora", "shape": 7, "type": "REMOTE_LORA", "link": null},
+        {"localized_name": "model_params", "name": "model_params", "shape": 7, "type": "VIDEO_PARAMS", "link": 3},
+        {"localized_name": "fast_h3", "name": "fast_h3", "shape": 7, "type": "FASTH3_DEPLOYMENT", "link": null},
+        {"localized_name": "url", "name": "url", "type": "STRING", "widget": {"name": "url"}, "link": null},
+        {"localized_name": "model", "name": "model", "type": "STRING", "widget": {"name": "model"}, "link": null},
+        {"localized_name": "prompt", "name": "prompt", "type": "STRING", "widget": {"name": "prompt"}, "link": null},
+        {"localized_name": "negative_prompt", "name": "negative_prompt", "type": "STRING", "widget": {"name": "negative_prompt"}, "link": null},
+        {"localized_name": "width", "name": "width", "type": "INT", "widget": {"name": "width"}, "link": null},
+        {"localized_name": "height", "name": "height", "type": "INT", "widget": {"name": "height"}, "link": null},
+        {"localized_name": "fps", "name": "fps", "type": "INT", "widget": {"name": "fps"}, "link": null},
+        {"localized_name": "duration", "name": "duration", "type": "FLOAT", "widget": {"name": "duration"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "video", "name": "video", "type": "VIDEO", "links": [4]}
+      ],
+      "properties": {"Node name for S&R": "VLLMOmniGenerateVideo"},
+      "widgets_values": [
+        "http://localhost:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "A cinematic product shot begins exactly from the supplied keyframe. The subject moves naturally through a dark studio while blue and amber rim lights shift across reflective surfaces. The camera makes a slow controlled orbit. Audio: subtle room tone, precise mechanical details, and a restrained electronic score.",
+        "",
+        1344,
+        768,
+        24,
+        5.167
+      ]
+    },
+    {
+      "id": 9,
+      "type": "SaveVideo",
+      "pos": [1650, 170],
+      "size": [374, 314],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "video", "name": "video", "type": "VIDEO", "link": 4},
+        {"localized_name": "filename_prefix", "name": "filename_prefix", "type": "STRING", "widget": {"name": "filename_prefix"}, "link": null},
+        {"localized_name": "format", "name": "format", "type": "COMBO", "widget": {"name": "format"}, "link": null},
+        {"localized_name": "codec", "name": "codec", "type": "COMFY_DYNAMICCOMBO_V3", "widget": {"name": "codec"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "video", "name": "video", "type": "VIDEO", "links": null}
+      ],
+      "properties": {},
+      "widgets_values": ["video/MiniMax_H3_I2V", "auto", "auto"]
+    }
+  ],
+  "links": [
+    [1, 1, 0, 8, 1, "IMAGE"],
+    [2, 4, 0, 8, 4, "SAMPLING_PARAMS"],
+    [3, 5, 0, 8, 6, "VIDEO_PARAMS"],
+    [4, 8, 0, 9, 0, "VIDEO"]
+  ],
+  "groups": [
+    {"id": 1, "title": "Portable first/last-frame inputs", "bounding": [10, -30, 760, 770], "color": "#3f789e", "flags": {}},
+    {"id": 2, "title": "H3 sampling and optional Turbo", "bounding": [320, 270, 830, 460], "color": "#8a6d3b", "flags": {}},
+    {"id": 3, "title": "Remote generation and audio-preserving save", "bounding": [1130, 30, 930, 700], "color": "#3f789e", "flags": {}}
+  ],
+  "config": {},
+  "extra": {"workflowRendererVersion": "LG", "ds": {"scale": 0.75, "offset": [180, 120]}},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/web/main.js

@@ -52,6 +52,7 @@ const FAMILY_BY_OUTPUT = {
     REMOTE_LORA: "deployment",
     FASTH3_DEPLOYMENT: "deployment",
     VIDEO_REFERENCES: "media",
+    LATENT_MASK_EDITING: "media",
 };
 
 function familyOf(nodeData) {

benchmarks/README.md

@@ -41,6 +41,17 @@ LingBot-Video MoE transformer.
 - **Dense pipeline**: decoded-video MAE, MSE, PSNR, latency, and optional steady-state timings
 - **MoE transformer**: bitwise router, sparse-block, shared-expert, and full-transformer parity
 
+### [LingBot-World](lingbot_world/README.md) — Realtime Streaming Cadence
+
+Client-side benchmark for a `WS /v1/realtime/video` world-model session, where one
+request is a whole rollout and one AR block streams out as one video chunk. It
+measures a cadence rather than a throughput: there is exactly one session, and what
+matters is whether the next chunk arrives before the viewer finishes the last one.
+
+- **Layout**: `lingbot_world/benchmark_lingbot_world_realtime.py` (WS client), `lingbot_world/workload.py` (rollout and metric math), `lingbot_world/configs/` (single-GPU eager and Ulysses-4 compiled deploy configs)
+- **Workload**: built-in image-conditioned rollout with a generated camera script, or a JSON spec with per-chunk actions and mid-rollout prompt updates
+- **Key metrics**: TTFC (time to first chunk), steady-state chunk inter-arrival percentiles, VIDEO_RTF (wall seconds per video second; lower is better), chunk-deadline attainment, and simulated playback underruns
+
 ### [Distributed](distributed/omni_connectors/README.md) — RDMA Connector Testing
 
 RDMA environment setup and transfer tests for `MooncakeTransferEngineConnector`, including pytest-based single-node checks and manual cross-node benchmarks.

benchmarks/kernels/mot_linear_benchmarks.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 # ruff: noqa: N803
 
 """MoT (Mixture-of-Tokens) GEMM kernel benchmark and auto-tuning.
@@ -10,7 +13,7 @@ Usage:
     python benchmarks/kernels/mot_linear_benchmarks.py \
         --model ByteDance-Seed/BAGEL-7B-MoT \
         --tp-size 1 --dtype w16a16 --tune \
-        --save-dir vllm_omni/diffusion/layers/mot/configs/
+        --save-dir vllm_omni/diffusion/models/bagel/mot/configs/
 
     # Auto-tune with local model path (offline clusters):
     python benchmarks/kernels/mot_linear_benchmarks.py \
@@ -43,7 +46,7 @@ from vllm.triton_utils import triton
 from vllm.utils.torch_utils import set_random_seed
 
 # NOTE: you should use the same naming syetem for the kernel to load properly
-from vllm_omni.diffusion.layers.mot.ops.mot_gemm import build_config_filename, get_device_name
+from vllm_omni.diffusion.models.bagel.mot.ops.mot_gemm import build_config_filename, get_device_name
 
 # clear the triton cache from time to time, usaully no need to change
 _CACHE_CLEAR_INTERVAL_ENV = "VLLM_MOT_TUNE_CACHE_CLEAR_INTERVAL"
@@ -428,7 +431,7 @@ def benchmark_config(
     cache_flusher: torch.Tensor | None = None,
 ) -> float:
     """Run a MoT GEMM with the given tile config and return avg latency (us)."""
-    from vllm_omni.diffusion.layers.mot.ops.mot_gemm import invoke_mot_gemm
+    from vllm_omni.diffusion.models.bagel.mot.ops.mot_gemm import invoke_mot_gemm
 
     text_indices, vae_indices, M = build_regular_indices(
         image_num=image_num,
@@ -553,7 +556,7 @@ class BenchmarkWorker:
         use_int8_w8a16: bool,
     ) -> tuple[dict[str, int], float]:
         set_random_seed(self.seed)
-        from vllm_omni.diffusion.layers.mot.ops.mot_gemm import (
+        from vllm_omni.diffusion.models.bagel.mot.ops.mot_gemm import (
             get_best_mot_config,
         )
 

benchmarks/lingbot_world/README.md

@@ -0,0 +1,224 @@
+# LingBot-World Benchmarks — Realtime Streaming Cadence
+
+`benchmark_lingbot_world_realtime.py` measures LingBot-World 2.0 as it is
+actually deployed: one `WS /v1/realtime/video` session that streams one video
+chunk per AR block, for as long as the world keeps running.
+
+## Why this is not the diffusion serving benchmark
+
+[`benchmarks/diffusion`](../diffusion/README.md) measures "one request, one
+finished video" and reports request throughput. LingBot-World has neither shape:
+
+- **One request is the whole rollout.** `prepare_encode` runs once, then every AR
+  block is four DMD steps plus one clean-KV commit forward, and each block
+  streams out as a chunk. Throughput in requests per second is meaningless.
+- **There is exactly one session.** The stepwise deploy config pins
+  `max_num_seqs: 1`, and the AR-Diffusion consumer rejects a stage with more than
+  one replica, because session-affine routing does not exist yet. There is no
+  concurrency to sweep.
+- **What matters is cadence.** A deployment is healthy when chunk *N + 1* arrives
+  before a viewer finishes watching chunk *N*. That is a latency distribution
+  against a wall clock, not a throughput number.
+
+## Quick start
+
+Serve the model, then point the benchmark at it. The benchmark never starts a
+server.
+
+```bash
+# Terminal 1 — one GPU, eager (portable smoke configuration)
+vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+  --omni \
+  --deploy-config benchmarks/lingbot_world/configs/single_gpu_eager.yaml \
+  --port 8000
+
+# Terminal 2
+pip install websockets
+python benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py \
+  --host 127.0.0.1 --port 8000 \
+  --num-chunks 16 \
+  --output-json /tmp/lingbot_world_realtime.json
+```
+
+With no `--image`, the first frame comes from the shared vLLM image asset, so the
+default workload needs no setup and this repository carries no benchmark binary.
+Pass `--image /path/to/frame.png` to condition the rollout on your own frame.
+
+### Realtime topology
+
+The single-GPU config is a smoke configuration, not a realtime one: one card runs
+all five forwards of every block serially and nothing is compiled. Use the
+four-GPU config for numbers that mean anything about realtime behavior.
+
+```bash
+vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+  --omni \
+  --deploy-config benchmarks/lingbot_world/configs/usp4_compiled.yaml \
+  --port 8000
+
+python benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py \
+  --port 8000 --num-chunks 40 --warmup-sessions 1 --sessions 3 \
+  --target-fps 12 --output-json /tmp/lingbot_world_repeated.json
+```
+
+`usp4_compiled.yaml` sets pure Ulysses sequence parallelism over four ranks and
+`enforce_eager: false`. The pipeline validates the parallel shape itself:
+`sequence_parallel_size` must equal `ulysses_degree`, `ring_degree` and
+`allgather_degree` must be 1, `ulysses_mode` must be `strict`, and pipeline,
+CFG, and VAE parallel sizes above 1 are rejected. This benchmark config sets
+`tensor_parallel_size: 1`; the pipeline validator does not reject larger TP sizes.
+
+Compiled mode can pay compilation and capture costs on the first rollout.
+Use `--warmup-sessions 1` before measured repetitions. Excluding the first six
+chunks addresses the attention-window ramp; it does not guarantee that
+compilation and capture have finished.
+
+## Metrics
+
+| Metric | Meaning |
+| --- | --- |
+| `ttfc_ms` | **Time to first chunk**: `session.start` sent until the first chunk's bytes arrive. Pays for prompt encode, first-frame VAE encode, and any compile or capture. |
+| `interval_all` | Inter-arrival latency across every chunk: mean, median, std, min, p90, p95, p99, max. |
+| `interval_steady` | The same summary over steady-state chunks only. |
+| `video_rtf` | Wall seconds spent per second of video, matching vLLM-Omni's own RTF (`generation_s / video_duration`). **Lower is better; below 1.0 is real time.** |
+| `video_rtfx` | The reciprocal, under the name this repo uses for it: video seconds per wall second, where higher is better. |
+| `steady_video_rtf` | `video_rtf` restricted to steady-state chunks. |
+| `chunk_deadline_ms` | Wall-clock budget one steady chunk must meet to hold real time: 12 frames ÷ `--target-fps`. Override directly with `--chunk-slo-ms`. |
+| `slo_attainment` | Fraction of steady chunks that met the deadline. |
+| `playback` | Underrun count, total stall, and stall ratio from a simulated viewer. |
+
+### Which frame rate counts as real time
+
+**The checkpoint declares no frame rate.** There is none in `model_index.json`, the
+scheduler config, or the transformer config, and the pipeline never reads one. A
+chunk is twelve pixel frames; how long those frames *last* is a property of how the
+output is played, not of the model.
+
+That choice moves the verdict without moving anything measured:
+
+| basis | video per chunk | deadline | RTF at a measured 1357 ms/chunk |
+| --- | --- | --- | --- |
+| 16 fps | 0.750 s | 750 ms | 1.809 |
+| 12 fps | 1.000 s | 1000 ms | 1.357 |
+
+So `--fps` (sent to the server, which uses it to label the muxed MP4) and
+`--target-fps` (the real-time basis for RTF, the deadline, and the playback
+simulation) are separate flags. `--target-fps` defaults to `--fps`; set it when the
+two differ, and the report prints both. State the basis whenever you quote a number
+— an RTF without its fps is not a measurement. (RTF is a cost here, so a bigger number is a
+worse result; a steady chunk's RTF is simply its interval divided by its video duration.)
+An RTF above 1 means the measured deployment is slower than real time at the
+stated target frame rate. The benchmark measures this failure as well as success.
+
+### Why warmup chunks are not optional
+
+The DiT attends over a sliding window of `sliding_window_num_frames = 18` latent
+frames with `sink_size = 9`, so the key count a block attends over keeps growing
+until eighteen latent frames of history exist — six chunks at three frames per
+block. **Chunk cost is not stationary before chunk six.** A mean over a short
+rollout is therefore lower than anything a running session will sustain, which is
+why `--warmup-chunks` defaults to 6 and steady-state metrics are reported
+separately. Keep `--num-chunks` comfortably above the warmup, or the benchmark
+tells you no chunk reached steady state. The terminal chunk skips next-chunk
+preparation, so it is excluded from steady metrics too; it remains in overall
+RTF, all-chunk intervals, and playback simulation.
+
+The default 16-chunk rollout provides only nine steady intervals. Percentiles
+are interpolated sample summaries; the report warns when fewer than 100
+intervals contribute, and even 100 samples do not guarantee a stable p99.
+Use longer rollouts for tail analysis and at least `--sessions 3` after
+`--warmup-sessions 1` for reported comparisons. Session warmup pays compilation
+cost; chunk warmup separately excludes the attention-window ramp. The command
+above yields 33 steady intervals per session (indices 6 through 38), or 99 across
+three sessions. Report each session's mean and the spread across sessions; this
+sample still does not establish a reliable p99.
+
+### Why playback is simulated
+
+A p99 interval alone cannot say whether a viewer saw a stall: the same p99 is
+invisible behind a three-chunk buffer and a visible freeze behind none. The
+benchmark replays chunk arrivals against a wall clock — playback starts once
+`--playback-buffer-chunks` chunks have landed and then consumes video at `--target-fps`
+— and reports every moment the player ran dry.
+
+### What the client-side number is worth
+
+On a four-GPU Ulysses run the client's mean inter-arrival over all fifteen
+post-first-chunk intervals was **1326.743 ms**, against the server's own
+`StageRequestStats.inter_output_latency_ms` of **1327.479 ms** for the same
+request — a difference of **0.74 ms**. This cross-check shows agreement in
+average output cadence for that run, including warmup and the terminal chunk.
+It does not measure encoding or transport latency: a fixed delivery delay can
+cancel out of inter-arrival differences. Attributing that latency requires
+per-chunk timestamps at both boundaries or server-side profiling.
+
+Compare matching interval populations when investigating discrepancies; both
+measurement bugs and delivery buffering can change the observed cadence.
+
+### What this benchmark cannot see
+
+Chunk metadata carries no server-side timestamps, so the benchmark cannot split
+DiT time from VAE decode time; use `--enable-diffusion-pipeline-profiler` on the
+server for that. It also reports no GPU memory, which is a server-side quantity.
+
+## Custom rollouts
+
+`--workload rollout.json` replaces the built-in workload, including mid-rollout
+prompt changes, which the CLI flags cannot express:
+
+```json
+{
+  "prompt": "The camera moves slowly forward through the scene.",
+  "image": "/path/to/first_frame.png",
+  "num_chunks": 24,
+  "camera_pattern": "orbit",
+  "fps": 16,
+  "seed": 42,
+  "prompt_updates": [
+    {"after_chunk": 8, "prompt": "Rain begins to fall", "transition_chunks": 2}
+  ]
+}
+```
+
+Supply `camera_action_script` directly instead of `num_chunks` to control every
+frame's action. It must hold exactly one three-entry action list per chunk; one
+AR block is three latent frames, and the server rejects any other shape.
+Explicit `--image`, `--prompt`, `--negative-prompt`, `--width`, `--height`,
+`--fps`, `--seed`, and `--flow-shift` flags override the matching workload-file
+values. `--num-chunks` and `--camera-pattern` apply only to the built-in
+workload; when using `--workload`, set those fields in the file instead.
+An explicit `camera_action_script` takes precedence over the file's
+`num_chunks` and `camera_pattern`. Each prompt update must use a distinct
+`after_chunk` boundary so its interaction event ID is unique.
+
+Camera actions change what the world does, not what it costs, so `--camera-pattern`
+(`forward`, `orbit`, `hold`) keeps a rollout representative rather than sweeping a
+cost dimension.
+
+## Options worth knowing
+
+| Flag | Purpose |
+| --- | --- |
+| `--target-fps` | Playback rate the real-time verdict is measured against, independent of the mux label. |
+| `--sessions N` | N sequential measured rollouts; the report adds spread across them. |
+| `--warmup-sessions N` | Unmeasured rollouts first, to pay compile and capture cost. |
+| `--print-chunks` | One line per chunk as it arrives. |
+| `--save-video PATH` | Remux the last measured session for eyeballing (needs vLLM-Omni). |
+| `--ping-interval` | `session.ping` cadence keeping the server's stall clock fresh during a slow first chunk. `0` disables. |
+| `--first-chunk-timeout` | Raise it when the server is still compiling or capturing graphs. |
+
+## Output
+
+`--output-json` writes the run's configuration, every chunk's arrival time and
+size, per-session metrics, and the aggregate, along with the git revision and
+hostname so a number can be traced back to what produced it.
+
+## Tests
+
+`tests/benchmarks/test_lingbot_world_realtime.py` covers the chunk and frame
+arithmetic against the server's own formula, the metric math, and the protocol
+handling against a scripted WebSocket server. It needs no GPU:
+
+```bash
+pytest tests/benchmarks/test_lingbot_world_realtime.py
+```

benchmarks/lingbot_world/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py

@@ -0,0 +1,642 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Realtime streaming benchmark for LingBot-World 2.0 over ``WS /v1/realtime/video``.
+
+Every other diffusion benchmark in this repository measures "one request, one
+finished video". LingBot-World is not that: one request is a whole rollout, and
+the thing a deployment lives or dies by is whether chunk *N + 1* keeps arriving
+before a viewer has finished watching chunk *N*. This benchmark therefore
+measures a **cadence**, not a throughput:
+
+* ``TTFC`` -- session.start to the first chunk's bytes, which pays for prompt
+  encode, the first-frame VAE encode, and any compile or capture the deploy
+  config asks for;
+* **inter-arrival latency** over all chunks and, separately, over the steady
+  state after the attention window saturates;
+* ``VIDEO_RTF`` -- wall seconds spent per second of video generated, matching how
+  vLLM-Omni computes RTF everywhere else (``generation_s / video_duration``).
+  **Lower is better and below 1.0 is real time**; ``VIDEO_RTFX`` is reported
+  alongside it as the reciprocal, under the repo's own name for that;
+* **playback continuity** -- a simulated viewer with a start buffer, because a
+  p99 that is harmless behind a three-chunk buffer is a visible stall behind
+  none.
+
+The server is not started here. Bring one up first (see the README), then point
+this script at it. Requires ``websockets``; ``--save-video`` additionally needs
+a vLLM-Omni installation for the remux helper.
+
+Example:
+
+    python benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py \\
+      --host 127.0.0.1 --port 8000 \\
+      --num-chunks 16 \\
+      --output-json /tmp/lingbot_world_realtime.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import base64
+import json
+import mimetypes
+import os
+import platform
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from benchmarks.lingbot_world.workload import (  # noqa: E402
+    CAMERA_PATTERNS,
+    DEFAULT_FLOW_SHIFT,
+    DEFAULT_FPS,
+    DEFAULT_HEIGHT,
+    DEFAULT_MODEL,
+    DEFAULT_PROMPT,
+    DEFAULT_SEED,
+    DEFAULT_WARMUP_CHUNKS,
+    DEFAULT_WIDTH,
+    ChunkRecord,
+    Workload,
+    aggregate_metrics,
+    compute_metrics,
+    default_workload,
+    load_workload,
+    steady_chunk_deadline_ms,
+)
+
+# The shared vLLM asset keeps the default workload runnable with no setup and
+# adds no binary to this repository. Any real photograph conditions a rollout.
+_DEFAULT_IMAGE_ASSET = "2560px-Gfp-wisconsin-madison-the-nature-boardwalk"
+_WS_PATH = "/v1/realtime/video"
+
+
+class BenchmarkError(RuntimeError):
+    """A session failed in a way that invalidates its numbers."""
+
+
+# --------------------------------------------------------------------------- #
+# Input resolution
+# --------------------------------------------------------------------------- #
+
+
+def resolve_image_reference(value: str | None) -> str:
+    """Return an ``http(s)`` or ``data:`` URL for the session's first frame."""
+    if value is None:
+        from vllm.assets.image import ImageAsset
+
+        value = str(ImageAsset(_DEFAULT_IMAGE_ASSET).get_path("jpg"))
+    if value.startswith(("http://", "https://", "data:")):
+        return value
+    path = Path(value).expanduser()
+    if not path.is_file():
+        raise BenchmarkError(f"--image is neither a URL nor a readable file: {value}")
+    media_type = mimetypes.guess_type(path.name)[0] or "image/png"
+    return f"data:{media_type};base64,{base64.b64encode(path.read_bytes()).decode()}"
+
+
+def build_workload(args: argparse.Namespace) -> Workload:
+    if args.workload is not None:
+        # The file names its own first frame, so resolving happens inside
+        # load_workload -- never eagerly, which would fetch the default asset a
+        # --workload run has no use for.
+        return load_workload(
+            args.workload,
+            image_resolver=lambda value: resolve_image_reference(value if args.image is None else args.image),
+            overrides={
+                "width": args.width_override,
+                "height": args.height_override,
+                "fps": args.fps_override,
+                "seed": args.seed_override,
+                "prompt": args.prompt_override,
+                "flow_shift": args.flow_shift,
+                "negative_prompt": args.negative_prompt,
+            },
+        )
+    return default_workload(
+        image_reference=resolve_image_reference(args.image),
+        num_chunks=args.num_chunks,
+        camera_pattern=args.camera_pattern,
+        prompt=args.prompt_override if args.prompt_override is not None else DEFAULT_PROMPT,
+        width=args.width_override if args.width_override is not None else DEFAULT_WIDTH,
+        height=args.height_override if args.height_override is not None else DEFAULT_HEIGHT,
+        fps=args.fps_override if args.fps_override is not None else DEFAULT_FPS,
+        seed=args.seed_override if args.seed_override is not None else DEFAULT_SEED,
+        flow_shift=args.flow_shift if args.flow_shift is not None else DEFAULT_FLOW_SHIFT,
+        negative_prompt=args.negative_prompt,
+    )
+
+
+# --------------------------------------------------------------------------- #
+# One streaming session
+# --------------------------------------------------------------------------- #
+
+
+@dataclass
+class SessionResult:
+    request_id: str | None
+    records: list[ChunkRecord]
+    media: bytes
+    ttfc_s: float | None
+    video_start_s: float | None
+    wall_s: float
+    done_event: dict[str, Any]
+
+
+async def _keepalive(websocket: Any, send_lock: asyncio.Lock, interval: float) -> None:
+    """Refresh the server's stall clock while a slow first chunk is generating.
+
+    A compiled deploy config records CUDA graphs on the first block, which can
+    outlast the server's ~60 s stall timeout. ``session.ping`` is the documented
+    way to say the client is still there.
+    """
+    while True:
+        await asyncio.sleep(interval)
+        async with send_lock:
+            await websocket.send(json.dumps({"type": "session.ping"}))
+
+
+async def run_session(
+    url: str,
+    model: str,
+    workload: Workload,
+    *,
+    first_chunk_timeout: float,
+    chunk_timeout: float,
+    ping_interval: float,
+    collect_media: bool,
+    print_chunks: bool,
+) -> SessionResult:
+    """Drive one rollout and timestamp every chunk as its bytes land."""
+    try:
+        from websockets.asyncio.client import connect
+    except ImportError:  # pragma: no cover - exercised only without websockets
+        try:
+            from websockets import connect  # type: ignore[attr-defined]
+        except ImportError as exc:
+            raise BenchmarkError("This benchmark needs the websockets package: pip install websockets") from exc
+
+    payload = workload.session_start_payload(model)
+    records: list[ChunkRecord] = []
+    media = bytearray()
+    pending: dict[str, Any] | None = None
+    request_id: str | None = None
+    ttfc_s: float | None = None
+    video_start_s: float | None = None
+    last_arrival: float | None = None
+    done_event: dict[str, Any] = {}
+    send_lock = asyncio.Lock()
+    keepalive: asyncio.Task[None] | None = None
+    # Fire remaining prompt updates in chunk order as their boundary passes.
+    updates = sorted(workload.prompt_updates, key=lambda update: update.after_chunk)
+    update_index = 0
+
+    async with connect(url, max_size=None, ping_interval=None) as websocket:
+        async with send_lock:
+            await websocket.send(json.dumps(payload, ensure_ascii=False))
+        started = time.perf_counter()
+        if ping_interval > 0:
+            keepalive = asyncio.create_task(_keepalive(websocket, send_lock, ping_interval))
+
+        try:
+            while True:
+                timeout = first_chunk_timeout if not records else chunk_timeout
+                try:
+                    deadline = started + (last_arrival if last_arrival is not None else 0.0) + timeout
+                    remaining = deadline - time.perf_counter()
+                    if remaining <= 0:
+                        raise asyncio.TimeoutError
+                    message = await asyncio.wait_for(websocket.recv(), timeout=remaining)
+                except asyncio.TimeoutError as exc:
+                    raise BenchmarkError(
+                        f"No media chunk for {timeout:g} s after {len(records)} chunk(s); "
+                        "the server may still be loading, compiling, or capturing graphs."
+                    ) from exc
+                now = time.perf_counter()
+
+                if isinstance(message, (bytes, bytearray)):
+                    if pending is None:
+                        raise BenchmarkError("Received a binary frame with no preceding video.chunk_metadata event.")
+                    if collect_media:
+                        media.extend(message)
+                    if pending.get("kind") == "media":
+                        index = int(pending["generation_chunk_index"])
+                        arrival = now - started
+                        if ttfc_s is None:
+                            ttfc_s = arrival
+                        records.append(
+                            ChunkRecord(
+                                index=index,
+                                arrival_s=arrival,
+                                inter_arrival_s=arrival - (last_arrival if last_arrival is not None else 0.0),
+                                byte_length=len(message),
+                                num_frames=int(pending["num_frames"]),
+                            )
+                        )
+                        last_arrival = arrival
+                        if print_chunks:
+                            print(
+                                f"  chunk {index:3d}  t={arrival:8.3f}s  "
+                                f"interval={records[-1].inter_arrival_s * 1000:8.1f}ms  "
+                                f"frames={records[-1].num_frames:3d}  bytes={len(message)}",
+                                flush=True,
+                            )
+                        while update_index < len(updates) and updates[update_index].after_chunk <= index:
+                            async with send_lock:
+                                await websocket.send(json.dumps(updates[update_index].to_payload(), ensure_ascii=False))
+                            update_index += 1
+                    pending = None
+                    continue
+
+                event = json.loads(message)
+                event_type = event.get("type")
+                if event_type == "video.chunk_metadata":
+                    pending = event
+                elif event_type == "video.start":
+                    request_id = event.get("request_id")
+                    video_start_s = now - started
+                elif event_type == "error":
+                    # Say what was asked for alongside what the server refused:
+                    # some builds cap a rollout's num_frames, and the operator
+                    # needs to know which --num-chunks would fit.
+                    raise BenchmarkError(
+                        f"Server rejected the session after {len(records)} chunk(s): "
+                        f"{event.get('message', event)} "
+                        f"(requested {workload.num_chunks} chunks = num_frames {workload.num_frames})"
+                    )
+                elif event_type == "session.done":
+                    done_event = event
+                    break
+        finally:
+            if keepalive is not None:
+                keepalive.cancel()
+                try:
+                    await keepalive
+                except asyncio.CancelledError:
+                    pass
+
+        wall_s = time.perf_counter() - started
+
+    if not records:
+        raise BenchmarkError("The session finished without delivering a single media chunk.")
+    if done_event.get("stopped"):
+        raise BenchmarkError(f"The session was stopped before it finished: {done_event}")
+    observed = [record.index for record in records]
+    if observed != list(range(len(records))):
+        raise BenchmarkError(f"Chunk indices are not contiguous from zero: {observed}")
+    if len(records) != workload.num_chunks:
+        raise BenchmarkError(
+            f"Expected {workload.num_chunks} chunks for num_frames={workload.num_frames}, received {len(records)}."
+        )
+
+    return SessionResult(
+        request_id=request_id,
+        records=records,
+        media=bytes(media),
+        ttfc_s=ttfc_s,
+        video_start_s=video_start_s,
+        wall_s=wall_s,
+        done_event=done_event,
+    )
+
+
+# --------------------------------------------------------------------------- #
+# Reporting
+# --------------------------------------------------------------------------- #
+
+
+def _git_revision() -> str | None:
+    try:
+        return subprocess.check_output(
+            ["git", "-C", str(REPO_ROOT), "rev-parse", "HEAD"],
+            text=True,
+            stderr=subprocess.DEVNULL,
+        ).strip()
+    except (OSError, subprocess.CalledProcessError):
+        return None
+
+
+def _print_row(label: str, value: str) -> None:
+    print(f"{label:<44}{value:>16}")
+
+
+def print_report(
+    metrics: dict[str, Any],
+    *,
+    workload: Workload,
+    endpoint: str,
+    sessions: int,
+    target_fps: float,
+) -> None:
+    header = " LingBot-World realtime streaming benchmark "
+    print("\n{s:{c}^{n}}".format(s=header, n=60, c="="))
+    _print_row("Endpoint:", endpoint)
+    _print_row("Geometry:", f"{workload.width}x{workload.height}")
+    _print_row("Chunks per session:", str(workload.num_chunks))
+    _print_row("Sessions:", str(sessions))
+    frames = workload.num_frames
+    _print_row("Real-time basis:", f"{target_fps:g} fps")
+    _print_row("Video seconds per session:", f"{frames / target_fps:.2f} ({frames} frames)")
+    if abs(target_fps - workload.fps) > 1e-9:
+        _print_row("Mux label sent to server:", f"{workload.fps} fps")
+
+    if sessions == 1:
+        print("{s:{c}^{n}}".format(s=" Cadence ", n=60, c="-"))
+        if metrics.get("ttfc_ms") is not None:
+            _print_row("TTFC (session.start -> first chunk):", f"{metrics['ttfc_ms']:.1f} ms")
+        interval_all = metrics.get("interval_all") or {}
+        interval_steady = metrics.get("interval_steady") or {}
+        for title, summary in (("All chunks", interval_all), ("Steady state", interval_steady)):
+            if not summary:
+                continue
+            print(f"  {title} (n={int(summary['count'])}):")
+            _print_row("    mean / median:", f"{summary['mean_ms']:.1f} / {summary['median_ms']:.1f} ms")
+            _print_row("    std / max:", f"{summary['std_ms']:.1f} / {summary['max_ms']:.1f} ms")
+            tail = f"{summary['p90_ms']:.0f} / {summary['p95_ms']:.0f} / {summary['p99_ms']:.0f} ms"
+            _print_row("    p90 / p95 / p99:", tail)
+            if summary["count"] < 100:
+                print(
+                    "    NOTE: fewer than 100 intervals; tail percentiles are descriptive, not reliable tail estimates."
+                )
+
+        print("{s:{c}^{n}}".format(s=" Real time ", n=60, c="-"))
+        _print_row("VIDEO_RTF (wall s / video s, <1 real time):", f"{metrics['video_rtf']:.3f}")
+        _print_row("VIDEO_RTFX (video s / wall s):", f"{metrics['video_rtfx']:.3f}")
+        if metrics.get("steady_video_rtf") is not None:
+            _print_row("VIDEO_RTF steady state:", f"{metrics['steady_video_rtf']:.3f}")
+        _print_row("Frames per second:", f"{metrics['frames_per_second']:.2f}")
+        _print_row("Chunk deadline:", f"{metrics['chunk_deadline_ms']:.1f} ms")
+        if metrics.get("slo_attainment") is not None:
+            _print_row(
+                "Steady chunks within deadline:",
+                f"{metrics['slo_attainment'] * 100:.1f}% ({metrics['slo_violations']} late)",
+            )
+        if not metrics.get("steady_state_reached", False):
+            print(
+                f"  NOTE: no chunk reached the steady state; the attention window needs "
+                f"{metrics['warmup_chunks']} chunks. Raise --num-chunks."
+            )
+
+        playback = metrics.get("playback") or {}
+        print("{s:{c}^{n}}".format(s=" Playback continuity ", n=60, c="-"))
+        _print_row("Start buffer:", f"{playback.get('buffer_chunks')} chunk(s)")
+        _print_row("Underruns:", str(playback.get("underrun_count")))
+        _print_row("Total stall:", f"{playback.get('total_stall_ms', 0.0):.1f} ms")
+        _print_row("Stall ratio:", f"{playback.get('stall_ratio', 0.0) * 100:.2f}%")
+        if playback.get("first_underrun_chunk") is not None:
+            _print_row("First underrun at chunk:", str(playback["first_underrun_chunk"]))
+    else:
+        print("{s:{c}^{n}}".format(s=" Across sessions ", n=60, c="-"))
+        if metrics.get("ttfc_ms_mean") is not None:
+            _print_row("TTFC mean / max:", f"{metrics['ttfc_ms_mean']:.1f} / {metrics['ttfc_ms_max']:.1f} ms")
+        if metrics.get("steady_interval_ms_mean") is not None:
+            _print_row("Steady interval mean:", f"{metrics['steady_interval_ms_mean']:.1f} ms")
+            _print_row("Steady interval spread:", f"{metrics['steady_interval_ms_spread']:.1f} ms")
+        _print_row("VIDEO_RTF mean (lower is better):", f"{metrics['video_rtf_mean']:.3f}")
+        _print_row(
+            "VIDEO_RTF best / worst:",
+            f"{metrics['video_rtf_best']:.3f} / {metrics['video_rtf_worst']:.3f}",
+        )
+        _print_row("Chunk deadline:", f"{metrics['chunk_deadline_ms']:.1f} ms")
+        _print_row("Underruns (all sessions):", str(metrics["underrun_count"]))
+    print("=" * 60)
+
+
+def save_video(media: bytes, path: Path, *, fps: int) -> None:
+    from vllm_omni.diffusion.utils.media_utils import finalize_streaming_video_bytes
+
+    playback_bytes = finalize_streaming_video_bytes(media, input_format="m4s", fps=fps)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp_path = path.with_suffix(path.suffix + ".tmp")
+    tmp_path.write_bytes(playback_bytes)
+    os.replace(tmp_path, path)
+    print(f"Saved {len(media)} streamed bytes -> {len(playback_bytes)} playback bytes at {path}")
+
+
+# --------------------------------------------------------------------------- #
+# CLI
+# --------------------------------------------------------------------------- #
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Benchmark LingBot-World 2.0 realtime streaming over WS /v1/realtime/video.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    endpoint = parser.add_argument_group("endpoint")
+    endpoint.add_argument("--host", default="127.0.0.1")
+    endpoint.add_argument("--port", type=int, default=8000)
+    endpoint.add_argument("--base-url", default=None, help="Full ws:// URL; overrides --host/--port.")
+    endpoint.add_argument("--model", default=DEFAULT_MODEL)
+
+    workload = parser.add_argument_group("workload")
+    workload.add_argument("--workload", default=None, help="JSON rollout spec; overrides the built-in workload.")
+    workload.add_argument("--image", default=None, help="First frame: path or URL. Defaults to the shared vLLM asset.")
+    workload.add_argument("--prompt", dest="prompt_override", default=None, help=f"Default: {DEFAULT_PROMPT!r}")
+    workload.add_argument("--negative-prompt", default=None)
+    workload.add_argument("--num-chunks", type=int, default=16, help="AR blocks per session; ignored with --workload.")
+    workload.add_argument(
+        "--camera-pattern",
+        choices=CAMERA_PATTERNS,
+        default="forward",
+        help="Camera pattern for the built-in workload only; ignored with --workload.",
+    )
+    workload.add_argument("--width", dest="width_override", type=int, default=None, help=f"Default: {DEFAULT_WIDTH}")
+    workload.add_argument("--height", dest="height_override", type=int, default=None, help=f"Default: {DEFAULT_HEIGHT}")
+    workload.add_argument("--fps", dest="fps_override", type=int, default=None, help=f"Default: {DEFAULT_FPS}")
+    workload.add_argument("--seed", dest="seed_override", type=int, default=None, help=f"Default: {DEFAULT_SEED}")
+    workload.add_argument("--flow-shift", type=float, default=None, help=f"Default: {DEFAULT_FLOW_SHIFT}")
+
+    run = parser.add_argument_group("run")
+    run.add_argument("--sessions", type=int, default=1, help="Sequential measured rollouts.")
+    run.add_argument("--warmup-sessions", type=int, default=0, help="Unmeasured rollouts first (pays compile cost).")
+    run.add_argument(
+        "--warmup-chunks",
+        type=int,
+        default=DEFAULT_WARMUP_CHUNKS,
+        help="Chunks excluded from steady state; the attention window saturates after 6.",
+    )
+    run.add_argument(
+        "--chunk-slo-ms",
+        type=float,
+        default=None,
+        help="Deadline a steady chunk must meet. Default: one chunk of video at --target-fps.",
+    )
+    run.add_argument(
+        "--target-fps",
+        type=float,
+        default=None,
+        help=(
+            "Playback rate used for real-time factor, the chunk deadline, and the playback "
+            "simulation. The checkpoint declares no frame rate, so this is a consumption "
+            "choice, not a model property. Default: the --fps sent to the server."
+        ),
+    )
+    run.add_argument("--playback-buffer-chunks", type=int, default=1, help="Chunks buffered before playback starts.")
+    run.add_argument("--first-chunk-timeout", type=float, default=900.0)
+    run.add_argument("--chunk-timeout", type=float, default=300.0)
+    run.add_argument("--ping-interval", type=float, default=20.0, help="session.ping cadence; 0 disables.")
+
+    output = parser.add_argument_group("output")
+    output.add_argument("--output-json", default=None)
+    output.add_argument("--save-video", default=None, help="Write the last measured session's video here.")
+    output.add_argument("--print-chunks", action="store_true")
+
+    args = parser.parse_args(argv)
+    if args.sessions < 1:
+        parser.error("--sessions must be at least 1.")
+    if args.warmup_sessions < 0:
+        parser.error("--warmup-sessions must be non-negative.")
+    if args.warmup_chunks < 0:
+        parser.error("--warmup-chunks must be non-negative.")
+    if args.playback_buffer_chunks < 1:
+        parser.error("--playback-buffer-chunks must be at least 1.")
+    if args.target_fps is not None and args.target_fps <= 0:
+        parser.error("--target-fps must be positive.")
+    if args.workload is None and args.num_chunks < 1:
+        parser.error("--num-chunks must be at least 1.")
+    return args
+
+
+def endpoint_url(args: argparse.Namespace) -> str:
+    if args.base_url is not None:
+        base = args.base_url.rstrip("/")
+        for prefix, replacement in (("http://", "ws://"), ("https://", "wss://")):
+            if base.startswith(prefix):
+                base = replacement + base[len(prefix) :]
+                break
+        return base if base.endswith(_WS_PATH) else base + _WS_PATH
+    return f"ws://{args.host}:{args.port}{_WS_PATH}"
+
+
+async def run_benchmark(args: argparse.Namespace) -> dict[str, Any]:
+    workload = build_workload(args)
+    url = endpoint_url(args)
+    # The mux label and the real-time basis are different questions; only the
+    # latter decides whether a run passes.
+    target_fps = args.target_fps if args.target_fps is not None else float(workload.fps)
+    if max(1, args.warmup_chunks) >= workload.num_chunks - 1:
+        print(
+            f"WARNING: --warmup-chunks={args.warmup_chunks} leaves no nonterminal steady-state chunk in a "
+            f"{workload.num_chunks}-chunk rollout; steady-state metrics will be empty.",
+            file=sys.stderr,
+        )
+
+    print(f"Connecting to {url}")
+    print(
+        f"Rollout: {workload.num_chunks} chunks, num_frames={workload.num_frames}, "
+        f"{workload.video_seconds:.2f}s of video"
+    )
+    print(
+        f"Real-time basis: {target_fps:g} fps -> a steady 12-frame chunk is "
+        f"{steady_chunk_deadline_ms(target_fps):.1f} ms of video"
+        + ("" if args.target_fps is None else f" (mux label stays {workload.fps} fps)")
+    )
+
+    for index in range(args.warmup_sessions):
+        print(f"Warmup session {index + 1}/{args.warmup_sessions} (not measured)...")
+        await run_session(
+            url,
+            args.model,
+            workload,
+            first_chunk_timeout=args.first_chunk_timeout,
+            chunk_timeout=args.chunk_timeout,
+            ping_interval=args.ping_interval,
+            collect_media=False,
+            print_chunks=False,
+        )
+
+    session_metrics: list[dict[str, Any]] = []
+    session_payloads: list[dict[str, Any]] = []
+    last_result: SessionResult | None = None
+    for index in range(args.sessions):
+        if args.sessions > 1:
+            print(f"Session {index + 1}/{args.sessions}...")
+        result = await run_session(
+            url,
+            args.model,
+            workload,
+            first_chunk_timeout=args.first_chunk_timeout,
+            chunk_timeout=args.chunk_timeout,
+            ping_interval=args.ping_interval,
+            collect_media=args.save_video is not None,
+            print_chunks=args.print_chunks,
+        )
+        last_result = result
+        metrics = compute_metrics(
+            result.records,
+            fps=target_fps,
+            warmup_chunks=args.warmup_chunks,
+            slo_ms=args.chunk_slo_ms,
+            playback_buffer_chunks=args.playback_buffer_chunks,
+            ttfc_s=result.ttfc_s,
+            session_wall_s=result.wall_s,
+        )
+        session_metrics.append(metrics)
+        session_payloads.append(
+            {
+                "session_index": index,
+                "request_id": result.request_id,
+                "video_start_ms": (result.video_start_s * 1000.0) if result.video_start_s is not None else None,
+                "chunks": [record.to_dict() for record in result.records],
+                "metrics": metrics,
+            }
+        )
+
+    aggregate = aggregate_metrics(session_metrics, fps=target_fps)
+    print_report(aggregate, workload=workload, endpoint=url, sessions=args.sessions, target_fps=target_fps)
+
+    if args.save_video is not None and last_result is not None:
+        save_video(last_result.media, Path(args.save_video), fps=workload.fps)
+
+    document = {
+        "benchmark": "lingbot_world_realtime",
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "endpoint": url,
+        "model": args.model,
+        "git_revision": _git_revision(),
+        "hostname": platform.node(),
+        "workload": workload.describe(),
+        "config": {
+            "sessions": args.sessions,
+            "warmup_sessions": args.warmup_sessions,
+            "warmup_chunks": args.warmup_chunks,
+            "camera_pattern": args.camera_pattern,
+            "chunk_slo_ms": args.chunk_slo_ms,
+            "target_fps": target_fps,
+            "mux_fps": workload.fps,
+            "playback_buffer_chunks": args.playback_buffer_chunks,
+            "workload_file": args.workload,
+        },
+        "sessions": session_payloads,
+        "metrics": aggregate,
+    }
+    if args.output_json is not None:
+        path = Path(args.output_json)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n")
+        print(f"Wrote {path}")
+    return document
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv)
+    try:
+        asyncio.run(run_benchmark(args))
+    except BenchmarkError as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmarks/lingbot_world/configs/single_gpu_eager.yaml

@@ -0,0 +1,45 @@
+# Benchmark deploy config: LingBot-World 2.0 on one GPU, eager.
+#
+# This is the portable smoke configuration -- it matches the shipped
+# vllm_omni/deploy/lingbot_world_v2_stepwise.yaml so a number measured here is
+# comparable to the default deployment. It is NOT a realtime configuration: one
+# GPU runs all five forwards of every AR block serially and nothing is compiled,
+# so expect a chunk interval several times the real-time deadline.
+#
+# Use configs/usp4_compiled.yaml for the realtime topology.
+#
+#   vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+#     --omni --deploy-config benchmarks/lingbot_world/configs/single_gpu_eager.yaml \
+#     --port 8000
+
+pipeline: lingbot_world
+async_chunk: false
+distributed_executor_backend: mp
+dtype: bfloat16
+
+stages:
+  - stage_id: 0
+    devices: "0"
+    # AR-Diffusion binds one session per runner invocation; the runner rejects
+    # anything larger.
+    max_num_seqs: 1
+    enforce_eager: true
+    model_class_name: LingBotWorldCausalDMDPipeline
+    engine_backend: vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine
+    step_execution: true
+    streaming_output: true
+    default_sampling_params:
+      # The causal-fast checkpoint pins a four-step DMD schedule; this sizes the
+      # step budget before the pipeline is reached.
+      num_inference_steps: 4
+      output_type: pil
+      extra_args:
+        flow_shift: 5.0
+    model_config:
+      # The AR cache geometry is fixed at load time, so a benchmark request must
+      # ask for exactly this resolution.
+      ar_diffusion_height: 480
+      ar_diffusion_width: 832
+      ar_diffusion_kv_config:
+        gpu_memory_fraction: 0.6
+        warmup_cudagraph: true

benchmarks/lingbot_world/configs/usp4_compiled.yaml

@@ -0,0 +1,60 @@
+# Benchmark deploy config: LingBot-World 2.0 on four GPUs, Ulysses SP-4, compiled.
+#
+# This is the realtime topology. Two things differ from the single-GPU config,
+# and both matter to the number the benchmark reports:
+#
+#   parallel_config   Pure Ulysses sequence parallelism over four ranks. The
+#                     pipeline validates this: LingBot requires
+#                     sequence_parallel_size == ulysses_degree, ring_degree and
+#                     allgather_degree of 1, ulysses_mode "strict", and
+#                     ulysses_a2a_permute disabled. Pipeline, CFG, and VAE
+#                     parallel sizes above 1 are rejected. This config uses
+#                     TP=1; the validator does not reject larger TP sizes.
+#   enforce_eager     false puts LingBotDMDBlockRunner on the compiled path
+#                     (regional torch.compile with cudagraph step marking). The
+#                     first rollout therefore pays compile and capture cost --
+#                     use --warmup-sessions 1 before measured repetitions.
+#                     Chunk warmup only excludes the attention-window ramp.
+#
+# Serve it under a GPU reservation, since it needs four cards:
+#
+#   gpu run --gpus 4 -- vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+#     --omni --deploy-config benchmarks/lingbot_world/configs/usp4_compiled.yaml \
+#     --port 8000
+#
+# `devices` is the rank->card mapping inside the reservation, so it stays
+# "0,1,2,3" no matter which physical cards CUDA_VISIBLE_DEVICES selected.
+
+pipeline: lingbot_world
+async_chunk: false
+distributed_executor_backend: mp
+dtype: bfloat16
+
+stages:
+  - stage_id: 0
+    devices: "0,1,2,3"
+    max_num_seqs: 1
+    enforce_eager: false
+    model_class_name: LingBotWorldCausalDMDPipeline
+    engine_backend: vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine
+    step_execution: true
+    streaming_output: true
+    parallel_config:
+      tensor_parallel_size: 1
+      sequence_parallel_size: 4
+      ulysses_degree: 4
+      ring_degree: 1
+    default_sampling_params:
+      num_inference_steps: 4
+      output_type: pil
+      extra_args:
+        flow_shift: 5.0
+    model_config:
+      ar_diffusion_height: 480
+      ar_diffusion_width: 832
+      ar_diffusion_kv_config:
+        # Each rank holds its own shard of the paged KV cache, so the per-card
+        # fraction can stay at the single-GPU value; lower it if the four-rank
+        # start OOMs on smaller cards.
+        gpu_memory_fraction: 0.6
+        warmup_cudagraph: true

benchmarks/lingbot_world/workload.py

@@ -0,0 +1,631 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Workload construction and metric math for the LingBot-World realtime benchmark.
+
+Nothing here imports torch, vllm, or websockets, so the arithmetic that decides
+what a benchmark number *means* is unit-testable without a GPU or a server.
+
+Two model properties drive every formula in this module, both read from the
+checkpoint's ``transformer/config.json`` rather than assumed:
+
+``num_frames_per_block = 3``
+    One AR block commits three latent frames, so one streamed chunk is three
+    latent frames and a request's ``camera_action_script`` carries exactly one
+    three-entry action list per chunk.
+
+``sliding_window_num_frames = 18`` with ``sink_size = 9``
+    Self-attention sees nine sink frames plus a nine-frame recent window, so the
+    key count a block attends over keeps growing until eighteen latent frames of
+    history exist -- six chunks. Chunk cost is therefore *not* stationary before
+    chunk six, and a mean taken over a short rollout reports a number no steady
+    session will ever produce. :data:`DEFAULT_WARMUP_CHUNKS` exists for that
+    reason and is not a round-number guess.
+
+The causal VAE expands the first latent frame to one pixel frame and every later
+latent frame to four, so chunk zero carries nine pixel frames and every chunk
+after it carries twelve. Video-duration math has to respect that asymmetry or
+real-time factor comes out wrong on short rollouts.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+# transformer.config.num_frames_per_block: latent frames committed per AR block.
+FRAMES_PER_BLOCK = 3
+# vae_scale_factor_temporal: latent frames -> pixel frames, after the first.
+TEMPORAL_COMPRESSION = 4
+# (sink_size 9 + recent window 9) / FRAMES_PER_BLOCK: chunks until the attention
+# window saturates and per-chunk cost becomes stationary.
+DEFAULT_WARMUP_CHUNKS = 6
+
+DEFAULT_MODEL = "robbyant/lingbot-world-v2-14b-causal-fast-diffusers"
+DEFAULT_PROMPT = "The camera moves slowly forward through the scene."
+# The AR KV cache geometry is fixed at load time, so a request must ask for
+# exactly the resolution the deploy config declares.
+DEFAULT_WIDTH = 832
+DEFAULT_HEIGHT = 480
+DEFAULT_FPS = 16
+DEFAULT_FLOW_SHIFT = 5.0
+DEFAULT_SEED = 42
+
+_HOLD = "hold"
+_ACTION_KEYS = ("w", "a", "s", "d")
+
+
+def num_frames_for_chunks(num_chunks: int) -> int:
+    """Pixel-frame count whose rollout is exactly ``num_chunks`` chunks.
+
+    Inverse of :func:`chunks_for_num_frames`, which is the server's own
+    ``((num_frames - 1) // 4 + 1) // 3``.
+    """
+    if num_chunks < 1:
+        raise ValueError("num_chunks must be at least 1.")
+    return (num_chunks * FRAMES_PER_BLOCK - 1) * TEMPORAL_COMPRESSION + 1
+
+
+def chunks_for_num_frames(num_frames: int) -> int:
+    """Chunks the server generates for ``num_frames``, mirroring its own formula."""
+    if num_frames < 1:
+        raise ValueError("num_frames must be at least 1.")
+    return ((num_frames - 1) // TEMPORAL_COMPRESSION + 1) // FRAMES_PER_BLOCK
+
+
+def pixel_frames_in_chunk(chunk_index: int) -> int:
+    """Pixel frames carried by one chunk.
+
+    The causal decoder expands the session's opening latent frame once and every
+    later latent frame four times, so chunk zero is short by three frames.
+    """
+    if chunk_index < 0:
+        raise ValueError("chunk_index must be non-negative.")
+    if chunk_index == 0:
+        return (FRAMES_PER_BLOCK - 1) * TEMPORAL_COMPRESSION + 1
+    return FRAMES_PER_BLOCK * TEMPORAL_COMPRESSION
+
+
+def camera_actions_for_chunk(pattern: str, chunk_index: int) -> list[list[str]]:
+    """One three-entry action list for ``chunk_index``.
+
+    Camera actions select camera embeddings; they change what the world does,
+    not how much arithmetic it costs. The pattern therefore exists to keep a
+    benchmark rollout representative rather than to sweep a cost dimension.
+    """
+    if chunk_index < 0:
+        raise ValueError("chunk_index must be non-negative.")
+    if pattern == "forward":
+        return [["w"] for _ in range(FRAMES_PER_BLOCK)]
+    if pattern == _HOLD:
+        return [[] for _ in range(FRAMES_PER_BLOCK)]
+    if pattern == "orbit":
+        key = _ACTION_KEYS[chunk_index % len(_ACTION_KEYS)]
+        return [[key] for _ in range(FRAMES_PER_BLOCK)]
+    raise ValueError(f"Unknown camera pattern {pattern!r}; expected one of {sorted(CAMERA_PATTERNS)}.")
+
+
+CAMERA_PATTERNS = ("forward", "orbit", _HOLD)
+
+
+def build_camera_script(num_chunks: int, pattern: str) -> list[list[list[str]]]:
+    """Full ``camera_action_script``: one action list per generated chunk."""
+    if num_chunks < 1:
+        raise ValueError("num_chunks must be at least 1.")
+    return [camera_actions_for_chunk(pattern, index) for index in range(num_chunks)]
+
+
+def _validate_camera_script(script: Sequence[Any]) -> list[list[list[str]]]:
+    validated: list[list[list[str]]] = []
+    for chunk_index, actions in enumerate(script):
+        if not isinstance(actions, Sequence) or isinstance(actions, (str, bytes)):
+            raise ValueError(f"camera_action_script[{chunk_index}] must be a list of three action lists.")
+        if len(actions) != FRAMES_PER_BLOCK:
+            raise ValueError(
+                f"camera_action_script[{chunk_index}] must hold exactly {FRAMES_PER_BLOCK} action lists "
+                f"(got {len(actions)}); one AR block is {FRAMES_PER_BLOCK} latent frames."
+            )
+        frames: list[list[str]] = []
+        for frame_index, frame in enumerate(actions):
+            if not isinstance(frame, Sequence) or isinstance(frame, (str, bytes)):
+                raise ValueError(f"camera_action_script[{chunk_index}][{frame_index}] must be a list of strings.")
+            if any(not isinstance(action, str) for action in frame):
+                raise ValueError(f"camera_action_script[{chunk_index}][{frame_index}] must contain only strings.")
+            frames.append([str(action) for action in frame])
+        validated.append(frames)
+    if not validated:
+        raise ValueError("camera_action_script must not be empty.")
+    return validated
+
+
+@dataclass(frozen=True)
+class PromptUpdate:
+    """A mid-rollout ``session.interaction`` scheduled on a chunk boundary."""
+
+    after_chunk: int
+    prompt: str
+    transition_chunks: int = 3
+
+    def __post_init__(self) -> None:
+        if self.after_chunk < 0:
+            raise ValueError("PromptUpdate.after_chunk must be non-negative.")
+        if not self.prompt.strip():
+            raise ValueError("PromptUpdate.prompt must contain non-whitespace text.")
+        if self.transition_chunks < 0:
+            raise ValueError("PromptUpdate.transition_chunks must be non-negative.")
+
+    def to_payload(self) -> dict[str, Any]:
+        return {
+            "type": "session.interaction",
+            "interaction": {
+                "event_id": f"chunk-{self.after_chunk}",
+                "event": {"prompt": self.prompt},
+                "transition_chunks": self.transition_chunks,
+            },
+        }
+
+
+@dataclass(frozen=True)
+class Workload:
+    """One rollout: what the benchmark asks the server to generate."""
+
+    prompt: str
+    image_reference: str
+    camera_script: list[list[list[str]]]
+    width: int = DEFAULT_WIDTH
+    height: int = DEFAULT_HEIGHT
+    fps: int = DEFAULT_FPS
+    seed: int = DEFAULT_SEED
+    flow_shift: float = DEFAULT_FLOW_SHIFT
+    negative_prompt: str | None = None
+    prompt_updates: tuple[PromptUpdate, ...] = ()
+    extra_params: Mapping[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if not self.prompt.strip():
+            raise ValueError("Workload.prompt must contain non-whitespace text.")
+        if not self.image_reference:
+            raise ValueError("Workload.image_reference is required; LingBot-World is image-conditioned.")
+        if self.width <= 0 or self.height <= 0 or self.width % 16 or self.height % 16:
+            raise ValueError("Workload width/height must be positive multiples of 16.")
+        if self.fps <= 0:
+            raise ValueError("Workload.fps must be positive.")
+        object.__setattr__(self, "camera_script", _validate_camera_script(self.camera_script))
+        seen_update_chunks: set[int] = set()
+        for update in self.prompt_updates:
+            if update.after_chunk in seen_update_chunks:
+                raise ValueError(f"Duplicate prompt update after chunk {update.after_chunk}; event IDs must be unique.")
+            seen_update_chunks.add(update.after_chunk)
+            if update.after_chunk >= self.num_chunks:
+                raise ValueError(
+                    f"Prompt update after chunk {update.after_chunk} never fires in a {self.num_chunks}-chunk rollout."
+                )
+
+    @property
+    def num_chunks(self) -> int:
+        return len(self.camera_script)
+
+    @property
+    def num_frames(self) -> int:
+        return num_frames_for_chunks(self.num_chunks)
+
+    @property
+    def video_seconds(self) -> float:
+        return self.num_frames / self.fps
+
+    def session_start_payload(self, model: str) -> dict[str, Any]:
+        """The ``session.start`` event for ``WS /v1/realtime/video``."""
+        extra_params: dict[str, Any] = {
+            "flow_shift": self.flow_shift,
+            "camera_action_script": self.camera_script,
+        }
+        extra_params.update(self.extra_params)
+        payload: dict[str, Any] = {
+            "type": "session.start",
+            "model": model,
+            "prompt": self.prompt,
+            "image_reference": {"image_url": self.image_reference},
+            "width": self.width,
+            "height": self.height,
+            "num_frames": self.num_frames,
+            "fps": self.fps,
+            "seed": self.seed,
+            "extra_params": extra_params,
+        }
+        if self.negative_prompt is not None:
+            payload["negative_prompt"] = self.negative_prompt
+        return payload
+
+    def describe(self) -> dict[str, Any]:
+        return {
+            "num_chunks": self.num_chunks,
+            "num_frames": self.num_frames,
+            "width": self.width,
+            "height": self.height,
+            "fps": self.fps,
+            "seed": self.seed,
+            "flow_shift": self.flow_shift,
+            "video_seconds": self.video_seconds,
+            "prompt_updates": [
+                {
+                    "after_chunk": update.after_chunk,
+                    "transition_chunks": update.transition_chunks,
+                }
+                for update in self.prompt_updates
+            ],
+        }
+
+
+def default_workload(
+    *,
+    image_reference: str,
+    num_chunks: int,
+    camera_pattern: str = "forward",
+    prompt: str = DEFAULT_PROMPT,
+    width: int = DEFAULT_WIDTH,
+    height: int = DEFAULT_HEIGHT,
+    fps: int = DEFAULT_FPS,
+    seed: int = DEFAULT_SEED,
+    flow_shift: float = DEFAULT_FLOW_SHIFT,
+    negative_prompt: str | None = None,
+) -> Workload:
+    """Built-in rollout: a fixed prompt and a generated camera script."""
+    return Workload(
+        prompt=prompt,
+        image_reference=image_reference,
+        camera_script=build_camera_script(num_chunks, camera_pattern),
+        width=width,
+        height=height,
+        fps=fps,
+        seed=seed,
+        flow_shift=flow_shift,
+        negative_prompt=negative_prompt,
+    )
+
+
+def load_workload(
+    path: str | Path,
+    *,
+    image_resolver: Any = None,
+    overrides: Mapping[str, Any] | None = None,
+) -> Workload:
+    """Read a rollout spec from JSON.
+
+    ``image_resolver`` turns the spec's ``image`` field into an ``http(s)`` or
+    ``data:`` URL; the caller owns that because inlining a local file is I/O.
+    Keys in ``overrides`` that are not ``None`` win over the file, so CLI flags
+    stay meaningful next to ``--workload``.
+    """
+    document = json.loads(Path(path).read_text())
+    if not isinstance(document, dict):
+        raise ValueError("A workload file must contain a JSON object.")
+
+    merged = dict(document)
+    for key, value in (overrides or {}).items():
+        if value is not None:
+            merged[key] = value
+
+    camera_script = merged.get("camera_action_script")
+    if camera_script is None:
+        num_chunks = merged.get("num_chunks")
+        if num_chunks is None:
+            raise ValueError("A workload file needs either camera_action_script or num_chunks.")
+        camera_script = build_camera_script(int(num_chunks), str(merged.get("camera_pattern", "forward")))
+
+    image = merged.get("image") or merged.get("image_reference")
+    if image is None:
+        raise ValueError("A workload file needs an image or image_reference field.")
+    image_reference = image_resolver(str(image)) if image_resolver is not None else str(image)
+
+    updates_field = merged.get("prompt_updates") or ()
+    prompt_updates = tuple(
+        PromptUpdate(
+            after_chunk=int(entry["after_chunk"]),
+            prompt=str(entry["prompt"]),
+            transition_chunks=int(entry.get("transition_chunks", 3)),
+        )
+        for entry in updates_field
+    )
+
+    extra_params = merged.get("extra_params") or {}
+    if not isinstance(extra_params, Mapping):
+        raise ValueError("Workload extra_params must be a JSON object.")
+
+    return Workload(
+        prompt=str(merged.get("prompt", DEFAULT_PROMPT)),
+        image_reference=image_reference,
+        camera_script=camera_script,
+        width=int(merged.get("width", DEFAULT_WIDTH)),
+        height=int(merged.get("height", DEFAULT_HEIGHT)),
+        fps=int(merged.get("fps", DEFAULT_FPS)),
+        seed=int(merged.get("seed", DEFAULT_SEED)),
+        flow_shift=float(merged.get("flow_shift", DEFAULT_FLOW_SHIFT)),
+        negative_prompt=merged.get("negative_prompt"),
+        prompt_updates=prompt_updates,
+        extra_params=dict(extra_params),
+    )
+
+
+# --------------------------------------------------------------------------- #
+# Metrics
+# --------------------------------------------------------------------------- #
+
+
+@dataclass
+class ChunkRecord:
+    """One media chunk as the client observed it."""
+
+    index: int
+    arrival_s: float
+    """Seconds from ``session.start`` being sent to this chunk's bytes arriving."""
+    inter_arrival_s: float
+    """Seconds since the previous chunk; for chunk zero this equals ``arrival_s``."""
+    byte_length: int
+    num_frames: int
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "index": self.index,
+            "arrival_s": self.arrival_s,
+            "inter_arrival_ms": self.inter_arrival_s * 1000.0,
+            "byte_length": self.byte_length,
+            "num_frames": self.num_frames,
+        }
+
+
+def percentile(values: Sequence[float], fraction: float) -> float:
+    """Linearly interpolated percentile, so a 10-sample run still reports a p95.
+
+    Implemented here rather than pulled from numpy to keep this module importable
+    in a bare environment; the benchmark's own numbers must not depend on which
+    optional package happens to be installed.
+    """
+    if not values:
+        raise ValueError("percentile() needs at least one value.")
+    if not 0.0 <= fraction <= 1.0:
+        raise ValueError("percentile fraction must be within [0, 1].")
+    ordered = sorted(values)
+    if len(ordered) == 1:
+        return ordered[0]
+    position = fraction * (len(ordered) - 1)
+    lower = int(position)
+    upper = min(lower + 1, len(ordered) - 1)
+    weight = position - lower
+    return ordered[lower] * (1.0 - weight) + ordered[upper] * weight
+
+
+def _stddev(values: Sequence[float]) -> float:
+    if len(values) < 2:
+        return 0.0
+    mean = sum(values) / len(values)
+    return (sum((value - mean) ** 2 for value in values) / (len(values) - 1)) ** 0.5
+
+
+def _latency_summary(values: Sequence[float]) -> dict[str, float]:
+    """Millisecond summary of a set of inter-arrival intervals."""
+    if not values:
+        return {}
+    milliseconds = [value * 1000.0 for value in values]
+    return {
+        "count": float(len(milliseconds)),
+        "mean_ms": sum(milliseconds) / len(milliseconds),
+        "median_ms": percentile(milliseconds, 0.5),
+        "std_ms": _stddev(milliseconds),
+        "min_ms": min(milliseconds),
+        "p90_ms": percentile(milliseconds, 0.90),
+        "p95_ms": percentile(milliseconds, 0.95),
+        "p99_ms": percentile(milliseconds, 0.99),
+        "max_ms": max(milliseconds),
+    }
+
+
+@dataclass(frozen=True)
+class PlaybackResult:
+    """What a viewer would have seen, given a start buffer and a playback rate."""
+
+    underrun_count: int
+    total_stall_s: float
+    playback_span_s: float
+    first_underrun_chunk: int | None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "underrun_count": self.underrun_count,
+            "total_stall_ms": self.total_stall_s * 1000.0,
+            "playback_span_s": self.playback_span_s,
+            "stall_ratio": (self.total_stall_s / self.playback_span_s) if self.playback_span_s > 0 else 0.0,
+            "first_underrun_chunk": self.first_underrun_chunk,
+        }
+
+
+def simulate_playback(
+    records: Sequence[ChunkRecord],
+    *,
+    fps: float,
+    buffer_chunks: int = 1,
+) -> PlaybackResult:
+    """Replay arrivals against a wall clock and count stalls.
+
+    Playback starts when ``buffer_chunks`` chunks have arrived and then consumes
+    video at ``fps``. A chunk that has not arrived by the moment the player needs
+    it is an underrun, and the player resumes only when the bytes land -- which
+    is what a viewer experiences, and what a single p99 number cannot tell you:
+    the same p99 is harmless behind a three-chunk buffer and fatal behind none.
+    """
+    if fps <= 0:
+        raise ValueError("fps must be positive.")
+    if buffer_chunks < 1:
+        raise ValueError("buffer_chunks must be at least 1.")
+    if not records:
+        return PlaybackResult(0, 0.0, 0.0, None)
+    if buffer_chunks > len(records):
+        raise ValueError(f"buffer_chunks={buffer_chunks} exceeds the {len(records)} chunks received.")
+
+    ordered = sorted(records, key=lambda record: record.index)
+    start = ordered[buffer_chunks - 1].arrival_s
+    cursor = start
+    underruns = 0
+    total_stall = 0.0
+    first_underrun: int | None = None
+
+    for record in ordered:
+        if record.arrival_s > cursor:
+            stall = record.arrival_s - cursor
+            underruns += 1
+            total_stall += stall
+            if first_underrun is None:
+                first_underrun = record.index
+            cursor = record.arrival_s
+        cursor += record.num_frames / fps
+
+    return PlaybackResult(
+        underrun_count=underruns,
+        total_stall_s=total_stall,
+        playback_span_s=cursor - start,
+        first_underrun_chunk=first_underrun,
+    )
+
+
+def steady_chunk_deadline_ms(fps: float) -> float:
+    """Wall-clock budget a steady chunk must meet to hold real time.
+
+    ``fps`` here is the rate the frames are *intended to be played at*, which the
+    checkpoint does not declare: nothing in ``model_index.json``, the scheduler
+    config, or the transformer config carries a frame rate, and the pipeline
+    never reads one. It is a property of how the output is consumed, so both
+    this deadline and real-time factor scale linearly with the choice and the
+    caller has to state it rather than inherit it from a muxing default.
+    """
+    if fps <= 0:
+        raise ValueError("fps must be positive.")
+    return pixel_frames_in_chunk(1) / fps * 1000.0
+
+
+def compute_metrics(
+    records: Sequence[ChunkRecord],
+    *,
+    fps: float,
+    warmup_chunks: int = DEFAULT_WARMUP_CHUNKS,
+    slo_ms: float | None = None,
+    playback_buffer_chunks: int = 1,
+    ttfc_s: float | None = None,
+    session_wall_s: float | None = None,
+) -> dict[str, Any]:
+    """Summarize one session.
+
+    ``warmup_chunks`` are excluded from the steady-state block because the
+    attention window has not saturated before then (see the module docstring).
+    The terminal chunk also stays out of steady metrics because it skips
+    next-chunk preparation. All chunks stay in the playback simulation, which is
+    what a session really costs end to end.
+    """
+    if not records:
+        raise ValueError("compute_metrics() needs at least one chunk record.")
+    if warmup_chunks < 0:
+        raise ValueError("warmup_chunks must be non-negative.")
+
+    ordered = sorted(records, key=lambda record: record.index)
+    deadline_ms = slo_ms if slo_ms is not None else steady_chunk_deadline_ms(fps)
+
+    all_intervals = [record.inter_arrival_s for record in ordered[1:]]
+    # The terminal chunk skips next-chunk preparation and is not representative
+    # of sustained generation. Keep it in end-to-end and playback metrics.
+    steady = [record for record in ordered[:-1] if record.index >= warmup_chunks]
+    steady_intervals = [record.inter_arrival_s for record in steady if record.index > 0]
+
+    total_frames = sum(record.num_frames for record in ordered)
+    video_seconds = total_frames / fps
+    wall_seconds = session_wall_s if session_wall_s is not None else ordered[-1].arrival_s
+
+    metrics: dict[str, Any] = {
+        "chunks_received": len(ordered),
+        "warmup_chunks": warmup_chunks,
+        "steady_chunks": len(steady_intervals),
+        "steady_state_reached": bool(steady_intervals),
+        "total_frames": total_frames,
+        "total_bytes": sum(record.byte_length for record in ordered),
+        "video_seconds": video_seconds,
+        "wall_seconds": wall_seconds,
+        # RTF is processing time over content duration, as vLLM-Omni computes it
+        # (`patch.py`: generation_s / video_duration; `compute_audio_rtf`: latency
+        # / audio_duration, "SLO red line < 1"). LOWER IS BETTER and below 1.0 is
+        # real time. RTFX is the reciprocal the repo reports under its own name,
+        # kept here because "fraction of real time achieved" reads better on a
+        # dashboard -- but the two must never be confused for one another.
+        "video_rtf": (wall_seconds / video_seconds) if video_seconds > 0 else 0.0,
+        "video_rtfx": (video_seconds / wall_seconds) if wall_seconds > 0 else 0.0,
+        "frames_per_second": (total_frames / wall_seconds) if wall_seconds > 0 else 0.0,
+        "chunk_deadline_ms": deadline_ms,
+        "interval_all": _latency_summary(all_intervals),
+        "interval_steady": _latency_summary(steady_intervals),
+    }
+    if ttfc_s is not None:
+        metrics["ttfc_ms"] = ttfc_s * 1000.0
+
+    if steady_intervals:
+        steady_frames = sum(record.num_frames for record in steady if record.index > 0)
+        steady_wall = sum(steady_intervals)
+        steady_video = steady_frames / fps
+        metrics["steady_video_rtf"] = (steady_wall / steady_video) if steady_video > 0 else 0.0
+        metrics["steady_video_rtfx"] = (steady_video / steady_wall) if steady_wall > 0 else 0.0
+        within = sum(1 for value in steady_intervals if value * 1000.0 <= deadline_ms)
+        metrics["slo_attainment"] = within / len(steady_intervals)
+        metrics["slo_violations"] = len(steady_intervals) - within
+
+    metrics["playback"] = simulate_playback(
+        ordered,
+        fps=fps,
+        buffer_chunks=playback_buffer_chunks,
+    ).to_dict()
+    metrics["playback"]["buffer_chunks"] = playback_buffer_chunks
+    return metrics
+
+
+def aggregate_metrics(sessions: Iterable[Mapping[str, Any]], *, fps: float) -> dict[str, Any]:
+    """Combine per-session metrics across sequential repeats.
+
+    Steady intervals are pooled rather than averaging per-session means, so a
+    session that happened to produce fewer steady chunks does not carry the same
+    weight as a full one.
+    """
+    entries = list(sessions)
+    if not entries:
+        raise ValueError("aggregate_metrics() needs at least one session.")
+    if len(entries) == 1:
+        single = dict(entries[0])
+        single["sessions"] = 1
+        return single
+
+    deadline_ms = entries[0]["chunk_deadline_ms"]
+    if any(entry["chunk_deadline_ms"] != deadline_ms for entry in entries):
+        raise ValueError("Cannot aggregate sessions with different chunk deadlines.")
+    ttfcs = [float(entry["ttfc_ms"]) for entry in entries if entry.get("ttfc_ms") is not None]
+    # Lower is better, so "max" is the worst session, not the best.
+    rtfs = [float(entry["video_rtf"]) for entry in entries]
+    steady_means = [
+        float(entry["interval_steady"]["mean_ms"]) for entry in entries if entry.get("interval_steady", {}).get("count")
+    ]
+    steady_count = sum(entry.get("interval_steady", {}).get("count", 0) for entry in entries)
+    steady_total_ms = sum(
+        entry["interval_steady"]["mean_ms"] * entry["interval_steady"]["count"]
+        for entry in entries
+        if entry.get("interval_steady", {}).get("count")
+    )
+    return {
+        "sessions": len(entries),
+        "chunks_received": sum(int(entry["chunks_received"]) for entry in entries),
+        "total_frames": sum(int(entry["total_frames"]) for entry in entries),
+        "video_seconds": sum(float(entry["video_seconds"]) for entry in entries),
+        "wall_seconds": sum(float(entry["wall_seconds"]) for entry in entries),
+        "video_rtf_mean": sum(rtfs) / len(rtfs),
+        "video_rtf_best": min(rtfs),
+        "video_rtf_worst": max(rtfs),
+        "ttfc_ms_mean": (sum(ttfcs) / len(ttfcs)) if ttfcs else None,
+        "ttfc_ms_max": max(ttfcs) if ttfcs else None,
+        "steady_interval_ms_mean": (steady_total_ms / steady_count) if steady_count else None,
+        "steady_interval_ms_spread": (max(steady_means) - min(steady_means)) if steady_means else None,
+        "underrun_count": sum(int(entry["playback"]["underrun_count"]) for entry in entries),
+        "chunk_deadline_ms": deadline_ms,
+    }

benchmarks/qwen3_omni/embedding_harness.py

@@ -0,0 +1,226 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Small fixtures for the real Qwen3 thinker embedding method and its oracle."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import asdict, dataclass
+from types import SimpleNamespace
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+SCENARIOS = ("text", "empty", "audio", "image", "video", "mixed", "interleaved", "vision_no_deepstack")
+
+
+@dataclass(frozen=True)
+class ModelShape:
+    vocab_size: int = 152064
+    hidden_size: int = 2048
+    deepstack_levels: int = 3
+    image_token_id: int = 151655
+    video_token_id: int = 151656
+    audio_token_id: int = 151675
+
+    def __post_init__(self):
+        if min(self.vocab_size, self.hidden_size, self.deepstack_levels) <= 0:
+            raise ValueError("vocab_size, hidden_size and deepstack_levels must be positive")
+        ids = (self.image_token_id, self.video_token_id, self.audio_token_id)
+        if len(set(ids)) != 3 or not all(0 < token < self.vocab_size for token in ids):
+            raise ValueError("Provide three distinct, positive, in-vocabulary modality IDs; use oov=True to test OOV")
+
+    def token_ids(self, oov=False):
+        if oov:
+            return dict(zip(("image", "video", "audio"), range(self.vocab_size, self.vocab_size + 3)))
+        return {name: getattr(self, f"{name}_token_id") for name in ("image", "video", "audio")}
+
+
+@dataclass
+class Feature:
+    modality: str
+    positions: torch.Tensor
+    main: torch.Tensor
+    deepstack: torch.Tensor | None
+    combined: torch.Tensor
+
+
+@dataclass
+class EmbeddingCase:
+    name: str
+    input_ids: torch.Tensor
+    is_multimodal: torch.Tensor | None
+    features: tuple[Feature, ...]
+    fingerprint: str
+
+    def fresh_embeddings(self):
+        # The thinker replaces list entries with main-scale views.
+        return None if self.name == "text" else [feature.combined for feature in self.features]
+
+
+def _tensor_bytes(tensor):
+    return tensor.detach().contiguous().view(torch.uint8).cpu().numpy().tobytes()
+
+
+def build_case(
+    name,
+    num_tokens,
+    num_mm_tokens,
+    shape,
+    *,
+    device="cuda",
+    dtype=torch.bfloat16,
+    mask_device="cpu",
+    seed=42,
+    oov=False,
+):
+    """Build known feature placements; no model merge is used to define them."""
+    if name not in SCENARIOS:
+        raise ValueError(f"Unknown scenario: {name}")
+    if num_tokens <= 0 or not 0 <= num_mm_tokens < num_tokens:
+        raise ValueError("Require N > 0 and 0 <= M < N")
+    if (name in {"text", "empty"}) != (num_mm_tokens == 0):
+        raise ValueError("Only text/empty scenarios accept M=0")
+    if name in {"mixed", "interleaved"} and (num_mm_tokens < 4 or num_mm_tokens % 2):
+        raise ValueError("Mixed/interleaved scenarios require an even M >= 4")
+    token_ids = shape.token_ids(oov)
+    generator = torch.Generator().manual_seed(seed)
+    ids = torch.randint(0, min(shape.vocab_size, *token_ids.values()), (num_tokens,), generator=generator)
+    mask = torch.zeros(num_tokens, dtype=torch.bool)
+    positions = torch.arange((num_tokens - num_mm_tokens) // 2, (num_tokens - num_mm_tokens) // 2 + num_mm_tokens)
+    groups = []
+    if name == "mixed":
+        groups = [("audio", positions[: num_mm_tokens // 2]), ("image", positions[num_mm_tokens // 2 :])]
+    elif name == "interleaved":
+        groups = [("video", positions[::2]), ("audio", positions[1::2])]
+    elif num_mm_tokens:
+        groups = [("image" if name == "vision_no_deepstack" else name, positions)]
+    features = []
+    for modality, indexes in groups:
+        ids[indexes] = token_ids[modality]
+        mask[indexes] = True
+        rows = torch.arange(len(indexes), dtype=torch.float32)[:, None].remainder(31) / 32
+        cols = torch.arange(shape.hidden_size, dtype=torch.float32)[None, :].remainder(7) / 16
+        main_cpu = rows + cols + {"audio": 2, "image": 4, "video": 6}[modality]
+        main = main_cpu.to(device=device, dtype=dtype)
+        deepstack = None
+        parts = [main]
+        if modality != "audio" and name != "vision_no_deepstack":
+            deepstack = torch.stack(
+                [(main_cpu + 16 + 4 * level).to(device=device, dtype=dtype) for level in range(shape.deepstack_levels)]
+            )
+            parts.extend(deepstack.unbind(0))
+        combined = torch.cat(parts, dim=-1)
+        # set_mm_embedding_modality simply attaches this attribute. Tag only
+        # after cat/to: views and copies do not preserve arbitrary attributes.
+        combined.modality = modality
+        features.append(Feature(modality, indexes, main, deepstack, combined))
+    digest = hashlib.sha256(json.dumps({"name": name, "shape": asdict(shape), "seed": seed, "oov": oov}).encode())
+    digest.update(_tensor_bytes(ids))
+    digest.update(_tensor_bytes(mask))
+    for feature in features:
+        digest.update(feature.modality.encode())
+        digest.update(_tensor_bytes(feature.combined))
+    return EmbeddingCase(
+        name,
+        ids.to(device=device),
+        None if name == "text" else mask.to(device=mask_device),
+        tuple(features),
+        digest.hexdigest(),
+    )
+
+
+class LanguageModelAdapter(nn.Module):
+    def __init__(self, embedding):
+        super().__init__()
+        self.embedding = embedding
+
+    def embed_input_ids(self, input_ids):
+        return self.embedding(input_ids)
+
+    def forward(self, input_ids):
+        return self.embed_input_ids(input_ids)
+
+
+def make_thinker(
+    shape,
+    *,
+    device="cuda",
+    dtype=torch.bfloat16,
+    seed=42,
+    backend="torch",
+    deepstack=True,
+    buffer_capacity=32,
+    oov=False,
+):
+    """Skip checkpoint construction, preserving the real class and its MRO."""
+    from tests.model_executor.helpers import bootstrap_vllm_layer_custom_op_modules
+
+    bootstrap_vllm_layer_custom_op_modules()
+    from vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker import (
+        Qwen3OmniMoeThinkerForConditionalGeneration,
+    )
+
+    if buffer_capacity <= 0:
+        raise ValueError("buffer_capacity must be positive")
+    device = torch.device(device)
+    if backend == "torch":
+        embedding = nn.Embedding(shape.vocab_size, shape.hidden_size, device=device, dtype=dtype)
+    elif backend == "vllm":
+        from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
+
+        with torch.device(device):
+            embedding = VocabParallelEmbedding(shape.vocab_size, shape.hidden_size, params_dtype=dtype, disable_tp=True)
+    else:
+        raise ValueError(f"Unknown embedding backend: {backend}")
+    with torch.no_grad():
+        embedding.weight.uniform_(-0.5, 0.5, generator=torch.Generator(device=device).manual_seed(seed))
+    thinker = object.__new__(Qwen3OmniMoeThinkerForConditionalGeneration)
+    nn.Module.__init__(thinker)
+    thinker.config = SimpleNamespace(text_config=SimpleNamespace(hidden_size=shape.hidden_size))
+    for modality, token_id in shape.token_ids(oov).items():
+        setattr(thinker.config, f"{modality}_token_id", token_id)
+    thinker.visual = SimpleNamespace(
+        deepstack_visual_indexes=list(range(shape.deepstack_levels)) if deepstack else None
+    )
+    thinker.language_model = LanguageModelAdapter(embedding)
+    thinker._has_oov_mm_tokens = oov
+    thinker.use_deepstack = deepstack
+    thinker.deepstack_num_level = shape.deepstack_levels if deepstack else 0
+    thinker.deepstack_input_embeds = [
+        torch.zeros(buffer_capacity, shape.hidden_size, device=device, dtype=dtype)
+        for _ in range(thinker.deepstack_num_level)
+    ]
+    thinker.deepstack_input_embeds_num_tokens = 0
+    return thinker.eval()
+
+
+@torch.inference_mode()
+def expected_outputs(model, case):
+    """Independent oracle: direct lookup and explicit positions, not merge helpers."""
+    ids = case.input_ids
+    if model._has_oov_mm_tokens and case.is_multimodal is not None:
+        ids = ids.masked_fill(case.is_multimodal.to(ids.device), 0)
+    main = F.embedding(ids, model.language_model.embedding.weight)
+    has_deepstack = any(feature.deepstack is not None for feature in case.features)
+    deepstack = main.new_zeros((model.deepstack_num_level, *main.shape)) if has_deepstack else None
+    for feature in case.features:
+        indexes = feature.positions.to(main.device)
+        main[indexes] = feature.main
+        if feature.deepstack is not None:
+            deepstack[:, indexes, :] = feature.deepstack
+    return main, deepstack
+
+
+@torch.inference_mode()
+def assert_case_output(model, case, result):
+    expected, deepstack = expected_outputs(model, case)
+    torch.testing.assert_close(result, expected, rtol=0, atol=0)
+    if deepstack is None:
+        assert model.deepstack_input_embeds_num_tokens == 0, "Unexpected stale DeepStack buffer"
+    else:
+        assert model.deepstack_input_embeds_num_tokens == case.input_ids.numel()
+        actual = torch.stack([buffer[: case.input_ids.numel()] for buffer in model.deepstack_input_embeds])
+        torch.testing.assert_close(actual, deepstack, rtol=0, atol=0)

docker/Dockerfile.rocm

@@ -2,7 +2,11 @@
 # docker/Dockerfile.ci. A stale base image can build successfully while failing
 # at runtime when vLLM-Omni imports APIs added by the newer vLLM release.
 ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.29.0
-FROM ${BASE_IMAGE} AS base
+
+# Keep the dependency-heavy portion independent from the source tree. The
+# registry cache is keyed on these inputs, while the final source copy can be
+# linked onto the cached image without materializing its large ROCm layers.
+FROM ${BASE_IMAGE} AS deps
 
 # NOTE: REMINDER to vLLM-Omni rebase maintainer
 # BASE_IMAGE pins the default prebuilt runtime when USE_NIGHTLY_BUILD=0.
@@ -64,9 +68,24 @@ RUN --mount=type=cache,target=/root/.cache/pip \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-# Step 3: Copy vllm-omni code and install without uv
-RUN mkdir -p ${COMMON_WORKDIR}/vllm-omni
-COPY . ${COMMON_WORKDIR}/vllm-omni
+# Step 3: Install project dependencies against a placeholder package tree.
+# The editable install keeps resolving imports from this directory after the
+# final COPY --link replaces the placeholder with the real source tree.
+WORKDIR ${COMMON_WORKDIR}/vllm-omni
+RUN mkdir -p vllm_omni && touch vllm_omni/__init__.py
+COPY pyproject.toml setup.py ./
+COPY requirements/ requirements/
+ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0
+
+RUN uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" \
+        --no-cache-dir -e ".[dev]" --no-build-isolation && \
+    python3 -m pip uninstall -y onnxruntime && \
+    python3 -m pip install --no-cache-dir --force-reinstall --no-deps \
+        "onnxruntime-rocm>=1.22.2" && \
+    python3 -c "import importlib.metadata as m; import onnxruntime as ort; assert 'ROCMExecutionProvider' in ort.get_available_providers(), ort.get_available_providers(); assert m.version('onnxruntime-rocm'); assert not any(d.metadata['Name'] == 'onnxruntime' for d in m.distributions())" && \
+    python3 -c "from torchcodec.decoders import VideoDecoder"
+
+RUN ln -sf /usr/bin/python3 /usr/bin/python
 
 # This is a workaround to ensure pytest exits with the correct status code in CI tests.
 RUN printf '%s\n' \
@@ -85,19 +104,13 @@ RUN printf '%s\n' \
     '    os._exit(_exit_code)' \
     > ${COMMON_WORKDIR}/vllm-omni/conftest.py
 
-# Some optional dependencies pull in the CPU ONNX Runtime distribution. The
-# CPU and ROCm wheels own the same import path and cannot coexist, so remove
-# the CPU wheel and reinstall the ROCm wheel after dependency resolution.
-RUN cd ${COMMON_WORKDIR}/vllm-omni && \
-    uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" \
-        --no-cache-dir ".[dev]" --no-build-isolation && \
-    python3 -m pip uninstall -y onnxruntime && \
-    python3 -m pip install --no-cache-dir --force-reinstall --no-deps \
-        "onnxruntime-rocm>=1.22.2" && \
-    python3 -c "import importlib.metadata as m; import onnxruntime as ort; assert 'ROCMExecutionProvider' in ort.get_available_providers(), ort.get_available_providers(); assert m.version('onnxruntime-rocm'); assert not any(d.metadata['Name'] == 'onnxruntime' for d in m.distributions())" && \
-    python3 -c "from torchcodec.decoders import VideoDecoder"
-
-RUN ln -sf /usr/bin/python3 /usr/bin/python
+# Keep source changes in one lightweight linked layer. Do not add RUN, COPY, or
+# ADD instructions that depend on the resulting filesystem after this point;
+# doing so would force BuildKit to download and extract the cached ROCm layers.
+FROM deps AS base
+ARG COMMON_WORKDIR=/app
+WORKDIR ${COMMON_WORKDIR}/vllm-omni
+COPY --link . .
 
 FROM base AS test
 

docker/Dockerfile.rocm.dockerignore

@@ -0,0 +1,35 @@
+# Keep the linked source layer small and deterministic.
+.git
+.venv
+/build
+dist
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+.mypy_cache
+
+# Distribution / packaging
+.Python
+/build/
+cmake-build-*/
+CMakeUserPresets.json
+develop-eggs/
+/dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+rust/target/

docker/Dockerfile.xpu

@@ -40,7 +40,6 @@ ENV VLLM_OMNI_TARGET_DEVICE=xpu
 # Keep the upstream oneCCL packages paired with the XPU torch runtime.
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install --no-build-isolation \
-    --extra-index-url https://mirrors.ustc.edu.cn/pypi/simple/ \
     ".[dev]" && \
     uv pip uninstall triton && \
     uv pip install triton-xpu==3.7.2 --reinstall

docs/.nav.yml

@@ -161,6 +161,7 @@ nav:
         - Cache-DiT: design/feature/cache_dit.md
         - TeaCache: design/feature/teacache.md
         - Diffusion Continuous Batching: design/feature/diffusion_continuous_batching.md
+        - Mooncake KV Transfer: design/feature/mooncake-kv-transfer.md
     - Infrastructure and Performance:
       - Prometheus Metrics: design/metrics.md
       - Speech Generation Performance Optimizations: design/qwen3_omni_tts_performance_optimization.md

docs/README.md

@@ -56,7 +56,7 @@ vLLM-Omni is flexible and easy to use with:
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
 - Omni-modality models (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
-- TTS models (e.g. Qwen3-TTS, IndexTTS 2.5, dots.tts, CosyVoice3)
+- TTS models (e.g. Qwen3-TTS, IndexTTS 2.5, CosyVoice3)
 - Diffusion models — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
 - Robot-policy and action models (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
 

docs/cli/bench/serve.md

@@ -23,7 +23,9 @@ You can use `vllm bench serve --omni --help=all` to get descriptions of all para
   The API endpoint exposed externally, to which clients send their requests.
 
 - `--dataset-name`
-  The name of the dataset used; random-mm indicates generating random multimodal inputs (images, videos, audio), while `omniinteract` replays official OmniInteract videos.
+  The name of the dataset used; `random-mm` generates random multimodal inputs (images, videos, audio),
+  `omniinteract` replays official OmniInteract videos, and `videomme` loads Video-MME MCQ videos
+  (`lmms-eval/Video-MME`, MiniCPM OmniEvalKit packing).
 
 - `--num-prompts`
   The total number of requests to send, an integer.
@@ -343,6 +345,57 @@ transcript, event, and result artifacts without errors. A valid LISTEN-only case
 chunks. Official-manifest eligibility is reported separately because clipped or cancelled output is a benchmark-quality
 signal, not a transport failure. This local performance test does not score answer accuracy.
 
+### Video-MME Benchmark
+
+Video-MME (`--dataset-name videomme`) scores multiple-choice video QA. Default packing is
+OmniEvalKit MiniCPM `minicpm-frames` (up to 96 sampled frames as `image_url`). Pass a local
+mirror with `--dataset-path` / `--videomme-parquet` + `--videomme-video-dir`, or a Hugging
+Face dataset id (`lmms-eval/Video-MME` by default; any `org/name` is accepted when
+`--dataset-name videomme` is explicit). Relative `--videomme-video-dir` values are resolved
+to absolute `file://` URLs. For those URLs, start the server with
+`--allowed-local-media-path` covering the video root; otherwise use `--videomme-inline-local-video`.
+
+For MiniCPM-o 4.5, the default deployment allows 64 images per request. To run the
+96-frame recipe, start the server with a stage 0 override (replace `/path/to/Video-MME`
+with the local video root shared by the benchmark client and server):
+
+```bash
+vllm serve openbmb/MiniCPM-o-4_5 --omni \
+  --allowed-local-media-path /path/to/Video-MME \
+  --stage-overrides '{"0":{"limit_mm_per_prompt":{"image":96,"audio":64,"video":1}}}'
+```
+
+The override applies to this server invocation only. Alternatively, use
+`--videomme-max-frames 64` in the benchmark to stay within the default image limit;
+this evaluates a different frame-sampling setting from the 96-frame recipe.
+
+Run the benchmark against that server:
+
+```bash
+vllm bench serve --omni \
+  --backend openai-chat-omni \
+  --dataset-name videomme \
+  --dataset-path /path/to/Video-MME \
+  --videomme-pack-mode minicpm-frames \
+  --videomme-max-frames 96 \
+  --model openbmb/MiniCPM-o-4_5 \
+  --endpoint /v1/chat/completions \
+  --num-prompts 8 \
+  --max-concurrency 1 \
+  --save-result
+```
+
+Accuracy keys (`videomme_accuracy`, per-duration / domain / task breakdowns) are written into
+the saved JSON. `videomme_accuracy` excludes HTTP failures; `videomme_accuracy_incl_http_fail`
+counts them as wrong. `videomme_submitted` / `videomme_unique_question_ids` show when
+oversampling made the request count differ from unique-question coverage.
+`videomme_skipped_rows` counts rows skipped during sampling because of missing or unreadable
+media or invalid question fields. The opt-in accuracy runner rejects skipped rows, HTTP
+failures, duplicate question IDs, and missing gold answers. This checks the sampled subset;
+it does not establish that a supplied local mirror contains the entire official dataset.
+Use `--videomme-save-eval-items` (or `VIDEOMME_SAVE_EVAL_ITEMS=1`) for per-request
+`videomme_eval_items` rows.
+
 ### Multi-Modal Benchmark
 
 <details class="admonition abstract" markdown="1">

docs/configuration/environment_variables.md

@@ -70,6 +70,7 @@ The lifecycle labels used below are:
 | `OMNI_DIFFUSION_PROMPT_EMBED_CACHE_SIZE` | Positive integer; default `32` entries | Each diffusion runner; resolved during model setup | A valid environment value overrides the explicit cache size. A non-integer logs a warning; a non-positive value is ignored. | Experimental |
 | `OMNI_DIFFUSION_SESSION_STATE_MANAGER` | Boolean with the same accepted spellings as the prompt cache; default disabled | Experimental diffusion session manager; model setup | A recognized environment value overrides the explicit enable setting. An unrecognized value is ignored. | Experimental |
 | `OMNI_DIFFUSION_SESSION_STATE_MANAGER_MAX_SESSIONS` | Positive integer; default `64` | Experimental diffusion session manager; model setup | A valid environment value overrides the explicit maximum. A non-integer logs a warning; a non-positive value is ignored. | Experimental |
+| `VLLM_OMNI_AR_DIFFUSION_KV_GATHER` | `1` enables; default `0` (off) | AR-diffusion KV manager at allocation and attention at dispatch; set before worker startup | Only the exact value `1` enables contiguous KV gathering. History staging additionally requires `ARDiffusionKVConfig.reuse_history_staging`. | Experimental |
 
 Backend names for `DIFFUSION_ATTENTION_BACKEND` are the members of
 `DiffusionAttentionBackendEnum`, such as `FLASH_ATTN`, `TORCH_SDPA`,
@@ -84,9 +85,10 @@ depends on the installed kernels and model path.
 | `SPEAKER_MAX_UPLOADED` | Integer; default `1000` | Speech server; read when speaker storage initializes | Environment-only setting. A non-integer logs a warning and uses `1000`; range is not otherwise validated. | Stable |
 | `VLLM_OMNI_ABORT_TIMEOUT` | Float seconds; default `2` | Engine abort wait for Videos DELETE and `generate()` cancel/error cleanup; read when `async_omni` imports | Environment-only setting. A non-float raises `ValueError` during import. | Experimental |
 | `VLLM_OMNI_ASYNC_OUTPUT_TIMEOUT` | Float seconds; default `600` | Diffusion engine async-output wait in `step_streaming`; resolved per call on the request path, not at import | Environment-only setting. A non-float or `<=0` value warns once and uses the default. | Experimental |
-| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `1`, `true`, `yes` or `on` enables; default `0` (off) | Orchestration loop and the serving-side final-output drain; read once when the `Orchestrator` is constructed | Environment-only setting. Values are stripped and case-normalized; any unrecognized value leaves the legacy poll loop selected. | Experimental |
+| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `1`, `true`, `yes` or `on` enables; defaults to on for Qwen3-TTS and off for other pipelines | Pipeline default computed from `pipeline_config.model_type` at engine initialization; env override resolved at `Orchestrator` construction and separately when the serving-side final-output drain starts | An explicit env value wins; otherwise both consumers use the engine's pipeline default. Values are stripped and case-normalized; any unrecognized value selects the legacy poll loop. Set before server startup. | Experimental |
 | `VLLM_OMNI_INPUT_WAIT_TIMEOUT_S` | Float seconds; default `600`; `<=0` disables | Full-payload input coordinator, not async-chunk transfer; read when the scheduler module imports in each worker | Environment-only setting. A non-float logs a warning and uses `600`. | Stable operational control |
 | `VLLM_OMNI_ORCH_MONITOR_PATH` | Filesystem path; default `<current-working-directory>/vllm_omni_orch_monitor_<timestamp>.json` | Orchestrator monitor enabled by `--enable-orch-monitor`; read when the monitor is created | Environment-only path override. Parent directories are created; write errors are logged. | Diagnostic |
+| `VLLM_OMNI_SPEAKER_REGISTRATION_POLICY` | `overwrite` or `immutable`; default `overwrite` | Speech server; read when speaker storage initializes | Environment-only setting. `immutable` rejects re-registering an existing uploaded voice name until it is deleted; any other value raises `ValueError` at startup. | Experimental |
 | `VLLM_OMNI_VIDEO_SYNC_TIMEOUT` | Float seconds; default `600` | Synchronous Videos API; read when the API server module imports | Environment-only setting. A non-float raises `ValueError` during import. | Experimental |
 | `VLLM_VIDEO_ASYNC_CHUNK` | `on` or `off`; default `on` | Streaming video output; read on attribute access | Environment-only setting. Values are trimmed and case-normalized; an invalid value warns once and uses `on`. | Experimental |
 | `VLLM_VIDEO_AUDIO_DELTA_MODE` | `fast` or `slow`; default `fast` | Streaming video audio deltas; read on attribute access | Environment-only setting. Values are trimmed and case-normalized; an invalid value warns once and uses `fast`. | Experimental |

docs/configuration/stage_configs.md

@@ -290,3 +290,96 @@ vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/depl
 
 !!! important
     We are actively iterating on the definition of deployment configurations, and we welcome feedback from users and developers.
+
+## Qwen3-TTS with Model Runner V2
+
+Qwen3-TTS runs the native CUDA Model Runner V2 pipeline by default on vLLM
+0.29.0. MRV2 is an experimental feature for this model: the bundled default
+profile selects it on CUDA only, and its scheduler and delivery paths are
+still being qualified. Set `model_runner: v1` in a copy of the deploy config
+to opt out; the `platforms:` sections of `qwen3_tts.yaml` keep V1 on NPU, XPU,
+ROCm and MUSA. Select one of these deployment profiles:
+
+| Profile | Runner | Code2Wav graph batches | Intended use |
+| --- | --- | --- | --- |
+| `qwen3_tts.yaml` | V2 (default) | Existing defaults | Shipped default; experimental |
+| `qwen3_tts_mrv2.yaml` | V2 | B1 | Explicit MRV2 profile (same runner selection as the default) |
+| `qwen3_tts_high_concurrency_mrv2.yaml` | V2 | B1, B2 | Opt-in throughput tuning |
+| `qwen3_tts_high_concurrency_mrv2_b4.yaml` | V2 | B1, B2, B3, B4 | Experimental throughput / buffered playback |
+| `qwen3_tts_high_concurrency.yaml` | V1 | Existing defaults | V1 high-concurrency control |
+
+```bash
+# MRV2 is the default; pass a copy with `model_runner: v1` to force V1.
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base --omni \
+  --deploy-config /path/to/qwen3_tts_v1.yaml
+```
+
+The V2 profiles bound Talker prefill to 512 tokens per step and select the
+Talker AR runner and the Code2Wav generation
+runner together. Native inter-stage delivery carries codec payloads directly;
+request-owned snapshots preserve buffers through asynchronous completion and
+CUDA graph reuse. Terminal completion waits for upstream stage metrics before
+releasing request state. Platform sections retain V1 on NPU, XPU, ROCm and MUSA;
+this change does not qualify MRV2 on those backends or enable other model families.
+Selecting V2 for a stage without `supports_native_mrv2_data_plane` emits a warning
+when pipeline and deployment settings are merged. That stage retains the legacy
+transport path; the warning does not establish support for that combination.
+
+MRV2 model hooks use capability declarations rather than architecture or stage
+names. Omni lifecycle flags select the model state, and `_returns_tuple` declares
+the capture output contract. The optional batched predictor hook is `mtp`, with
+an explicit `mtp_output_key` (a string or two-part payload key), optional
+`mtp_validity_key`, and `mtp_graph_safe`/`mtp_disable_graph` capture controls.
+Its inputs are token IDs, embeddings, previous hidden states and per-row
+conditioning; it returns updated embeddings and prediction codes. Models may
+supply `mtp_sampling_params` and `get_mtp_seed(sampling_params)` for model-local explicit seeds, and declare
+`mtp_accepts_per_row_generators`, `mtp_accepts_req_infos`, or `mtp_sample_uniforms`
+(with `mtp_sample_steps` and `mtp_sample_vocab_size`) as needed. Qwen3-TTS retains
+its existing `talker_mtp` entry point for V1.
+
+### Included performance work
+
+- Request snapshots have a fast path for immutable scalar leaves, including
+  waveform lists. Mutable containers, aliases and cycles keep deep-copy
+  semantics. This is not the historical shallow `dict(prompt)` experiment.
+- The API reference cache holds owned float32 arrays with a default capacity
+  of 1024 entries and a 512 MiB waveform-payload budget. Cache hits return
+  independent lists. Qwen3-TTS model artifact caches also default to 1024
+  entries. These caches have different owners and eviction policies; equal
+  entry limits do not make them a single coherent cache.
+- Talker state stays on GPU where its lifecycle allows it. BOS/EOS projections
+  are cached in their projection dtype and invalidated on weight loading.
+- Code2Wav packs CPU codec inputs before device transfer. Optional B2/B4 graph
+  buckets batch compatible requests without sharing their per-request state.
+- Native output materialization runs in a bounded worker and drains before
+  closing the data plane. The existing control-only shortcut avoids launching
+  model work for a step that has only lifecycle events.
+
+MTP prefix re-prefill remains disabled in the high-concurrency V2 profile.
+Adding more graph shapes increases compilation cost and has not established a
+stable end-to-end gain for this extracted version.
+
+### Decoder batches and playback
+
+Changing `decode_cudagraph_batch_sizes` selects the captured batch buckets.
+Keep `decode_batch_max_size` consistent with the intended maximum too; the
+stateful graph path currently groups according to the captured buckets, while
+the stateless path also uses the explicit maximum.
+
+B4 remains experimental. First-packet latency alone does not establish
+uninterrupted playback. Validate inter-chunk arrival times, buffering, WER and
+speaker similarity before adopting either batching preset for a production
+workload. Floating-point decoder outputs can differ across batch sizes; this PR
+does not claim bitwise or quality equivalence.
+
+### Optional MPS deployment
+
+NVIDIA MPS is an optional operator setting for colocated CUDA processes, not a
+YAML option or a library default. This PR does not establish a throughput or
+first-packet latency benefit from MPS. Measure the exact deployment with and
+without MPS before enabling it.
+
+Use only assigned GPUs and an independent MPS pipe directory. A private MPS
+server does not provide exclusive GPU ownership or MIG isolation. For a
+single-GPU deployment, explicitly place both stages on that GPU; the supplied
+high-concurrency profile places its two stages on different GPUs by default.

docs/contributing/ci/test_execution_guide.md

@@ -177,10 +177,10 @@ Failed jobs: 1/2
     cd tests
 
     # Stability: Qwen3-Omni
-    pytest -s -v dfx/stability/scripts/test_stability_qwen3_omni.py -m slow
+    pytest -s -v dfx/stability/scripts/run_stability_qwen3_omni.py -m slow
 
     # Stability: Wan2.2 (v1/videos diffusion benchmark loop)
-    pytest -s -v dfx/stability/scripts/test_stability_wan22.py -m slow
+    pytest -s -v dfx/stability/scripts/run_stability_wan22.py -m slow
 
     # Reliability: Qwen3-Omni (H100 × 2)
     pytest -s -v dfx/reliability/test_reliability_qwen3_omni.py -m slow

docs/contributing/ci/test_writing_guide.md

@@ -555,7 +555,7 @@ L5 level testing focuses on the performance of model services under ***long-runn
 
 - ***Trigger Timing***: **`Weekly`** (weekly) or **`Days before Release`** (several days before a major release). Due to long execution times, the frequency is lower.
 - ***Run Command***:
-    - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_<model>.py` alongside a matching JSON config)
+    - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/run_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/run_stability_wan22.py` (or add `run_stability_<model>.py` alongside a matching JSON config)
     - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_<model>.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`).
 - ***Script Example***:
 
@@ -564,7 +564,7 @@ L5 level testing focuses on the performance of model services under ***long-runn
 
 ##### Stability
 
-When you want to add L5-level stability test cases, add or extend the appropriate JSON file under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` for Omni bench traffic, or `test_wan22.json` for diffusion `/v1/videos` workloads). Pair the JSON with `tests/dfx/stability/scripts/test_stability_<model>.py`. The following illustrates the Qwen3-Omni shape:
+When you want to add L5-level stability test cases, add or extend the appropriate JSON file under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` for Omni bench traffic, or `test_wan22.json` for diffusion `/v1/videos` workloads). Pair the JSON with `tests/dfx/stability/scripts/run_stability_<model>.py`. The following illustrates the Qwen3-Omni shape:
 
 ```json
 {

docs/contributing/profiling.md

@@ -22,7 +22,7 @@ profiler_config:
 Supported fields:
 
 | Field | Description |
-|---|---|
+| --- | --- |
 | `profiler` | Profiler backend. Supported values: `torch`, `cuda`. Use `torch` for `trace.json`, Excel operator tables, and optional memory snapshots. Use `cuda` for Nsight Systems only. |
 | `torch_profiler_dir` | Output directory for torch-profiler artifacts. Required when `profiler: torch`. |
 | `torch_profiler_use_gzip` | Compress `trace_rank*.json` into `trace_rank*.json.gz`. |
@@ -287,6 +287,52 @@ Recommended viewers:
 
 For upstream background on the underlying vLLM profiling infrastructure, see the [vLLM profiling guide](https://docs.vllm.ai/en/stable/contributing/profiling/).
 
+## 6. Profiling Omni AR/TTS stages
+
+For Omni AR and TTS stages, put `profiler_config` under each stage that should
+be captured. The serving API then controls a narrow window with
+`POST /start_profile` and `POST /stop_profile`; model initialization and
+warmup stay outside the trace.
+
+Example stage override for a host-stack torch trace:
+
+```bash
+PROFILE='{"profiler":"torch","torch_profiler_dir":"/tmp/qwen3_tts_profile",'\
+'"torch_profiler_with_stack":true,"torch_profiler_record_shapes":false,'\
+'"torch_profiler_use_gzip":true,"torch_profiler_dump_cuda_time_total":true}'
+
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni \
+  --deploy-config vllm_omni/deploy/qwen3_tts_high_concurrency.yaml \
+  --stage-overrides "{\"0\":{\"profiler_config\":$PROFILE},\"1\":{\"profiler_config\":$PROFILE}}"
+```
+
+After one or more warmup requests, capture one short measured request:
+
+```bash
+curl -sS -X POST http://127.0.0.1:8000/start_profile \
+  -H 'Content-Type: application/json' -d '{"stages":[0,1]}'
+# send one fixed /v1/audio/speech or /v1/chat/completions request here
+curl -sS -X POST http://127.0.0.1:8000/stop_profile \
+  -H 'Content-Type: application/json' -d '{"stages":[0,1]}'
+```
+
+For an Nsight Systems CUDA timeline, use the same stage override with
+`{"profiler":"cuda"}` and wrap the server:
+
+```bash
+nsys profile --trace=cuda,nvtx,osrt --sample=process-tree \
+  --cpuctxsw=process-tree --trace-fork-before-exec=true \
+  --capture-range=cudaProfilerApi --capture-range-end=stop-shutdown \
+  --cuda-graph-trace=node -o /tmp/qwen3_tts_nsys \
+  vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni ...
+```
+
+For Omni AR workers, `profiler="cuda"` requires the native CUDA profiler
+wrapper and uses `cudaProfilerStart/Stop`; the report is only valid if the
+`/start_profile` and `/stop_profile` calls both succeed. Use `nsys stats` for
+CUDA API, GPU kernel, OS runtime, and NVTX summaries. Treat profiler latency as
+diagnostic only; use a separate no-profiler run for performance numbers.
+
 ## 7. Orchestrator Monitor
 
 > **Warning:** Diagnostic only. Adds lightweight counters on the orchestrator poll loop and per-replica queue depth. Disable in production unless you are actively debugging orchestrator saturation.
@@ -314,9 +360,11 @@ If unset, the monitor writes `./vllm_omni_orch_monitor_<MMDDHHMM>.json` at shutd
 Each 1-second window records:
 
 | Series | Description |
-|---|---|
+| --- | --- |
 | `windows.duration_s` | Wall time covered by the window |
 | `windows.loop_idle` / `windows.loop_active` | Orchestrator poll-loop iterations with no work vs. active forwarding |
+| `windows.dispatch_queue_size` | Ready-queue size sampled at each monitor window boundary |
+| `windows.dispatch_queue_high_water` | Maximum Event Driven ready-queue size observed in each window |
 | `replicas.<stage,replica>.outputs_queue_size` | MP client `outputs_queue` backlog for that replica |
 | `replicas.<stage,replica>.inflight` | Requests currently bound/routed to the replica |
 
@@ -331,12 +379,16 @@ orchestrator wakes only once per 0.5 s reconcile timeout, while a busy one still
 records one iteration per routed output. Window counts and the `loop_active_pct`
 summary are therefore not comparable across the two modes.
 
+For Event Driven runs, use `dispatch_queue_high_water` rather than only the
+window-boundary `dispatch_queue_size`: a serial dispatcher can drain a burst
+before the window closes, hiding a transient ready-queue backlog in the latter.
+
 ### Relationship to other diagnostics
 
 This monitor is intentionally separate from the existing profiling tools:
 
 | Tool | Scope | Output |
-|---|---|---|
+| --- | --- | --- |
 | `--enable-diffusion-pipeline-profiler` | Diffusion stage functions (`vae.decode`, `diffuse`, …) | Per-request `stage_durations` / logs |
 | `--enable-ar-profiler` | AR stage generation time | Per-request `stage_durations` |
 | `profiler_config` (`torch` / `cuda`) | GPU/CPU kernels inside a stage worker | `trace.json`, Nsight, operator tables |

docs/design/feature/cache_dit.md

@@ -180,17 +180,52 @@ cache_dit.enable_cache(
 
 > **Note:** For single transformer with multiple block lists, `refresh_context` works the same as standard models.
 
+### Example 3: Sequential-CFG Diffusion Pipeline (MammothModa2)
+
+MammothModa2's DiT stage runs on the diffusion runner, which enables the
+configured backend at startup and transfers ownership to the pipeline through
+the request-scoped protocol (`adopt_cache_dit_backend` / `is_cache_dit_enabled`);
+`forward()` then reconciles per-request state through
+`RequestScopedCacheDiTRuntime` (MiniMax H3 precedent).
+
+**Key difference:** The enabler caches only the repeated main-layer stack and
+marks the pipeline as requiring paired CFG forwards, because cache-dit's
+separate-CFG accounting distinguishes cond/uncond passes purely by forward
+parity:
+
+```python
+pipeline._cache_dit_requires_paired_cfg = True
+block_adapter = BlockAdapter(
+    transformer=pipeline.gen_transformer,
+    blocks=[pipeline.gen_transformer.layers],  # Refiners stay outside
+    forward_pattern=[ForwardPattern.Pattern_3],
+    has_separate_cfg=True,
+    check_forward_pattern=True,
+)
+```
+
+Per request, the pipeline calls `runtime.prepare(spec)` for CFG requests
+(refreshing the step context) and `runtime.prepare(None)` for `guidance=1.0`
+requests (disabling hooks — their single forward per step cannot be expressed
+by the parity accounting).
+
 ### Registering Custom Implementations
 
-After writing your custom enabler, register it in `CUSTOM_DIT_ENABLERS` in
+After writing your custom enabler, register it in
+`CUSTOM_DIT_ENABLERS` via `register_custom_dit_enablers()` in
 `vllm_omni/diffusion/cache/cachedit/model_specific.py`:
 
 ```python
-CUSTOM_DIT_ENABLERS = {
-    "Wan22Pipeline": enable_cache_for_wan22,
-    "LongCatImagePipeline": enable_cache_for_longcat_image,
-    "YourCustomPipeline": enable_cache_for_your_model,  # Add here
-}
+CUSTOM_DIT_ENABLERS.update(
+    {
+        "Wan22Pipeline": enable_cache_for_wan22,
+        "Cosmos3OmniDiffusersPipeline": enable_cache_for_cosmos3,
+        "Krea2Pipeline": enable_cache_for_krea2,
+        "Magi2Pipeline": enable_cache_for_magi2,
+        "MammothModa2DiTPipeline": enable_cache_for_mammothmoda2,
+        "YourCustomPipeline": enable_cache_for_your_model,  # Add here
+    }
+)
 ```
 
 ---
@@ -274,6 +309,7 @@ Complete examples in the codebase:
 | **Wan2.2** | `cachedit.model_specific::enable_cache_for_wan22` | Single or dual-transformer | Auto-detects mode based on transformer_2 presence |
 | **LongCat** | `cachedit.config::CacheDiTAdapterConfig` | Declarative block adapter | Two block lists in one transformer |
 | **BAGEL** | `cachedit.model_specific::BagelCachedAdapter` | Custom cached adapter | Complex architecture |
+| **MammothModa2** | `cachedit.model_specific::enable_cache_for_mammothmoda2` | Sub-stack + paired CFG | Generation-runner pipeline; lifecycle owned by `pipeline_mammothmoda2_dit.py` |
 
 ---
 

docs/design/feature/mooncake-kv-transfer.md

@@ -0,0 +1,89 @@
+# Mooncake KV Transfer
+
+## PR summary
+
+This change adds an opt-in AR-to-DiT KV handoff using vLLM's native
+`KVTransferConfig`, Scheduler-owned diffusion KV pages, and
+`MooncakeConnector`. The existing `OmniKVTransferManager` path remains the
+fallback when `kv_transfer_config` is absent.
+
+The orchestrator creates one transfer ticket per request and resolves the
+bound AR replica's Mooncake endpoint after generation. DiT allocates the
+destination pages, waits for every worker rank to finish loading, and releases
+the source and destination pages only after the connector lifecycle completes.
+Request registration failures are rolled back before Worker dispatch. Receive
+timeouts and cancellation terminate the request without recycling in-flight
+pages: the scheduler retains them until every CFG row has completed on every
+rank. Late completion is polled only while such requests remain. If a peer
+never completes, its pages remain reserved until shutdown; other requests can
+continue using the remaining capacity. Invalid pages, missing Worker ranks,
+and failures without a proven rollback still stop the replica before teardown.
+The example sets `transfer_timeout: 60.0` seconds explicitly. Cancellation
+queued during a synchronous receive is processed when that receive completes
+or times out; it does not interrupt the remote write.
+
+## Reference configuration
+
+| Stage | Replicas | Devices | Parallelism | KV cache |
+| --- | ---: | --- | --- | ---: |
+| AR | 2 | 0-3 | TP=2 per replica | 2 GiB per worker |
+| DiT | 2 | 4-7 | TP=2 per replica, EP enabled | 1 GiB per worker |
+
+The acceptance workload uses 512x512 images, two denoising steps, guidance
+scale 2.0, and `max_num_seqs=1`.
+
+For a four-GPU debug run, set both stages to one replica and assign AR TP=2 to
+devices `0,1` and DiT TP=2 to devices `2,3`.
+
+## Experiment
+
+The final comparison used one node with eight NVIDIA RTX PRO 6000 Blackwell
+Server Edition GPUs, TCP Mooncake transport, the same 50-prompt/seed T2I
+dataset, concurrency 8, and two independent 100-request runs per
+implementation. Each side completed 200/200 requests without an OOM or request
+failure.
+
+| Implementation | Run 1 (req/s) | Run 2 (req/s) | Mean (req/s) |
+| --- | ---: | ---: | ---: |
+| `OmniKVTransferManager` | 4.4103 | 4.4497 | 4.4300 |
+| Native `MooncakeConnector` | 4.4457 | 4.4699 | 4.4578 |
+
+Native throughput is **0.63% higher** than the legacy mean. The absolute
+performance difference is below the 1% acceptance threshold.
+
+For accuracy, a non-zero transferred prefix and the resulting attention output
+were compared bit-for-bit with the legacy manager path:
+
+```text
+prefix_bits_equal    True
+attention_bits_equal True
+max_abs_diff         0.0
+```
+
+The one-AR/one-DiT TP=2 debug topology also completed 20/20 sequential
+requests. A separate real-TCP transport check transferred both CFG rows from a
+TP=1 producer to a strict-Ulysses SP=2 consumer with exact values on both
+target ranks.
+
+## Scope and limitations
+
+- The fully validated end-to-end topology is 2x AR TP=2 plus 2x DiT TP=2.
+- Heterogeneous TP=1 to SP=2 reshaping is transport-tested, but arbitrary mixed
+  TP/CFG/SP end-to-end combinations have not been exhaustively validated.
+- CFG-parallel end-to-end deployment is not part of this acceptance run.
+- Native AR-to-DiT transfer currently requires `async_chunk: false`.
+- TCP is covered by the acceptance run. The supplementary single-node RDMA
+  measurements use a separate short workload; they do not establish RDMA
+  support for multi-node deployment, which remains out of scope.
+- The native path currently validates the two-stage `AR(0) -> DiT(1)` graph. Other stage graphs are rejected during startup until
+  their producer/consumer edge mapping is implemented.
+- Sleep mode is disabled for the native paged path because registered KV pages
+  must remain mapped for the connector lifetime.
+
+- Ascend NPU adaptation and its performance validation are maintained in a
+  separate follow-up PR; the GPU acceptance results above do not validate NPU.
+- The homogeneous piecewise CUDA batching optimization is retained. A native
+  paged-attention CUDA graph capture/replay regression covers two CFG rows,
+  strided QKV, mixed causal/image spans, changed inputs, and output stability.
+  It must be run on CUDA hardware; CPU unit tests do not establish graph or
+  end-to-end performance acceptance.

docs/design/feature/offloader/README.md

@@ -84,10 +84,22 @@ Block topology comes from the pipeline `OffloadPlan` first: `block_attrs` maps
 each DiT path to its ordered block containers, `encoder_block_attrs` declares
 streamable encoder stacks, `on_demand_component_paths` marks pipeline-managed
 residency, `resident_dit_paths` marks the DiTs that may hold resident layers,
-and `encoder_dlo_weight_replication` marks the encoders whose loader-produced
-weights are safe for AllGather. DiTs absent from the plan fall back to
-`_layerwise_offload_blocks_attrs` (including the deprecated singular-name
-compatibility path).
+`encoder_dlo_weight_replication` marks the encoders whose loader-produced
+weights are safe for AllGather, and `offload_submodules` maps a DiT submodule
+that owns its residency to its own block container. DiTs absent from the plan
+fall back to `_layerwise_offload_blocks_attrs` (including the deprecated
+singular-name compatibility path).
+
+An undeclared DiT submodule joins the plan only when it is large enough that
+keeping it resident would defeat streaming; its block container is then found
+by scanning well-known attribute names, which warns once per submodule class.
+Size is computed from parameter shapes and dtypes, including meta parameters,
+so resolving before mmap loading preserves the same residency decision.
+Block ownership validation includes nested components: a block shared with a
+parent, sibling, or encoder is rejected before any hooks or storage are changed.
+A submodule with no block container must implement the `load_to_device` /
+`offload_to_cpu` lifecycle, and the resolver rejects it before any component of
+the pipeline is placed or hooked.
 
 The resolver is pure: it moves no tensor, installs no hook, writes no module
 attribute, and reads no process group — multi-rank facts come from

docs/design/feature/omni_async_output_materialization.md

@@ -318,18 +318,18 @@ The same behavior applies to the Base and VoiceDesign Qwen3-TTS checkpoints
 because they use the same Talker implementation.
 
 !!! warning
-    Do not pass `--no-async-chunk` or enable prefix caching when you want this
-    optimization. Either change causes the runner to fall back to synchronous
-    output construction. Enabling async chunk alone cannot activate the feature
-    for a model that has not opted into async Omni output.
+    Do not pass `--no-async-chunk` when you want this optimization. Disabling
+    async chunk causes the runner to fall back to synchronous output
+    construction. Enabling async chunk alone cannot activate the feature for a
+    model that has not opted into async Omni output.
 
 !!! note "Prefix cache compatibility"
-    Async Omni output materialization and Omni prefix caching cannot currently
-    run together. `_should_use_async_omni_output()` returns `False` whenever
-    `self.omni_prefix_cache` is present, so prefix caching selects synchronous
-    output materialization. Supporting both features requires snapshotting or
-    otherwise synchronizing prefix-cache merge and update state before the
-    background output path can consume it safely.
+    Prefix cache and async Omni output can run together. `save_outputs` returns
+    a `step_id`; the output builder later calls `materialize(sid, req_ids)` with
+    the save-time request list, which may be a step late relative to the live
+    `input_batch`. `_should_use_async_omni_output()` does not disable itself
+    when the cache is present. See
+    [Automatic Prefix Caching](prefix_caching.md#implementation).
 
 ## Compatibility and Fallbacks
 
@@ -341,10 +341,9 @@ runtime conditions hold:
 | AR async scheduling is enabled | The optimization relies on the scheduler advancing while the prior output is materialized |
 | `async_chunk` is enabled | The feature targets incremental downstream Omni payloads |
 | The model stage opts in with `use_async_omni_output` | Models must declare that their output lifecycle is safe to defer |
-| Omni prefix cache is disabled | Prefix-cache merge and update ordering currently requires synchronous materialization |
 | Speculative decoding is disabled | Speculative output state is not included in this deferred path |
 | Routed-expert output is disabled | Routed-expert extraction currently requires the synchronous path |
-| Postprocess is absent or explicitly runs eagerly | State needed by the next decode step must be updated before the runner returns |
+| Postprocess is absent or explicitly runs eagerly | State needed by the next decode step must be updated before the runner returns. Eager postprocess sees the scheduled slice (`None, None` combined tensors), not the prefix-cache-merged full prompt; it must only consume the tail |
 
 These checks are evaluated per stage on runners based on `GPUARModelRunner`.
 An unsupported combination does not prevent serving; it only falls back to
@@ -380,6 +379,8 @@ so Omni payload materialization remains synchronous.
 - `tests/worker/test_gpu_ar_model_runner.py`: Snapshot, guard, connector
   ordering, and background error-propagation tests.
 - [Async Chunk](async_chunk.md): Inter-stage chunking and scheduling design.
+- [Automatic Prefix Caching](prefix_caching.md#implementation): Prefix-cache
+  `step_id` / `materialize` contract used by the deferred output builder.
 - [Qwen3-Omni optimization blog](https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization):
   Optimization context and controlled performance results.
 - [PR #4476](https://github.com/vllm-project/vllm-omni/pull/4476): Feature

docs/design/feature/prefix_caching.md

@@ -1,6 +1,5 @@
 # Automatic Prefix Caching in Omni Models
 
-
 ---
 
 ## Table of Contents
@@ -9,6 +8,8 @@
 - [High-Level Approach](#high-level-approach)
 - [Example](#example)
 - [What About Multimodal Inputs?](#what-about-multimodal-inputs)
+- [Implementation](#implementation)
+- [Related Files](#related-files)
 
 ---
 
@@ -27,6 +28,7 @@ vLLM implements automatic prefix caching for managing its kv-cache, which is bes
     This document describes vLLM-Omni's mechanism for caching tensor outputs that are meant to be passed between stages, when requests have common prefixes, similar to the way in which vLLM has prefix caching for the kv-cache. This works in conjunction with vLLM's multimodal encoder caching, but is distinct. See the final section for a concrete example for how they tie together in practice.
 
 ### High-Level Approach
+
 !!! note "Note 2"
     Prior to reading this section, it's recommended to take a look at the design documents in vLLM for [Automatic Prefix Caching](https://docs.vllm.ai/en/latest/features/automatic_prefix_caching/), which will make some of the concepts more clear.
 
@@ -38,8 +40,10 @@ The main focus of vLLM-Omni's approach to prefix caching stage outputs is to bui
 
 With this in mind, consider the set of blocks in a 2D layout, where the row represents the index of blocks being considered, and the columns represent the slots corresponding to tokens within each block. Since we know the `num_blocks` and `block_size` from our kv cache config, if we want to cache a tensor with feature size `D`, we can preallocate a CPU tensor of size `(num_blocks, block_size, D)`, and use the same block index and slot mapping to retrieve the corresponding feature vector.
 
+Host footprint: each cached key costs `num_blocks × block_size × D × dtype_bytes` of **pinned** CPU memory (pinned so device→host can overlap compute), allocated on the first `save_outputs` that sees the key — the first real request pays the `cudaHostAlloc`. Measured on a Qwen3-Omni deployment: thinker `__hidden_states__` `[15092, 16, 2048]` bf16 ≈ 0.92 GiB (plus the same again for each `hidden_states.layer_*` key a model exposes), Qwen3-TTS talker `codes.audio` `[16180, 16, 16]` ≈ 33 MiB. Budget host RAM for the stage accordingly.
 
 ### Example
+
 !!! note "Note 3"
     Prefix caching in vLLM-Omni currently is only supported on AutoRegressive stages with one kv-cache group. Configure it with the pipeline-wide `enable_prefix_caching` field in the deploy config.
 
@@ -56,7 +60,7 @@ The prefix cache flow is then outlined below.
 
 2. Say we process the request `The quick brown fox was tired and slept beneath the shady tree`, which is 12 tokens and evenly divides into 3 blocks as shown below.
 
-```
+```text
          [  The quick brown fox  ] [  was tired and slept ] [beneath the shady tree ]
 Block 1: |<--- block tokens ---->|
 Block 2: |<------- prefix ------>| |<--- block tokens --->|
@@ -65,10 +69,9 @@ Block 3: |<------------------ prefix -------------------->| |<--- block tokens -
 
 When the request processes, we inspect the multimodal outputs and identify the `mm_feature` tensor, which will be of shape `(seq_len, feature_dim)`, i.e., `(12, 16)` in this example. We note that the first axis is dependent on the `seq_len` and add a new cache_tensor of shape `(num_blocks, block_size, feature_dim)` to our multimodal cache for tensors.
 
+1. If we lay out the cache as a 2D tensor of shape (`num_blocks`, `block_size`), we'll have something like the following:
 
-3. If we lay out the cache as a 2D tensor of shape (`num_blocks`, `block_size`), we'll have something like the following:
-
-```
+```text
 0: [  The quick brown fox  ]
 1: [  was tired and slept  ]
 2: [beneath the shady tree ]
@@ -78,7 +81,8 @@ When the request processes, we inspect the multimodal outputs and identify the `
 ```
 
 Or, if we flatten it down to 1D,
-```
+
+```text
 0: The
 1: quick
 2: brown
@@ -90,7 +94,8 @@ Or, if we flatten it down to 1D,
 ```
 
 which we can think of as row indices into the hidden states tensor if we view it as the 2D shape `(num_blocks x block_size, feature_dim)`. That is, the analogous flattened (from 3D -> 2D) mapping of the cache for hidden states becomes the following.
-```
+
+```text
 0: <hidden states vector of len 2 corresponding to 'The'>
 1: <hidden states vector of len 2 corresponding to 'quick'>
 2: <hidden states vector of len 2 corresponding to 'brown'>
@@ -103,10 +108,9 @@ which we can think of as row indices into the hidden states tensor if we view it
 
 Similarly, for the multimodal outputs cache, the flattened coordinates are the same, but the `mm_feature` maps to vectors of length `16` instead of the hidden size of `2`. Note that in practice, we may have multiple  multimodal output tensors per forward pass, which may have different names and different feature dimensions.
 
+1. Now, say that we receive a new request `The quick brown fox jumped over the dog`.
 
-4. Now, say that we receive a new request `The quick brown fox jumped over the dog`.
-
-```
+```text
          [  The quick brown fox  ] [  jumped over the dog ]
 Block 1: |<--- block tokens ---->|
 Block 2: |<------- prefix ------>| |<--- block tokens --->|
@@ -116,7 +120,7 @@ Here, we will have a cache hit for `Block 1` which will be detected by vLLM base
 
 Since we have the block indices / slot mappings from the kv cache manager, we can simply mirror the mappings and leverage the same indices for the cached hidden states and multimodal outputs. This allows us to look up the correct tensors from our externally maintained 3D caches.
 
-```
+```text
 0: [  The quick brown fox  ] < already in the cache
 1: [  was tired and slept  ]
 2: [beneath the shady tree ]
@@ -129,11 +133,11 @@ Since we have the block indices / slot mappings from the kv cache manager, we ca
 
 Finally, to pass the full hidden states and multimodal outputs to the next stage, we simply concatenate the cached contents with the corresponding new tensors computed from the current forward call.
 
-
 ### What About Multimodal Inputs?
+
 It's also useful to consider the case about how Omni prefix caching is handled when we have multimodal inputs that don't cleanly end on block boundaries, as well as how this works with multimodal encoder caching in vLLM. For example:
 
-```
+```text
          [   Im0  Im1  Im2  Im3  ] [ Im4  Im5 foo <empty> ]
 Block 1: |<--- block tokens ---->|
 Block 2: |<------- prefix ------>| |<--- block tokens --->|
@@ -147,9 +151,9 @@ In reality, this isn't a big problem for correctness, because vLLM also maintain
 - The hash describing the image data starting at position 0 and with length 6
 - In vLLM's encoder cache, a mapping from the image hash above to the encoder output
 
-
 To understand what happens, say we get the following input as a second request:
-```
+
+```text
          [   Im0  Im1  Im2  Im3  ] [  Im4  Im5 bar  baz  ]
 Block 1: |<--- block tokens ---->|
 Block 2: |<------- prefix ------>| |<--- block tokens --->|
@@ -162,3 +166,162 @@ Because we have multimodal data in a scheduled span that isn't fully precomputed
 When we pass our multimodal tensors to the language model component in the same stage, we'll then expect the same outputs, because the prefix caching behaviors in vLLM-Omni / vLLM match, so the LLM will use vLLM's KV cache manager's prefix caching to correctly handle the attention information for `Block 1` while calculating the outputs for `Block 2`, giving us the correct results for processing `Block 2` with the context of `Block 1`.
 
 Finally, we look up the output hidden states/multimodal tensors corresponding to the prefix cache hit `Block 1` and concatenate it with the forward pass result to get the final result, which is expected to be identical to the full hidden states when prefix caching is disabled.
+
+### Implementation
+
+The block/slot model is `vllm_omni/core/prefix_cache/`.
+`OmniPrefixCacheManager` owns slot occupancy, the request-task table,
+hit spans, and merge.
+`OmniPrefixCacheController` moves data: a reusable `StagingBufferPool` for
+this step's device→host copy, and writes into the durable `PrefixBlockPool`.
+The state lock covers those tables only.
+
+Miss is not an error (this step's forward slice only). A hit span whose
+hidden rows are absent is fatal. For mm keys the rule is looser: a model may
+emit a key only for some requests, so a hit span with no rows behind an mm key
+reads zeros from the pool for those positions rather than raising. Abort still
+writes: once a hash entered this step's batch it must land in the cache.
+
+`enable_prefix_caching` is refused on KV-consumer stages (`kv_role` of
+`kv_consumer` or `kv_both`). KV received from a producer is reported as
+`num_computed_tokens` too, and the manager cannot tell it from a local hit.
+Producer-only stages are unaffected. Pooling stages never save, so they get
+no cache. This gate is one function, `stage_prefix_cache_config`, called by
+both the GPU and the NPU model runner at kv-cache init.
+
+Which stages may set `enable_prefix_caching: true`:
+
+| Stage | `enable_prefix_caching` | Why |
+| --- | --- | --- |
+| AR stage with one full-attention kv group whose hidden states / per-token mm feed the next stage (Qwen3-Omni thinker and talker) | supported | The case the cache is built for: full-prompt hidden states are merged from the pool on a hit. |
+| AR stage that sets `requires_full_prefix_cached_hidden_states = False`, optionally with `deferred_prefix_cache_mm_keys` (Qwen3-TTS talker, Higgs v3 talker) | supported | Hidden is not cached; deferred codec rows are written once on finish. |
+| Pooling stage | ignored | Never saves; the gate returns no config. |
+| `kv_role: kv_consumer` / `kv_both` | refused at kv-cache init (`OmniPrefixCacheUnmatchError`) | Producer KV shows up as `num_computed_tokens` and is indistinguishable from a local hit. |
+| Speculative decoding on the stage | refused at kv-cache init | Under async scheduling vLLM keeps `num_computed_tokens_cpu` optimistic (all drafts accepted) during the forward and corrects it afterwards; `step_slots_cpu` would mirror rows at the wrong slots. Not verified; refused as a whole. |
+| `prefix_match_unit` smaller than `block_size` | refused at kv-cache init | Sub-block hits make `num_computed_tokens` unaligned; the hit registry only mirrors whole blocks. |
+| Hybrid / sliding-window / multi-group kv cache (e.g. a talker with `attention_type: sliding_recompute`) | refused at kv-cache init | The cache mirrors exactly one full-attention block table. |
+| Attention backend whose kernel block size differs from `--block-size` (FlashInfer / FlashMLA / CutlassMLA with a block size they do not list natively), or decode context parallel | refused at first step (`FullAttentionGroupView`) | `step_slots_cpu` computes `table[req, pos // block_size] * block_size + pos % block_size` over allocator block ids; hybrid kernel blocks and DCP token striping change that row layout. FlashAttention / Triton accept any multiple of 16 and never split blocks. |
+| Codec decoder / Code2Wav stages (Qwen3-Omni stage 2, Qwen3-TTS stage 1) | keep `false` | Nothing downstream consumes their hidden states; the cache would only add device→host copies. Not validated. |
+| Diffusion stages | n/a | No vLLM KV cache to mirror. |
+
+Hit spans come from `scheduled_new_reqs` only, as in the pre-refactor cache:
+
+- A new request with a (partial) prefix hit is the normal path: the hit
+  blocks are read from the pool, the rest is this step's rows, and the
+  divergent tail is written under its own blocks. The gather starts on the
+  prefetch thread at `new_step_starts`; a same-step hit (vLLM hashes blocks
+  at schedule time, so `b` can hit blocks `a` computes in the same forward)
+  cannot plan until `a`'s write is registered and starts at `save_outputs`
+  instead. Either way it runs before the next step can hand those blocks
+  to a new tenant. vLLM frees blocks one step before `finished_req_ids`
+  arrives, so a finished request whose last step materializes late may
+  still find a hit slot reassigned. Each `(slot, key)` carries a write
+  version, bumped whenever a new write claims it. A planned read captures
+  that version and is registered in `_pending_reads`; a later write that
+  reclaims those slots copy-on-writes the still-`COMMITTED` rows into the
+  ref before it overwrites the pool, so a delayed fetch serves the original
+  tenant. A version mismatch with no preserved copy raises for live and
+  finished alike (the pool rows are a newer tenant's). Production defaults
+  stay opt-in until preempt/resume hit spans are reconstructed.
+- `async_chunk` continuation: when the next upstream chunk arrives, the same
+  request id re-enters `scheduled_new_reqs` with `num_computed_tokens` equal
+  to what it already computed itself. Ids already in `live_reqs` are skipped
+  for hit marking: those rows were delivered in earlier steps and re-emitting
+  them would duplicate output. A `delivered_upto` span for this case is
+  Phase 2.
+- Preemption + reschedule: vLLM resets `num_computed_tokens` to 0 on
+  preemption and re-runs prefix matching on resume, so the resumed request
+  can come back with a fresh hit. With the V1 model runner it arrives
+  through `scheduled_cached_reqs` (id in `resumed_req_ids`, `new_block_ids`
+  replaces the table); with the V2 runner it re-enters `scheduled_new_reqs`
+  while still in `live_reqs`. Neither path marks an omni hit span: the
+  resumed request gets only the rows it recomputes, and its still-open
+  deferred write keeps appending (a slot written twice keeps the later
+  chunk). Cache integrity holds either way — the hit blocks already have
+  rows, from this request or the one it hit. Stages that need full prompt
+  hidden states should be sized so preemption does not occur while prefix
+  caching is on. Same as before this refactor; tracked for Phase 2.
+
+Two write paths, split by `ModelCachePolicy.deferred_keys`:
+
+- Immediate (`JOIN_NEXT_STEP`): save launches a whole-step device→host into
+  a staging slot; the committer waits that event and writes the CPU pool.
+  The next save waits `done` (the pool write): a reused slot must never
+  leave a pending pool write behind, or a delayed hit read of the old rows
+  would find nothing recoverable.
+- Deferred (`JOIN_ON_FINISH`): mm whose first dim is this step's token count
+  stays on a per-request GPU clone; `_WriteChunk`s append across steps;
+  finish/abort (or GPU-byte-budget pressure) forces the copy. One open
+  WriteTask per request; a budget flush may close it mid-request, in which
+  case the next save opens a new one (`write_n` + 1) and a hit reads both,
+  so one long request cannot pin the whole budget.
+
+A `WriteTask` moves through `TaskState` only via `transition()`, one step at
+a time along a strict chain:
+`PENDING` (registered, not queued) → `QUEUED` (on a copy queue) →
+`COPYING` (one thread owns the copy stage) → `HOST_READY` (host rows ready) →
+`WRITTEN` (in the CPU pool); `FAILED` is reachable from any non-terminal
+state. Skipping a step raises. The worker claims `COPYING` in the same `_wake`
+critical section as the queue pop, so `escalate` never re-queues a task it can
+see is already claimed. `host_ready` / `done` are wait primitives set by the
+transitions into `HOST_READY` / `WRITTEN` / `FAILED`. GPU-clone bytes are
+charged once per clone on a `_BudgetTicket` pinned by every task that views
+it, and uncharged when the last holder releases (idempotent per tid).
+
+Mm whose first dim equals the unpadded scheduled length *or* the CUDA-graph
+padded length is registered on first sighting. Talker `codes.audio` is a
+cat of scheduled rows and stays unpadded while hidden is padded; both must
+open a pool key, whether immediate or deferred.
+Leftover mm (lists, `codes.ref`, any tensor whose first dim is not this
+step's token count) is copied to CPU at save without truncating that first
+dim. That leftover copy is the async-builder read replica for this step; it
+does not write the pool or carry abort/preempt occupancy.
+
+```python
+cache.register_policy(ModelCachePolicy.from_model(model))   # load_model
+cache.new_step_starts(scheduler_output)   # before _update_states
+sid = cache.save_outputs(hidden, mm_outputs, num_tokens_unpadded=n,
+                         num_tokens_padded=n_pad)
+outs = cache.materialize(sid, req_ids)    # or discard_step(sid)
+```
+
+Each step id is consumed exactly once. `req_ids` must be a subset of the save
+snapshot. At most `staging_depth` unused step ids may exist at once: every
+`save_outputs` claims one staging slot, including saves with only leftover
+mm that copy no device→host page. A slot is also held by each immediate
+write that views it, until the committer's pool write. A later save waits
+for `materialize`/`discard_step` or that pool write to free a slot;
+`staging_claim_timeout_s` then errors with the unused ids and the task count.
+`join`/`join_host_ready` use the same bound and raise with the stuck task's
+id and state instead of hanging the caller.
+`materialize` may run on the async output builder after the engine has
+entered the next step; leftover mm (not written to the pool) is copied
+to CPU at `save_outputs` so the builder never reads live CUDA-graph
+buffers. See
+[Async Omni Output Materialization](omni_async_output_materialization.md).
+
+The cache is constructed only on the last pipeline-parallel rank
+(`_ensure_omni_prefix_cache`). Other ranks skip it: they never call
+`save_outputs`, so a hit table there would resolve to absent slots.
+
+Threads, locks, and what each may block on:
+
+| Thread | Role | May block on | Must not hold while blocked |
+| --- | --- | --- | --- |
+| Engine | `new_step_starts`, `save_outputs` | previous-step `join` (`done`) / finished-write `join_host_ready`; `reserve()` GPU-byte flush; staging-slot claim; `dispatch()` / finish-abort `escalate()` (eager mode: the copy + pool write run inline) | `_state_lock` |
+| Async output builder | `materialize` (may overlap the next engine step) | this step's `step_d2h_event`; `join` (`done`); deferred `fetch_host` | `_state_lock` |
+| Committer | `_worker_loop`: wait device→host / deferred copy / pool write | `_wake.wait`; `step_d2h_event` or copy-stream sync | never takes `_state_lock` |
+| Prefetch pool | hit-span gather during forward | `join` (`done`); deferred `fetch_host` | `_state_lock` |
+
+| Lock | Covers | Does not cover |
+| --- | --- | --- |
+| manager `_state_lock` | occupancy tables, step contexts, hit spans, task registration, pool-key publish (`install_key`) | join, GPU-byte flush, copy, `step_d2h_event` wait, pool-key allocation, eager `dispatch()` / `escalate()` |
+| controller `_lock` / `_wake` | task registry, queues, GPU-clone byte budget | device→host / pool-write body (released before `synchronize`) |
+| `WriteTask.lock` | `state`, `reassigned`, `append_chunk` | waiting on `host_ready` / `done` (those are events) |
+
+### Related Files
+
+- `vllm_omni/core/prefix_cache/`
+- `vllm_omni/worker/gpu_model_runner.py` (`_ensure_omni_prefix_cache`)
+- `tests/core/test_prefix_cache.py`
+- [Async Omni Output Materialization](omni_async_output_materialization.md)

docs/design/fullduplex-personaplex.md

@@ -271,3 +271,111 @@ The required evidence is:
 
 Audio that is empty, all zero, non-finite, or only a protocol `listen` event is
 not a successful end-to-end result.
+
+### Paced multi-session load driver
+
+The existing driver keeps its two-session admission and slot-reuse validation
+when `--sessions` is omitted. Passing `--sessions N` selects a separate,
+bounded load mode; it does not replace the lifecycle test or change server
+admission capacity. Start an existing PersonaPlex server configured to admit
+the requested number of sessions, then run from a source checkout with the
+vLLM-Omni client dependencies installed:
+
+```bash
+python -m tests.e2e.online_serving.personaplex_realtime_duplex \
+  --model nvidia/personaplex-7b-v1 \
+  --input-wav /path/to/input.wav \
+  --sessions 2 --load-frames 1000 --drain-s 2 \
+  --server-revision SERVER_COMMIT --server-hardware A100-80GB \
+  --output-dir /tmp/personaplex-load-n2
+```
+
+Use a new output directory for every run. The driver refuses to overwrite an
+existing `load-result.json`. The server labels are supplied by the operator,
+not detected or verified by the client. Record the server command, package
+versions, checkpoint revision and hardware alongside a published measurement.
+
+All attempted sessions finish admission before admitted sessions share a
+common input start. A rejected session remains a failed row in the report;
+it is not removed from the denominator. Each session receives at most
+`--load-frames` input frames, without looping the WAV, and uses 80 ms pacing.
+A stalled send cannot cause a catch-up burst. The timeline records send
+schedule lateness so an overloaded client is visible rather than mistaken
+for a faster server. This mode measures steady concurrency, not join/leave
+interference; the original lifecycle mode still checks slot reuse.
+Load mode does not probe an additional N+1 session for `resource_exhausted`;
+the lifecycle mode owns that admission-limit check. A per-session
+`client_pacing_warning` and the summary's `client_pacing_warning_sessions`
+identify send-lateness p99 above one nominal frame (80 ms). This is a warning,
+not an extra acceptance gate: read it alongside server-facing measurements
+so a slow client is not mistaken for a slow server.
+
+The fixed drain window is part of the measurement configuration. Audio
+arriving during cleanup is excluded, so closing cannot rescue a session that
+missed its observation deadline. Cleanup is attempted for admitted sessions
+even when audio validation fails. Protocol errors, missing/invalid audio,
+shared response IDs and cleanup failures make the report fail and the CLI
+exit nonzero. Cancellation closes transports and propagates to the caller.
+`--cleanup-timeout-s` (default 5 s) bounds session-close send and acknowledgement
+together, independently of `--timeout-s`. The subsequent WebSocket close
+handshake has the same separate bound; an unresponsive close acknowledgement
+cannot hold cancellation for the full inference timeout.
+
+`load-result.json` retains per-session send and audio-packet timelines using
+the client's monotonic clock. It does not export handshake credentials or
+raw audio payloads. Metric definitions are explicit:
+
+Both modes validate each nonempty audio packet's strict base64 encoding, PCM16
+sample alignment, 24 kHz sample rate and response identity. Valid aggregate
+audio cannot hide a malformed packet. In load mode, rejected packets make the session fail
+and appear in `invalid_audio_packets` with their event index, receive time and
+fixed diagnostic code; they do not contribute samples or timing intervals.
+Empty audio deltas remain legal and contribute no samples. This check does
+not change the public client's decoding. The lifecycle scenario and dispatch
+remain the same, but malformed packets that aggregate checks once tolerated
+now fail that mode as well.
+
+The probe accepts both the current OpenAI Realtime audio event name
+`response.output_audio.delta` and the legacy `response.audio.delta`. Validation
+uses the wire payload directly rather than relying on a particular client
+library's alias table, so a server-side event-name migration cannot silently
+turn real audio into a zero-output measurement.
+
+| Field | Meaning |
+| --- | --- |
+| `client_audio_packet_interval_ms` | Consecutive nonempty packet arrivals: median, linear p99, max, and zero-based index of the first maximal interval. Interval index 0 ends at packet 1. One packet can contain multiple codec frames; this is not server tick latency. |
+| `client_first_audio_after_stream_start_ms` | First audio receipt minus first input send start; admission is excluded. |
+| `client_stream_rtf` | First input send start to last audio receipt, divided by received audio duration. This includes real-time input pacing and is not an inference-only RTF. |
+| `client_send_lateness_ms` | Send start minus its original shared schedule, clipped at zero. |
+| `frame_deficit` | Integer input-frame minus output-frame count from the shared session validation; missing when valid whole-frame accounting is unavailable. |
+| `output_frames`, `voiced_frames`, `silent_frames`, `frame_coverage_ratio`, `audio_rms`, `audio_chunks` | Session-validation statistics retained on success and on acceptance failure once computed. |
+| `acceptance_check` | Failed local check, with numeric diagnostics in `error`; other exceptions retain phase/type only, without remote handshake or metadata text. |
+| `audio_underrun_s`, `audio_underrun_event_count`, `audio_continuity_ok` | Shared `compute_continuity_stats` on valid PCM16 packet arrivals, with its default 100 ms underrun threshold; absent audio gives missing values rather than a continuity pass. |
+
+No output gives missing latency/RTF values, not zero. By default RTF is
+diagnostic; `--max-client-rtf` adds an explicit ceiling. The existing frame
+coverage, audible-frame and scheduler-data-plane checks still apply. A longer
+`--drain-s` can demonstrate eventual completion under backlog, but it must not
+be reported as real-time capacity when the fixed-window run failed or client
+RTF exceeded the stated target. Likewise, the first live session can pay
+one-time model/codec/JIT costs; either use the server's configured duplex
+warmup path or label that run as cold-start evidence before comparing steady
+capacity. Passing client-only metrics does not certify model quality or a
+real-time capacity target on another GPU. Begin with N=1 and N=2; higher N is a
+capacity experiment, not a promised property of an A100 or of this patch.
+Continuity is a client-arrival diagnostic, not proof of speaker playback and
+not an additional pass/fail gate. Packet-interval and send-lateness summaries
+retain their documented linear p99; the older client interval helper reports
+rounded nearest-rank p95 instead and is not substituted for that schema.
+
+Deterministic driver and localhost WebSocket checks need no model or GPU:
+
+```bash
+python -m pytest tests/e2e/online_serving/test_personaplex_load_driver.py \
+  -m 'core_model and cpu' --run-level core_model -q
+```
+
+These checks test the driver against controlled protocol responses. Run the
+load-driver command against real weights separately before reporting model-serving
+performance. Server tick timestamps and generic TTFT/TPOT aggregation remain
+the responsibility of the existing duplex metrics paths, not this driver.

docs/design/fullduplex.md

@@ -9,6 +9,12 @@ the owner document for `vllm_omni/engine/duplex/**`,
 `vllm_omni/entrypoints/duplex_omni.py`, `vllm_omni/entrypoints/duplex/**`,
 `vllm_omni/clients/**` and the per-model `duplex/` packages.
 
+`vllm_omni/protocol/realtime/**` (the OpenAI Realtime API) and
+`vllm_omni/protocol/duplex/**` (vLLM-Omni's extension of it) are the shared
+wire codec duplex is built on. Duplex is its first consumer but not its owner: the codec
+is deliberately runtime-agnostic so a second Realtime surface can reuse it
+(RFC #6592 P0a).
+
 The public wire contract lives in
 [`docs/serving/realtime_duplex_api.md`](../serving/realtime_duplex_api.md);
 this page describes how it is produced.
@@ -124,6 +130,40 @@ plugin and the session runtime config to `DuplexOrchestrator` directly.
    attachment/resume/replay bookkeeping; it holds no session state.
 6. **No `typing.Protocol`** in the duplex surfaces: the plugin, data plane,
    session state, PCM buffer, stage port and client transport seams are ABCs.
+7. **The wire codec is not duplex, and a wire type is not a mailbox message.**
+   The event and command *vocabulary* plus its wire rendering, the audio format
+   negotiation, the conversation-item rules and the error envelope are model-
+   and runtime-agnostic, and split along the three tiers
+   [`docs/serving/realtime_duplex_api.md`](../serving/realtime_duplex_api.md)
+   already defines: **Tier 1** (identical to OpenAI) is
+   `vllm_omni/protocol/realtime/**`; **Tier 2** (OpenAI names carrying our
+   extensions) and **Tier 3** (ours alone) are `vllm_omni/protocol/duplex/**`.
+   A Tier 2 class *subclasses* its Tier 1 twin and declares only what it adds,
+   so the OpenAI surface stays honestly pure --- `session.created` without
+   `resume_token`, `input_audio_buffer.append` without `video_frames`. Our
+   error **codes** are Tier 3 (`protocol/duplex/errors.py`); only the envelope
+   shape and OpenAI's three `error.type` classes are Tier 1.
+   What stays engine-side is everything that is *internal representation*
+   rather than contract: `DuplexCommand.payload()` renders the session runner's
+   mailbox dictionary, whose channel genuinely differs from the client event
+   (`session.update` and the three `conversation.item.*` commands all travel on
+   `turn.signal`); `engine/duplex/realtime_commands.py` decides which command a
+   decoded event becomes; `engine/duplex/realtime_events.py` holds the
+   session's projection state. The codec may not import the engine, the
+   entrypoints, `model_executor` or the clients
+   (`tests/protocol/realtime/test_protocol_import_boundary.py`), and there is
+   exactly one implementation of each codec behaviour
+   (`tests/protocol/realtime/test_realtime_codec_single_source.py`).
+   The dependency chain is `protocol/realtime` <- `protocol/duplex` <- engine /
+   entrypoints / clients, and a duplex consumer uses **only** the middle link:
+   `protocol/duplex/**` re-exports the Tier 1 names it does not extend, so a
+   helper that later needs a duplex-specific version (`convert_input_audio_with_rate`
+   resamples to MiniCPM-o's 16 kHz rather than the client's rate) is overridden
+   in one file instead of at every call site
+   (`tests/protocol/duplex/test_duplex_protocol_facade.py`).
+   `DuplexEvent` is therefore a plain alias of `RealtimeEvent` --- an event has
+   no engine-internal half --- while `DuplexCommand` is a real class, because a
+   command does.
 
 ## Package layout
 
@@ -142,6 +182,31 @@ vllm_omni/
 │   │   ├── chat_completions.py      DuplexChatCompletionsAdapter (/v1/chat/completions on a session per request)
 │   │   └── websocket.py             websocket send/close/receive helpers
 │   └── openai/api_server.py         builds DuplexOmni for duplex models; session-backed app state
+├── protocol/                        SHARED WIRE CODEC (no engine / no model / no transport)
+│   ├── realtime/                    the OpenAI Realtime API itself
+│   │   ├── events.py                RealtimeEvent + OpenAI's 30 server events (22 ship as Tier 1;
+│   │   │                            8 have a Tier 2 twin in protocol/duplex)
+│   │   ├── commands.py              RealtimeCommand + OpenAI's 10 client commands (8 Tier 1,
+│   │   │                            2 with a Tier 2 twin) --- wire_type + fields, no payload()
+│   │   ├── formats.py               audio format spellings, what is supported, per-session validation
+│   │   ├── session.py               session-object readers; RealtimeInputDefaults (append defaults)
+│   │   ├── items.py                 conversation item shape, truncation, transcripts, camera frames
+│   │   ├── audio_input.py           decode_audio_append -> RealtimeAudioAppend; client speech hints
+│   │   ├── audio.py                 PCM / G.711 / WAV decode, resample, re-encode
+│   │   ├── errors.py                RealtimeProtocolError (the envelope's exception type)
+│   │   └── capabilities.py          RealtimeProtocolCapabilities: what one consumer can serve
+│   └── duplex/                      Tier 2 + Tier 3, and the one door a duplex consumer uses
+│       ├── __init__.py              re-exports the Tier 1 helper functions (audio, formats,
+│       │                            items, session, capabilities) so the engine never
+│       │                            imports protocol/realtime directly
+│       ├── events.py                the whole 42-event vocabulary: 22 Tier-1 re-exports,
+│       │                            8 Tier-2 subclasses (session.created + resume_token, ...),
+│       │                            12 Tier-3 events (listen/speak, playback, resume, ...);
+│       │                            DuplexEvent = RealtimeEvent
+│       ├── commands.py              the whole 17-command vocabulary: 8 Tier-1 re-exports,
+│       │                            2 Tier-2 subclasses (append + hints, commit + final),
+│       │                            7 Tier-3 commands (barge_in, playback.ack, ...)
+│       └── errors.py                our error-code vocabulary (Tier 3) + RealtimeProtocolError
 ├── engine/
 │   ├── omni_engine_base.py          OmniEngineBase (stage processes, orchestrator thread, queues, RPC)
 │   ├── async_omni_engine.py         AsyncOmniEngine (turn-based request building)
@@ -149,16 +214,20 @@ vllm_omni/
 │   ├── orchestrator.py              OrchestratorBase + Orchestrator
 │   ├── duplex_orchestrator.py       DuplexOrchestrator (+ DuplexOrchestratorRequestState; implements DuplexStagePort)
 │   └── duplex/
-│       ├── commands.py              DuplexCommand dataclasses, command_from_realtime
-│       ├── realtime_commands.py     Realtime client event -> DuplexCommand translation
-│       ├── events.py                DuplexEvent dataclasses (+ to_realtime), REALTIME_ERROR_TYPES_BY_CODE
+│       ├── commands.py              mailbox half: DuplexCommand.payload() + the mailbox `type`,
+│       │                            paired with each protocol command; command_from_realtime
+│       ├── realtime_commands.py     duplex binding of the codec: decoded event -> DuplexCommand,
+│       │                            DUPLEX_REALTIME_CAPABILITIES, duplex_response_format
+│       ├── events.py                re-export shim (DuplexEvent = RealtimeEvent) + the runner's
+│       │                            epoch-filter sets DOMAIN_TERMINAL_EVENTS / MODEL_OUTPUT_EVENTS
 │       ├── realtime_events.py       RealtimeProjectionState: internal event -> typed events
 │       ├── messages.py              queue envelopes (Open/Close/Resume/Touch/Command/Result/Event), DuplexSessionError
 │       ├── config.py                DuplexSessionConfig, DuplexCapabilities, ResponseCreateOptions
 │       ├── contracts.py             DuplexFence (session_id, epoch, turn_id), stage request records, DuplexStagePort
 │       ├── plugin.py                DuplexModelPlugin, DuplexModelSessionState, DuplexDataPlane, PcmAppendBuffer ABCs
 │       ├── turn_detection.py        server-side VAD turn detector used by the session
-│       ├── audio.py / vad.py / intermediate.py
+│       ├── audio.py                 compatibility re-export, via protocol/duplex
+│       ├── vad.py / intermediate.py
 │       └── session/                 one engine-resident session and everything that runs it
 │           ├── engine_session.py    DuplexEngineSession: ledgers, lease, fence, stage resources, append sequencing
 │           ├── runner.py            DuplexSessionRunner (per-session mailbox on the orchestrator loop)

docs/design/module/engine_orchestration.md

@@ -79,7 +79,7 @@ the boundary affected by the in-flight stage client/process refactor in
 responsibilities proposed only by that PR are not current contracts.
 
 The orchestration loop also has an opt-in event-driven mode
-(`VLLM_OMNI_EVENT_DRIVEN_ORCH=1`, default off) proposed in
+(`VLLM_OMNI_EVENT_DRIVEN_ORCH=1`, default off except for Qwen3-TTS) proposed in
 [#5221](https://github.com/vllm-project/vllm-omni/pull/5221). It changes poll
 cadence only: the routing, ordering, and terminal-state contracts below hold
 identically on both loops.

docs/design/module/entrypoints.md

@@ -30,6 +30,7 @@ primary_code_paths:
   - vllm_omni/entrypoints/utils.py
   - vllm_omni/entrypoints/duplex/**
   - vllm_omni/clients/**
+  - vllm_omni/protocol/**
 primary_path_exceptions:
   - path: vllm_omni/entrypoints/openai/errors.py
     owner: error_contracts.md
@@ -53,6 +54,7 @@ depends_on:
   - error_contracts.md
   - engine_orchestration.md
 validation_paths:
+  - tests/protocol/**
   - tests/entrypoints/test_omni_entrypoints.py
   - tests/entrypoints/test_async_omni.py
   - tests/entrypoints/test_async_omni_pause_sleep_routing.py
@@ -106,6 +108,31 @@ placement, payload implementation, or semantic error classification.
 `entrypoints/openai/errors.py` is an explicit primary-path exception owned by
 `error_contracts.md`.
 
+It also owns `vllm_omni/protocol/**`, the shared wire codec promoted by RFC
+[#6592](https://github.com/vllm-project/vllm-omni/issues/6592) P0a. It follows the three tiers
+`docs/serving/realtime_duplex_api.md` already defines. `protocol/realtime/**`
+is Tier 1, identical to OpenAI: 30 server events, 10 client commands, their
+typed fields and pure wire rendering, audio-format negotiation,
+conversation-item rules and the error-envelope shape. `protocol/duplex/**` is
+Tier 2 (OpenAI names carrying vLLM-Omni extensions, each a subclass of its
+Tier 1 twin declaring only what it adds) and Tier 3 (ours alone, including our
+error-code vocabulary), kept separate so a GA-only consumer is not handed a
+vocabulary its clients never send. That package sits
+outside `entrypoints/` on purpose --- the engine depends on it, so it belongs
+to neither layer --- but deciding what a Realtime client may put on the wire is
+this document's subject, and the same reasoning already places
+`vllm_omni/errors.py`, `inputs/` and `outputs/` under contract documents rather
+than under a consumer.
+
+It is deliberately **not** owned by `../fullduplex.md`, even though duplex is
+the codec's only consumer today. RFC #6592 asks for a named owner precisely so
+a second Realtime consumer does not need duplex review to change the codec
+(#6592 open question 8). Ownership here is review scope, not a dependency: the
+codec may not import the engine, the entrypoints or the model code, which
+`tests/protocol/realtime/test_protocol_import_boundary.py` enforces. When a
+second consumer lands (#6592 P0b) the package is expected to graduate to its
+own module document with owners drawn from both consumers.
+
 ## Candidate invariants
 
 These identifiers are proposals while the document is `draft`.
@@ -115,6 +142,22 @@ These identifiers are proposals while the document is `draft`.
 **Rule:** Entrypoints MUST NOT implement cross-stage routing or stage lifecycle
 policy.
 
+### ENTRY-INV-002: The shared Realtime codec does not depend on its consumers
+
+**Rule:** `vllm_omni/protocol/**` MUST NOT import `vllm_omni.engine`,
+`vllm_omni.entrypoints`, `vllm_omni.model_executor`, `vllm_omni.worker` or
+`vllm_omni.clients`; `protocol/duplex/**` MAY depend on `protocol/realtime/**`
+but never the reverse, and a Tier 1 class MUST NOT carry a vLLM-Omni extension
+field; and each protocol/codec behavior MUST have exactly one
+implementation that every consumer uses. A wire type MUST NOT carry a
+consumer's internal representation: the duplex mailbox rendering
+(`DuplexCommand.payload()`, whose channel differs from the client event for
+`session.update` and the `conversation.item.*` commands) stays engine-side.
+A duplex consumer (`engine/duplex/**`, `entrypoints/duplex/**`, the duplex
+clients) MUST import `vllm_omni.protocol.duplex` and MUST NOT import
+`vllm_omni.protocol.realtime` directly, so the tier boundary has exactly one
+extension point.
+
 ### ENTRY-INV-100: Public requests are normalized once
 
 **Rule:** Public protocol values MUST be validated and converted to internal

docs/features/custom_pipeline.md

@@ -1,7 +1,7 @@
 # Custom Pipeline Extension Guide
 
 Transformer already support Custom Pipeline via
-https://github.com/huggingface/diffusers/blob/main/docs/source/en/using-diffusers/custom_pipeline_overview.md
+<https://github.com/huggingface/diffusers/blob/main/docs/source/en/using-diffusers/custom_pipeline_overview.md>
 
 This guide demonstrates how to use the newly added features for extending vLLM-Omni's diffusion pipeline with custom functionality.
 
@@ -20,8 +20,9 @@ Three main features enable custom pipeline extension:
 `WorkerWrapperBase` is a wrapper class that creates `DiffusionWorker` instances with optional extension support. It enables dynamic inheritance, allowing you to add custom methods and functionality to workers without modifying the base worker class.
 
 **Key capabilities:**
+
 - Dynamic worker class extension via `worker_extension_cls`
-- Support for custom pipeline initialization via `custom_pipeline_args`
+- Support for custom pipeline initialization via `custom_pipeline_args["pipeline_class"]`
 - Method delegation to underlying worker
 - Attribute access forwarding
 
@@ -42,6 +43,7 @@ The `diffusion_load_format` parameter controls the initial diffusion model load.
 `CustomPipelineWorkerExtension` is a mixin class that extends `DiffusionWorker` with the ability to re-initialize the pipeline with a custom implementation.
 
 **Key method:**
+
 - `re_init_pipeline(custom_pipeline_args)`: Re-initializes the pipeline with custom arguments, properly cleaning up the old pipeline first
 
 **Location:** `vllm_omni/diffusion/worker/diffusion_worker.py`
@@ -150,6 +152,7 @@ omni = Omni(
     },
     worker_extension_cls=MyCustomExtension,
     # Note: worker_extension_cls is an internal parameter
-    # CustomPipelineWorkerExtension will automatically init pipeline when custom_pipeline_args is provided
+    # CustomPipelineWorkerExtension will automatically init pipeline when
+    # custom_pipeline_args includes "pipeline_class".
 )
 ```

docs/features/sleep_mode.md

@@ -9,9 +9,11 @@ This feature is inherited from [vLLM’s Sleep Mode](https://blog.vllm.ai/2025/1
 ## 1. Feature Documentation
 
 ### Overview
+
 Omni Sleep Mode provides a mechanism to "sleep" specific model stages. When a stage enters sleep, its physical VRAM is reclaimed by the system, while the process state is preserved for rapid "wake-up" without full re-initialization.
 
 ### Sleep Levels
+
 We support two levels of hibernation to balance recovery speed and memory efficiency:
 
 | Level | Name | Mechanism | Recovery Speed | Memory Freed |
@@ -29,15 +31,67 @@ Omni Sleep Mode is optimized for high-performance computing backends:
 * **Huawei NPU**: Supported via Ascend memory scavenging.
 
 ### Hardware Requirements
+
 * **Memory Considerations**: System RAM must be sufficient to hold offloaded weights during sleep.
 * **TP Support**: Tensor Parallel groups synchronize sleep/wake transitions across all workers.
 
----
+### Quiescing a busy diffusion stage: `pause_generation(mode="keep")`
 
+`sleep()` on a diffusion stage is a worker-level RPC: it offloads memory but does not stop the
+stage's scheduler, so a stage that still has queued requests must be made quiet first. Aborting
+discards the batch that is running. `pause_generation(mode="keep")` keeps it instead:
+
+* The batch that is already running finishes on the weights it started with and its outputs are
+  delivered normally.
+* Requests that have not started stay queued and run after `resume_generation()`, on whatever
+  weights are loaded by then. A request never continues denoising across a weight change.
+* The call returns only after every replica has acknowledged: the batch that was running has
+  finished executing, its asynchronous output copies are done, and the device is synchronized; its
+  outputs are then delivered on the normal path. On a non-preemptible backend that acknowledgement
+  arrives at the next batch boundary, so expect it to take as long as the remaining batch.
+* Control RPCs (`list_loras`, `sleep`, `wake_up`, `resume_generation`) keep working while paused.
+
+Recommended order for a weight update on a diffusion stage:
+
+```python
+await engine.pause_generation(mode="keep", clear_cache=False)
+await engine.sleep(level=1)          # optional: free memory for the trainer
+await engine.wake_up()               # admission stays closed until resume
+# install new weights here
+await engine.resume_generation()
+```
+
+Support matrix and limits:
+
+| Configuration | `mode="keep"` |
+| :--- | :--- |
+| Request-level execution (`step_execution=False`) with the in-process (`uni`) executor | Supported |
+| Request-level execution with the multi-process (`mp`) executor, including asynchronous output | Supported; the acknowledgement waits for the output copies |
+| Request-level execution with asynchronous KV prefetch | Supported; the acknowledgement joins any prefetch still in flight |
+| Step-level execution (`step_execution=True` / streaming output) | Not supported: the call raises `NotImplementedError` |
+| `mode="abort"` / `mode="wait"` / `wait_for_inflight_requests=True` on a diffusion stage | Unchanged: frontend admission is paused only |
+
+Behavior notes:
+
+* Callers that already passed `mode="keep"` for a diffusion stage used to get an immediate return
+  that only closed frontend admission. They now also stop the backend and wait for the ACK.
+* A timeout or error from `pause_generation` means the pause did **not** complete; the scheduler
+  stays closed and the caller must retry the pause or call `resume_generation()` explicitly. Do not
+  sleep or mutate weights after a failed pause.
+* `resume_generation(stage_ids=...)` that leaves another paused stage closed (an AR stage, or a
+  keep-paused diffusion stage) keeps frontend admission closed as well, so new requests cannot
+  queue on a stage that will not schedule them. Admission reopens once every stage paused by
+  `pause_generation` has been resumed.
+* `clear_cache` has no diffusion-specific effect.
+* Without a pause call nothing changes: no barrier runs and batching, abort and output ordering are
+  as before.
+
+---
 
 ## 2. Usage Examples
 
 ### Python API Example
+
 You can programmatically control the lifecycle of stages using the `AsyncOmni` engine.
 
 ```python
@@ -65,6 +119,7 @@ if __name__ == "__main__":
 ```
 
 ### server command Example
+
 Start the server with sleep mode enabled:
 
 The first method
@@ -92,9 +147,6 @@ python3 -m vllm_omni.entrypoints.openai.api_server \
 
 ```
 
-
-
-
 ### Test Scenarios & Commands
 
 #### Scenario 1: LLM Engine Sleep
@@ -113,9 +165,8 @@ curl -X POST http://localhost:8000/v1/omni/sleep \
 
 Tip: Open a new terminal and run rocm-smi or nvidia-smi or to observe the immediate drop in VRAM usage.
 
-
-
 #### Scenario 2: Diffusion Sleep
+
 Objective: Verify VRAM reclamation for Stage 1 (Diffusion).
 
 Trigger sleep (Level 1 or Level 2) via client:
@@ -128,9 +179,8 @@ curl -X POST http://localhost:8000/v1/omni/sleep \
 
 ```
 
-
-
 #### Scenario 3: Multi-Stage Coordinated Stress Test
+
 Objective: Test concurrent sleep and rapid wake-up across multiple stages.
 
 Concurrent Sleep (Stage 0 & 1):
@@ -143,7 +193,6 @@ curl -X POST http://localhost:8000/v1/omni/sleep \
 
 ```
 
-
 Rapid Wake-up:
 
 ```
@@ -154,8 +203,8 @@ curl -X POST http://localhost:8000/v1/omni/wakeup \
 
 ```
 
-
 #### Scenario 4: Full Lifecycle Memory Audit & Functional Integrity
+
 Objective: Audit the complete flow from Sleep to Wake-up followed by an Inference validation.
 
 Check Initial State: Observe baseline VRAM usage.
@@ -196,20 +245,16 @@ curl -X POST http://localhost:8000/v1/images/generations \
 
 ```
 
-
-
-
 ## 3. API Reference
 
-
 ### Methods
 
 | Method | Arguments | Return Type | Description |
 | :--- | :--- | :--- | :--- |
 | **sleep** | `stage_ids: List[int], level: int` | `List[OmniACK]` | Triggers hibernation for specified stages. |
 | **wake_up** | `stage_ids: List[int]` | `List[OmniACK]` | Reloads weights and re-maps memory. |
-
-
+| **pause_generation** | `mode: str, stage_ids: List[int]` | `None` | Stops admission; with `mode="keep"` also stops diffusion schedulers and returns after their ACK. |
+| **resume_generation** | `stage_ids: List[int]` | `None` | Reopens paused schedulers, then admission. |
 
 ### OmniACK Dataclass Fields
 
@@ -233,7 +278,8 @@ The metadata field is a dynamic dictionary containing hardware-specific telemetr
 }
 ```
 
-#### Core Utility:
+#### Core Utility
+
 **VRAM Reclamation Audit (total_freed_gib)**: Converts raw freed_bytes into human-readable GiB. It serves as the primary metric to verify that Level 2 sleep has successfully purged model weights from VRAM.
 
 **Residual & Fragmentation Monitoring (rank_residual_gib)**: Reports the remaining VRAM footprint after memory de-mapping. A low residual value (e.g., 2.07 GiB) confirms a successful "clean" state, ensuring the device is ready for high-memory co-located tasks like training or diffusion pipelines.

docs/models/supported_models.md

@@ -32,6 +32,7 @@ th {
 | `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2509 | `Qwen/Qwen-Image-Edit-2509` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `QwenImageLayeredPipeline` | Qwen-Image-Layered | `Qwen/Qwen-Image-Layered` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2511 | `Qwen/Qwen-Image-Edit-2511` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `QwenImage21Pipeline` | Qwen-Image-2.1 (T2I + image-conditioned generation, up to 4 condition images) | `Qwen/Qwen-Image-2.1` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen-Image-2.1.md) |
 | `GlmImagePipeline` | GLM-Image | `zai-org/GLM-Image` | ✅︎ | ✅︎ | | | — |
 | `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Krea2Pipeline` | Krea 2 (Raw + Turbo) | `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | ✅︎ | | | ✅︎ | — |
@@ -58,6 +59,7 @@ th {
 | `Magi2Pipeline` | MAGI-2 native Preview-stage T2VA and I2VA (272p/540p) | `sand-ai/MAGI-2-preview` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SandAI/MAGI-2-preview-L20X.md) |
 | `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Pi0Pipeline` | π0 (Pi-Zero) VLA | `lerobot/pi0_base` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/lerobot/Pi0.md) |
+| `Pi05Pipeline` | π0.5 VLA | `lerobot/pi05_base` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/lerobot/Pi05.md) |
 | `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | | — |
 | `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
@@ -72,7 +74,6 @@ th {
 | `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | ✅︎ | — |
 | `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | ✅︎ | — |
 | `BreezeForConditionalGeneration` | Breeze-TTS-2 (TTS, voice clone, voice direction) | `BreezeBlue/Breeze-TTS-2` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/BreezeBlue/Breeze-TTS-2.md) |
-| `DotsTTSForConditionalGeneration` | dots.tts | `dots-studio/dots.tts-soar` | ✅︎ | | | | — |
 | `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | | — |
 | `MammothModa2ForConditionalGeneration` | MammothModa2-Dev (AR-only image understanding) | `bytedance-research/MammothModa2-Dev` | ✅︎ | | | | — |
 | `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
@@ -106,12 +107,12 @@ th {
 | `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | | — |
 | `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | | — |
 | `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | | — |
-| `DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | ✅︎ | — |
 | `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/OpenBMB/MiniCPM-o-4_5.md) |
 | `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `GepardTalkerForConditionalGeneration` | Gepard-1.0 | `nineninesix/gepard-1.0` | ✅︎ | | | ✅︎ | — |
 | `HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | | — |
 | `HiDreamO1ImagePipeline` | HiDream-O1-Image | `HiDream-ai/HiDream-O1-Image` | ✅︎ | | | | — |
+| `AnimaPipeline` | Anima | `circlestone-labs/Anima` | ✅︎ | ✅︎ | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/circlestone-labs/Anima.md) |
 
 The `Recipe` column links the corresponding deployment recipe. `Published` is
 preferred when a corresponding `recipes.vllm.ai` page exists; otherwise the

docs/serving/realtime_duplex_api.md

@@ -8,12 +8,41 @@ history stays honest, and a dropped connection can resume the same session.
 This page covers how to run a duplex deployment, how to drive it from Python
 with `vllm_omni.clients.duplex.DuplexClient`, and the complete wire contract.
 
-The endpoint is served for models that ship a duplex plugin — currently
-MiniCPM-o 4.5 only — and just for deploy configurations that declare
+The endpoint is served for models that ship a duplex plugin — including
+MiniCPM-o 4.5 and Qwen3-Omni — and just for deploy configurations that declare
 `session_mode: duplex`. PersonaPlex and Nemotron VoiceChat are ported to the
 plugin contract in follow-up PRs and are not served over this endpoint yet. The runtime architecture
 is described in [Full-Duplex Runtime (MiniCPM-o 4.5)](../design/fullduplex.md).
 
+## Qwen3-Omni conversation history
+
+Qwen3-Omni keeps incoming conversation items independently addressable by item
+ID. Before applying the model's chat template, it merges consecutive user
+items into one multimodal user message. An assistant message, including an
+empty message reserved for an interrupted or unheard answer, separates turns.
+The final user message places media before text instructions, following the
+[official Qwen3-Omni demo's `format_history()`](https://github.com/QwenLM/Qwen3-Omni/blob/e4235853125589c789f06a2dd83e9f4126df5e9d/web_demo.py#L84-L179).
+
+A camera image or text item may arrive after an audio commit and before
+`response.create`. Those items belong to the same pending user turn: the
+committed audio remains associated with its history item even when that item
+is no longer last. Prompt preparation snapshots the selected inputs; later
+items do not modify an already prepared request.
+
+The duplex adapter retains its existing budgets: at most four audio payloads
+(including the current input), an 8 MiB encoded-audio history budget (keeping
+at least the newest payload), and at most eight prompt images. The recent
+16-message window is rounded outwards to a complete turn. If any audio in an
+older turn is no longer retained, the entire turn's model context, including
+its images/text and assistant replies, is omitted. Images from that turn are
+therefore **not** permanent session-wide model context. These limits differ
+from the demo's one image-bearing turn and five audio-bearing turns; the
+shared rule is grouping before pruning, rather than copying its UI budgets.
+
+Grouping and pruning affect only the model-input view. They do not merge or
+delete the source conversation items exposed to clients, and do not change
+playback acknowledgement or interruption handling.
+
 ## Quick Start
 
 ### Start the Server
@@ -768,7 +797,10 @@ events with `server_event_seq > 40` are replayed in order)
 ```
 
 with camera frames (omni video; rides the append that closes a 1 s model
-unit; each entry is a **bare** base64 JPEG/PNG — no `data:` URL prefix)
+unit; each entry is a **bare** base64 JPEG/PNG — no `data:` URL prefix).
+Only a model whose Stage 0 interleaves a frame track at unit boundaries takes
+this field; a turn model has no such boundary and rejects it, expecting
+`conversation.item.create` with `input_image` content instead.
 
 ```json
 {
@@ -945,6 +977,10 @@ deferred commit during an active response (`event.response_create_deferred`)
 {"type": "conversation.item.truncated", "item_id": "item_resp_01", "content_index": 0, "audio_end_ms": 1850, "event": {"…": "…"}}
 ```
 
+After this acknowledgement, `conversation.item.retrieve` returns the truncated
+transcript. Without text/audio alignment marks, its prefix is estimated from
+the requested position and audio duration; this is not exact word alignment.
+
 #### Response lifecycle
 
 `C→S response.create`

docs/serving/speech_api.md

@@ -289,6 +289,19 @@ Upload a new voice sample for voice cloning in Base task TTS requests.
 
 Fields `ref_text` and `speaker_description` are omitted when not provided at upload time.
 
+**Naming rules:**
+
+- Names that collide with one of the model's built-in or precomputed voices are rejected (400). Voice
+  files already on disk under such a name are ignored at startup with a warning, so an upload can never
+  shadow a built-in voice.
+- Re-uploading an existing uploaded name overwrites it in place (the previous audio file is
+  deleted). Set `VLLM_OMNI_SPEAKER_REGISTRATION_POLICY=immutable` on the server to reject duplicates instead,
+  requiring an explicit `DELETE /v1/audio/voices/{name}` before re-registering — useful when the
+  endpoint is reachable by multiple writers and silent overwrites are a risk.
+- The voice registry has no per-user ownership: any client that can reach the endpoint can
+  overwrite (default policy) or delete any uploaded voice. For multi-tenant deployments, add
+  authentication at a proxy and namespace voice names per user (e.g. `{user}.{name}`).
+
 **Usage Example:**
 
 ```bash
@@ -521,6 +534,15 @@ are cached in-process with a shared LRU so repeated requests with the same
 all TTS model types; deleting a voice invalidates every model-type slot at
 once.
 
+Decoded reference waveforms use a separate LRU cache of owned, contiguous
+float32 arrays at the source sampling rate. Its byte budget counts numeric
+buffers (four bytes per mono sample), excluding cache metadata. Numeric storage
+avoids retaining a Python float object for every sample during garbage
+collection. MOSS reference encoding consumes these arrays directly; other
+list-based interfaces, including MOSS Nano, receive temporary lists that are
+not retained in the resolve cache. The MOSS reference encoder also releases
+completed batch inputs before waiting for more work.
+
 ### Precomputed Custom Voices
 
 Qwen3-TTS Base and VoxCPM2 can load offline-precomputed voices at startup.
@@ -568,6 +590,7 @@ by `GET /v1/audio/voices`. Valid precomputed voices can be used in
 | ---------- | --------- | ------------- |
 | `SPEAKER_SAMPLES_DIR` | `~/.cache/vllm-omni/speakers` | Directory for persisted uploaded speakers (`.safetensors` files). |
 | `SPEAKER_MAX_UPLOADED` | `1000` | Maximum number of uploaded speakers kept on disk. Upload requests past the cap return 400. |
+| `VLLM_OMNI_SPEAKER_REGISTRATION_POLICY` | `overwrite` | `immutable` rejects re-uploading an existing uploaded name (400) until it is deleted; any other value fails startup. |
 
 The in-memory LRU has a fixed 512 MiB byte budget.
 
@@ -948,22 +971,24 @@ Use `/v1/audio/voices` to list available voices for the loaded model.
 ## Orchestration Loop (experimental)
 
 Multi-stage omni deployments route stage outputs through a single orchestrator
-loop. By default that loop polls every stage replica on a 1 ms cadence. An
-opt-in event-driven mode replaces the poll with one reader task per live stage
+loop. The legacy loop polls every stage replica on a 1 ms cadence. The
+event-driven mode replaces the poll with one reader task per live stage
 replica awaiting its client directly, and switches the serving-side
-final-output drain to a condition-variable wakeup at the same time.
+final-output drain to a condition-variable wakeup at the same time. Qwen3-TTS
+uses this mode by default; other pipelines keep the legacy poll unless enabled.
 
 **Configuration (environment variables):**
 
 | Variable | Default | Description |
 | --- | --- | --- |
-| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `0` (off) | Switches the orchestration loop and the final-output drain from the legacy 1 ms poll to event-driven wakeups. Enabled by `1`, `true`, `yes`, or `on`, matched case-insensitively after surrounding whitespace is stripped; any other value leaves it off. |
+| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | On for Qwen3-TTS; off for other pipelines | Switches the orchestration loop and the final-output drain from the legacy 1 ms poll to event-driven wakeups. An explicit value wins; otherwise the pipeline default computed at engine initialization is used. The override is resolved at orchestrator construction and separately when the final-output drain starts. `1`, `true`, `yes`, or `on` enables it, ignoring case and surrounding whitespace; other values select the legacy poll loop. |
 
-Set it on the process that runs the orchestrator (stage 0 of an omni
-deployment) before starting the server:
+Set it on the process that runs the orchestrator (stage 0 of an omni deployment)
+before starting the server when overriding the pipeline default. For example,
+explicitly disable event-driven orchestration for Qwen3-TTS:
 
 ```bash
-export VLLM_OMNI_EVENT_DRIVEN_ORCH=1
+VLLM_OMNI_EVENT_DRIVEN_ORCH=0 \
 vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base \
     --omni \
     --port 8091
@@ -973,8 +998,9 @@ The server logs the selected loop mode and its reader/poller counts once at
 startup, so you can confirm which loop is live.
 
 Routing, output ordering, and terminal-state behavior are identical on both
-loops; only the poll cadence changes. Leaving the variable unset keeps the
-legacy poll loop, which is the supported default.
+loops; only the poll cadence changes. Leaving the variable unset selects the
+pipeline default: event-driven for Qwen3-TTS, and legacy polling for other
+pipelines, including Qwen3-Omni.
 
 **Known limitations:**
 

docs/serving/streaming_video_output_api.md

@@ -41,7 +41,7 @@ bytes.
 | Server to client | `video.start` | Confirms the request ID, format, and accepted configuration |
 | Server to client | `video.chunk_metadata` | Describes the next binary media frame |
 | Server to client | Binary frame | Carries one fragmented MP4 chunk |
-| Client to server | `session.interaction` | Queues an optional prompt update during generation |
+| Client to server | `session.interaction` | Queues an optional interaction (e.g., prompt. camera action) during generation |
 | Server to client | `session.interaction.queued` | Confirms that an interaction was queued |
 | Client to server | `session.ping` | Refreshes the stall clock |
 | Server to client | `session.pong` | Acknowledges a ping |
@@ -83,10 +83,39 @@ Models that implement interaction updates can change the active prompt:
 }
 ```
 
+Models that register the camera modality (for example LingBot-World) also
+accept structural SE3 payloads on `event.multi_modal_data.camera`. Coordinates
+follow Unity semantics (`+X` right, `+Y` up, `+Z` forward; same as LingBot's
+native camera frame). WASD key tokens are not admitted by the engine:
+
+```json
+{
+  "type": "session.interaction",
+  "interaction": {
+    "event_id": "forward-1",
+    "event": {
+      "multi_modal_data": {
+        "camera": {
+          "mode": "velocity",
+          "data": {
+            "translation": [0.0, 0.0, 0.05],
+            "rotation": [0.0, 0.0, 0.0, 1.0]
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+`mode` is `target` (lerp to an absolute pose over `transition_chunks`) or
+`velocity` (hold a per-frame SE3 delta until replaced). Prompt and camera may
+be combined in one interaction event.
+
 This feature is pipeline-dependent. An enabled endpoint does not imply that
-the loaded model supports midway prompt updates.
+the loaded model supports midway prompt or camera updates.
 
 The default start timeout is 10 seconds and the server reports a stall after
 about 60 seconds without generation progress or a `session.ping`. See the
 [streaming video generation example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/streaming_video_generation)
-for prompt-update scheduling and browser playback.
+for prompt/camera-update scheduling and browser playback.

docs/serving/videos_api.md

@@ -73,6 +73,10 @@ curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4
 | `image_reference` | string | null | JSON-encoded reference image payload; do not combine with `input_reference` or `video_reference` |
 | `video_reference` | string | null | JSON-encoded reference video payload; do not combine with `input_reference` or `image_reference` |
 | `audio_reference` | string | null | JSON-encoded audio reference for speech-to-video: `{"audio_url": "..."}` — supports HTTP(s) URLs or base64 data URLs |
+| `source_video` | file | null | MiniMax H3 latent-edit source video (`.mp4` or `.mov`, up to 512 MiB) |
+| `source_audio` | file | null | Optional MiniMax H3 latent-edit source audio (`.wav` or `.mp3`, up to 512 MiB) |
+| `video_noise_mask` | file | null | UTF-8 JSON MiniMax H3 video mask; `0` preserves and `1` regenerates a token |
+| `audio_noise_mask` | file | null | UTF-8 JSON MiniMax H3 audio mask; `0` preserves and `1` regenerates a token |
 | `width` | integer | model default | Output video width |
 | `height` | integer | model default | Output video height |
 | `num_frames` | integer | 1 | Number of generated frames |
@@ -214,6 +218,49 @@ upstream vLLM's media URL allowlist protection. Deployments should therefore
 treat remote media URLs as untrusted and choose this setting as part of their
 URL access policy.
 
+### MiniMax H3 Latent-Mask Editing
+
+MiniMax H3 accepts request-scoped source media and video/audio noise masks.
+At least one mask is required. A nontrivial `video_noise_mask` requires
+`source_video`, while a nontrivial `audio_noise_mask` requires either
+`source_audio` or a `source_video` with an audio stream. Mask values are in
+`[0, 1]`: `0` preserves the source, `1` regenerates it, and fractional values
+blend the two behaviors. Exact all-one masks are no-ops and do not require a
+source. Source uploads without a mask are rejected. Masks may be a JSON scalar
+or arrays matching the H3 latent/token grid; pixel-resolution masks must be
+resized or pooled by the client before upload.
+
+For an aligned output of `F` frames at `W x H`, the video latent grid is
+`[Tv, H/16, W/16]`, where `Tv = 2 + 5 * ((F - 5) / 17)`. The video mask may be
+a scalar, a flat token vector, `[Tv, H/32, W/32]`, or the full latent grid. For
+the model input, timestep, and velocity, a full-grid mask is max-pooled over
+each 2x2 spatial token and fractional values are rounded upward to 1/256
+levels. The final x0 restore uses the original, unquantized mask, so full-grid
+masks retain cell-level preservation inside a model token. The audio length is
+`Ta = round(F * 40 / 24)`, and its mask may be a scalar, `[Ta]`, `[2, Ta]`, or
+a flat `2 * Ta` vector. If source audio is short, its missing latent tail is
+forced to mask value `1` so H3 generates that portion. Each mask must be sent
+as a UTF-8 JSON file part and is limited to 8 MiB. A file may contain either a
+JSON scalar or an array.
+
+```bash
+curl -s http://localhost:8091/v1/videos/sync \
+  -F "prompt=Partially restyle the complete clip and soundtrack" \
+  -F 'extra_params={"task":"t2va","duration":4.0,"aspect_ratio":"16:9"}' \
+  -F "source_video=@source.mp4;type=video/mp4" \
+  -F "source_audio=@source.wav;type=audio/wav" \
+  -F "video_noise_mask=@video-mask.json;type=application/json" \
+  -F "audio_noise_mask=@audio-mask.json;type=application/json" \
+  -o edited.mp4
+```
+
+For example, each mask file in the request above may contain the JSON scalar
+`0.5`.
+
+The server streams source files to temporary request-scoped storage and removes
+them after synchronous or asynchronous generation finishes. Models that do not
+declare latent-mask editing support reject these fields.
+
 ### Speech-to-Video
 
 For models that support audio-driven generation (e.g., Wan2.2-S2V), pass both
@@ -251,6 +298,7 @@ These `extra_params` control how the server turns decoded frames into MP4 bytes.
 | Field | Type | Default | Description |
 | --- | --- | --- | --- |
 | `preencode_mp4` | boolean | false | Encode the MP4 on the worker while the VAE is still decoding, instead of after the full video is materialized |
+| `preencode_batch_frames` | positive integer | 17 (H3, Wan T2V/I2V); 1 (Wan S2V) | Minimum accumulated frames per worker transfer/encoding batch; used only with `preencode_mp4=true` |
 | `video_codec_options` | object | null | Encoder options passed through to the H.264 encoder, such as `{"preset": "ultrafast", "threads": "0"}` |
 
 With `preencode_mp4` enabled, each committed VAE chunk leaves the accelerator and
@@ -261,16 +309,29 @@ unchanged: the same complete MP4, byte-for-byte equivalent frames.
 ```bash
 curl -X POST http://localhost:8091/v1/videos/sync \
   -F "prompt=A small robot walking through a neon city" \
-  -F 'extra_params={"preencode_mp4": true, "video_codec_options": {"preset": "ultrafast"}}' \
+  -F 'extra_params={"preencode_mp4": true, "preencode_batch_frames": 33, "video_codec_options": {"preset": "ultrafast"}}' \
   -o output.mp4
 ```
 
+Set `preencode_batch_frames` in `extra_params` (or `extra_args` for offline
+sampling) to tune batching. The worker accumulates complete VAE chunks until
+it has at least this many frames, then transfers and encodes them together.
+It always flushes the final partial batch. This is a threshold, not an exact
+chunk length: a value of 1 submits every native chunk immediately, and a value
+smaller than a native chunk does not split it. Larger values reduce transfers
+but retain more frames on the accelerator and delay encoding. The VAE decode
+window and output frame count stay unchanged. Wan S2V keeps its existing
+per-clip behavior by default. Zero, negative, fractional, boolean, string, and
+null values are rejected when pre-encoding is enabled.
+
 `preencode_mp4` applies to the complete-MP4 response paths only. The
 `/v1/realtime/video` WebSocket endpoint rejects it, because that path already
-overlaps encoding through its own incremental fragmented-MP4 encoder.
+overlaps encoding through its own incremental fragmented-MP4 encoder. Wan also
+rejects it together with `enable_frame_interpolation`, which needs the decoded
+frames the pre-encoded path no longer materializes.
 
-Support is per model: MiniMax-H3 implements it, and other models ignore the flag
-and take the full-decode path.
+Support is per model: MiniMax-H3 and Wan 2.2 (T2V, I2V, and S2V) implement it,
+and other models ignore the flag and take the full-decode path.
 
 ## Storage
 

docs/user_guide/diffusion/attention_backends/rainfusion.md

@@ -78,3 +78,35 @@ Protected video tails require a compatible MindIE-SD release.
 For common configuration and selector behavior, see the
 [attention backend overview](../attention_backends.md) and the
 [backend selection design](../../../design/feature/attention_backend_selection.md).
+
+## Wan2.2 T2V quantization on Ascend
+
+`RAINFUSION_ATTN` accepts `diffusion_kv_cache_dtype: fp8` or `mxfp4` for
+Wan2.2 T2V A14B.
+MindIE-SD must expose `sparse_attention` with `rf_v3` and an explicit `precision`
+argument. Omni passes the selected precision directly to this API; the installed
+native operators must support that precision. Unsupported inputs or an
+incompatible Python API raise an actionable error; Omni does not change BSA
+precision automatically. Update `diffusion_kv_cache_dtype` to a precision
+supported by the installed MindIE-SD, or unset it. Native execution errors also
+propagate without retry.
+BSA FP8 and MXFP4 retain MindIE-SD's default rotation policy.
+
+`diffusion_kv_cache_skip_layers`/`diffusion_kv_cache_skip_steps` explicitly
+select floating-point BSA and keep attention sparse. The existing `block_sparse`
+warmup/layer exclusions and short-sequence threshold instead select Dense
+attention, using the configured Dense precision. Leave `block_sparse.precision`
+at its default when using `diffusion_kv_cache_dtype` as the shared Dense/BSA
+precision setting.
+
+Configure BSA quantization in the model's existing deployment YAML with
+`backend: RAINFUSION_ATTN` and stage-level `diffusion_kv_cache_dtype: fp8` or
+`mxfp4`. The precision and skip selectors are siblings of
+`diffusion_attention_config`; they are not per-role `quant` fields.
+For 40 denoising steps, `diffusion_kv_cache_skip_steps: "0,1,38,39"` and
+`diffusion_kv_cache_skip_layers: "0,39"` are recommended starting points. These
+fixed step indices do not automatically follow a changed step count. Selected
+forwards remain floating-point BSA; unsupported BSA precision is an error rather
+than an automatic fallback. See
+[quantized attention](../../quantization/quantized_kvcache.md#wan22-t2v-quantized-attention-on-ascend)
+for the shared configuration structure and index semantics.

docs/user_guide/diffusion/cache_acceleration/cache_dit.md

@@ -72,6 +72,39 @@ omni = Omni(
 
 ## Example Script
 
+### Multi-Stage Models (MammothModa2)
+
+MammothModa2 is a multi-stage AR→DiT pipeline; the diffusion runner owns the
+Cache-DiT lifecycle of its DiT stage. Configure it on the **DiT stage entry**
+of the deploy YAML so the acceleration only applies to the denoising stage:
+
+```yaml
+# deploy YAML (see vllm_omni/deploy/mammoth_moda2.yaml)
+stages:
+  - stage_id: 1
+    # ... other DiT stage settings ...
+    cache_backend: cache_dit
+    cache_config:
+      Fn_compute_blocks: 1
+      Bn_compute_blocks: 0
+      max_warmup_steps: 4
+      residual_diff_threshold: 0.24
+      max_continuous_cached_steps: 3
+    enable_cache_dit_summary: true   # log skip-ratio stats per request
+```
+
+Behavior notes:
+
+- Only the repeated main-layer stack is cached; the Q-Former refiners always run.
+- Sequential-CFG parity requires the unconditional pass on every denoise step,
+  so the `cfg_range` skip optimization is not used: outside the interval CFG is
+  neutralized with `scale=1.0` instead of skipping the uncond forward.
+- Requests with `text_guidance_scale = 1.0` run with cache hooks disabled.
+- `residual_diff_threshold` defaults to `0.24` (measured ~1.7-2.2x at 50 steps,
+  PSNR 27-34 dB vs uncached). For 20-step generation — especially with a
+  partial `cfg_range` — use `0.12` for noticeably closer parity at a modest
+  speed cost.
+
 ### Offline Inference
 
 Use the example script under `examples/offline_inference/text_to_image`:

docs/user_guide/diffusion/mot_config.md

@@ -28,7 +28,7 @@ and their optimal Triton tile configurations:
 
 Config loading order (3-tier, see ops/mot_gemm.py):
     1. $VLLM_TUNED_CONFIG_FOLDER/<filename>  (env override)
-    2. This directory: vllm_omni/diffusion/layers/mot/configs/<filename>
+    2. This directory: vllm_omni/diffusion/models/bagel/mot/configs/<filename>
     3. Conservative default config (compiles everywhere, sub-optimal perf)
 
 If no config file matches the current device, a warning is printed with
@@ -39,7 +39,7 @@ To generate configs for your hardware:
     python benchmarks/kernels/mot_linear_benchmarks.py \
         --model ByteDance-Seed/BAGEL-7B-MoT \
         --tp-size 1 --dtype w16a16 --tune \
-        --save-dir vllm_omni/diffusion/layers/mot/configs/
+        --save-dir vllm_omni/diffusion/models/bagel/mot/configs/
 
 For multi-GPU tuning (uses Ray for parallel search):
 

docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md

@@ -353,9 +353,14 @@ modes can resolve the same `None` value to different schedules.
   Cache-DiT on any selected component. Prompt-embedding cache is also
   incompatible when the text encoder uses `allgather`. Rank-local cache hits
   could otherwise make ranks enter different weight collectives.
-- Per-tensor online FP8 linears use the ordinary loader and can run with either
-  DiT transfer path. With DiT `allgather`, every rank temporarily materializes
-  the complete FP8 model in host memory before DLO retains only its shard.
+- Per-tensor online FP8, INT8, and MXFP8 linears use the ordinary loader and
+  can run with either DiT transfer path. With DiT `allgather`, every rank
+  temporarily materializes the complete quantized model in host memory before
+  DLO retains only its shard. Layers too wide for the NPU quantization kernel
+  (`npu_quant_matmul` rejects an output dimension past 65535) stay
+  unquantized and load straight into host memory under offload-after-quant —
+  their runtime layout is a plain contiguous bf16 weight, the same layout DLO
+  shards on the ordinary path, so they are allowed under `allgather` too.
   Other online quantization methods require rank-local transfer for the
   affected component until their runtime layouts are validated.
 - Resident leading layers require DiT `rank-local` transfer and a model

docs/user_guide/diffusion_features.md

@@ -130,7 +130,7 @@ The following tables show which models support each feature:
 | **Krea 2**               |     ❌     |     ✅      |           ❌           |       ❌        |         ❌         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **LongCat-Image**        |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **LongCat-Image-Edit**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
-| **MammothModa2(T2I)**    |     ❌     |     ❌      |           ❌           |       ❌        |         ❌         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
+| **MammothModa2(T2I)**    |     ❌     |     ✅      |           ❌           |       ❌        |         ❌         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **Nextstep_1(T2I)**      |     ❓     |     ❓      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **OmniGen2**             |     ❌     |     ✅      |           ✅           |       ❌        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **Ovis-Image**           |     ❌     |     ✅      |           ❌           |       ✅        |         ❌         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
@@ -139,6 +139,7 @@ The following tables show which models support each feature:
 | **Qwen-Image-Edit**      |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Qwen-Image-Edit-2509** |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |        ✅ (decode)         |          ✅           |       ❌        |        ❌         |
 | **Qwen-Image-Layered**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
+| **Qwen-Image-2.1**       |     ❌     |     ❌      |   ✅ (Ulysses only)    |       ✅        |         ❓         |          ❌          |   ❓    |             ❓             |      ✅ (decode)      |       ✅        |        ✅         |
 | **SenseNova-U1 / U1.5**  |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **Stable-Diffusion-XL**  |     ❌     |     ❌      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Stable-Diffusion3.5**  |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
@@ -154,6 +155,8 @@ The following tables show which models support each feature:
 > 4. Krea 2 currently supports single-GPU inference plus LoRA, Cache-DiT, HSDP, CPU/layerwise offload, and VAE-patch-parallel (decode). TP/SP/CFG-Parallel are not yet wired. The few-step distilled (Turbo) checkpoint uses `is_distilled=true` (fixed timestep shift `mu=1.15`); generate at 2048x2048 by default with `num_inference_steps≈8` and `guidance_scale=0`. The Raw checkpoint uses 1024x1024, `num_inference_steps=28`, and `guidance_scale=4.5`.
 > 5. HunyuanImage3 supports step execution. Multi-request step execution requires `TORCH_SDPA`; see [Diffusion Execution Modes](diffusion/execution_modes.md#step-execution).
 > 6. BAGEL step execution supports image generation with `bagel.yaml`, `bagel_think.yaml`, and `bagel_single_stage.yaml`; two-stage Thinker execution and explicit single-stage text output remain on their existing complete-request paths. Image requests require `num_inference_steps >= 2`. BAGEL step execution cannot currently be combined with sequence parallelism or a diffusion cache backend; see [Diffusion Execution Modes](diffusion/execution_modes.md#step-execution).
+> 7. MammothModa2 runs its DiT stage on the diffusion runner (`StageExecutionType.DIFFUSION`); Cache-DiT is enabled through the standard diffusion-stage knobs on the stage entry of the deploy YAML (`cache_backend: cache_dit`, plus optional `cache_config` / `enable_cache_dit_summary`). The runner installs the backend at startup and the pipeline adopts it per request. Only the repeated main-layer stack is cached; requests with `text_guidance_scale = 1.0` bypass the cache hooks.
+> 8. Qwen-Image-2.1 (`QwenImage21Pipeline`) handles both text-to-image and image-conditioned generation (up to 4 condition images) and supports LoRA through `QwenImageLoraLoaderMixin`. Sequence parallelism is Ulysses-only (ring attention is not supported), and cache acceleration (TeaCache / Cache-DiT) is unavailable; its built-in KV cache for the text/condition prefix is a separate feature from those cache backends.
 
 ### VideoGen
 

docs/user_guide/examples/offline_inference/text_to_speech.md

@@ -2,7 +2,6 @@
 
 Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech>.
 
-
 vLLM-Omni supports several autoregressive TTS models. They share a common
 CLI shape (`--text`, `--ref-audio`, `--ref-text`, `--output-dir`) and live
 together in this hub. Each model has its own subdirectory containing a
@@ -15,7 +14,7 @@ list of supported architectures across all modalities, see
 ## Supported Models
 
 | Model | HuggingFace repo | Stages | Voice cloning | Streaming | Special modes | Sample rate |
-|---|---|---|---|---|---|---|
+| --- | --- | --- | --- | --- | --- | --- |
 | VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | — | continuation (`--ref-audio` + `--ref-text`) | 48 kHz |
 | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ (`async_chunk: true` default) | — | 22.05 kHz |
 | Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ (`--streaming`) | — | 44.1 kHz |
@@ -45,13 +44,15 @@ python examples/offline_inference/text_to_speech/<model>/end2end.py \
 Single-stage native AR TTS at 48 kHz. Pipeline: `feat_encoder → MiniCPM4 → FSQ → residual_lm → LocDiT → AudioVAE`.
 
 ### Prerequisites
+
 ```bash
 pip install voxcpm
 # or, for a local source checkout:
-export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/voxcpm
+pip install -e /path/to/voxcpm
 ```
 
 ### Quick start
+
 ```bash
 python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
     --model openbmb/VoxCPM2 \
@@ -59,7 +60,9 @@ python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
 ```
 
 ### Voice cloning
+
 Pass a reference audio for isolated cloning, or both `--ref-audio` + `--ref-text` for prompt continuation:
+
 ```bash
 python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
     --text "Hello, this is a voice clone demo." \
@@ -68,6 +71,7 @@ python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
 ```
 
 ### Notes
+
 - Output: 48 kHz mono WAV.
 - Deploy config: `vllm_omni/deploy/voxcpm2.yaml` (default).
 
@@ -78,12 +82,14 @@ python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
 2-stage TTS pipeline (`talker` + `code2wav`) at 22.05 kHz.
 
 ### Prerequisites
+
 ```bash
 uv pip install -e .
 # Includes soundfile, onnxruntime, x-transformers, einops via requirements.
 ```
 
 Download the model snapshot:
+
 ```python
 from huggingface_hub import snapshot_download
 snapshot_download('FunAudioLLM/Fun-CosyVoice3-0.5B-2512',
@@ -91,15 +97,18 @@ snapshot_download('FunAudioLLM/Fun-CosyVoice3-0.5B-2512',
 ```
 
 If your downloaded checkpoint lacks `config.json`, add it:
+
 ```json
 {
     "model_type": "cosyvoice3",
     "architectures": ["CosyVoice3Model"]
 }
 ```
+
 This is required because `AutoConfig.register("cosyvoice3", CosyVoice3Config)` only registers the class mapping; the loader still reads `model_type` from `config.json` to select the class.
 
 ### Quick start
+
 ```bash
 python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
     --model pretrained_models/Fun-CosyVoice3-0.5B \
@@ -107,7 +116,9 @@ python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
 ```
 
 ### Voice cloning
+
 Pass a reference audio. Note that CosyVoice3's `--prompt-text` is a system-style prompt for the GPT stage, not a reference transcript:
+
 ```bash
 python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
     --model pretrained_models/Fun-CosyVoice3-0.5B \
@@ -117,6 +128,7 @@ python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
 ```
 
 ### Notes
+
 - Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform.
 - Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config <path>` to override.
 - `async_chunk: true` is the default; pass `--no-async-chunk` to switch to the legacy synchronous path.
@@ -129,12 +141,14 @@ python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
 No extra packages are required; the DAC codec is vendored in vLLM-Omni.
 
 ### Quick start
+
 ```bash
 python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
     --text "Hello, this is a test of the Fish Speech text to speech system."
 ```
 
 ### Voice cloning
+
 ```bash
 python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
     --text "Hello, this is a cloned voice." \
@@ -143,14 +157,17 @@ python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
 ```
 
 ### Streaming
+
 ```bash
 python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
     --text "Hello, this is a streaming test." \
     --streaming
 ```
+
 Streaming requires `async_chunk: true` in the deploy config.
 
 ### Notes
+
 - Output: 44.1 kHz mono WAV.
 - DAC codec weights (`codec.pth`) are loaded lazily from the model directory.
 
@@ -161,6 +178,7 @@ Streaming requires `async_chunk: true` in the deploy config.
 2-stage TTS pipeline (AR + DiT flow-matching) at 24 kHz. Every request requires reference audio and its transcript for zero-shot voice cloning.
 
 ### Quick start
+
 ```bash
 python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
     --model zai-org/GLM-TTS \
@@ -171,12 +189,14 @@ python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
 ```
 
 ### Architecture
-```
+
+```text
 Text → [Stage 0: AR] → Speech Tokens → [Stage 1: DiT + HiFT] → Audio (24 kHz)
         (Llama-based)    (32k vocab)      (Flow Matching)
 ```
 
 ### Notes
+
 - `--ref-audio` and `--ref-text` are **required** together; GLM-TTS does not support text-only synthesis.
 - Reference audio should be 3-10 seconds.
 - First run may be slow due to lazy loading of WhisperVQ tokenizer and CampPlus ONNX speaker embedder.
@@ -190,12 +210,15 @@ Text → [Stage 0: AR] → Speech Tokens → [Stage 1: DiT + HiFT] → Audio (24
 Zero-shot multilingual TTS supporting 600+ languages, with three modes (auto / clone / design).
 
 ### Prerequisites
+
 ```bash
 huggingface-cli download k2-fsa/OmniVoice
 ```
+
 Voice cloning requires `transformers>=5.3.0`. Auto and design modes work with `transformers>=4.57.0`.
 
 ### Quick start (auto voice)
+
 ```bash
 python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
     --model k2-fsa/OmniVoice \
@@ -203,6 +226,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
 ```
 
 ### Voice cloning
+
 ```bash
 python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
     --model k2-fsa/OmniVoice \
@@ -212,6 +236,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
 ```
 
 ### Voice design
+
 ```bash
 python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
     --model k2-fsa/OmniVoice \
@@ -220,6 +245,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
 ```
 
 ### Language hint
+
 ```bash
 python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
     --model k2-fsa/OmniVoice \
@@ -228,6 +254,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
 ```
 
 ### Notes
+
 - Stage 0 (Generator): Qwen3-0.6B with 32-step iterative unmasking.
 - Stage 1 (Decoder): HiggsAudioV2 RVQ + DAC at 24 kHz.
 
@@ -238,13 +265,16 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
 3-task-variant TTS with 24 kHz output. Has its own argparse surface (this script does not follow the common `--text` / `--ref-audio` shape).
 
 ### Prerequisites
+
 For ROCm builds, replace `onnxruntime` with `onnxruntime-rocm`:
+
 ```bash
 pip uninstall onnxruntime
 pip install onnxruntime-rocm
 ```
 
 ### Task variants
+
 - `CustomVoice`: predefined speaker (speaker ID) with optional style instruction.
 - `VoiceDesign`: text + descriptive instruction designs a new voice.
 - `Base`: voice cloning from reference audio + transcript.
@@ -264,16 +294,20 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-ty
 ```
 
 ### Streaming
+
 ```bash
 python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
     --query-type CustomVoice \
     --streaming \
     --output-dir /tmp/out_stream
 ```
+
 Streaming requires `async_chunk: true` in the deploy config.
 
 ### Batched decoding
+
 The Code2Wav stage supports batched decoding through the SpeechTokenizer. Configure both stages with `max_num_seqs > 1` via `--stage-overrides` and pass multiple prompts via `--txt-prompts`:
+
 ```bash
 python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
     --query-type CustomVoice \
@@ -281,9 +315,11 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
     --batch-size 4 \
     --stage-overrides '{"0":{"max_num_seqs":4,"gpu_memory_utilization":0.2},"1":{"max_num_seqs":4,"gpu_memory_utilization":0.2}}'
 ```
+
 `--batch-size` must match a CUDA-graph capture size (1, 2, 4, 8, 16…).
 
 ### Notes
+
 - Run `--help` for the full argument surface.
 - See `qwen3_tts/end2end.py` for the prompt-length-estimation logic the Talker uses.
 
@@ -294,12 +330,15 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
 Voxtral-4B-TTS (Mistral). Has its own argparse surface; uses voice presets and the `mistral_common` `SpeechRequest` protocol.
 
 ### Prerequisites
+
 Latest `mistral_common` with `SpeechRequest` support:
+
 ```bash
 pip install -e /path/to/mistral-common  # or upgrade from PyPI when available
 ```
 
 ### Quick start (voice preset)
+
 ```bash
 python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
     --write-audio --voice cheerful_female \
@@ -308,6 +347,7 @@ python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
 ```
 
 ### Voice cloning (capability gated upstream)
+
 ```bash
 python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
     --write-audio \
@@ -317,15 +357,18 @@ python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
 ```
 
 ### Streaming + concurrency
+
 ```bash
 python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
     --num-prompts 32 --concurrency 8 --streaming --write-audio --voice neutral_female \
     --model mistralai/Voxtral-4B-TTS-2603 \
     --text "..."
 ```
+
 Available voice presets are listed on the HF model card (`mistralai/Voxtral-4B-TTS-2603`).
 
 ### Notes
+
 - `--num-prompts N` replicates the prompt for performance measurement.
 - `--concurrency M` requires `--streaming` and must evenly divide `--num-prompts`.
 - Run `--help` for the full argument surface.

docs/user_guide/examples/online_serving/aura_omni.md

@@ -6,29 +6,29 @@
 ASR -> AURA -> Qwen3-TTS Talker -> Code2Wav
 ```
 
-Qwen3-TTS remains two engine stages so the pipeline reuses the existing native
-Talker and Code2Wav implementation.
+**Supported online path:** Realtime duplex at `/v1/realtime?duplex=1`.
+
+The default deploy profile (`vllm_omni/deploy/aura_omni.yaml`) sets
+`session_mode: duplex`. The `aura_omni` pipeline declares `duplex_plugin`, and
+`DuplexOmniEngine` requires that mode (same pattern as MiniCPM-o duplex
+deploys). Turn-based `chat/completions` is not a supported AURA online mode
+against this profile.
+
+`async_chunk` defaults to `false` in that file (safer if a turn-based
+`OmniOrchestrator` ever loads it). Duplex Realtime Stage2→3 still needs
+async chunks — use the smoke deploy below, or set `async_chunk: true`
+yourself.
 
 ```bash
-vllm serve aurateam/AURA \
-  --omni \
-  --deploy-config vllm_omni/deploy/aura_omni.yaml \
-  --served-model-name aurateam/AURA \
-  --trust-remote-code
+bash examples/online_serving/aura_omni/run_duplex_smoke_serve.sh
+python examples/online_serving/aura_omni/smoke_duplex_realtime_client.py
 ```
 
-Configure local checkpoints by editing per-stage `model` values in
-`vllm_omni/deploy/aura_omni.yaml`. The deploy file sets
-`pipeline: aura_omni`, so the four-stage topology is used even if the
-command-line `--model` points at one of the component checkpoints.
+Configure local checkpoints by editing per-stage `model` values in the deploy
+or smoke YAML. The file sets `pipeline: aura_omni`, so the four-stage topology
+is used even if the command-line `--model` points at one component checkpoint.
 
-Send requests with `"model": "aurateam/AURA"`. The ASR, AURA, and Qwen3-TTS
-checkpoint paths are internal stage models from the deploy YAML, not the
-OpenAI-facing served model name.
-
-The AURA stage can emit `<|silent|>`. Silent outputs are treated as a gate:
-they produce no Qwen3-TTS Talker input, so no audio is synthesized for that
-turn.
+Silent Stage1 outputs (`<|silent|>` / id `151669`) skip TTS for that turn.
 
 ## GPU Utilization Recommendation
 
@@ -40,26 +40,21 @@ VRAM each stage can reserve. Start with this split for a single GPU:
 - Stage 2 (Qwen3-TTS Talker): `0.20`
 - Stage 3 (Qwen3-TTS Code2Wav): `0.20`
 
-## TTS Modes
+## TTS modes (stage extras)
 
-`aura_omni` can pass AURA text to Qwen3-TTS in two task modes:
+When the duplex session supplies TTS extras, AURA text can feed Qwen3-TTS as:
 
-- `Base`: voice clone from `tts_ref_audio` with ICL enabled in the AURA
-  pipeline. Provide both `tts_ref_audio` and `tts_ref_text`. Set
-  `tts_x_vector_only_mode=true` to disable ICL and use speaker embedding only.
-- `CustomVoice`: predefined speaker mode. Use a Qwen3-TTS CustomVoice
-  checkpoint for stages 2 and 3 in `aura_omni.yaml`, then pass
-  `tts_task_type=CustomVoice` and `tts_speaker`.
+- `Base`: voice clone from `tts_ref_audio` (optional x-vector-only mode)
+- `CustomVoice`: predefined speaker (`tts_speaker`) with a CustomVoice checkpoint
+  on stages 2 and 3
 
-By default, AURA responses are passed to Qwen3-TTS as text. Set
-`additional_information.tts_pass_token_ids=true` to pass AURA-generated
-assistant token ids directly instead. Even when token passthrough is disabled,
-the stage processor uses AURA token ids when available to estimate the Talker
-prefill length, so scheduling does not rely on raw character length.
+Optional `tts_pass_token_ids` passes AURA assistant token ids into Talker
+instead of detokenized text.
 
-The example client exposes this as:
+## Old chat / Gradio / curl scripts (unsupported)
 
-```bash
-python examples/online_serving/aura_omni/openai_chat_completion_client.py \
-  --tts-pass-token-ids
-```
+`examples/online_serving/aura_omni/` still contains chat-completions, curl, and
+Gradio helpers from an older turn-based path. They do **not** work against the
+shipped duplex profile. That path was an incomplete stand-in before streaming
+I/O and session history were available; it is not a supported AURA mode. Use
+the Realtime duplex smoke client for online verification.

docs/user_guide/examples/online_serving/bagel.md

@@ -247,12 +247,18 @@ curl http://localhost:8091/v1/chat/completions \
   -d @payload.json
 ```
 
-BAGEL img2img derives its generated image size from the input image. It
-preserves the input aspect ratio, aligns dimensions to the latent stride, and
-applies the checkpoint size limit; explicit `height` and `width` values do not
-override this model-specific resize policy. In step mode, the effective size is
-resolved before scheduler admission so requests with different resulting
-shapes are not combined.
+When no output size is requested (`size=auto` on `/v1/images/edits`, or no
+`height`/`width` in a chat request), BAGEL img2img derives the canvas from the
+input image: it preserves the input aspect ratio, aligns dimensions to the
+latent stride (16 px), and applies the checkpoint size limit. An explicit
+`size`, `height`, or `width` is honored instead, in both the single-stage and
+the two-stage deployment; the input image is still resized to a stride-aligned
+size for the VAE/ViT prefill. The latent grid needs multiples of 16, so a
+requested side that is not one is floored to the next multiple (1000x700
+generates 992x688) and the server logs a warning; the response reports the
+generated size. A side above the checkpoint limit is rejected with HTTP 400.
+In step mode, the effective size is resolved before scheduler admission so
+requests with different resulting shapes are not combined.
 
 ### Image to Text (img2text)
 
@@ -318,8 +324,8 @@ curl http://localhost:8091/v1/chat/completions \
 | `--server` / `-s` | `http://localhost:8091` | Server URL |
 | `--image-url` / `-i` | `None` | Input image URL or local path (img2img/img2text) |
 | `--modality` / `-m` | `text2img` | `text2img`, `img2img`, `img2text`, `text2text` |
-| `--height` | `512` | Text-to-image output height; BAGEL img2img derives its size from the input image |
-| `--width` | `512` | Text-to-image output width; BAGEL img2img derives its size from the input image |
+| `--height` | `512` | Output height, a multiple of 16; omit both `--height` and `--width` to let img2img derive the size from the input image |
+| `--width` | `512` | Output width, a multiple of 16; omit both `--height` and `--width` to let img2img derive the size from the input image |
 | `--steps` | `25` | Number of inference steps |
 | `--seed` | `42` | Random seed |
 | `--negative` | `None` | Negative prompt for CFG |

docs/user_guide/quantization/fp8.md

@@ -1,4 +1,4 @@
-# FP8 Quantization
+# FP8 Quantization
 
 ## Overview
 
@@ -85,6 +85,7 @@ warmup_quack_fp8([(14040, 2048, 6144), (14040, 2048, 2048)])
 | Model | HF models | Online | Pre-calibrated | Recommendation | `ignored_layers` | Text-Encoder quantization |
 |-------|-----------|:-------:|:------:|----------------|------------------|------------------|
 | Qwen-Image | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes | Yes | Skip sensitive image-stream MLPs when quality regresses | `img_mlp` | |
+| Qwen-Image-2.1 | `Qwen/Qwen-Image-2.1` | Yes | Not validated | Skip image-stream MLPs for best fidelity; all-layer FP8 is loadable but composition drifts on some prompts. On Blackwell GB200 FP8 saves ~17% peak memory but is not faster than BF16 at 1024x1024. Text encoder FP8 covers the language model only — the vision tower and LM head stay BF16 | `img_mlp` | ✅︎ (language model only) |
 | Wan2.2 | Wan2.2 diffusion pipelines | Not validated | Not validated | Validate against BF16 before documenting as supported | TBD | |
 | LTX-2 | `Lightricks/LTX-2`, `rootonchair/LTX-2-19b-distilled` | Yes | Not validated | Transformer only; use dynamic phase LoRA for ordinary two-stage | None | |
 | LTX-2.3 | `diffusers/LTX-2.3-Diffusers`, `diffusers/LTX-2.3-Distilled-Diffusers` | Yes | Not validated | Transformer only; use dynamic phase LoRA for ordinary two-stage | None | |

docs/user_guide/quantization/overview.md

@@ -13,7 +13,7 @@ For the internal architecture and backend extension points, see the
 | Mode | Guide | Description | Methods |
 | ------ | ------- | ------------- | --------- |
 | Online quantization | [Online Quantization](online.md) | vLLM-Omni computes quantized weights and scales while loading the model. | FP8 W8A8, Int8 W8A8, BitsAndBytes W4, MXFP8 W8A8, MXFP4 W4A4 |
-| Runtime attention quantization | [Quantized KV Cache](quantized_kvcache.md) | vLLM-Omni dynamically quantizes eligible diffusion Flash Attention tensors during inference. | FP8 FA |
+| Runtime attention quantization | [Quantized KV Cache](quantized_kvcache.md) | vLLM-Omni dynamically quantizes eligible diffusion attention tensors during inference. | FP8, MXFP8, MXFP4 |
 | Pre-quantized checkpoints | Method-specific guides | The checkpoint or an offline quantizer provides quantized weights and scales before serving. | ModelOpt, AutoRound, TorchAO, msModelSlim, serialized Int8, offline MXFP8, offline MXFP4 DualScale |
 
 ## Hardware Support

docs/user_guide/quantization/quantized_kvcache.md

@@ -2,41 +2,36 @@
 
 ## Overview
 
-In DiT-based image and video generation, Flash Attention can take a large share
-of denoising time, especially for high-resolution or long-frame workloads.
-vLLM-Omni supports online FP8 quantization for eligible diffusion Flash
-Attention (FA) to reduce FA latency while keeping model weights in their
-original dtype.
+In DiT-based image and video generation, attention can take a large share of
+denoising time, especially for high-resolution or long-frame workloads.
+vLLM-Omni supports online Q/K/V quantization for eligible diffusion attention
+paths while keeping model weights in their original dtype.
 
-This feature is configured through `diffusion_kv_cache_dtype` on
-`OmniDiffusionConfig` (CLI: `--diffusion-kv-cache-dtype`). It is intentionally
-**not** the same as vLLM's `--kv-cache-dtype`, which controls autoregressive
-language-model KV cache storage and defaults to `"auto"`. Diffusion FA
-quantization uses the dedicated diffusion flags so omni serve does not inherit
-that default.
+Runtime attention precision is configured through `diffusion_kv_cache_dtype`
+on `OmniDiffusionConfig` (CLI: `--diffusion-kv-cache-dtype`). This setting is
+separate from vLLM's `--kv-cache-dtype`, which controls autoregressive
+language-model KV cache storage.
 
 In vLLM-Omni diffusion pipelines, this is a runtime FA path: Q/K/V tensors are
 dynamically quantized before the attention operator. It does not quantize model
 weights and is separate from [FP8 W8A8](fp8.md), [Int8 W8A8](int8.md), or
 pre-quantized checkpoint formats.
 
-If `diffusion_kv_cache_dtype` is not set, behavior is unchanged and attention
-runs in the native dtype.
+If `diffusion_kv_cache_dtype` is not set, attention runs in the native dtype.
 
 ## Hardware Support
 
-| Device | FP8 FA |
-|--------|--------|
-| Ascend NPU | ✅ |
-| NVIDIA GPU | ❌ |
-| AMD ROCm | ❌ |
-| Intel XPU | ❌ |
+| Device | Dense FP8 | Dense MXFP8/MXFP4 | BSA FP8/MXFP4 |
+|--------|------------|-------------------|----------------|
+| Ascend NPU | ✅ | ✅ | ✅ |
+| NVIDIA GPU | ❌ | ❌ | ❌ |
+| AMD ROCm | ❌ | ❌ | ❌ |
+| Intel XPU | ❌ | ❌ | ❌ |
 
 Legend: `✅` supported, `❌` unsupported.
 
-FP8 FA is currently implemented only for the NPU Flash Attention backend. Other
-backends do not support `diffusion_kv_cache_dtype="fp8"` for diffusion attention
-and fall back to native dtype execution.
+These runtime methods are implemented only by the NPU Flash Attention and BSA
+backends. Other backends reject an incompatible explicit configuration.
 
 ## Model Type Support
 
@@ -44,7 +39,7 @@ and fall back to native dtype execution.
 
 | Model | Scope | Status | Notes |
 |-------|-------|--------|-------|
-| Wan2.2 | Eligible DiT full-attention FA on Ascend NPU | Tested | Compare quality and latency against a BF16 baseline before production use |
+| Wan2.2 | Self-attention on Ascend NPU | NPU interface and T2V smoke tested | Compare quality and latency against a BF16 baseline before production use |
 | Other diffusion models | Eligible DiT full-attention FA on Ascend NPU | Not tested | You can try `diffusion_kv_cache_dtype="fp8"`; tune `diffusion_kv_cache_skip_steps` and `diffusion_kv_cache_skip_layers` when higher precision is needed |
 
 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
@@ -105,12 +100,13 @@ The legacy keyword aliases `kv_cache_dtype`, `kv_cache_skip_steps`, and
 
 | Parameter | Type | Default | Description |
 |-----------|------|---------|-------------|
-| `diffusion_kv_cache_dtype` | str \| None | `None` | Set to `"fp8"` to enable dynamic FP8 FA on supported attention backends |
+| `diffusion_kv_cache_dtype` | str \| None | `None` | Runtime attention method. Dense FA supports `fp8`, `mxfp8`, and `mxfp4`; BSA supports `fp8` and `mxfp4` |
 | `diffusion_kv_cache_skip_steps` | str \| None | `None` | Denoising step selector to keep in native dtype, for example `"0,1,4-6"` |
 | `diffusion_kv_cache_skip_layers` | str \| None | `None` | Transformer layer selector to keep in native dtype, for example `"0-2,10"` |
 
 Selectors use comma-separated integers and inclusive ranges. Listed steps or
-layers skip FP8 FA; all other eligible full-attention forwards use the FP8 path.
+layers run floating-point attention through the selected Dense or BSA backend;
+all other eligible forwards use `diffusion_kv_cache_dtype`.
 
 ## Validation and Notes
 
@@ -123,3 +119,60 @@ layers skip FP8 FA; all other eligible full-attention forwards use the FP8 path.
 4. Report both latency and quality results when enabling this option for a new
    model. For image or video models, include visual comparison and quantitative
    metrics when available, such as PSNR or SSIM.
+
+## Wan2.2 T2V quantized attention on Ascend
+
+Wan2.2 T2V A14B supports quantized self-attention through MindIE-SD. Model
+weights are unchanged, and model-level opt-out keeps cross-attention in its
+native dtype.
+
+| Attention path | self-attention `backend` | Supported `diffusion_kv_cache_dtype` |
+| --- | --- | --- |
+| Dense FA | `FLASH_ATTN` | `fp8`, `mxfp8`, `mxfp4` |
+| BSA | `RAINFUSION_ATTN` | `fp8`, `mxfp4` |
+
+Configure the method and fallback selectors once at the diffusion stage. The
+per-role attention config only selects the self-attention backend:
+
+```yaml
+stages:
+  - stage_id: 0
+    diffusion_kv_cache_dtype: "mxfp8"
+    diffusion_kv_cache_skip_layers: "0,39"
+    diffusion_kv_cache_skip_steps: "0,1,38,39"
+    diffusion_attention_config:
+      per_role:
+        self:
+          backend: FLASH_ATTN  # Use RAINFUSION_ATTN with fp8 or mxfp4 for BSA.
+```
+
+For 40 denoising steps, `diffusion_kv_cache_skip_steps: "0,1,38,39"` and
+`diffusion_kv_cache_skip_layers: "0,39"` are recommended starting points. Steps
+are zero-based across the complete request and do not reset when Wan switches
+transformers; layer indices are zero-based and local to each transformer.
+Selectors accept comma-separated indices and inclusive ranges such as
+`"0,3-5"`. A selected forward uses floating-point attention while preserving
+the Dense or BSA path.
+
+Unsupported precision/input combinations raise an error and must be corrected
+in the configuration; operator errors are not retried with another precision.
+
+For BSA, change the backend to `RAINFUSION_ATTN` and set
+`diffusion_kv_cache_dtype` to `fp8` or `mxfp4`.
+See [RainFusion attention](../diffusion/attention_backends/rainfusion.md) for
+sparse-path configuration and behavior.
+
+### Requirement and interface test
+
+Until these APIs are available in a MindIE-SD release, use revision
+[`8637b5333b0225381b215390fd09a8732e671cc4`](https://gitcode.com/zqxu/MindIE-SD/commit/8637b5333b0225381b215390fd09a8732e671cc4).
+Build the Python package, PyTorch plugin, and custom operators from that same
+checkout. With CANN sourced, run the NPU interface tests from the Omni checkout:
+
+```bash
+python -m pytest tests/platforms/npu/quant/test_kv_quant_npu.py \
+    -k real_npu -vv -s -o addopts=''
+```
+
+A skipped NPU test is not a pass. Validate generated video quality against a
+floating-point run with the same prompt, seed, resolution, and step count.

examples/model_display_names.yml

@@ -2,7 +2,6 @@
 # The docs generator combines these names with the serving mode for each page H1.
 aura_omni: AURA Omni
 covo_audio: Covo-Audio-Chat
-dynin_omni: Dynin-Omni
 lance: Lance
 mimo_audio: MiMo-Audio
 ming_flash_omni: Ming-flash-omni 2.0

examples/offline_inference/dynin_omni/README.md

@@ -1,110 +0,0 @@
-# Dynin-Omni: Offline inference
-
-This folder contains a unified offline inference entrypoint:
-
-- `end2end.py`
-
-## 1. Environment Setup
-
-Run from repository root:
-
-```bash
-cd <REPO_ROOT>
-```
-
-If needed, install this repo in editable mode:
-
-```bash
-pip install -e .
-```
-
-## 2. Extra Dependencies (EMOVA)
-
-Install the following packages for EMOVA-related components:
-
-```bash
-pip install \
-  "phonemizer==3.3.0" \
-  "Unidecode==1.4.0" \
-  "hydra-core==1.3.2" \
-  "pytorch-lightning==1.1.0" \
-  "wget==3.2" \
-  "wrapt==2.1.1" \
-  "onnx==1.20.1" \
-  "frozendict==2.4.7" \
-  "inflect==7.5.0" \
-  "braceexpand==0.1.7" \
-  "webdataset==1.0.2" \
-  "torch-stft==0.1.4" \
-  "editdistance==0.8.1"
-```
-
-## 3. Hardware and VRAM Requirements
-
-This example uses a 3-stage pipeline on one GPU by default
-([`dynin_omni.yaml`](../../../vllm_omni/deploy/dynin_omni.yaml)):
-
-- Stage-0 (`token2text`): `gpu_memory_utilization: 0.5`
-- Stage-1 (`token2image`): `gpu_memory_utilization: 0.1`
-- Stage-2 (`token2audio`): `gpu_memory_utilization: 0.1`
-
-### Requested GPU Memory Budget from `gpu_memory_utilization`
-
-| Stage | Utilization | A100 80GB | H200 141GB |
-| :-- | :-- | :-- | :-- |
-| Stage-0 (token2text) | 0.5 | ~40.0 GB | ~70.5 GB |
-| Stage-1 (token2image) | 0.1 | ~8.0 GB | ~14.1 GB |
-| Stage-2 (token2audio) | 0.1 | ~8.0 GB | ~14.1 GB |
-| Total requested budget | 0.7 | ~56.0 GB | ~98.7 GB |
-
-### Observed Runtime Signal (from your log)
-
-- Stage-0 reported: `Model loading took 15.12 GiB memory` (weights footprint signal).
-- Stages 1/2 can still add runtime memory depending on task path and backend allocations.
-- Keep extra headroom for CUDA/PyTorch overhead and temporary allocations.
-
-### GPU Compatibility
-
-- Confirmed target GPUs for this setup: **NVIDIA H200**, **NVIDIA A100**.
-- CI/e2e coverage in this repo also includes CUDA **L4** markers for Dynin tests.
-
-## 4. End2End Run Examples
-
-```bash
-# t2t
-python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
-  --task t2t --model snu-aidas/Dynin-Omni --text <INSTRUCTION_TEXT>
-
-# i2t
-python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
-  --task i2t --model snu-aidas/Dynin-Omni --image <IMAGE_PATH> --text "Please describe this image in detail."
-
-# s2t
-python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
-  --task s2t --model snu-aidas/Dynin-Omni --audio <AUDIO_PATH> --text "Transcribe the given audio."
-
-# t2i
-python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
-  --task t2i --model snu-aidas/Dynin-Omni --text <INSTRUCTION_TEXT>
-
-# v2t
-python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
-  --task v2t --model snu-aidas/Dynin-Omni --video <VIDEO_PATH> --text "Describe this video in detail."
-
-# i2i
-python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
-  --task i2i --model snu-aidas/Dynin-Omni --image <IMAGE_PATH> --text <INSTRUCTION_TEXT>
-
-# t2s
-python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
-  --task t2s --model snu-aidas/Dynin-Omni --text <INSTRUCTION_TEXT>
-```
-
-## 5. Notes
-
-- Outputs are saved under task-specific directories in `/tmp` by default.
-- You can override output path with `--output-dir`.
-- If you want to force local config resolution, pass `--dynin-config-path <PATH_TO_DYNIN_OMNI_YAML>`.
-- If you see the warning
-  `max_num_batched_tokens (32768) exceeds max_num_seqs * max_model_len (4096)`,
-  reduce `max_num_batched_tokens` in the deploy config (for example, `4096` in the CI config).

examples/offline_inference/image_to_image/image_edit.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
 """
 Example script for image editing with OmniGen2.
@@ -209,14 +209,14 @@ def parse_profiler_config(value: str) -> dict[str, Any]:
 
 
 def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Edit an image with Qwen-Image-Edit.")
+    parser = argparse.ArgumentParser(description="Edit images with vLLM-Omni.")
     parser.add_argument(
         "--model",
         default="Qwen/Qwen-Image-Edit",
         help=(
             "Diffusion model name or local path. "
-            "For multiple image inputs, use Qwen/Qwen-Image-Edit-2509 or Qwen/Qwen-Image-Edit-2511"
-            "which supports QwenImageEditPlusPipeline."
+            "For multiple image inputs, use Qwen/Qwen-Image-Edit-2509, "
+            "Qwen/Qwen-Image-Edit-2511, or Qwen/Qwen-Image-2.1."
         ),
     )
     parser.add_argument(
@@ -368,7 +368,7 @@ def parse_args() -> argparse.Namespace:
         "--color-format",
         type=str,
         default="RGB",
-        help="For Qwen-Image-Layered, set to RGBA.",
+        help="For Qwen-Image-Layered or Qwen-Image-2.1, set to RGBA to preserve transparency.",
     )
 
     # Cache-DiT specific parameters
@@ -647,6 +647,7 @@ def main():
 
     diffusion_params = OmniDiffusionSamplingParams(
         generator=generator,
+        seed=args.seed,
         true_cfg_scale=args.cfg_scale,
         guidance_scale=args.guidance_scale,
         guidance_scale_2=args.guidance_scale_2,

examples/offline_inference/image_to_image/run_qwen_image_21.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+set -euo pipefail
+
+python "$(dirname "${BASH_SOURCE[0]}")/image_edit.py" \
+    --model Qwen/Qwen-Image-2.1 \
+    --color-format RGBA \
+    --seed 42 \
+    --image qwen_bear.png \
+    --prompt "Let this mascot dance under the moon, surrounded by floating stars and poetic bubbles such as 'Be Kind'" \
+    --output qwen_image_21_edit.png \
+    --num-inference-steps 50 \
+    --cfg-scale 1.0 \
+    "$@"

examples/offline_inference/text_to_image/README.md

@@ -19,23 +19,23 @@ This folder provides several entrypoints for experimenting with text-to-image di
 
 ### Supported Models
 
-| Model | Image Shape  | Peak VRAM (GiB) * | Model Weights (GiB) |
+| Model | Image Shape | Peak VRAM (GiB) * | Model Weights (GiB) |
 | ----- | ----------- | ----------- | ----------------- |
 | `Qwen/Qwen-Image` | 1024 x 1024 | 60.0 | 53.7 |
-| `Qwen/Qwen-Image-2512` |1024 x 1024 | 60.0 | 53.7 |
+| `Qwen/Qwen-Image-2512` | 1024 x 1024 | 60.0 | 53.7 |
 | `Tongyi-MAI/Z-Image-Turbo` | 1024 x 1024 | 24.8 | 19.2 |
 | `stepfun-ai/NextStep-1.1` | 512 x 512 | 71.8 | 28.1 |
 | `meituan-longcat/LongCat-Image` | 1024 x 1024 | 71.2 | 27.3 |
 | `AIDC-AI/Ovis-Image-7B` | 1024 x 1024 | 71.8 | 17.1 |
-| `OmniGen2/OmniGen2` |  1024 x 1024 | 20.1 | 14.7 |
+| `OmniGen2/OmniGen2` | 1024 x 1024 | 20.1 | 14.7 |
 | `stabilityai/stable-diffusion-3.5-medium` | 1024 x 1024 | 20.1 | 15.6 |
 | `black-forest-labs/FLUX.1-dev` | 1024 x 1024 | 33.9 | 31.4 |
 | `black-forest-labs/FLUX.1-schnell` | 1024 x 1024 | 33.9 | 31.4 |
 | `black-forest-labs/FLUX.2-klein-4B` | 1024 x 1024 | 72.7 | 14.9 |
 | `black-forest-labs/FLUX.2-klein-9B` | 1024 x 1024 | 37.1 | 32.3 |
 | `black-forest-labs/FLUX.2-dev` | 1024 x 1024 | 65.7 | >80 (CPU offload required) |
-| `HunyuanImage-3.0` | 1024 x 1024 | 80.0 (TP≥3)  | 160 |
-| `HiDream-I1-Full` | 1024 x 1024 | 63.7  | 57.7 |
+| `HunyuanImage-3.0` | 1024 x 1024 | 80.0 (TP≥3) | 160 |
+| `HiDream-I1-Full` | 1024 x 1024 | 63.7 | 57.7 |
 | `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | 1024 (Raw) / 2048 (Turbo) | — | ~30 |
 
 !!! info
@@ -112,9 +112,9 @@ python text_to_image.py \
 | `--enable-cpu-offload` | flag | off | Enable CPU offloading for diffusion models |
 | `--lora-path` | str | — | Path to PEFT LoRA adapter folder or checkpoint file |
 | `--lora-scale` | float | `1.0` | Scale factor for LoRA weights |
-| `--lora-backend` | str |`"peft"`| LoRA backend for loading LoRA adapters. Default: peft. Available options: peft, distill |
-| `--use-system-prompt` | str | `None` | System prompt preset: `en_unified`, `en_vanilla`, `en_recaption`, `en_think_recaption`, `dynamic`, `None`, or custom text. Recommended: `en_unified`. Only for HunyuanImage-3.0.|
-| `--system-prompt` | str | `None` | Custom system prompt text. Only used when `--use-system-prompt` is set to `custom`. Only for HunyuanImage-3.0.|
+| `--lora-backend` | str | `"peft"` | LoRA backend for loading LoRA adapters. Default: peft. Available options: peft, distill |
+| `--use-system-prompt` | str | `None` | System prompt preset: `en_unified`, `en_vanilla`, `en_recaption`, `en_think_recaption`, `dynamic`, `None`, or custom text. Recommended: `en_unified`. Only for HunyuanImage-3.0. |
+| `--system-prompt` | str | `None` | Custom system prompt text. Only used when `--use-system-prompt` is set to `custom`. Only for HunyuanImage-3.0. |
 | `--auxiliary-text-encoder` | str | `None` | Supplementary auxiliary text encoder parameters model name or path (especially for Hidream-l1-full). |
 
 **NextStep-1.1 specific arguments:**
@@ -127,7 +127,7 @@ python text_to_image.py \
 | `--use-norm` | flag | off | Apply layer normalization to sampled tokens |
 
 > If you encounter OOM errors, try using `--vae-use-slicing` and `--vae-use-tiling` to reduce memory usage.
-
+>
 > Qwen-Image currently publishes best-effort presets at `1328x1328`, `1664x928`, `928x1664`, `1472x1140`, `1140x1472`, `1584x1056`, and `1056x1584`. Adjust `--height/--width` accordingly for the most reliable outcomes.
 
 ## More CLI Examples
@@ -216,6 +216,26 @@ python examples/offline_inference/text_to_image/text_to_image.py \
   --output /output.png
 ```
 
+### Anima Single-File Checkpoints
+
+Anima uses the official model dimensions by default. A checkpoint may provide a JSON file with the same stem (for example, `anima.json` beside `anima.safetensors`) containing `transformer` and `text_conditioner` configuration objects. These override the default dimensions and must match the checkpoint weights. The shared tiny-model tests use this to load small random-weight models. Components in `text_encoder/`, `vae/`, `tokenizer/`, `t5_tokenizer/`, and `scheduler/` beside the checkpoint are discovered automatically.
+
+To load Anima, point `--model` to the single-file checkpoint path, pass the native pipeline class name using `--model-class-name`, and supply the converted components directory using `--custom-pipeline-args`:
+
+```bash
+python examples/offline_inference/text_to_image/text_to_image.py \
+  --model /path/to/models/anima-official/split_files/diffusion_models/anima-base-v1.0.safetensors \
+  --model-class-name AnimaPipeline \
+  --custom-pipeline-args '{"components_path": "/path/to/models/anima-components"}' \
+  --prompt "A cinematic close-up of a glass teapot on a wooden table." \
+  --seed 42 \
+  --guidance-scale 4.0 \
+  --num-inference-steps 50 \
+  --height 1024 \
+  --width 1024 \
+  --output anima_output.png
+```
+
 ### Krea 2
 
 Krea 2 is published as diffusers-format checkpoints: `krea/Krea-2-Turbo` (few-step distilled) and `krea/Krea-2-Raw`.
@@ -336,7 +356,7 @@ python text_to_image.py \
 
 LoRA adapters must be in PEFT format. A typical adapter directory structure:
 
-```
+```text
 lora_adapter/
 ├── adapter_config.json
 └── adapter_model.safetensors

examples/offline_inference/text_to_image/text_to_image.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
 import argparse
 import functools
@@ -58,6 +58,7 @@ def parse_json_object(value: str, flag_name: str = "argument") -> dict[str, Any]
 
 
 parse_profiler_config = functools.partial(parse_json_object, flag_name="--profiler-config")
+parse_custom_pipeline_args = functools.partial(parse_json_object, flag_name="--custom-pipeline-args")
 
 
 def build_text_to_image_prompt(prompt: str, negative_prompt: str | None) -> dict[str, Any]:
@@ -88,7 +89,8 @@ def parse_args() -> argparse.Namespace:
         "--model",
         default="Qwen/Qwen-Image",
         help="Diffusion model name or local path. Supported models: "
-        "Qwen/Qwen-Image, Tongyi-MAI/Z-Image-Turbo, Qwen/Qwen-Image-2512, stepfun-ai/NextStep-1.1, "
+        "Qwen/Qwen-Image, Tongyi-MAI/Z-Image-Turbo, Qwen/Qwen-Image-2512, Qwen/Qwen-Image-2.1, "
+        "stepfun-ai/NextStep-1.1, "
         "black-forest-labs/FLUX.1-dev, black-forest-labs/FLUX.2-klein-9B, "
         "black-forest-labs/FLUX.2-dev, tencent/HunyuanImage-3.0-Instruct, "
         "HiDream-ai/HiDream-O1-Image, meituan-longcat/LongCat-Image, OvisAI/Ovis-Image, "
@@ -156,7 +158,11 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--enable-cache-dit-summary",
         action="store_true",
-        help="Enable cache-dit summary logging after diffusion forward passes.",
+        default=None,
+        help=(
+            "Enable cache-dit summary logging after diffusion forward passes. "
+            "Default: unset (defer to the deploy YAML's enable_cache_dit_summary)."
+        ),
     )
     parser.add_argument(
         "--ulysses-degree",
@@ -379,6 +385,18 @@ def parse_args() -> argparse.Namespace:
         default=None,
         help="Supplementary auxiliary text encoder parameters model name or path (especially for Hidream-l1-full).",
     )
+    parser.add_argument(
+        "--model-class-name",
+        type=str,
+        default=None,
+        help="Override the diffusion pipeline class name (e.g. AnimaPipeline).",
+    )
+    parser.add_argument(
+        "--custom-pipeline-args",
+        type=parse_custom_pipeline_args,
+        default=None,
+        help='JSON object passed to native/custom pipelines (e.g. \'{"components_path": "/path"}\').',
+    )
     parser.add_argument(
         "--trust-remote-code",
         action="store_true",
@@ -558,7 +576,6 @@ def main():
         "vae_patch_parallel_size": args.vae_patch_parallel_size,
         "enable_expert_parallel": args.enable_expert_parallel,
         "enable_cpu_offload": args.enable_cpu_offload,
-        "mode": "text-to-image",
         "log_stats": args.log_stats,
         "enable_diffusion_pipeline_profiler": args.enable_diffusion_pipeline_profiler,
         "profiler_config": args.profiler_config,
@@ -576,9 +593,13 @@ def main():
         omni_kwargs["trust_remote_code"] = True
     if args.deploy_config:
         omni_kwargs["deploy_config"] = args.deploy_config
-    if use_nextstep:
+    if args.model_class_name:
+        omni_kwargs["model_class_name"] = args.model_class_name
+    elif use_nextstep:
         # NextStep-1.1 requires explicit pipeline class
         omni_kwargs["model_class_name"] = "NextStep11Pipeline"
+    if args.custom_pipeline_args is not None:
+        omni_kwargs["custom_pipeline_args"] = args.custom_pipeline_args
     # Cosmos3 loads its (gated) guardrail models at build time, so the guardrails
     # gate is an engine-level config (offline analog of the server's --no-guardrails).
     if args.extra_body and "guardrails" in args.extra_body:
@@ -614,6 +635,10 @@ def main():
         print(f"  LoRA: scale={args.lora_scale}")
     if args.deploy_config:
         print(f"  deploy-config: {args.deploy_config}")
+    if args.model_class_name:
+        print(f"  Model class name: {args.model_class_name}")
+    if args.custom_pipeline_args is not None:
+        print(f"  Custom pipeline args: {args.custom_pipeline_args}")
     print(f"{'=' * 60}\n")
 
     # Build LoRA request when --lora-path is set
@@ -723,7 +748,10 @@ def main():
     # stop-token-ids declaratively from the plain prompt + extra_body, so this
     # example stays model-agnostic. Models without one are untouched.
     ar_input_builder = get_ar_input_builder(model_class_name)
-    if ar_input_builder is not None:
+    # A model can also be deployed with only its diffusion stage. Keep those
+    # requests on the string-prompt path, as in the single-stage images API.
+    has_ar_stage = any(not isinstance(params, OmniDiffusionSamplingParams) for params in sampling_params_list)
+    if ar_input_builder is not None and has_ar_stage:
         _apply_ar_stage_inputs(
             ar_input_builder,
             model=args.model,

examples/offline_inference/text_to_speech/README.md

@@ -13,7 +13,7 @@ list of supported architectures across all modalities, see
 ## Supported Models
 
 | Model | HuggingFace repo | Stages | Voice cloning | Streaming | Special modes | Sample rate |
-|---|---|---|---|---|---|---|
+| --- | --- | --- | --- | --- | --- | --- |
 | AuK | `tencent/AuK`, `tencent/AuK-Flash` (assembled with `tools/prepare_auk_checkpoint.py`) | 2 (thinker encoder + DiT) | ✓ | — | instruction-driven editing, enhancement, separation; no example script (use `vllm_omni.model_extras.auk`) | 24 kHz |
 | Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | dual-AR | ✓ | ✓ | 11 languages | 44.1 kHz |
 | Breeze-TTS-2 | `BreezeBlue/Breeze-TTS-2` | 2 (talker + codec) | ✓ | ✓ (async chunk) | voice design / voice direction (`--instruction`) | 24 kHz |
@@ -27,7 +27,6 @@ list of supported architectures across all modalities, see
 | OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design, language hint | 24 kHz |
 | Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | 2 (talker + code2wav) | ✓ (Base) | ✓ | 3 task variants | 24 kHz |
 | VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | ✓ (online) | continuation | 48 kHz |
-| dots.tts | `dots-studio/dots.tts-soar` | single (native AR) | — (text-only) | ✓ (online) | no-reference synthesis | 48 kHz |
 | IndexTTS-2 | `IndexTeam/IndexTTS-2` | 2 (AR talker + S2Mel DiT + BigVGAN) | ✓ (required) | — | emotion control (`--emo-audio`, `--emo-text`, `--emo-vector`) | 22.05 kHz |
 | IndexTTS-2.5 | native `checkpoints/` bundle | 2 (AR talker + EnhancedCodec + S2Mel DiT + BigVGAN) | ✓ (required) | — | multilingual (`--lang`) + emotion control | 22.05 kHz |
 | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | varies | ✓ | ✓ | voice presets | 24 kHz |
@@ -54,32 +53,38 @@ codec: a slow AR transformer predicts one semantic token per codec frame, and a
 4-layer fast AR predicts that frame's remaining 9 codebooks.
 
 ### Prerequisites
+
 No extra packages: the codec architecture ships in tree and its weights
 (`codec.pth`) come with the checkpoint.
 
 ### Quick start
+
 ```bash
 python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
     --text "Welcome to Audio8 TTS, a compact multilingual text to speech model."
 ```
 
 ### Voice cloning
+
 ```bash
 python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
     --text "Welcome to Audio8 TTS." \
     --ref-audio /path/to/reference.wav \
     --ref-text  "The exact transcript of the reference recording."
 ```
+
 The reference transcript must match what is actually spoken in the clip;
 a mismatch degrades both stability and speaker similarity.
 
 ### Streaming
+
 ```bash
 python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
     --text "Welcome to Audio8 TTS." --streaming
 ```
 
 ### Notes
+
 - Output: 44.1 kHz mono WAV; the codec emits ~21.5 frames/s (2048 samples per frame).
 - Context is 2048 packed text+audio positions, so `max_model_len` is 2048.
 - Greedy decoding (`temperature: 0`) is degenerate for this checkpoint — it
@@ -92,6 +97,7 @@ python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
 - `--num-prompts 4` smoke-tests concurrency (stage 0 runs `max_num_seqs: 4`).
 
 ### Measured behaviour (1x H20, shared GPU)
+
 `--streaming` on a 58-character English sentence: TTFA ~0.74 s (4-frame first
 chunk), 7 chunks, 1.55 s wall for 3.16 s of audio (RTF ~0.49). Treat these as
 smoke-test magnitudes, not a benchmark.
@@ -103,12 +109,14 @@ smoke-test magnitudes, not a benchmark.
 2-stage TTS pipeline (`talker` + `code2wav`) at 24 kHz.
 
 ### Prerequisites
+
 ```bash
 uv pip install -e .
 # Includes soundfile, onnxruntime, x-transformers, einops via requirements.
 ```
 
 Download the model snapshot:
+
 ```python
 from huggingface_hub import snapshot_download
 snapshot_download('FunAudioLLM/Fun-CosyVoice3-0.5B-2512',
@@ -116,15 +124,18 @@ snapshot_download('FunAudioLLM/Fun-CosyVoice3-0.5B-2512',
 ```
 
 If your downloaded checkpoint lacks `config.json`, add it:
+
 ```json
 {
     "model_type": "cosyvoice3",
     "architectures": ["CosyVoice3Model"]
 }
 ```
+
 This is required because `AutoConfig.register("cosyvoice3", CosyVoice3Config)` only registers the class mapping; the loader still reads `model_type` from `config.json` to select the class.
 
 ### Quick start
+
 ```bash
 python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
     --model pretrained_models/Fun-CosyVoice3-0.5B \
@@ -132,9 +143,11 @@ python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
 ```
 
 ### Voice cloning
+
 If `--ref-audio` is omitted, the script downloads the upstream
 [`zero_shot_prompt.wav`](https://github.com/FunAudioLLM/CosyVoice/blob/main/asset/zero_shot_prompt.wav) from the CosyVoice repo into the current directory.
 To use your own clip, pass `--ref-audio /path/to/reference.wav`, and modify `--prompt-text` correspondingly.
+
 ```bash
 python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
     --model pretrained_models/Fun-CosyVoice3-0.5B \
@@ -144,9 +157,11 @@ python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
 ```
 
 ### Streaming
+
 Streaming is enabled by default via `async_chunk: true` in `vllm_omni/deploy/cosyvoice3.yaml`. Pass `--no-async-chunk` on `vllm serve` to switch to the legacy synchronous path.
 
 ### Notes
+
 - Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform.
 - Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config <path>` to override.
 
@@ -157,6 +172,7 @@ Streaming is enabled by default via `async_chunk: true` in `vllm_omni/deploy/cos
 2-stage TTS pipeline (AR + DiT flow-matching) at 24 kHz. Every request requires reference audio and its transcript for zero-shot voice cloning.
 
 ### Quick start
+
 ```bash
 python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
     --model zai-org/GLM-TTS \
@@ -167,12 +183,14 @@ python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
 ```
 
 ### Architecture
+
 ```text
 Text → [Stage 0: AR] → Speech Tokens → [Stage 1: DiT + HiFT] → Audio (24 kHz)
         (Llama-based)    (32k vocab)      (Flow Matching)
 ```
 
 ### Notes
+
 - `--ref-audio` and `--ref-text` are **required** together; GLM-TTS does not support text-only synthesis.
 - Reference audio should be 3-10 seconds.
 - First run may be slow due to lazy loading of WhisperVQ tokenizer and CampPlus ONNX speaker embedder.
@@ -187,12 +205,14 @@ Text → [Stage 0: AR] → Speech Tokens → [Stage 1: DiT + HiFT] → Audio (24
 No extra packages are required; the DAC codec is vendored in vLLM-Omni.
 
 ### Quick start
+
 ```bash
 python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
     --text "Hello, this is a test of the Fish Speech text to speech system."
 ```
 
 ### Voice cloning
+
 ```bash
 python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
     --text "Hello, this is a cloned voice." \
@@ -201,14 +221,17 @@ python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
 ```
 
 ### Streaming
+
 ```bash
 python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
     --text "Hello, this is a streaming test." \
     --streaming
 ```
+
 Streaming requires `async_chunk: true` in the deploy config.
 
 ### Notes
+
 - Output: 44.1 kHz mono WAV.
 - DAC codec weights (`codec.pth`) are loaded lazily from the model directory.
 
@@ -219,6 +242,7 @@ Streaming requires `async_chunk: true` in the deploy config.
 Dense 0.5B two-stage TTS pipeline (`AR + flow` + audio VAE) at 44.1 kHz. The example covers style, IP voice, music-only generation, text-to-audio events, emotion, dialect, zero-shot cloning, podcast, speech+BGM, and speech+environment-sound cases.
 
 ### Quick start
+
 ```bash
 python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
     --case style \
@@ -227,6 +251,7 @@ python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
 ```
 
 ### Voice cloning
+
 ```bash
 python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
     --case zero_shot \
@@ -237,6 +262,7 @@ python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
 ```
 
 ### Streaming
+
 ```bash
 python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
     --case basic \
@@ -247,6 +273,7 @@ python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
 ```
 
 ### Notes
+
 - `style`, `ip`, `bgm`, and `tta` do not require reference audio.
 - Reference-audio cases use `--ref-audio`; `zero_shot` also requires `--ref-text`.
 - `podcast` uses multiple references via `--ref-audio-paths`.
@@ -259,14 +286,17 @@ python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
 Standalone talker-only deployment of Ming-flash-omni-2.0 at 44.1 kHz. Voice is controlled through caption fields (`风格` / `IP` / `语速`/`基频`/`音量`) rather than reference audio.
 
 ### Prerequisites
+
 The example calls into `vllm_omni.model_executor.models.ming_flash_omni.prompt_utils` for the default prompt and instruction builder; no extra pip install on top of the base vLLM-Omni install.
 
 ### Quick start
+
 ```bash
 python examples/offline_inference/text_to_speech/ming_flash_omni_tts/end2end.py --case style
 ```
 
 ### Cases
+
 ```bash
 # ASMR-style whisper (caption-driven)
 python examples/offline_inference/text_to_speech/ming_flash_omni_tts/end2end.py --case style
@@ -281,6 +311,7 @@ python examples/offline_inference/text_to_speech/ming_flash_omni_tts/end2end.py
 Override the default text per case with `--text`, write to a custom path with `--output`.
 
 ### Notes
+
 - Talker-only deployment — for the multimodal Ming-flash-omni example, see [`examples/offline_inference/ming_flash_omni/`](../../ming_flash_omni/).
 - Deploy config: `vllm_omni/deploy/ming_flash_omni_tts.yaml` (single GPU, `enforce_eager`, `max_num_seqs: 1`).
 - Decode defaults from the Ming cookbook: `max_decode_steps=200`, `cfg=2.0`, `sigma=0.25`, `temperature=0.0`, `use_zero_spk_emb=True`.
@@ -294,6 +325,7 @@ Single-stage 0.1B AR LM + MOSS-Audio-Tokenizer-Nano codec at 48 kHz mono (mixed
 > **No built-in speaker presets.** `--ref-audio` is required on every call. Default `--mode voice_clone` matches upstream's recommended workflow; `--mode continuation` is exposed for completeness but upstream's continuation-with-prompt path emits very short / near-silent output, so it is rarely useful in practice. Sample reference clips ship in the upstream repo under [`assets/audio/`](https://github.com/OpenMOSS/MOSS-TTS-Nano/tree/main/assets/audio) (e.g. `zh_1.wav`, `en_2.wav`, `jp_2.wav`).
 
 ### Quick start
+
 ```bash
 # Fetch a sample reference clip (one-off, user-scoped cache).
 REF_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/moss-tts-nano"
@@ -305,9 +337,11 @@ python examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py \
     --text "你好，这是MOSS-TTS-Nano的语音合成演示。" \
     --ref-audio "$REF_DIR/zh_1.wav"
 ```
+
 The first run downloads `OpenMOSS-Team/MOSS-TTS-Nano` and `OpenMOSS-Team/MOSS-Audio-Tokenizer-Nano` from Hugging Face.
 
 ### Reproducible runs
+
 ```bash
 python examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py \
     --text "Deterministic test." \
@@ -316,6 +350,7 @@ python examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py \
 ```
 
 ### Notes
+
 - Output: 48 kHz mono WAV (the tokenizer is internally stereo at 48 kHz; the wrapper averages to mono before reaching the engine).
 - Deploy config: `vllm_omni/deploy/moss_tts_nano.yaml` (auto-loaded; override with `--deploy-config`).
 - Default `--max-new-frames 375` ≈ 14 s of audio; raise for longer outputs.
@@ -329,12 +364,15 @@ python examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py \
 Zero-shot multilingual TTS supporting 600+ languages, with three modes (auto / clone / design).
 
 ### Prerequisites
+
 ```bash
 huggingface-cli download k2-fsa/OmniVoice
 ```
+
 Voice cloning requires `transformers>=5.3.0`. Auto and design modes work with `transformers>=4.57.0`.
 
 ### Quick start (auto voice)
+
 ```bash
 python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
     --model k2-fsa/OmniVoice \
@@ -342,6 +380,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
 ```
 
 ### Voice cloning
+
 ```bash
 python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
     --model k2-fsa/OmniVoice \
@@ -351,6 +390,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
 ```
 
 ### Voice design
+
 ```bash
 python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
     --model k2-fsa/OmniVoice \
@@ -359,6 +399,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
 ```
 
 ### Language hint
+
 ```bash
 python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
     --model k2-fsa/OmniVoice \
@@ -367,6 +408,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
 ```
 
 ### Seed for Reproducibility
+
 ```bash
 python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
     --model k2-fsa/OmniVoice \
@@ -375,6 +417,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
 ```
 
 ### Notes
+
 - Stage 0 (Generator): Qwen3-0.6B with 32-step iterative unmasking.
 - Stage 1 (Decoder): HiggsAudioV2 RVQ + DAC at 24 kHz.
 
@@ -385,13 +428,16 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
 3-task-variant TTS with 24 kHz output. Has its own argparse surface (this script does not follow the common `--text` / `--ref-audio` shape).
 
 ### Prerequisites
+
 For ROCm builds, replace `onnxruntime` with `onnxruntime-rocm`:
+
 ```bash
 pip uninstall onnxruntime
 pip install onnxruntime-rocm
 ```
 
 ### Task variants
+
 - `CustomVoice`: predefined speaker (speaker ID) with optional style instruction.
 - `VoiceDesign`: text + descriptive instruction designs a new voice.
 - `Base`: voice cloning from reference audio + transcript.
@@ -415,22 +461,27 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-ty
 ```
 
 ### Streaming
+
 ```bash
 python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
     --query-type CustomVoice \
     --streaming \
     --output-dir /tmp/out_stream
 ```
+
 Streaming requires `async_chunk: true` in the deploy config.
 
 ### Word Timestamps
+
 Word-level timestamps are currently a serving-path feature: launch
 `vllm-omni serve` with `--forced-aligner` and request `word_timestamps`
 (see `examples/online_serving/text_to_speech/README.md`). An offline
 example is not provided in this release.
 
 ### Batched decoding
+
 The Code2Wav stage supports batched decoding through the SpeechTokenizer. Pass multiple prompts via `--txt-prompts` and set `--batch-size` accordingly. To raise `max_num_seqs` on either stage, create an overlay based on `vllm_omni/deploy/qwen3_tts.yaml` and pass it with `--deploy-config`:
+
 ```bash
 python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
     --query-type CustomVoice \
@@ -438,9 +489,11 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
     --batch-size 4 \
     --deploy-config /path/to/qwen3_tts_batched.yaml
 ```
+
 `--batch-size` must match a CUDA-graph capture size (1, 2, 4, 8, 16…).
 
 ### Notes
+
 - Run `--help` for the full argument surface.
 - See `qwen3_tts/end2end.py` for the prompt-length-estimation logic the Talker uses.
 
@@ -451,6 +504,7 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
 Single-stage native AR TTS at 22.05 kHz. Pipeline: `Qwen3.5 backbone → 32 FSQ codebook heads (one frame/step) → NeMo NanoCodec`. Backbone runs under vLLM paged attention; the 32-head sampling + learned embedding feedback ride the native-AR runner hooks. `enforce_eager` (CUDA graph is a perf follow-up).
 
 ### Prerequisites
+
 The NanoCodec decoder needs NeMo (NVIDIA Open Model License), installed separately:
 
 ```bash
@@ -480,15 +534,18 @@ failure rather than an environment one. The PyTorch sampler it falls back to cos
 here.
 
 ### Quick start (zero-shot, default voice)
+
 ```bash
 python examples/offline_inference/text_to_speech/gepard/end2end.py \
     --text "Hello, this is a Gepard demo."
 ```
 
 ### Voice cloning
+
 Not yet — PR1 is zero-shot only (the learned `null_prefix` default voice). Reference-audio cloning (`ref_compressor` speaker prefix) is a follow-up PR.
 
 ### Notes
+
 - Output: 22.05 kHz mono WAV.
 - First run downloads two checkpoints, not one: the model itself, and the NanoCodec decoder it names in `codec_id` (`nvidia/nemo-nano-codec-22khz-1.89kbps-21.5fps` by default). Both are fetched automatically; expect the first startup to be correspondingly slower.
 - Offline and online: `examples/offline_inference/text_to_speech/gepard/end2end.py` and `/v1/audio/speech` (see the [online Gepard section](../../online_serving/text_to_speech/README.md#gepard-10)).
@@ -502,13 +559,15 @@ Not yet — PR1 is zero-shot only (the learned `null_prefix` default voice). Ref
 Single-stage native AR TTS at 48 kHz. Pipeline: `feat_encoder → MiniCPM4 → FSQ → residual_lm → LocDiT → AudioVAE`.
 
 ### Prerequisites
+
 ```bash
 pip install voxcpm
 # or, for a local source checkout:
-export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/voxcpm
+pip install -e /path/to/voxcpm
 ```
 
 ### Quick start
+
 ```bash
 python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
     --model openbmb/VoxCPM2 \
@@ -516,7 +575,9 @@ python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
 ```
 
 ### Voice cloning
+
 Pass a reference audio for isolated cloning, or both `--ref-audio` + `--ref-text` for prompt continuation:
+
 ```bash
 python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
     --text "Hello, this is a voice clone demo." \
@@ -525,39 +586,16 @@ python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
 ```
 
 ### Streaming
+
 Streaming is exposed through the online OpenAI Speech API (`stream=true`). See [`examples/online_serving/text_to_speech/voxcpm2/gradio_demo.py`](../../online_serving/text_to_speech/voxcpm2/gradio_demo.py) for an AudioWorklet-based gapless streaming player; the offline `end2end.py` script does not expose a streaming path.
 
 ### Notes
+
 - Output: 48 kHz mono WAV.
 - Deploy config: `vllm_omni/deploy/voxcpm2.yaml` (auto-loaded by HF `model_type`).
 
 ---
 
-## dots.tts
-
-Single-stage native AR TTS at 48 kHz (rednote-hilab). Pipeline: `Qwen2.5-1.5B base LM → DiT (10-step Euler flow matching) → patch_encoder AR loopback → AudioVAE (streaming decode)`. Same "vLLM-native base LM + side-path computation" pattern as VoxCPM2, with a plain Qwen2 backbone instead of MiniCPM4 and no FSQ / residual LM stage.
-
-### Quick start
-```bash
-python examples/offline_inference/text_to_speech/dots_tts/end2end.py \
-    --model dots-studio/dots.tts-soar \
-    --text "Hello, this is a test of dots TTS running on vLLM Omni."
-```
-
-### Voice cloning
-Not wired in this release. Generation is text-only and does not consume reference audio, a named voice, or a precomputed speaker embedding. The CAM++ x-vector speaker encoder weights load, but `end2end.py` has no `--ref-audio`/`--ref-text` flags yet.
-
-### Streaming
-The AudioVAE decoder emits incremental audio through its internal streaming path (`init_stream_state` / `stream_step` / `stream_flush`). Online streaming is exposed through the OpenAI-compatible `/v1/audio/speech` endpoint with `stream=true`; see the [online serving guide](../../online_serving/text_to_speech/README.md#dotstts). The offline `end2end.py` script returns the consolidated waveform.
-
-### Notes
-- Output: 48 kHz mono WAV.
-- Deploy config: `vllm_omni/deploy/dots_tts.yaml` (auto-loaded by HF `model_type`).
-- Checkpoints: `dots-studio/dots.tts-soar` is the validated default. `dots.tts-base` shares the same architecture but is unvalidated in this repo. `dots.tts-mf` (MeanFlow, 2-4 step) is not supported yet.
-- Known limitation: no CUDA graph capture and no batched side-path computation yet, so concurrent requests do not currently scale (each request's DiT Euler steps run serially). See `recipes/rednote-hilab/dots.tts.md` for details and the roadmap.
-
----
-
 ## IndexTTS-2 and IndexTTS-2.5
 
 Both versions use a 2-stage TTS pipeline and produce 22.05 kHz mono audio. Stage 0 is a GPT AR talker. IndexTTS-2 Stage 1 uses RepCodec semantic embeddings, GPT latent, S2Mel CFM/DiT, and BigVGAN. IndexTTS-2.5 replaces RepCodec with EnhancedCodec and uses the official code-only path (`use_gpt_latent=false`). Every request requires reference audio for zero-shot voice cloning.
@@ -592,6 +630,7 @@ python examples/offline_inference/text_to_speech/indextts2/end2end.py \
 ```
 
 ### Emotion control
+
 ```bash
 # Emotion from reference audio
 python examples/offline_inference/text_to_speech/indextts2/end2end.py \
@@ -616,6 +655,7 @@ python examples/offline_inference/text_to_speech/indextts2/end2end.py \
 ```
 
 ### Notes
+
 - `--ref-audio` is **required** — neither version provides text-only synthesis.
 - IndexTTS-2.5 Stage 0 (AR Talker) generates mel codes from text + reference audio using plain vLLM sampling. It does not reproduce the official default `num_beams=3` beam search. For parity comparisons, run upstream with `num_beams=1`; output quality can differ from the official beam-search result.
 - IndexTTS-2.5 accepts language codes such as `zh`, `en`, `zhen` (mixed Chinese/English), `ja`, and `yue`. `Mandarin` is a vLLM-Omni convenience alias for `zh`. Japanese (`ja`) uses `fugashi` tokenization and produces audio, but it does not automatically expand numbers, dates, or percentages; write those inputs as readable Japanese text before calling the model. Use `--no-text-normalization` only when the input is already normalized.
@@ -631,12 +671,15 @@ python examples/offline_inference/text_to_speech/indextts2/end2end.py \
 Voxtral-4B-TTS (Mistral). Has its own argparse surface; uses voice presets and the `mistral_common` `SpeechRequest` protocol.
 
 ### Prerequisites
+
 Latest `mistral_common` with `SpeechRequest` support:
+
 ```bash
 pip install -e /path/to/mistral-common  # or upgrade from PyPI when available
 ```
 
 ### Quick start (voice preset)
+
 ```bash
 python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
     --write-audio --voice cheerful_female \
@@ -645,6 +688,7 @@ python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
 ```
 
 ### Voice cloning (capability gated upstream)
+
 ```bash
 python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
     --write-audio \
@@ -654,12 +698,14 @@ python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
 ```
 
 ### Streaming + concurrency
+
 ```bash
 python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
     --num-prompts 32 --concurrency 8 --streaming --write-audio --voice neutral_female \
     --model mistralai/Voxtral-4B-TTS-2603 \
     --text "..."
 ```
+
 Available voice presets are listed on the HF model card (`mistralai/Voxtral-4B-TTS-2603`).
 
 ### Notes

examples/offline_inference/text_to_speech/dots_tts/end2end.py

@@ -1,125 +0,0 @@
-"""Offline dots.tts inference example (single-stage native AR pipeline).
-
-Uses the single-stage native AR config (vllm_omni/deploy/dots_tts.yaml).
-Reference checkpoint: dots-studio/dots.tts-soar.
-"""
-
-from __future__ import annotations
-
-import os
-import time
-from pathlib import Path
-
-import soundfile as sf
-import torch
-
-from vllm_omni import Omni
-from vllm_omni.utils.tracking_parser import TrackingArgumentParser
-
-REPO_ROOT = Path(__file__).resolve().parents[4]
-SAMPLE_RATE = 48_000
-
-
-def parse_args():
-    parser = TrackingArgumentParser(description="Offline dots.tts native AR inference")
-    parser.add_argument(
-        "--model",
-        type=str,
-        default="dots-studio/dots.tts-soar",
-        help="dots.tts model path or HuggingFace repo ID.",
-    )
-    parser.add_argument(
-        "--text",
-        type=str,
-        default="Hello, this is a test of dots TTS running on vLLM Omni.",
-        help="Text to synthesize.",
-    )
-    parser.add_argument(
-        "--output-dir",
-        type=str,
-        default="output_audio",
-        help="Directory for output WAV files.",
-    )
-    parser.add_argument(
-        "--deploy-config",
-        type=str,
-        default=None,
-        help="Override the deploy config path. If unset, auto-loads "
-        "vllm_omni/deploy/dots_tts.yaml based on the HF model_type.",
-    )
-    return parser.parse_args()
-
-
-def extract_audio(multimodal_output: dict) -> torch.Tensor:
-    """Extract the final complete audio tensor from multimodal output.
-
-    The output processor concatenates per-step delta tensors under
-    ``model_outputs``.  Falls back to ``audio`` for backwards compat.
-    """
-    audio = multimodal_output.get("model_outputs")
-    if audio is None:
-        audio = multimodal_output.get("audio")
-    if audio is None:
-        raise ValueError(f"No audio key in multimodal_output: {list(multimodal_output.keys())}")
-
-    if isinstance(audio, list):
-        valid = [torch.as_tensor(a).float().cpu().reshape(-1) for a in audio if a is not None]
-        if not valid:
-            raise ValueError("Audio list is empty or all elements are None.")
-        return torch.cat(valid, dim=0) if len(valid) > 1 else valid[0]
-
-    return torch.as_tensor(audio).float().cpu().reshape(-1)
-
-
-def main():
-    args = parse_args()
-
-    output_dir = Path(args.output_dir)
-    output_dir.mkdir(parents=True, exist_ok=True)
-
-    engine = Omni(
-        model=args.model,
-        deploy_config=args.deploy_config,
-    )
-
-    from transformers import AutoTokenizer
-
-    from vllm_omni.model_executor.models.dots_tts.dots_tts_prompt import (
-        build_dots_tts_prompt,
-    )
-
-    tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
-
-    prompt = build_dots_tts_prompt(
-        tokenizer=tokenizer,
-        text=args.text,
-    )
-
-    print(f"Model       : {args.model}")
-    print(f"Text        : {args.text}")
-    print(f"Output dir  : {output_dir}")
-    print(f"Prompt len  : {len(prompt['prompt_token_ids'])} tokens")
-
-    t_start = time.perf_counter()
-    outputs = engine.generate([prompt])
-    elapsed = time.perf_counter() - t_start
-
-    request_output = outputs[0]
-    mm = request_output.outputs[0].multimodal_output
-    audio = extract_audio(mm)
-
-    duration = audio.numel() / SAMPLE_RATE
-    rtf = elapsed / duration if duration > 0 else float("inf")
-
-    output_path = output_dir / "output.wav"
-    sf.write(str(output_path), audio.numpy(), SAMPLE_RATE, format="WAV")
-
-    print(f"Saved       : {output_path}")
-    print(f"Duration    : {duration:.2f}s")
-    print(f"Inference   : {elapsed:.2f}s")
-    print(f"RTF         : {rtf:.3f}")
-
-
-if __name__ == "__main__":
-    os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-    main()

examples/offline_inference/text_to_speech/voxcpm2/end2end.py

@@ -1,7 +1,10 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 """Offline VoxCPM2 inference example (native AR pipeline).
 
 Uses the single-stage native AR config (voxcpm2.yaml).
-Requires the `voxcpm` package or VLLM_OMNI_VOXCPM_CODE_PATH env var.
+Requires the `voxcpm` package (pip install voxcpm).
 """
 
 from __future__ import annotations

examples/online_serving/aura_omni/README.md

@@ -6,139 +6,89 @@
 Qwen3-ASR -> AURA/Qwen3-VL -> Qwen3-TTS Talker -> Qwen3-TTS Code2Wav
 ```
 
-The pipeline has three semantic modules, but four engine stages because the
-existing Qwen3-TTS implementation is natively split into Talker and Code2Wav.
+**AURA's supported online path is Realtime duplex** (`/v1/realtime?duplex=1`).
+The default deploy profile sets `session_mode: duplex` because the pipeline
+declares `duplex_plugin` and `DuplexOmniEngine` requires that mode.
 
-Start the server with the deploy profile:
+This is AURA v1 / Qwen3-VL (silent / ChatML ids `151669` / `151645`), not
+AURA v2 / Qwen3.5-VL.
+
+## Duplex Realtime (only supported online path)
+
+Stage2→3 codec handoff needs `async_chunk: true`. The default
+`vllm_omni/deploy/aura_omni.yaml` keeps `async_chunk: false` as a safer
+baseline (turn-based `OmniOrchestrator` never sees the duplex async-chunk
+gate). For Realtime, use the smoke deploy (already `async_chunk: true`) or
+set that flag yourself:
 
 ```bash
+bash examples/online_serving/aura_omni/run_duplex_smoke_serve.sh
+python examples/online_serving/aura_omni/smoke_duplex_realtime_client.py
+```
+
+Smoke deploy file: `examples/online_serving/aura_omni/aura_omni_duplex_smoke.yaml`.
+
+Equivalent serve with the default topology and async chunks enabled:
+
+```bash
+# Copy or edit so async_chunk: true, then:
 vllm serve aurateam/AURA \
   --omni \
   --port 8091 \
-  --deploy-config vllm_omni/deploy/aura_omni.yaml \
+  --deploy-config examples/online_serving/aura_omni/aura_omni_duplex_smoke.yaml \
   --served-model-name aurateam/AURA \
   --trust-remote-code
 ```
 
-The deploy file sets per-stage model repos:
+Connect clients to `/v1/realtime?duplex=1`. Silent Stage1 outputs gate TTS (no
+audio for that turn). Overlapped input and vision-follow are AURA duplex
+capabilities.
+
+### Browser UI
+
+AURA has no client VAD: control is **push-to-talk** on the shared shell from
+[#7585](https://github.com/vllm-project/vllm-omni/pull/7585)
+([`examples/online_serving/realtime_web/`](../realtime_web/README.md)), profile
+`aura-ptt` (hold = `is_speech=true` PCM + sticky frames; release = `commit`;
+silent+frames for vision-follow). Thin wrapper:
+
+```bash
+python -m examples.online_serving.aura_omni.realtime_web \
+    --ws-backend ws://127.0.0.1:8099 --model aurateam/AURA --port 7862
+```
+
+Do **not** fork a separate `aura_omni/realtime_web` asset tree; MiniCPM / Qwen
+profiles are unchanged and hide the PTT control.
+
+### Per-stage models
+
+Edit `model` on each stage in `vllm_omni/deploy/aura_omni.yaml` (or the smoke
+YAML) for local checkpoints:
 
 - Stage 0 ASR: `Qwen/Qwen3-ASR-1.7B`
 - Stage 1 AURA: `aurateam/AURA`
-- Stage 2/3 TTS: `Qwen/Qwen3-TTS-12Hz-1.7B-Base`
+- Stage 2/3 TTS: `Qwen/Qwen3-TTS-12Hz-1.7B-Base` (or CustomVoice)
 
-For local weights, edit the `model` value on each stage in
-`vllm_omni/deploy/aura_omni.yaml`. The deploy profile includes
-`pipeline: aura_omni`, so the server uses this four-stage topology even when
-the command-line model path points at one component checkpoint.
+### GPU utilization
 
-Expected request shape:
-
-- Send microphone audio as the Stage 0 multimodal audio input.
-- Include video frames in the original request `multi_modal_data`; the
-  `asr2aura` processor carries them forward to AURA.
-- Optional `additional_information` keys:
-  - `aura_system_prompt`
-  - `tts_task_type`
-  - `tts_language`
-  - `tts_speaker`
-  - `tts_instruct`
-  - `tts_ref_audio`
-  - `tts_ref_text`
-  - `tts_x_vector_only_mode`
-  - `tts_pass_token_ids`
-
-If AURA emits `<|silent|>`, the `aura2tts` processor returns no TTS request, so
-the TTS stages are skipped for that turn.
-
-## GPU Utilization Recommendation
-
-Tune `gpu_memory_utilization` per stage in `vllm_omni/deploy/aura_omni.yaml`.
-Recommended baseline on one GPU for H200
+Tune `gpu_memory_utilization` per stage. Baseline on one large GPU:
 
 - Stage 0 (ASR): `0.10`
-- Stage 1 (AURA): `0.4`
-- Stage 2 (Qwen3-TTS Talker): `0.20`
-- Stage 3 (Qwen3-TTS Code2Wav): `0.20`
+- Stage 1 (AURA): `0.40`
+- Stage 2 (Talker): `0.20`
+- Stage 3 (Code2Wav): `0.20`
 
-## Python Client
+## Old chat / Gradio / curl scripts (unsupported)
 
-```bash
-python examples/online_serving/aura_omni/openai_chat_completion_client.py \
-  --host localhost \
-  --port 8091 \
-  --model aurateam/AURA \
-  --modalities text,audio
-```
+`openai_chat_completion_client.py`, `run_curl_multimodal_generation.sh`, and
+`run_gradio_demo.sh` target the older turn-based Omni path. They do **not**
+work against the shipped `session_mode: duplex` profile.
 
-Use local media:
-
-```bash
-python examples/online_serving/aura_omni/openai_chat_completion_client.py \
-  --audio-path /path/to/input.wav \
-  --video-path /path/to/video.mp4 \
-  --output-dir output_aura_omni_online
-```
-
-Base voice clone mode (default, recommended as x-vector while debugging ICL):
-
-```bash
-python examples/online_serving/aura_omni/openai_chat_completion_client.py \
-  --tts-task-type Base \
-  --tts-ref-audio vllm-omni/tests/assets/qwen3_tts/clone_2.wav \
-  --tts-ref-text "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
-```
-
-Enable AURA token-id passthrough explicitly:
-
-```bash
-python examples/online_serving/aura_omni/openai_chat_completion_client.py \
-  --tts-pass-token-ids
-```
-
-CustomVoice mode requires stages 2 and 3 in `aura_omni.yaml` to point at a
-Qwen3-TTS CustomVoice checkpoint:
-
-```bash
-python examples/online_serving/aura_omni/openai_chat_completion_client.py \
-  --tts-task-type CustomVoice \
-  --tts-speaker Vivian
-```
-
-By default, AURA responses are passed to Qwen3-TTS as text. Set
-`tts_pass_token_ids=true` to pass AURA-generated assistant token ids directly
-to Qwen3-TTS instead. The processor still uses AURA token ids, when available,
-to estimate the Talker prompt length in the default text path.
-
-## Curl
-
-```bash
-cd examples/online_serving/aura_omni
-bash run_curl_multimodal_generation.sh
-```
-
-Set `PORT`, `MODEL`, or `OUTPUT_DIR` to override defaults:
-
-```bash
-PORT=8666 MODEL=aurateam/AURA bash run_curl_multimodal_generation.sh
-TTS_PASS_TOKEN_IDS=true PORT=8666 MODEL=aurateam/AURA bash run_curl_multimodal_generation.sh
-```
-
-## Gradio
-
-Launch the server and Gradio UI together:
-
-```bash
-cd examples/online_serving/aura_omni
-bash run_gradio_demo.sh
-```
-
-If the server is already running:
-
-```bash
-python examples/online_serving/aura_omni/gradio_demo.py \
-  --model aurateam/AURA \
-  --api-base http://localhost:8091/v1
-```
+That turn-based path was an incomplete June-era stand-in: vLLM-Omni did not
+yet support streaming I/O for this stack, and adding session history then would
+have required a much larger orchestrator change. It is not a supported AURA
+mode. Do not run those scripts against the default deploy. Prefer the duplex
+smoke client above.
 
 ## Offline
 

examples/online_serving/aura_omni/aura_omni_duplex_smoke.yaml

@@ -0,0 +1,119 @@
+# AURA duplex Realtime smoke (local weights, session_mode: duplex).
+# Stage1 = /workspace/models/AURA + AuraQwen3VLForConditionalGeneration (AURA v1 / Qwen3-VL).
+pipeline: aura_omni
+# Qwen3-TTS Stage2→3 codec handoff requires async_chunk=true (same as native /
+# duplex explore). async_chunk=false reaches talker/code_predictor then never
+# delivers audio to Stage3/client.
+async_chunk: true
+session_mode: duplex
+duplex_session:
+  max_sessions: 1
+
+connectors:
+  connector_of_shared_memory:
+    name: SharedMemoryConnector
+    extra:
+      codec_streaming: true
+      connector_get_sleep_s: 0.01
+      connector_get_max_wait_first_chunk: 3000
+      connector_get_max_wait: 300
+      codec_chunk_frames: 25
+      codec_left_context_frames: 72
+      # Match native/duplex explore. initial=1 can stall Code2Wav on the first
+      # undersized chunk (observed hang after "Code2Wav codec: frames=1").
+      initial_codec_chunk_frames: 8
+      decode_cudagraph_capture_sizes: [25, 73, 97, 169, 325]
+
+stages:
+  - stage_id: 0
+    max_num_seqs: 1
+    gpu_memory_utilization: 0.08
+    enforce_eager: true
+    trust_remote_code: true
+    enable_prefix_caching: true
+    async_scheduling: true
+    max_model_len: 4096
+    max_num_batched_tokens: 2048
+    devices: "0"
+    model: /workspace/models/Qwen3-ASR-1.7B
+    default_sampling_params:
+      temperature: 0.0
+      top_p: 1.0
+      top_k: -1
+      max_tokens: 256
+
+  - stage_id: 1
+    max_num_seqs: 1
+    gpu_memory_utilization: 0.55
+    enforce_eager: true
+    trust_remote_code: true
+    enable_prefix_caching: true
+    async_scheduling: true
+    max_model_len: 32768
+    max_num_batched_tokens: 8192
+    devices: "0"
+    model: /workspace/models/AURA
+    model_arch: AuraQwen3VLForConditionalGeneration
+    default_sampling_params:
+      temperature: 0.0
+      top_p: 1.0
+      top_k: -1
+      max_tokens: 256
+      repetition_penalty: 1.0
+      stop_token_ids: [151669, 151645]
+
+  - stage_id: 2
+    # Allow prior TTS to drain while the next Talker starts (not assumed 1).
+    max_num_seqs: 2
+    gpu_memory_utilization: 0.12
+    enforce_eager: true
+    trust_remote_code: true
+    enable_prefix_caching: false
+    async_scheduling: true
+    max_num_batched_tokens: 4096
+    max_model_len: 4096
+    devices: "0"
+    # Stage1→2 is orchestrator-fed via aura2tts (custom_process_input_func).
+    # A from_stage_1 SHM edge parks Talker in WAITING_FOR_CHUNK and clears
+    # additional_information.text → Missing Qwen3-TTS text conditioning.
+    # Only Stage2→3 codec handoff needs a connector.
+    output_connectors:
+      to_stage_3: connector_of_shared_memory
+    model: /workspace/models/Qwen3-TTS-12Hz-1.7B-CustomVoice
+    default_sampling_params:
+      temperature: 0.5
+      top_k: 50
+      # 12Hz × 240 ≈ 20s. Stay under the smoke 25s audio gate so a missed
+      # 2150 (English/quotes humming) cannot overlap the next vision-follow.
+      max_tokens: 240
+      min_tokens: 8
+      repetition_penalty: 1.05
+      # Qwen3-TTS Talker codec EOS — without this Talker can run to max_tokens
+      # (~26s garbage audio while Stage1 text was correct).
+      stop_token_ids: [2150]
+    subtalker_sampling_params:
+      do_sample: true
+      temperature: 0.5
+      top_k: 50
+      top_p: 1.0
+
+  - stage_id: 3
+    max_num_seqs: 2
+    gpu_memory_utilization: 0.12
+    enforce_eager: true
+    trust_remote_code: true
+    enable_prefix_caching: false
+    async_scheduling: true
+    max_num_batched_tokens: 16384
+    max_model_len: 16384
+    devices: "0"
+    input_connectors:
+      from_stage_2: connector_of_shared_memory
+    model: /workspace/models/Qwen3-TTS-12Hz-1.7B-CustomVoice
+    default_sampling_params:
+      temperature: 0.0
+      top_p: 1.0
+      top_k: -1
+      max_tokens: 16384
+      seed: 42
+      repetition_penalty: 1.0

examples/online_serving/aura_omni/realtime_web/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""AURA push-to-talk browser call UI."""

examples/online_serving/aura_omni/realtime_web/__main__.py

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+from examples.online_serving.realtime_web.server import main
+
+if __name__ == "__main__":
+    main(default_profile="aura-ptt")

examples/online_serving/aura_omni/run_duplex_smoke_serve.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Minimal AURA duplex Realtime server for smoke.
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+cd "$REPO_ROOT"
+
+PORT="${PORT:-8099}"
+MODEL="${MODEL:-/workspace/models/AURA}"
+DEPLOY_CONFIG="${DEPLOY_CONFIG:-$REPO_ROOT/examples/online_serving/aura_omni/aura_omni_duplex_smoke.yaml}"
+
+# AURA v1 / Qwen3-VL ChatML ids (do not use AURA v2 248070/248046 here).
+export VLLM_AURA_SILENT_TOKEN_ID="${VLLM_AURA_SILENT_TOKEN_ID:-151669}"
+export VLLM_AURA_IM_END_TOKEN_ID="${VLLM_AURA_IM_END_TOKEN_ID:-151645}"
+export VLLM_AURA_IM_START_TOKEN_ID="${VLLM_AURA_IM_START_TOKEN_ID:-151644}"
+export VLLM_AURA_ASSISTANT_TOKEN_ID="${VLLM_AURA_ASSISTANT_TOKEN_ID:-77091}"
+# Multi-stage colocated init can make earlier stages' Process.sentinel spuriously
+# readable; prefer spawn for stage engine subprocesses (override if needed).
+export VLLM_WORKER_MULTIPROC_METHOD="${VLLM_WORKER_MULTIPROC_METHOD:-spawn}"
+# Do NOT set TORCHDYNAMO_DISABLE here: it breaks Stage0/1 AOT when those
+# stages are not enforce_eager. Stage2/3 use enforce_eager; code_predictor
+# honors that and skips its own torch.compile.
+
+exec "$REPO_ROOT/.venv/bin/vllm-omni" serve "$MODEL" \
+  --omni \
+  --deploy-config "$DEPLOY_CONFIG" \
+  --trust-remote-code \
+  --host 0.0.0.0 \
+  --port "$PORT"

examples/online_serving/aura_omni/smoke_duplex_realtime_client.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""Minimal AURA duplex Realtime smoke via DuplexClient."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import base64
+import json
+import sys
+import time
+from io import BytesIO
+from pathlib import Path
+
+from PIL import Image
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from vllm_omni.clients.duplex import (  # noqa: E402
+    DuplexClient,
+    EventCollector,
+    SessionConfig,
+    read_pcm16_wav,
+)
+
+PROGRESS_TYPES = {
+    "response.created",
+    "response.done",
+    "response.audio.delta",
+    "response.output_audio.delta",
+    "response.audio_transcript.delta",
+    "response.output_text.delta",
+    "response.text.delta",
+    "response.listen",
+    "response.model_listen",
+    "error",
+}
+
+
+def _tiny_jpeg_b64() -> str:
+    img = Image.new("RGB", (64, 64), color=(200, 40, 40))
+    buf = BytesIO()
+    img.save(buf, format="JPEG", quality=85)
+    return base64.b64encode(buf.getvalue()).decode("ascii")
+
+
+async def run(url: str, model: str, wav: Path, timeout_s: float) -> int:
+    pcm16 = read_pcm16_wav(wav)
+    if not pcm16:
+        print("empty wav", file=sys.stderr)
+        return 2
+
+    frame = _tiny_jpeg_b64()
+    collector = EventCollector()
+    config = SessionConfig(
+        modalities=("text", "audio"),
+        instructions="You are AURA. Reply briefly in Chinese.",
+        extra_body={
+            "aura_system_prompt": "You are AURA. Reply briefly in Chinese.",
+            "tts_task_type": "CustomVoice",
+            "tts_language": "Chinese",
+            "tts_speaker": "Vivian",
+        },
+    )
+
+    started = time.time()
+    client = DuplexClient(
+        url,
+        model=model,
+        config=config,
+        reconnect=None,
+        heartbeat_interval_s=None,
+        handshake_timeout_s=min(60.0, timeout_s),
+    )
+
+    async with client:
+        consume_task = asyncio.create_task(collector.consume(client))
+        collector.add({"type": "session.created", "session": getattr(client, "session_info", {})})
+
+        await client.append_audio(pcm16, is_speech=True, video_frames=[frame])
+        await client.commit(create_response=True)
+
+        progress = False
+        got_done = False
+        while time.time() - started < timeout_s:
+            types = [e.get("type") for e in collector.events]
+            if any(t in PROGRESS_TYPES for t in types):
+                progress = True
+            if "response.done" in types or "response.completed" in types:
+                got_done = True
+                progress = True
+                break
+            for e in collector.events:
+                meta = e.get("metadata") if isinstance(e.get("metadata"), dict) else {}
+                if meta.get("model_listen") or meta.get("listen_source") == "aura_silent":
+                    progress = True
+                    break
+            if collector.count("error") > 0 or collector.count("session.closed") > 0:
+                break
+            # Keep draining until response.done so TTS is not truncated.
+            await asyncio.sleep(0.2)
+
+        try:
+            await client.close(timeout_s=30.0)
+        except Exception as exc:  # noqa: BLE001
+            print(f"close_warn={exc}", file=sys.stderr)
+        try:
+            await asyncio.wait_for(consume_task, timeout=5.0)
+        except (TimeoutError, asyncio.CancelledError):
+            consume_task.cancel()
+
+    event_types = [e.get("type") for e in collector.events]
+    errors = [json.dumps(e, ensure_ascii=False)[:500] for e in collector.errors()]
+    result = {
+        "ok": progress and (got_done or collector.count("response.done") > 0),
+        "got_response_progress": progress,
+        "got_response_done": got_done or collector.count("response.done") > 0,
+        "errors": errors,
+        "event_types": event_types,
+        "n_events": len(event_types),
+        "audio_bytes": len(collector.audio_bytes()),
+        "llm_text": collector.response_text(collector.response_ids[0]) if collector.response_ids else "",
+        "elapsed_s": round(time.time() - started, 2),
+    }
+    print(json.dumps(result, ensure_ascii=False, indent=2))
+    log_path = Path("/tmp/aura_duplex_smoke/client_events.jsonl")
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    log_path.write_text(
+        "".join(json.dumps(e, ensure_ascii=False) + "\n" for e in collector.events),
+        encoding="utf-8",
+    )
+    print(f"events_log={log_path}", file=sys.stderr)
+    return 0 if progress else 1
+
+
+def main() -> int:
+    p = argparse.ArgumentParser()
+    p.add_argument("--url", default="ws://127.0.0.1:8099/v1/realtime")
+    p.add_argument("--model", default="/workspace/models/AURA")
+    p.add_argument(
+        "--wav",
+        default=str(REPO_ROOT / "tests/assets/minicpmo_4_5/response_required_16k.wav"),
+    )
+    p.add_argument("--timeout-s", type=float, default=300.0)
+    args = p.parse_args()
+    return asyncio.run(run(args.url, args.model, Path(args.wav), args.timeout_s))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

examples/online_serving/dynin_omni/README.md

@@ -1,97 +0,0 @@
-# Dynin-Omni: Online serving
-
-## Installation
-
-Please refer to [README.md](../../../README.md).
-
-## Launch the Server
-
-First, find the `transformers_modules` path:
-
-```bash
-python - <<'PY'
-from transformers.utils.hub import HF_MODULES_CACHE
-print(HF_MODULES_CACHE)
-PY
-```
-
-Then export it for both `PYTHONPATH` and `HF_MODULES_CACHE`:
-
-```bash
-export PYTHONPATH=<transformers_modules_path>:$PYTHONPATH
-export HF_MODULES_CACHE=<transformers_modules_path>
-```
-
-Run from repository root. The 3-stage pipeline is resolved automatically from
-the model type (`vllm_omni/deploy/dynin_omni.yaml`), so no config path is
-required. Pass `--deploy-config` only to override the defaults:
-
-```bash
-vllm-omni serve snu-aidas/Dynin-Omni \
-  --omni \
-  --port 8091
-```
-
-If `vllm-omni` is not in PATH, run:
-
-```bash
-PYTHONPATH="$(pwd)" python -m vllm_omni.entrypoints.cli.main serve snu-aidas/Dynin-Omni \
-  --omni \
-  --port 8091
-```
-
-Wait until the server logs show both `All stages initialized successfully` and
-`Application startup complete.` before sending requests.
-
-## Send Requests via Python Client
-
-Move to the example directory:
-
-```bash
-cd examples/online_serving/dynin_omni
-```
-
-### Text -> Image
-
-```bash
-python openai_chat_completion_client_for_multimodal_generation.py \
-  --query-type t2i \
-  --prompt "A realistic indoor living room with natural daylight."
-```
-
-### Image -> Image
-
-```bash
-python openai_chat_completion_client_for_multimodal_generation.py \
-  --query-type i2i \
-  --image-path ../../offline_inference/dynin_omni/data/image/sofa_under_water.jpg \
-  --prompt "Transform this surreal underwater setting into a realistic indoor living room while preserving the sofa layout."
-```
-
-### Text -> Speech
-
-```bash
-python openai_chat_completion_client_for_multimodal_generation.py \
-  --query-type t2s \
-  --prompt "Hello. This is Dynin-omni."
-```
-
-## CLI Arguments
-
-- `--query-type` (`t2i|t2s|i2i`)
-- `--model` (default: `snu-aidas/Dynin-Omni`)
-- `--host` / `--port` (OpenAI-compatible vLLM endpoint)
-- `--prompt` (custom text)
-- `--image-path` (required for `i2i`)
-- `--modalities` (optional output modalities override)
-- `--output-dir` (default: `/tmp/dynin_online_outputs`)
-
-## Notes
-
-- This client currently supports only `t2i`, `t2s`, and `i2i`.
-- `t2t` is intentionally not exposed in this online example.
-- This example intentionally uses the OpenAI-compatible chat completion endpoint.
-- Task routing for non-text outputs relies on Dynin task trigger tokens (`<|t2i|>`, `<|i2i|>`, `<|t2s|>`) injected by the client.
-- Outputs are saved under `/tmp/dynin_online_outputs` by default.
-- Dynin stage-0 warmup can take a while on first startup; do not send requests before startup completes.
-- Dynin itself can execute text-returning tasks such as `t2t`, `s2t`, `i2t`, and `v2t`, but this online serving example currently runs stage-0 in `generation` mode. In that path, the generation worker does not surface the final text as `output.text`, so OpenAI chat responses for those text-output tasks may complete internally but still return empty text.

examples/online_serving/dynin_omni/openai_chat_completion_client_for_multimodal_generation.py

@@ -1,342 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-import argparse
-import base64
-import json
-import mimetypes
-import os
-import time
-from pathlib import Path
-from typing import Any
-
-DEFAULT_MODEL = "snu-aidas/Dynin-Omni"
-DEFAULT_OUTPUT_DIR = "/tmp/dynin_online_outputs"
-
-QUERY_CHOICES = ("t2i", "t2s", "i2i")
-DEFAULT_PROMPT_BY_QUERY = {
-    "t2i": "A high quality detailed living room interior photo.",
-    "t2s": "Please read this sentence naturally: Hello from Dynin-Omni online serving.",
-    "i2i": "Transform this image into a realistic indoor living room while preserving layout.",
-}
-DEFAULT_MODALITIES_BY_QUERY = {
-    "t2i": ["image"],
-    "t2s": ["audio"],
-    "i2i": ["image"],
-}
-OFFLINE_PARITY_STAGE_COUNT = 3
-OFFLINE_PARITY_STAGE_SAMPLING = {
-    "max_tokens": 1,
-    "temperature": 0.0,
-    "top_p": 1.0,
-    "detokenize": False,
-}
-
-
-def _infer_mime_type(path: Path) -> str:
-    mime_type, _ = mimetypes.guess_type(str(path))
-    return mime_type or "application/octet-stream"
-
-
-def _encode_file_as_data_url(path: Path) -> str:
-    mime_type = _infer_mime_type(path)
-    raw = path.read_bytes()
-    encoded = base64.b64encode(raw).decode("utf-8")
-    return f"data:{mime_type};base64,{encoded}"
-
-
-def _to_image_url(path_or_url: str) -> str:
-    value = str(path_or_url)
-    if value.startswith(("http://", "https://", "data:image/")):
-        return value
-    path = Path(value).expanduser().resolve()
-    if not path.exists():
-        raise FileNotFoundError(f"Image file not found: {path}")
-    return _encode_file_as_data_url(path)
-
-
-def _build_user_content(query_type: str, prompt: str, image_path: str | None) -> list[dict[str, Any]]:
-    if query_type == "t2i":
-        return [{"type": "text", "text": f"<|t2i|> {prompt}"}]
-
-    if query_type == "t2s":
-        return [{"type": "text", "text": f"<|t2s|> {prompt}"}]
-
-    if query_type == "i2i":
-        if not image_path:
-            raise ValueError("--image-path is required for query type i2i")
-        return [
-            {"type": "text", "text": f"<|i2i|> {prompt}"},
-            {"type": "image_url", "image_url": {"url": _to_image_url(image_path)}},
-        ]
-
-    raise ValueError(f"Unsupported query_type: {query_type}")
-
-
-def _collect_text_from_content(content: Any) -> list[str]:
-    texts: list[str] = []
-    if isinstance(content, str):
-        stripped = content.strip()
-        if stripped:
-            texts.append(stripped)
-        return texts
-
-    if isinstance(content, dict):
-        for key in ("text", "content", "value", "output_text"):
-            text_value = content.get(key)
-            if isinstance(text_value, str) and text_value.strip():
-                texts.append(text_value.strip())
-        return texts
-
-    if isinstance(content, list):
-        for item in content:
-            texts.extend(_collect_text_from_content(item))
-        return texts
-
-    content_text = getattr(content, "text", None)
-    if isinstance(content_text, str) and content_text.strip():
-        texts.append(content_text.strip())
-    content_value = getattr(content, "content", None)
-    if isinstance(content_value, str) and content_value.strip():
-        texts.append(content_value.strip())
-    output_text = getattr(content, "output_text", None)
-    if isinstance(output_text, str) and output_text.strip():
-        texts.append(output_text.strip())
-    return texts
-
-
-def _extract_text_outputs(chat_completion: Any) -> list[str]:
-    texts: list[str] = []
-    for choice in getattr(chat_completion, "choices", []) or []:
-        message = getattr(choice, "message", None)
-        if message is None:
-            continue
-        content = getattr(message, "content", None)
-        texts.extend(_collect_text_from_content(content))
-        reasoning_content = getattr(message, "reasoning_content", None)
-        if isinstance(reasoning_content, str) and reasoning_content.strip():
-            texts.append(reasoning_content.strip())
-        choice_text = getattr(choice, "text", None)
-        if isinstance(choice_text, str) and choice_text.strip():
-            texts.append(choice_text.strip())
-    top_level_output_text = getattr(chat_completion, "output_text", None)
-    if isinstance(top_level_output_text, str) and top_level_output_text.strip():
-        texts.append(top_level_output_text.strip())
-    return texts
-
-
-def _extract_image_data_urls(chat_completion: Any) -> list[str]:
-    urls: list[str] = []
-    for choice in getattr(chat_completion, "choices", []) or []:
-        message = getattr(choice, "message", None)
-        if message is None:
-            continue
-        content = getattr(message, "content", None)
-        if not isinstance(content, list):
-            continue
-        for item in content:
-            if not isinstance(item, dict):
-                continue
-            if item.get("type") != "image_url":
-                continue
-            image_url = (item.get("image_url") or {}).get("url")
-            if isinstance(image_url, str) and image_url.startswith("data:image"):
-                urls.append(image_url)
-    return urls
-
-
-def _extract_audio_payloads(chat_completion: Any) -> list[bytes]:
-    payloads: list[bytes] = []
-    for choice in getattr(chat_completion, "choices", []) or []:
-        message = getattr(choice, "message", None)
-        if message is None:
-            continue
-        message_audio = getattr(message, "audio", None)
-        if message_audio is None:
-            continue
-        data_b64 = getattr(message_audio, "data", None)
-        if isinstance(data_b64, str) and data_b64:
-            try:
-                payloads.append(base64.b64decode(data_b64))
-            except Exception:
-                continue
-    return payloads
-
-
-def _decode_data_url(data_url: str) -> tuple[bytes, str]:
-    header, data = data_url.split(",", 1)
-    mime_type = "image/png"
-    if ";" in header and ":" in header:
-        mime_type = header.split(":", 1)[1].split(";", 1)[0]
-    return base64.b64decode(data), mime_type
-
-
-def _image_extension_from_mime(mime_type: str) -> str:
-    if mime_type == "image/jpeg":
-        return ".jpg"
-    if mime_type == "image/webp":
-        return ".webp"
-    if mime_type == "image/gif":
-        return ".gif"
-    return ".png"
-
-
-def _save_outputs(
-    *,
-    query_type: str,
-    chat_completion: Any,
-    output_dir: Path,
-) -> None:
-    output_dir.mkdir(parents=True, exist_ok=True)
-    stamp = time.strftime("%Y%m%d_%H%M%S")
-
-    text_outputs = _extract_text_outputs(chat_completion)
-    image_data_urls = _extract_image_data_urls(chat_completion)
-    audio_payloads = _extract_audio_payloads(chat_completion)
-
-    if text_outputs:
-        text_path = output_dir / f"{query_type}_{stamp}.txt"
-        text_path.write_text("\n\n".join(text_outputs) + "\n", encoding="utf-8")
-        print(f"[dynin-online] text saved: {text_path}")
-        print(text_outputs[0])
-
-    for idx, image_url in enumerate(image_data_urls):
-        image_bytes, mime_type = _decode_data_url(image_url)
-        ext = _image_extension_from_mime(mime_type)
-        image_path = output_dir / f"{query_type}_{stamp}_{idx}{ext}"
-        image_path.write_bytes(image_bytes)
-        print(f"[dynin-online] image saved: {image_path}")
-
-    for idx, audio_bytes in enumerate(audio_payloads):
-        audio_path = output_dir / f"{query_type}_{stamp}_{idx}.wav"
-        audio_path.write_bytes(audio_bytes)
-        print(f"[dynin-online] audio saved: {audio_path}")
-
-    if not text_outputs and not image_data_urls and not audio_payloads:
-        print("[dynin-online] no output extracted from response")
-        raw_path = output_dir / f"{query_type}_{stamp}_raw_response.json"
-        try:
-            if hasattr(chat_completion, "model_dump_json"):
-                serialized = chat_completion.model_dump_json(indent=2)
-            else:
-                if hasattr(chat_completion, "model_dump"):
-                    raw_payload: Any = chat_completion.model_dump(mode="json")
-                else:
-                    raw_payload = chat_completion
-                try:
-                    serialized = json.dumps(raw_payload, ensure_ascii=False, indent=2)
-                except Exception:
-                    serialized = json.dumps({"repr": repr(raw_payload)}, ensure_ascii=False, indent=2)
-            raw_path.write_text(serialized + "\n", encoding="utf-8")
-            print(f"[dynin-online] raw response saved: {raw_path}")
-        except Exception:
-            pass
-
-
-def _build_offline_parity_sampling_params_list() -> list[dict[str, Any]]:
-    return [dict(OFFLINE_PARITY_STAGE_SAMPLING) for _ in range(OFFLINE_PARITY_STAGE_COUNT)]
-
-
-def run_request(args: argparse.Namespace) -> None:
-    from openai import OpenAI
-
-    client = OpenAI(
-        api_key="EMPTY",
-        base_url=f"http://{args.host}:{args.port}/v1",
-    )
-    prompt = args.prompt.strip() if args.prompt else DEFAULT_PROMPT_BY_QUERY[args.query_type]
-    user_content = _build_user_content(
-        query_type=args.query_type,
-        prompt=prompt,
-        image_path=args.image_path,
-    )
-    if args.modalities:
-        modalities = [item.strip() for item in args.modalities.split(",") if item.strip()]
-    else:
-        modalities = DEFAULT_MODALITIES_BY_QUERY[args.query_type]
-
-    extra_body = {
-        "sampling_params_list": _build_offline_parity_sampling_params_list(),
-    }
-    chat_completion = client.chat.completions.create(
-        model=args.model,
-        messages=[{"role": "user", "content": user_content}],
-        modalities=modalities,
-        extra_body=extra_body,
-    )
-    _save_outputs(
-        query_type=args.query_type,
-        chat_completion=chat_completion,
-        output_dir=Path(args.output_dir).expanduser(),
-    )
-
-
-def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Dynin-Omni online chat completion client")
-    parser.add_argument(
-        "--query-type",
-        "-q",
-        type=str,
-        default="t2i",
-        choices=QUERY_CHOICES,
-        help="Dynin query type",
-    )
-    parser.add_argument(
-        "--model",
-        "-m",
-        type=str,
-        default=DEFAULT_MODEL,
-        help="Model name/path",
-    )
-    parser.add_argument(
-        "--host",
-        type=str,
-        default="localhost",
-        help="Host/IP of the vLLM Omni API server",
-    )
-    parser.add_argument(
-        "--port",
-        type=int,
-        default=8091,
-        help="Port of the vLLM Omni API server",
-    )
-    parser.add_argument(
-        "--prompt",
-        "-p",
-        type=str,
-        default="",
-        help="Custom prompt text",
-    )
-    parser.add_argument(
-        "--image-path",
-        "-i",
-        type=str,
-        default=None,
-        help="Image path/URL for i2i",
-    )
-    parser.add_argument(
-        "--modalities",
-        type=str,
-        default="",
-        help="Comma-separated output modalities override (e.g., text,image,audio)",
-    )
-    parser.add_argument(
-        "--output-dir",
-        "-o",
-        type=str,
-        default=DEFAULT_OUTPUT_DIR,
-        help="Directory to save outputs",
-    )
-    return parser.parse_args()
-
-
-def main() -> None:
-    args = parse_args()
-    os.environ.setdefault("HF_HUB_DISABLE_XET", "1")
-    run_request(args)
-
-
-if __name__ == "__main__":
-    main()

examples/online_serving/minicpmo/README.md

@@ -316,3 +316,12 @@ them unless `--allow-invalid-clock` is explicit.
   [`examples/offline_inference/minicpmo/`](../../offline_inference/minicpmo/)
 - Recipe:
   [`recipes/OpenBMB/MiniCPM-o-4_5.md`](../../../recipes/OpenBMB/MiniCPM-o-4_5.md)
+
+### Shared realtime UI implementation
+
+The `python -m examples.online_serving.minicpmo.realtime_web` command remains
+available with the same MiniCPM defaults and required `--ref-audio`. Its assets
+now live in [the shared realtime UI](../realtime_web/README.md), with a dedicated
+`minicpm-native` profile preserving native duplex input, playback ACKs and camera
+frames. Qwen3 uses a separate profile in the same shell, supporting manual STT
+turns or Server VAD turns with speech interruption.

examples/online_serving/minicpmo/realtime_web/app/static/app.js

@@ -1,669 +0,0 @@
-(() => {
-  'use strict';
-
-  const config = window.FULL_DUPLEX_CONFIG || {};
-  const callButton = document.getElementById('callButton');
-  const muteButton = document.getElementById('muteButton');
-  const cameraButton = document.getElementById('cameraButton');
-  const cameraPreview = document.getElementById('cameraPreview');
-  const promptPreset = document.getElementById('promptPreset');
-  const systemPromptInput = document.getElementById('systemPrompt');
-  const connectionState = document.getElementById('connectionState');
-  const modelState = document.getElementById('modelState');
-  const playbackState = document.getElementById('playbackState');
-  const sessionTimer = document.getElementById('sessionTimer');
-  const meterFill = document.getElementById('meterFill');
-  const conversation = document.getElementById('conversation');
-  const emptyConversation = document.getElementById('emptyConversation');
-  const eventLog = document.getElementById('eventLog');
-  const eventCount = document.getElementById('eventCount');
-  const runtimeDetail = document.getElementById('runtimeDetail');
-  const clearLogButton = document.getElementById('clearLogButton');
-
-  const INPUT_RATE = 16000;
-  const OUTPUT_RATE = 24000;
-  const SEND_INTERVAL_MS = 200;
-  const ECHO_GUARD_MS = 300;
-  const INITIAL_PLAYBACK_BUFFER_MS = 400;
-  const SESSION_CLOSE_TIMEOUT_MS = 1000;
-
-  // Default prompts mirroring the official MiniCPM-o-Demo presets
-  // (assets/presets/{omni,audio_duplex}/*.yaml).
-  const PROMPT_PRESETS = {
-    omni: 'Streaming Omni Conversation.',
-    chinese_call: '扮演一个具有以上声音特征的助手。请认真、高质量地回复用户的问题。'
-      + '请用高自然度的方式和用户聊天。你处于双工模式，可以一边听、一边说。'
-      + '你是由面壁智能开发的人工智能助手：面壁小钢炮。',
-    english_call: 'Replicate the tone and style from the input audio. Your task is to be '
-      + 'a helpful assistant using this voice pattern. Please answer the user\'s questions '
-      + 'seriously and in a high quality. Please chat with the user in a high naturalness '
-      + 'style. You are in duplex mode, where you can listen and speak at the same time.',
-  };
-
-  let socket = null;
-  let mediaStream = null;
-  let captureContext = null;
-  let captureNode = null;
-  let playbackContext = null;
-  let playbackNode = null;
-  let sendTimer = null;
-  let clockTimer = null;
-  let startedAt = 0;
-  let running = false;
-  let muted = false;
-  let assistantActive = false;
-  let captureRate = INPUT_RATE;
-  let cameraStream = null;
-  let cameraTimer = null;
-  let cameraPendingFrame = null;
-  const cameraCanvas = document.createElement('canvas');
-  let playbackRate = OUTPUT_RATE;
-  let pendingCapture = [];
-  let currentResponseId = null;
-  let responseHasAudio = false;
-  let logCount = 0;
-  let liveUserTurn = null;
-  let liveAssistantTurn = null;
-  let sessionCloseResolver = null;
-
-  function staticAssetUrl(path) {
-    const version = String(config.appVersion || '').trim();
-    return version ? `${path}?v=${encodeURIComponent(version)}` : path;
-  }
-
-  if (promptPreset && systemPromptInput) {
-    promptPreset.addEventListener('change', () => {
-      const preset = PROMPT_PRESETS[promptPreset.value];
-      if (preset !== undefined) systemPromptInput.value = preset;
-    });
-    systemPromptInput.addEventListener('input', () => {
-      promptPreset.value = 'custom';
-    });
-  }
-
-  function realtimeUrl() {
-    const url = new URL(config.realtimePath, window.location.href);
-    url.protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
-    url.searchParams.set('duplex', '1');
-    url.searchParams.set('model', config.model || 'openbmb/MiniCPM-o-4_5');
-    url.searchParams.set('autostart', '0');
-    return url.toString();
-  }
-
-  function setConnection(label, kind) {
-    connectionState.textContent = label;
-    connectionState.className = `status status-${kind}`;
-  }
-
-  function setModel(label) {
-    modelState.textContent = label;
-  }
-
-  function setPlayback(label) {
-    playbackState.textContent = label;
-  }
-
-  function compactEvent(event) {
-    const fields = [];
-    const payload = event.event || event;
-    const responseId = responseIdOf(event);
-    if (responseId) fields.push(`response=${responseId}`);
-    if (event.item_id) fields.push(`item=${event.item_id}`);
-    if (event.code) fields.push(`code=${event.code}`);
-    if (event.response && event.response.status) fields.push(`status=${event.response.status}`);
-    if (payload.played_ms !== undefined) fields.push(`played=${payload.played_ms}ms`);
-    if (payload.committed_ms !== undefined) fields.push(`committed=${payload.committed_ms}ms`);
-    if (payload.history_committed !== undefined) fields.push(`history=${payload.history_committed}`);
-    return fields.join(' ');
-  }
-
-  function appendLog(message, error = false) {
-    const time = new Date().toLocaleTimeString([], { hour12: false });
-    const line = document.createElement('span');
-    if (error) line.className = 'log-error';
-    line.textContent = `${time}  ${message}\n`;
-    eventLog.appendChild(line);
-    eventLog.scrollTop = eventLog.scrollHeight;
-    logCount += 1;
-    eventCount.textContent = `${logCount} ${logCount === 1 ? 'event' : 'events'}`;
-  }
-
-  function appendEventLog(event) {
-    const detail = compactEvent(event);
-    appendLog(`${event.type || 'unknown'}${detail ? `  ${detail}` : ''}`, event.type === 'error');
-  }
-
-  function responseIdOf(event) {
-    return event.response_id || (event.response && event.response.id) || null;
-  }
-
-  function ensureTurn(role) {
-    const existing = role === 'user' ? liveUserTurn : liveAssistantTurn;
-    if (existing) return existing;
-    if (emptyConversation) emptyConversation.remove();
-    const row = document.createElement('div');
-    row.className = `turn turn-${role} turn-live`;
-    const label = document.createElement('div');
-    label.className = 'turn-role';
-    label.textContent = role === 'user' ? 'You' : 'Assistant';
-    const text = document.createElement('div');
-    text.className = 'turn-text';
-    row.append(label, text);
-    conversation.appendChild(row);
-    conversation.scrollTop = conversation.scrollHeight;
-    const turn = { row, text, value: '' };
-    if (role === 'user') liveUserTurn = turn;
-    else liveAssistantTurn = turn;
-    return turn;
-  }
-
-  function addTranscript(role, delta) {
-    if (!delta) return;
-    const turn = ensureTurn(role);
-    turn.value += delta;
-    turn.text.textContent = turn.value;
-    conversation.scrollTop = conversation.scrollHeight;
-  }
-
-  function finishTranscript(role, finalText = '') {
-    const turn = role === 'user' ? liveUserTurn : liveAssistantTurn;
-    if (!turn && !finalText) return;
-    const current = turn || ensureTurn(role);
-    if (finalText) {
-      current.value = finalText;
-      current.text.textContent = finalText;
-    }
-    current.row.classList.remove('turn-live');
-    if (role === 'user') liveUserTurn = null;
-    else liveAssistantTurn = null;
-  }
-
-  function bytesToBase64(bytes) {
-    let binary = '';
-    const chunkSize = 0x8000;
-    for (let offset = 0; offset < bytes.length; offset += chunkSize) {
-      binary += String.fromCharCode(...bytes.subarray(offset, offset + chunkSize));
-    }
-    return btoa(binary);
-  }
-
-  function int16ToBase64(pcm) {
-    return bytesToBase64(new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength));
-  }
-
-  function base64ToBytes(encoded) {
-    const binary = atob(encoded);
-    const bytes = new Uint8Array(binary.length);
-    for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index);
-    return bytes;
-  }
-
-  function resampleInt16(input, sourceRate, targetRate) {
-    if (sourceRate === targetRate) return input;
-    const ratio = sourceRate / targetRate;
-    const output = new Int16Array(Math.floor(input.length / ratio));
-    for (let index = 0; index < output.length; index += 1) {
-      const start = Math.floor(index * ratio);
-      const end = Math.max(start + 1, Math.min(input.length, Math.floor((index + 1) * ratio)));
-      let sum = 0;
-      for (let source = start; source < end; source += 1) sum += input[source];
-      output[index] = sum / (end - start);
-    }
-    return output;
-  }
-
-  async function decodeAudioDelta(event) {
-    const encoded = event.delta || (event.response && event.response.audio);
-    if (!encoded) return null;
-    const bytes = base64ToBytes(encoded);
-    const format = String(event.format || event.audio_format || 'pcm16').toLowerCase();
-    const sourceRate = Number(event.sample_rate_hz || event.sample_rate || OUTPUT_RATE);
-    if (format.includes('f32')) {
-      const floats = new Float32Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 4));
-      const pcm = new Int16Array(floats.length);
-      for (let index = 0; index < floats.length; index += 1) {
-        const sample = Math.max(-1, Math.min(1, floats[index]));
-        pcm[index] = sample < 0 ? sample * 32768 : sample * 32767;
-      }
-      return { pcm, sourceRate };
-    }
-    if (format.includes('wav')) {
-      const decoded = await playbackContext.decodeAudioData(bytes.buffer.slice(0));
-      const channel = decoded.getChannelData(0);
-      const pcm = new Int16Array(channel.length);
-      for (let index = 0; index < channel.length; index += 1) {
-        const sample = Math.max(-1, Math.min(1, channel[index]));
-        pcm[index] = sample < 0 ? sample * 32768 : sample * 32767;
-      }
-      return { pcm, sourceRate: decoded.sampleRate };
-    }
-    return {
-      pcm: new Int16Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 2)),
-      sourceRate,
-    };
-  }
-
-  function updateMeter(pcm) {
-    let peak = 0;
-    for (let index = 0; index < pcm.length; index += 8) peak = Math.max(peak, Math.abs(pcm[index]));
-    meterFill.style.width = `${Math.min(100, (peak / 32768) * 150).toFixed(0)}%`;
-  }
-
-  function microphoneUploadEnabled() {
-    return running && !muted;
-  }
-
-  function flushCapture() {
-    if (!socket || socket.readyState !== WebSocket.OPEN || pendingCapture.length === 0) return;
-    if (!microphoneUploadEnabled()) {
-      pendingCapture = [];
-      return;
-    }
-    const length = pendingCapture.reduce((total, chunk) => total + chunk.length, 0);
-    const merged = new Int16Array(length);
-    let offset = 0;
-    for (const chunk of pendingCapture) {
-      merged.set(chunk, offset);
-      offset += chunk.length;
-    }
-    pendingCapture = [];
-    const pcm = resampleInt16(merged, captureRate, INPUT_RATE);
-    const appendEvent = {
-      type: 'input_audio_buffer.append',
-      audio: int16ToBase64(pcm),
-      format: 'pcm16',
-      sample_rate_hz: INPUT_RATE,
-    };
-    // Omni duplex: ~1 fps camera frame rides the audio append (official
-    // MiniCPM-o-Demo contract: one base64 JPEG per ~1 s chunk).
-    if (cameraPendingFrame) {
-      appendEvent.video_frames = [cameraPendingFrame];
-      cameraPendingFrame = null;
-    }
-    socket.send(JSON.stringify(appendEvent));
-  }
-
-  function beginAssistant(responseId) {
-    currentResponseId = responseId || currentResponseId;
-    responseHasAudio = false;
-    assistantActive = true;
-    setModel('Speaking');
-  }
-
-  function feedPlayback(decoded, responseId) {
-    if (!decoded || !decoded.pcm || decoded.pcm.length === 0 || !playbackNode) return;
-    const pcm = resampleInt16(decoded.pcm, decoded.sourceRate, playbackRate);
-    responseHasAudio = true;
-    assistantActive = true;
-    setPlayback('Buffering');
-    playbackNode.port.postMessage({
-      type: 'audio',
-      pcm,
-      responseId: responseId || currentResponseId,
-      initialBufferMs: INITIAL_PLAYBACK_BUFFER_MS,
-    }, [pcm.buffer]);
-  }
-
-  function requestPlaybackDrain(responseId) {
-    if (!playbackNode) return;
-    playbackNode.port.postMessage({ type: 'drain', responseId: responseId || currentResponseId });
-  }
-
-  function sendPlaybackAck(responseId, playedMs) {
-    if (!responseId || !socket || socket.readyState !== WebSocket.OPEN || playedMs <= 0) {
-      if (!responseId && playedMs > 0) appendLog('playback ack skipped: missing response id', true);
-      return;
-    }
-    socket.send(JSON.stringify({
-      type: 'playback.ack',
-      response_id: responseId,
-      item_id: `item_${responseId}`,
-      played_ms: playedMs,
-      committed_ms: playedMs,
-    }));
-  }
-
-  function playbackDrained(message) {
-    const responseId = message.responseId || currentResponseId;
-    sendPlaybackAck(responseId, Number(message.playedMs) || 0);
-    setPlayback('Idle');
-    if (message.underrunMs > 0) {
-      appendLog(`playback underrun ${message.underrunMs} ms`);
-    }
-    window.setTimeout(() => {
-      assistantActive = false;
-      currentResponseId = null;
-      responseHasAudio = false;
-      if (running) setModel('Listening');
-    }, ECHO_GUARD_MS);
-  }
-
-  function handleEvent(event) {
-    appendEventLog(event);
-    const responseId = responseIdOf(event);
-    switch (event.type) {
-      case 'session.created':
-      case 'session.updated':
-        setConnection('Connected', 'online');
-        setModel('Listening');
-        break;
-      case 'response.listen':
-        assistantActive = false;
-        setModel('Listening');
-        break;
-      case 'response.created':
-      case 'response.speak':
-        beginAssistant(responseId);
-        break;
-      case 'response.output_audio.delta':
-        currentResponseId = responseId || currentResponseId;
-        assistantActive = true;
-        setModel('Speaking');
-        decodeAudioDelta(event)
-          .then((decoded) => feedPlayback(decoded, responseId))
-          .catch((error) => appendLog(`audio decode failed: ${error.message || error}`, true));
-        break;
-      case 'response.output_audio.done':
-        requestPlaybackDrain(responseId);
-        break;
-      case 'response.output_audio_transcript.delta':
-        addTranscript('assistant', event.delta || '');
-        break;
-      case 'response.output_audio_transcript.done':
-        finishTranscript('assistant', event.transcript || '');
-        break;
-      case 'conversation.item.input_audio_transcription.delta':
-        addTranscript('user', event.delta || '');
-        break;
-      case 'conversation.item.input_audio_transcription.completed':
-        finishTranscript('user', event.transcript || '');
-        break;
-      case 'response.done':
-        finishTranscript('assistant');
-        if (!responseHasAudio) requestPlaybackDrain(responseId);
-        break;
-      case 'playback.acknowledged':
-        {
-          const acknowledgement = event.event || event;
-          runtimeDetail.textContent = `Playback committed ${acknowledgement.committed_ms || 0} ms`;
-        }
-        break;
-      case 'session.closed':
-        if (sessionCloseResolver) sessionCloseResolver();
-        sessionCloseResolver = null;
-        break;
-      case 'error':
-        setConnection('Error', 'error');
-        runtimeDetail.textContent = String(event.error || event.code || 'Server error');
-        break;
-      default:
-        break;
-    }
-  }
-
-  async function openPlayback() {
-    playbackContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: OUTPUT_RATE });
-    playbackRate = playbackContext.sampleRate;
-    await playbackContext.audioWorklet.addModule(staticAssetUrl('static/playback_worklet.js'));
-    playbackNode = new AudioWorkletNode(playbackContext, 'fullduplex-pcm-playback');
-    playbackNode.port.onmessage = (message) => {
-      if (message.data.type === 'playback-started') setPlayback('Playing');
-      else if (message.data.type === 'playback-drained') playbackDrained(message.data);
-      else if (message.data.type === 'playback-underrun') {
-        runtimeDetail.textContent = `Playback underrun ${message.data.underrunMs || 0} ms`;
-      }
-    };
-    playbackNode.connect(playbackContext.destination);
-    await playbackContext.resume();
-  }
-
-  async function openCapture() {
-    mediaStream = await navigator.mediaDevices.getUserMedia({
-      audio: {
-        channelCount: 1,
-        echoCancellation: true,
-        noiseSuppression: true,
-        autoGainControl: true,
-        sampleRate: { ideal: INPUT_RATE },
-      },
-    });
-    try {
-      captureContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: INPUT_RATE });
-    } catch (_error) {
-      captureContext = new (window.AudioContext || window.webkitAudioContext)();
-    }
-    captureRate = captureContext.sampleRate;
-    await captureContext.audioWorklet.addModule(staticAssetUrl('static/pcm_worklet.js'));
-    const source = captureContext.createMediaStreamSource(mediaStream);
-    captureNode = new AudioWorkletNode(captureContext, 'fullduplex-pcm-capture');
-    captureNode.port.onmessage = (message) => {
-      const pcm = new Int16Array(message.data);
-      updateMeter(pcm);
-      if (microphoneUploadEnabled()) pendingCapture.push(pcm);
-    };
-    const silentSink = captureContext.createGain();
-    silentSink.gain.value = 0;
-    source.connect(captureNode);
-    captureNode.connect(silentSink).connect(captureContext.destination);
-    await captureContext.resume();
-  }
-
-  function openSocket() {
-    return new Promise((resolve, reject) => {
-      const url = realtimeUrl();
-      socket = new WebSocket(url);
-      let settled = false;
-      socket.onopen = () => {
-        settled = true;
-        const extraBody = {
-          auto_response: true,
-        };
-        const session = {
-          modalities: ['audio', 'text'],
-          voice: 'default',
-          extra_body: extraBody,
-        };
-        // Reference voice for TTS cloning, provided by the server via
-        // --ref-audio (mirrors the official demo's default ref audio).
-        if (config.refAudio) session.ref_audio = config.refAudio;
-        const instructions = systemPromptInput ? systemPromptInput.value.trim() : '';
-        if (instructions) session.instructions = instructions;
-        socket.send(JSON.stringify({ type: 'session.update', session }));
-        runtimeDetail.textContent = `${captureRate} Hz capture / ${playbackRate} Hz playback`;
-        appendLog(`websocket open  ${url}`);
-        resolve();
-      };
-      socket.onmessage = (message) => {
-        if (typeof message.data !== 'string') return;
-        try {
-          handleEvent(JSON.parse(message.data));
-        } catch (error) {
-          appendLog(`invalid server event: ${error.message || error}`, true);
-        }
-      };
-      socket.onerror = () => {
-        if (!settled) {
-          settled = true;
-          reject(new Error(`WebSocket connection failed: ${url}`));
-        }
-      };
-      socket.onclose = (event) => {
-        appendLog(`websocket closed  code=${event.code}`);
-        if (running) {
-          setConnection('Disconnected', 'error');
-          stopSession();
-        }
-      };
-    });
-  }
-
-  function formatElapsed(seconds) {
-    const minutes = Math.floor(seconds / 60);
-    return `${String(minutes).padStart(2, '0')}:${String(seconds % 60).padStart(2, '0')}`;
-  }
-
-  function startClock() {
-    startedAt = Date.now();
-    clockTimer = window.setInterval(() => {
-      sessionTimer.textContent = formatElapsed(Math.floor((Date.now() - startedAt) / 1000));
-    }, 1000);
-  }
-
-  async function startSession() {
-    if (running) return;
-    callButton.disabled = true;
-    setConnection('Connecting', 'connecting');
-    runtimeDetail.textContent = 'Requesting microphone access';
-    try {
-      await openPlayback();
-      await openCapture();
-      await openSocket();
-      running = true;
-      muted = false;
-      assistantActive = false;
-      sendTimer = window.setInterval(flushCapture, SEND_INTERVAL_MS);
-      startClock();
-      callButton.textContent = 'End session';
-      callButton.classList.add('is-active');
-      muteButton.disabled = false;
-      cameraButton.disabled = false;
-      setConnection('Connected', 'online');
-      setModel('Listening');
-      appendLog('session started');
-    } catch (error) {
-      appendLog(`start failed: ${error.message || error}`, true);
-      setConnection('Error', 'error');
-      runtimeDetail.textContent = String(error.message || error);
-      await stopSession();
-    } finally {
-      callButton.disabled = false;
-    }
-  }
-
-  async function startCamera() {
-    if (cameraStream) return;
-    cameraStream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false });
-    cameraPreview.srcObject = cameraStream;
-    cameraPreview.style.display = '';
-    await cameraPreview.play().catch(() => {});
-    // Official omni-duplex cadence: one JPEG (quality 0.7) per ~1 s chunk,
-    // no client-side resize (the server normalizes at scale_resolution=448).
-    cameraTimer = window.setInterval(() => {
-      if (!cameraStream || cameraPreview.videoWidth === 0) return;
-      cameraCanvas.width = cameraPreview.videoWidth;
-      cameraCanvas.height = cameraPreview.videoHeight;
-      cameraCanvas.getContext('2d').drawImage(cameraPreview, 0, 0);
-      cameraPendingFrame = cameraCanvas.toDataURL('image/jpeg', 0.7).split(',')[1];
-    }, 1000);
-    cameraButton.textContent = 'Camera off';
-    cameraButton.classList.add('is-active');
-    appendLog('camera on (1 fps omni frames)');
-  }
-
-  function stopCamera() {
-    if (cameraTimer !== null) clearInterval(cameraTimer);
-    cameraTimer = null;
-    if (cameraStream) {
-      for (const track of cameraStream.getTracks()) track.stop();
-    }
-    cameraStream = null;
-    cameraPendingFrame = null;
-    cameraPreview.srcObject = null;
-    cameraPreview.style.display = 'none';
-    cameraButton.textContent = 'Camera';
-    cameraButton.classList.remove('is-active');
-  }
-
-  cameraButton.addEventListener('click', () => {
-    if (cameraStream) {
-      stopCamera();
-      appendLog('camera off');
-      return;
-    }
-    startCamera().catch((error) => appendLog(`camera failed: ${error.message || error}`, true));
-  });
-
-  function waitForSessionClosed(targetSocket, timeoutMs) {
-    if (!targetSocket || targetSocket.readyState !== WebSocket.OPEN) return Promise.resolve();
-    return new Promise((resolve) => {
-      let done = false;
-      const finish = () => {
-        if (done) return;
-        done = true;
-        if (sessionCloseResolver === finish) sessionCloseResolver = null;
-        resolve();
-      };
-      sessionCloseResolver = finish;
-      window.setTimeout(finish, timeoutMs);
-    });
-  }
-
-  async function stopSession({ terminal = true } = {}) {
-    running = false;
-    assistantActive = false;
-    pendingCapture = [];
-    if (sendTimer !== null) clearInterval(sendTimer);
-    if (clockTimer !== null) clearInterval(clockTimer);
-    sendTimer = null;
-    clockTimer = null;
-    if (socket) {
-      const closingSocket = socket;
-      socket = null;
-      closingSocket.onclose = null;
-      if (terminal && closingSocket.readyState === WebSocket.OPEN) {
-        const closed = waitForSessionClosed(closingSocket, SESSION_CLOSE_TIMEOUT_MS);
-        closingSocket.send(JSON.stringify({ type: 'session.close' }));
-        await closed;
-      }
-      closingSocket.close(1000, 'client stop');
-    }
-    if (playbackNode) playbackNode.port.postMessage({ type: 'clear' });
-    if (mediaStream) {
-      for (const track of mediaStream.getTracks()) track.stop();
-    }
-    mediaStream = null;
-    stopCamera();
-    cameraButton.disabled = true;
-    if (captureContext) await captureContext.close().catch(() => {});
-    if (playbackContext) await playbackContext.close().catch(() => {});
-    captureContext = null;
-    captureNode = null;
-    playbackContext = null;
-    playbackNode = null;
-    currentResponseId = null;
-    responseHasAudio = false;
-    meterFill.style.width = '0%';
-    sessionTimer.textContent = '00:00';
-    callButton.textContent = 'Start session';
-    callButton.classList.remove('is-active');
-    muteButton.textContent = 'Mute';
-    muteButton.classList.remove('is-active');
-    muteButton.disabled = true;
-    setConnection('Offline', 'offline');
-    setModel('Idle');
-    setPlayback('Idle');
-    if (runtimeDetail.textContent.startsWith('Playback committed')) return;
-    if (!runtimeDetail.textContent.startsWith('start failed')) runtimeDetail.textContent = 'No active connection';
-  }
-
-  function toggleMute() {
-    if (!running) return;
-    muted = !muted;
-    pendingCapture = [];
-    muteButton.textContent = muted ? 'Unmute' : 'Mute';
-    muteButton.classList.toggle('is-active', muted);
-    appendLog(muted ? 'microphone muted' : 'microphone unmuted');
-  }
-
-  callButton.addEventListener('click', () => {
-    if (running) stopSession();
-    else startSession();
-  });
-  muteButton.addEventListener('click', toggleMute);
-  clearLogButton.addEventListener('click', () => {
-    eventLog.textContent = '';
-    logCount = 0;
-    eventCount.textContent = '0 events';
-  });
-  window.addEventListener('beforeunload', () => { stopSession({ terminal: false }); });
-})();

examples/online_serving/minicpmo/realtime_web/server.py

@@ -1,185 +1,23 @@
-"""Static host and same-origin Realtime proxy for the full-duplex demo."""
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
-from __future__ import annotations
+"""Backward-compatible MiniCPM entry point for the shared realtime UI."""
 
-import argparse
-import asyncio
-import base64
-import contextlib
-import hashlib
-import json
-import logging
-from pathlib import Path
-from urllib.parse import urlencode
+from examples.online_serving.realtime_web.server import (
+    APP_DIR,
+    STATIC_DIR,
+    _join_ws_url,
+    build_app,
+)
+from examples.online_serving.realtime_web.server import (
+    main as _main,
+)
 
-import uvicorn
-import websockets
-from fastapi import FastAPI, WebSocket, WebSocketDisconnect
-from fastapi.responses import HTMLResponse, Response
-from fastapi.staticfiles import StaticFiles
-
-logger = logging.getLogger(__name__)
-APP_DIR = Path(__file__).parent / "app"
-STATIC_DIR = APP_DIR / "static"
-
-
-def _join_ws_url(base: str, path: str, query: str) -> str:
-    return base.rstrip("/") + path + (("?" + query) if query else "")
-
-
-async def _pump_client_to_backend(client: WebSocket, backend) -> None:
-    try:
-        while True:
-            message = await client.receive()
-            if message["type"] == "websocket.disconnect":
-                await backend.close()
-                return
-            if message.get("text") is not None:
-                await backend.send(message["text"])
-            elif message.get("bytes") is not None:
-                await backend.send(message["bytes"])
-    except (WebSocketDisconnect, websockets.ConnectionClosed):
-        with contextlib.suppress(Exception):
-            await backend.close()
-
-
-async def _pump_backend_to_client(client: WebSocket, backend) -> None:
-    try:
-        async for message in backend:
-            if isinstance(message, bytes):
-                await client.send_bytes(message)
-            else:
-                await client.send_text(message)
-    except (WebSocketDisconnect, websockets.ConnectionClosed):
-        return
-    except RuntimeError as exc:
-        if "websocket.send" in str(exc) and "websocket.close" in str(exc):
-            return
-        raise
-
-
-def _expected_proxy_close(exc: BaseException) -> bool:
-    if isinstance(exc, (WebSocketDisconnect, websockets.ConnectionClosed, asyncio.CancelledError)):
-        return True
-    return isinstance(exc, RuntimeError) and "websocket.send" in str(exc) and "websocket.close" in str(exc)
-
-
-def build_app(
-    *,
-    ws_backend: str = "ws://127.0.0.1:8099",
-    model: str = "openbmb/MiniCPM-o-4_5",
-    public_realtime_url: str | None = None,
-    ref_audio: str | None = None,
-) -> FastAPI:
-    app = FastAPI(title="Experimental Full-Duplex Web Demo")
-    index_path = APP_DIR / "index.html"
-    app_version_hash = hashlib.sha256()
-    for asset_path in (
-        STATIC_DIR / "app.js",
-        STATIC_DIR / "pcm_worklet.js",
-        STATIC_DIR / "playback_worklet.js",
-    ):
-        app_version_hash.update(asset_path.read_bytes())
-    app_version = app_version_hash.hexdigest()[:12]
-
-    ref_audio_uri: str | None = None
-    if ref_audio:
-        ref_path = Path(ref_audio)
-        if not ref_path.is_file():
-            raise SystemExit(f"--ref-audio not found: {ref_audio}")
-        encoded = base64.b64encode(ref_path.read_bytes()).decode("ascii")
-        ref_audio_uri = f"data:audio/wav;base64,{encoded}"
-
-    @app.get("/", response_class=HTMLResponse)
-    def index() -> HTMLResponse:
-        config = json.dumps(
-            {
-                "model": model,
-                "realtimePath": public_realtime_url or "v1/realtime",
-                "refAudio": ref_audio_uri,
-                "appVersion": app_version,
-            },
-            ensure_ascii=True,
-        )
-        html = (
-            index_path.read_text(encoding="utf-8")
-            .replace("__FULL_DUPLEX_CONFIG__", config)
-            .replace("__FULL_DUPLEX_APP_VERSION__", app_version)
-        )
-        return HTMLResponse(html, headers={"Cache-Control": "no-store"})
-
-    @app.get("/healthz")
-    def healthz() -> Response:
-        return Response(content="ok", media_type="text/plain")
-
-    @app.websocket("/v1/realtime")
-    async def realtime_proxy(websocket: WebSocket) -> None:
-        await websocket.accept()
-        query = urlencode(websocket.query_params.multi_items())
-        backend_url = _join_ws_url(ws_backend, "/v1/realtime", query)
-        logger.info("Proxying Realtime WebSocket to %s", backend_url)
-        try:
-            async with websockets.connect(
-                backend_url,
-                max_size=64 * 1024 * 1024,
-            ) as backend:
-                tasks = {
-                    asyncio.create_task(_pump_client_to_backend(websocket, backend)),
-                    asyncio.create_task(_pump_backend_to_client(websocket, backend)),
-                }
-                done, pending = await asyncio.wait(
-                    tasks,
-                    return_when=asyncio.FIRST_COMPLETED,
-                )
-                for task in pending:
-                    task.cancel()
-                await asyncio.gather(*pending, return_exceptions=True)
-                for result in await asyncio.gather(*done, return_exceptions=True):
-                    if isinstance(result, BaseException) and not _expected_proxy_close(result):
-                        raise result
-        except (WebSocketDisconnect, websockets.ConnectionClosed):
-            return
-        except Exception:
-            logger.exception("Realtime WebSocket proxy failed")
-            with contextlib.suppress(Exception):
-                await websocket.close(code=1011)
-
-    app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
-    return app
+__all__ = ["APP_DIR", "STATIC_DIR", "_join_ws_url", "build_app", "main"]
 
 
 def main() -> None:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--host", default="0.0.0.0")
-    parser.add_argument("--port", type=int, default=7862)
-    parser.add_argument("--ws-backend", default="ws://127.0.0.1:8099")
-    parser.add_argument(
-        "--public-realtime-url",
-        help="Browser-visible ws:// or wss:// Realtime URL; defaults to the same-origin proxy.",
-    )
-    parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
-    parser.add_argument(
-        "--ref-audio",
-        required=True,
-        help=(
-            "Reference voice wav for TTS voice cloning, e.g. the "
-            "official MiniCPM-o-Demo assets/ref_audio/ref_minicpm_signature.wav"
-        ),
-    )
-    args = parser.parse_args()
-
-    logging.basicConfig(level=logging.INFO)
-    uvicorn.run(
-        build_app(
-            ws_backend=args.ws_backend,
-            model=args.model,
-            public_realtime_url=args.public_realtime_url,
-            ref_audio=args.ref_audio,
-        ),
-        host=args.host,
-        port=args.port,
-        log_level="info",
-    )
+    _main(default_profile="minicpm-native")
 
 
 if __name__ == "__main__":

examples/online_serving/qwen3_omni/README.md

@@ -320,24 +320,17 @@ python openai_realtime_client.py \
 Server VAD uses the pinned Silero v6.2 ONNX artifact from `istupakov/silero-vad-onnx` and never downloads model files while processing a session. Make the artifact available in the Hugging Face cache before startup, or configure a local artifact in the deployment YAML:
 
 ```yaml
+base_config: /path/to/vllm-omni/vllm_omni/deploy/qwen3_omni_duplex.yaml
 duplex_session:
   server_vad_model_path: /models/silero_vad.onnx
 ```
 
-Start from the bundled Qwen3-Omni deployment YAML, add the fields above at the top level, and serve that complete configuration:
-
-```bash
-cp vllm_omni/deploy/qwen3_omni_moe.yaml /path/to/qwen3_omni_server_vad.yaml
-# Edit /path/to/qwen3_omni_server_vad.yaml and add duplex_session.
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \
-  --omni \
-  --port 8091 \
-  --deploy-config /path/to/qwen3_omni_server_vad.yaml
-```
-
-Keep Qwen's default `session_mode: turn`; `duplex_session` enables the Realtime handler without changing scheduler
-semantics. Bare `/v1/realtime` selects that handler. The client uses `?duplex=0` only for the existing non-Server-VAD
-wire flow; `?duplex=1` remains a supported compatibility alias.
+For the engine-owned plugin, use an overlay based on
+`vllm_omni/deploy/qwen3_omni_duplex.yaml`, with `duplex_session.server_vad_model_path`
+pointing to your local artifact. See the [complete duplex setup](../realtime_web/README.md#qwen3-with-server-vad-automatic-turns).
+The default `qwen3_omni_moe.yaml` remains a turn deployment. Duplex mode serves
+`/v1/realtime?duplex=1` and `/v1/chat/completions`; the legacy STT handler requires
+a turn deployment.
 
 The Python client supports the following command-line arguments:
 
@@ -584,3 +577,53 @@ The gradio script supports the following arguments:
 - `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
 - `--port`: Port for Gradio server (default: 7861)
 - `--share`: Share the Gradio demo publicly (creates a public link)
+
+## Browser voice call (shared realtime UI)
+
+Run these commands from the repository root with the vLLM-Omni environment activated.
+
+### Without VAD: manually submit each turn
+
+Start the backend:
+
+```bash
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
+```
+
+In another terminal, check backend readiness and start the UI:
+
+```bash
+curl --fail http://127.0.0.1:8091/health
+python -m examples.online_serving.qwen3_omni.realtime_web \
+    --backend ws://127.0.0.1:8091 --stt --port 7863
+```
+
+Open `http://localhost:7863`, start a session, speak, and press **Send turn**.
+`--stt` is the default. Each turn uses a separate connection without shared model
+history; the conversation shown in the browser is a local log.
+
+### With VAD: automatically submit after silence
+
+Start the backend with a Server VAD deployment overlay and a compatible Silero
+ONNX artifact, following the
+[complete VAD setup](../realtime_web/README.md#qwen3-with-server-vad-automatic-turns).
+After its health check succeeds, start the UI:
+
+```bash
+python -m examples.online_serving.qwen3_omni.realtime_web \
+    --backend ws://127.0.0.1:8091 --vad --port 7863
+```
+
+Speak and pause for 500 ms to submit automatically. `--vad` configures the client;
+it does not load a VAD model or change the backend deployment by itself.
+
+VAD mode keeps uploading audio during replies and supports speech interruption.
+The engine-owned Qwen duplex plugin maintains conversation history and tracks
+played replies through playback acknowledgements. Each committed utterance starts
+a new generation request; the model does not use native streaming-input KV decoding.
+STT mode pauses microphone upload while the reply is generated and played.
+In VAD mode, click **Camera** to upload sampled frames with your spoken question;
+see [camera setup and limits](../realtime_web/README.md#qwen-vad-camera-input).
+STT remains audio-only. To switch back to manual turns, use the default turn
+deployment, restart the UI with `--stt`, refresh the page, and reconnect. Stop the existing UI before reusing port 7863.
+See the [shared UI guide](../realtime_web/README.md) for HTTPS access and testing.

examples/online_serving/qwen3_omni/openai_realtime_client.py

@@ -130,7 +130,7 @@ async def run_client(
                                 "prefix_padding_ms": 300,
                                 "silence_duration_ms": 500,
                                 "create_response": True,
-                                "interrupt_response": False,
+                                "interrupt_response": True,
                             },
                         }
                     },
@@ -233,6 +233,7 @@ async def run_client(
                 continue
 
             if event_type == "response.done" and server_vad:
+                await ws.send(json.dumps({"type": "session.close"}))
                 break
 
             if event_type == "error":

examples/online_serving/qwen3_omni/realtime_web/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Qwen3-Omni turn-based browser call UI."""

examples/online_serving/qwen3_omni/realtime_web/__main__.py

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+from examples.online_serving.realtime_web.server import main
+
+if __name__ == "__main__":
+    main(default_profile="qwen3-turn")

examples/online_serving/realtime_web/README.md

@@ -0,0 +1,255 @@
+# Shared Realtime Web UI
+
+One browser shell provides microphone capture, speaker playback, a level meter,
+conversation text, and an event log. Model profiles own the wire protocol.
+This implements [RFC #7222](https://github.com/vllm-project/vllm-omni/issues/7222).
+
+| Profile | Turn control | Camera | Playback acknowledgement |
+| --- | --- | --- | --- |
+| `minicpm-native` | Model-controlled listen/speak, continuous audio input | Frames accompany audio | Yes |
+| `qwen3-turn --stt` (default) | User presses **Send turn** | No | No |
+| `qwen3-turn --vad` | Server detects trailing silence; speech interrupts replies | Sampled frames with each spoken turn | Yes |
+| `aura-ptt` | Hold **Hold to talk**; release commits. Hold also stops local playback | Sticky frames with speech; vision-follow every 2 frames only while unlocked (locked from PTT release until text-final / listen) | Yes |
+
+Qwen3 VAD uses an engine-owned duplex plugin: microphone upload continues
+while replies stream, and speech can interrupt generation and playback. The
+checkpoint still generates committed turns; this is not native streaming-input
+KV decoding. STT remains explicit-turn and audio-only.
+
+## MiniCPM: existing command stays valid
+
+Start your MiniCPM backend using the [existing deployment instructions](../minicpmo/README.md), then:
+
+```bash
+python -m examples.online_serving.minicpmo.realtime_web \
+    --ws-backend ws://127.0.0.1:8099 \
+    --ref-audio /path/to/ref_minicpm_signature.wav
+```
+
+The compatibility wrapper serves the shared assets with `minicpm-native`.
+The current native query, `extra_body`, reference voice, continuous microphone
+upload, camera frames and playback acknowledgements are retained.
+
+## AURA: push-to-talk duplex
+
+AURA has no client VAD. Start the duplex serve, then the thin wrapper:
+
+```bash
+bash examples/online_serving/aura_omni/run_duplex_smoke_serve.sh
+
+python -m examples.online_serving.aura_omni.realtime_web \
+    --ws-backend ws://127.0.0.1:8099 --model aurateam/AURA --port 7862
+```
+
+Or select the profile on the shared host:
+
+```bash
+python -m examples.online_serving.realtime_web --profile aura-ptt \
+    --ws-backend ws://127.0.0.1:8099 --model aurateam/AURA --port 7862
+```
+
+Open the UI, start a session, optionally enable **Camera**, then **hold**
+**Hold to talk** to stream `is_speech=true` PCM (with sticky camera frames).
+**Release** to `input_audio_buffer.commit`. Holding the button stops playback
+in the browser only; the in-flight response is not cancelled on the server.
+With the camera on, frames are still captured at 2 fps. While the button is
+up they are committed once per two frames (`is_speech=false` + silent PCM).
+Both frames of that commit are sent and packed as one video (`<|video_pad|>`).
+Turns still open from this client commit, not from a server-side auto trigger.
+but only after the prior spoken turn unlocks (text-final or `response.listen`).
+Release itself locks vision immediately so a follow-up frame cannot abort the
+just-committed speech turn on Stage0/1 (`max_num_seqs=1`). Audio still playing
+does not hold the next vision turn once text is done.
+MiniCPM / Qwen profiles do not set `pushToTalk`, so the PTT control stays hidden.
+
+## Qwen3: explicit-turn STT adapter
+
+Start the backend, then the UI in a second terminal:
+
+```bash
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
+
+python -m examples.online_serving.qwen3_omni.realtime_web \
+    --backend ws://127.0.0.1:8091 --stt --port 7863
+```
+
+Open `http://localhost:7863`, start a session, speak, and press **Send turn**.
+Wait for the answer to finish playing before speaking again. The UI opens a
+fresh STT connection for each turn; the displayed conversation is a local log,
+and **previous turns are not provided as model history**. The STT endpoint
+accepts only model selection, so system-prompt controls are hidden in this mode.
+
+The adapter follows the shipped `qwen3_omni/openai_realtime_client.py` and
+`vllm_omni/entrypoints/openai/realtime_connection.py`:
+
+
+1. Explicit `duplex=0` selects the legacy STT handler on a turn deployment.
+   It does not enable STT on a duplex deployment.
+2. Send `{type: "session.update", model: ...}` and `commit(final=false)`.
+3. Stream mono PCM16 at 16 kHz; **Send turn** flushes buffered audio, then sends `commit(final=true)`.
+4. `response.output_audio.delta.audio` contains PCM; `response.output_audio.done`
+   terminates this adapter, including a response with no audio.
+5. The local implementation emits **model output** on `transcription.*`, so the
+   UI displays it as assistant text. This differs from the early RFC's suggested
+   user-transcript mapping. The VAD adapter also supports `response.output_text.*`
+   and audio-transcript events without duplicating the answer.
+
+## Qwen3 with Server VAD (automatic turns)
+
+Run commands from the repository root with the vLLM-Omni environment activated.
+Prepare the pinned Silero v6.2 ONNX artifact as described in the
+[Qwen3 Server VAD instructions](../qwen3_omni/README.md#realtime-websocket-client-openai_realtime_clientpy).
+The backend requires ONNX Runtime and a compatible local artifact; setting
+`--vad` on the UI alone does not enable VAD on the backend.
+
+Create an overlay named `qwen3_vad.yaml`, replacing both paths with absolute paths
+on the backend host:
+
+```yaml
+base_config: /path/to/vllm-omni/vllm_omni/deploy/qwen3_omni_duplex.yaml
+duplex_session:
+  server_vad_model_path: /path/to/silero_vad.onnx
+```
+
+The base configuration enables duplex mode and sets the per-prompt audio and image limits.
+
+Start the backend:
+
+```bash
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
+    --deploy-config qwen3_vad.yaml
+```
+
+In a second terminal, wait until the health check succeeds, then start the UI:
+
+```bash
+curl --fail http://127.0.0.1:8091/health
+python -m examples.online_serving.qwen3_omni.realtime_web \
+    --backend ws://127.0.0.1:8091 --vad --port 7863
+```
+
+Open `http://localhost:7863` and start a session. Speak and pause: the server
+commits the turn automatically after 500 ms of silence. No **Send turn** action
+is needed. Defaults are a speech threshold of 0.5 and 300 ms of prefix padding.
+Speak again to interrupt an answer. The engine owns input buffering, VAD,
+request cancellation, playback acknowledgements, and conversation history.
+Each committed utterance starts a normal Thinker → Talker → Code2Wav request;
+it does not append input to a live Qwen KV cache. The capability response reports
+`implementation_level: turn_based_duplex` and `supports_core_resumable_request: false`.
+
+The plugin retains up to four audio inputs (including the current turn), pruning
+older audio at an 8 MiB base64 budget; one current utterance is bounded by the
+engine's pending-input limit. The UI reports playback progress both on normal
+drain and on interruption. Qwen does not provide text/audio alignment, so assistant
+text enters history only after the complete audio response has been generated and
+acknowledged as played. A partially played answer contributes no guessed text
+prefix. Image conversation items remain available until deleted. Model context limits still apply.
+The bundled duplex deploy allows four audio inputs and eight images per prompt.
+Use headphones to avoid speaker audio triggering VAD interruption. Reference
+voices and tool calls are not supported by this plugin.
+
+### Switching between VAD and manual turns
+
+| Mode | Backend | UI flag | Submit a turn |
+| --- | --- | --- | --- |
+| Without VAD | Default Qwen turn deployment | `--stt` (default) | Press **Send turn** |
+| With VAD | Server VAD configuration and Silero artifact | `--vad` | Pause after speaking |
+
+Stop the existing UI process before starting another on the same port, then
+refresh the browser and reconnect. To use STT, start a backend with the default
+turn deployment and restart the UI with `--stt`. The duplex backend exposes
+Realtime duplex and ChatCompletion; it does not expose the legacy STT handler.
+To enable VAD on a default backend, first restart that backend with the overlay.
+Make `--backend` match the actual backend port; the UI and backend use separate
+ports. `Connection refused` means the target backend is unavailable: check its
+startup log and `/health` before connecting.
+
+VAD connects with `duplex=1` and sends nested `session.audio.input` configuration
+with `create_response: true` and `interrupt_response: true`. It sends playback
+ACKs when speaker audio drains. `response.audio.done` / `response.output_audio.done`
+only drain playback; `response.done` terminates a turn. Backpressure displays a
+notice without pausing the microphone. Failed handshakes time out after 15 seconds;
+Silero initialization failures display a specific error.
+
+After updating the backend Python code, restart the backend, refresh the UI, and
+start a new session so the plugin and browser assets use the updated code. A
+successful `/health` check confirms readiness, not which deployment mode is active.
+
+## Qwen VAD camera input
+
+After starting a session with `--vad`, click **Camera** and allow access. The UI
+sends one JPEG per second, resized to a longest side of 448 pixels, independently
+of microphone uploads (including while muted). Each frame is a user conversation
+item using the [OpenAI Realtime image input](https://developers.openai.com/api/docs/guides/realtime-conversations#image-inputs) shape:
+
+```json
+{"type":"conversation.item.create","item":{"id":"camera_1","type":"message","role":"user","content":[{"type":"input_image","image_url":"data:image/jpeg;base64,..."}]}}
+```
+
+Images become conversation context immediately; they do not start a response.
+Speak and let VAD commit your audio, or send `response.create` after adding an
+image/text item. A manual `input_audio_buffer.commit` only commits audio; follow
+it with `response.create` to generate. Images persist across audio commits,
+interruptions, and audio-buffer clears. Remove them with `conversation.item.delete`.
+The browser retains eight camera items and deletes the oldest before adding a
+ninth. Camera-off stops capture but leaves those items in the conversation.
+
+This deployment accepts JPEG/PNG Base64 data URLs with a total image-context
+budget of eight images and 4 MiB; external image URLs are not supported. Clients
+must delete old image items when reaching the limit. There is no precise
+video/audio timestamp alignment or native streaming-video KV update.
+
+### Protocol compatibility boundary
+
+Qwen VAD uses `session.update`, 24 kHz PCM16 `input_audio_buffer.append`,
+`conversation.item.create` with `input_image`/`input_text`, and `response.create`.
+The engine resamples input to Qwen's 16 kHz processing rate. Qwen rejects the
+custom `video_frames` field on an append: it answers whole turns, so it has no
+audio unit for a frame track to align to. MiniCPM keeps that field for its
+native camera protocol, where Stage 0 interleaves frames at unit boundaries.
+
+This is a supported subset, not full OpenAI Realtime API compatibility.
+`duplex=1`, `overlap_policy`, `playback.ack`, and `session.close` remain vLLM-Omni
+extensions. The legacy `--stt` mode is unchanged and is not the standard Realtime
+conversation API. This demo does not add WebRTC, semantic VAD, or tool calling.
+
+## Shared host options
+
+The shared entry point can select a profile explicitly:
+
+```bash
+python -m examples.online_serving.realtime_web --profile qwen3-turn \
+    --backend ws://127.0.0.1:8091 --stt --port 7863
+```
+
+- `--backend` / `--ws-backend`: backend WebSocket origin (not the full `/v1/realtime` URL).
+- `--model`: override the wrapper's model name.
+- `--public-realtime-url`: optional browser-visible WebSocket URL; otherwise the
+  static host proxies `/v1/realtime` on the same origin.
+- `--host`, `--port`: UI bind address and port (default port 7862).
+- `--ref-audio`: required for MiniCPM; rejected for Qwen3 and AURA.
+
+Microphone and camera access require `localhost` or HTTPS. For a remote backend,
+use an SSH tunnel to the UI host or an HTTPS reverse proxy with WebSocket support.
+An explicit `wss://` public URL retains its scheme.
+
+## Validation
+
+```bash
+node --test tests/examples/test_realtime_web_profiles.cjs
+pytest tests/examples/test_minicpmo_realtime_web_server.py \
+       tests/examples/test_minicpmo_realtime_web_static.py \
+       tests/examples/test_realtime_web_profiles.py
+pytest -m cpu tests/model_executor/models/qwen3_omni/test_duplex_plugin.py \
+       tests/engine/duplex/test_session_runner.py \
+       tests/engine/test_duplex_orchestrator.py
+```
+
+These CPU tests cover profile messages, two STT turns, VAD terminal/drain ordering,
+backpressure, native capture during playback, server config injection, and shared
+worklets. Backend regressions cover current audio inclusion in history-based
+prompts, delayed playback ACK ordering, and no extra generation during silence
+after a completed VAD turn. Hardware acceptance still requires a MiniCPM call
+with camera/barge-in and two Qwen turns against each enabled backend, including
+speech interruption in VAD mode; simulated tests do not
+establish model quality or hardware latency.