Merge main and align text encoder TP regression documentation

Signed-off-by: Sy03 <1370724210@qq.com>

.buildkite/amd/scripts/build-ci-image.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+set -euo pipefail
+
+# Build and push the ROCm CI image with a registry-backed BuildKit cache. The
+# AMD builders are ephemeral, so a local Docker cache cannot survive between
+# jobs. Keeping the cache beside the runtime image also reuses the registry
+# credentials already configured on the amd-cpu agents.
+
+REGISTRY="${ROCM_CI_IMAGE_REGISTRY:-rocm/vllm-omni}"
+DOCKERFILE="docker/Dockerfile.rocm"
+BUILDER_NAME="vllm-omni-rocm-builder"
+ROCM_ARCH="${PYTORCH_ROCM_ARCH:-gfx942}"
+
+if [[ -z "${BUILDKITE_COMMIT:-}" ]]; then
+    echo "ERROR: BUILDKITE_COMMIT is not set" >&2
+    exit 1
+fi
+
+dependency_files=(
+    "$DOCKERFILE"
+    "${DOCKERFILE}.dockerignore"
+    pyproject.toml
+    setup.py
+    tools/install_torchcodec_rocm.sh
+)
+while IFS= read -r dependency_file; do
+    dependency_files+=("$dependency_file")
+done < <(find requirements -maxdepth 1 -type f -name '*.txt' -print | LC_ALL=C sort)
+
+for dependency_file in "${dependency_files[@]}"; do
+    if [[ ! -f "$dependency_file" ]]; then
+        echo "ERROR: cache-key input does not exist: $dependency_file" >&2
+        exit 1
+    fi
+done
+
+CACHE_KEY=$(
+    {
+        printf 'rocm-arch=%s\n' "$ROCM_ARCH"
+        for dependency_file in "${dependency_files[@]}"; do
+            printf 'path=%s\n' "$dependency_file"
+            sha256sum "$dependency_file"
+        done
+    } | sha256sum | cut -c1-16
+)
+CACHE_REF="${REGISTRY}:rocm-deps-cache-${CACHE_KEY}"
+IMAGE_REF="${REGISTRY}:${BUILDKITE_COMMIT}"
+
+echo "ROCm dependency cache: ${CACHE_REF}"
+echo "ROCm CI image: ${IMAGE_REF}"
+
+# The docker-container driver supports exporting the full cache to a registry;
+# the default docker driver on CI does not.
+if ! docker buildx inspect "$BUILDER_NAME" >/dev/null 2>&1; then
+    docker buildx create --name "$BUILDER_NAME" --driver docker-container
+fi
+docker buildx use "$BUILDER_NAME"
+docker buildx inspect --bootstrap
+
+docker buildx build --push --pull --progress=plain \
+    --target test \
+    --build-arg "ARG_PYTORCH_ROCM_ARCH=${ROCM_ARCH}" \
+    --cache-from "type=registry,ref=${CACHE_REF}" \
+    --cache-to "type=registry,ref=${CACHE_REF},mode=max,compression=zstd,ignore-error=true" \
+    --file "$DOCKERFILE" \
+    --tag "$IMAGE_REF" .

.buildkite/amd/test-amd-merge.yml

@@ -9,15 +9,26 @@ steps:
   steps:
     - label: "Simple · Model Executor Test · Shard %N/%t"
       agent_pool: mi300_1
-      parallelism: 2
+      parallelism: 3
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      timeout_in_minutes: 90
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+        - "pytest -sv tests/model_executor -m 'core_model and cpu and not omni' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+
+    - label: "Simple · Omni Processor Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 60
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/model_executor/models/test_omni_processing.py -m 'core_model and cpu and omni'"
 
     - label: "Simple · Diffusion Test · Shard %N/%t"
       agent_pool: mi300_1
@@ -25,6 +36,7 @@ steps:
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      timeout_in_minutes: 45
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM

.buildkite/amd/test-amd-ready.yml

@@ -9,26 +9,39 @@ steps:
   steps:
     - label: "Simple · Model Executor Test · Shard %N/%t"
       agent_pool: mi300_1
-      parallelism: 2
+      parallelism: 3
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      timeout_in_minutes: 90
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+        - "pytest -sv tests/model_executor -m 'core_model and cpu and not omni' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
 
-    - label: "Simple · Diffusion Test"
+    - label: "Simple · Omni Processor Test"
       agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      timeout_in_minutes: 60
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/model_executor/models/test_omni_processing.py -m 'core_model and cpu and omni'"
+
+    - label: "Simple · Diffusion Test · Shard %N/%t"
+      agent_pool: mi300_1
+      parallelism: 4
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 45
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "pytest -sv tests/diffusion -m 'core_model and cpu'  --ignore=tests/diffusion/cache/test_teacache_extractors.py"
+        - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
 
     - label: "Simple · Engine&Entrypoints Test"
       agent_pool: mi300_1
@@ -69,6 +82,27 @@ steps:
 
 - group: ":card_index_dividers: Diffusion Test"
   steps:
+    # Keep real FlashAttention execution out of the CPU-marked Simple lane.
+    # AITER may need to build several kernels on a cold worker, so run one
+    # pytest worker with job-local build directories and a bounded timeout.
+    - label: "ROCm · FlashAttention/AITER GPU Coverage"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: NonBlocking
+      timeout_in_minutes: 60
+      commands:
+        - export VLLM_ROCM_USE_AITER=1
+        - export AITER_JIT_DIR="/tmp/vllm-omni-aiter-$$BUILDKITE_JOB_ID"
+        - export TORCH_EXTENSIONS_DIR="/tmp/vllm-omni-torch-extensions-$$BUILDKITE_JOB_ID"
+        - mkdir -p "$$AITER_JIT_DIR" "$$TORCH_EXTENSIONS_DIR"
+        - >-
+          timeout --signal=TERM --kill-after=2m 55m
+          pytest -s -v -n 1
+          tests/diffusion/attention/
+          -m 'core_model and rocm and MI325 and cards_1'
+          --run-level "core_model"
+
     - label: "Diffusion · Batch Test"
       agent_pool: mi300_1
       depends_on: amd-build
@@ -299,6 +333,12 @@ steps:
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      # A ROCm hardware hang aborts the interpreter with 134. Let Buildkite
+      # create one fresh pod instead of retrying on the poisoned GPU process.
+      retry:
+        automatic:
+          - exit_status: 134
+            limit: 1
       commands:
         - pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"
 

.buildkite/amd/test-template-amd-omni.j2

@@ -3,7 +3,8 @@
    Last synced: 2026-08-04
    Modifications: Use the shared MI300 queues and Omni command-passing contract
 #}
-{% set docker_image_amd = "rocm/vllm-omni:$BUILDKITE_COMMIT" %}
+{% set docker_registry_amd = "rocm/vllm-omni" %}
+{% set docker_image_amd = docker_registry_amd ~ ":$BUILDKITE_COMMIT" %}
 {% set default_working_dir = "/app/vllm-omni" %}
 
 {# MI300 runs tests natively in Kubernetes; Docker-in-Docker is unavailable. #}
@@ -74,11 +75,12 @@ steps:
         depends_on: ~
         soft_fail: false
         commands:
-          - "docker build -f docker/Dockerfile.rocm -t {{ docker_image_amd }} --target test --build-arg ARG_PYTORCH_ROCM_ARCH=gfx942 --progress plain ."
-          - "docker push {{ docker_image_amd }}"
+          - "bash .buildkite/amd/scripts/build-ci-image.sh"
         key: "amd-build"
         env:
           DOCKER_BUILDKIT: "1"
+          PYTORCH_ROCM_ARCH: "gfx942"
+          ROCM_CI_IMAGE_REGISTRY: "{{ docker_registry_amd }}"
         retry:
           automatic:
             - exit_status: -1  # Agent was lost
@@ -156,6 +158,14 @@ steps:
 {% for path in step.artifact_paths %}
           - "{{ path }}"
 {% endfor %}
+{% endif %}
+{% if step.retry %}
+        retry:
+          automatic:
+{% for retry_rule in step.retry.automatic %}
+            - exit_status: {{ retry_rule.exit_status }}
+              limit: {{ retry_rule.limit }}
+{% endfor %}
 {% endif %}
         {% if step.timeout_in_minutes %}
         timeout_in_minutes: {{ step.timeout_in_minutes }}
@@ -224,6 +234,14 @@ steps:
 {% for path in step.artifact_paths %}
       - "{{ path }}"
 {% endfor %}
+{% endif %}
+{% if step.retry %}
+    retry:
+      automatic:
+{% for retry_rule in step.retry.automatic %}
+        - exit_status: {{ retry_rule.exit_status }}
+          limit: {{ retry_rule.limit }}
+{% endfor %}
 {% endif %}
     {% if step.timeout_in_minutes %}
     timeout_in_minutes: {{ step.timeout_in_minutes }}

.buildkite/common/ci_source_file_dependencies.yml

@@ -23,6 +23,16 @@ x-qwen3-tts: &qwen3_tts
   - vllm_omni/entrypoints/openai/serving_speech.py
   - vllm_omni/entrypoints/openai/tts_adapters/
   - vllm_omni/engine/stage_pool.py
+  - vllm_omni/worker_v2/
+  - vllm_omni/core/sched/
+  - vllm_omni/distributed/omni_connectors/
+  - vllm_omni/config/stage_config.py
+  - vllm_omni/config/omni_config.py
+  - vllm_omni/engine/orchestrator.py
+  - vllm_omni/engine/stage_engine_core_proc.py
+  - vllm_omni/deploy/qwen3_tts_mrv2.yaml
+  - vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2.yaml
+  - vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2_b4.yaml
 
 x-minicpmo-4-5: &minicpmo_4_5
   - vllm_omni/engine/async_omni_engine.py
@@ -44,6 +54,7 @@ x-minicpmo-4-5-duplex: &minicpmo_4_5_duplex
   - vllm_omni/entrypoints/duplex/
   - vllm_omni/entrypoints/duplex_omni.py
   - vllm_omni/entrypoints/async_omni_base.py
+  - vllm_omni/protocol/
   - vllm_omni/clients/
   - vllm_omni/outputs/duplex.py
   - vllm_omni/model_executor/duplex_sampling.py
@@ -99,6 +110,7 @@ x-hunyuan-image3: &hunyuan_image3
 
 x-minimax-h3: &minimax_h3
   - vllm_omni/diffusion/models/minimax_h3/
+  - vllm_omni/diffusion/layers/indexed_modulation.py
 
 x-hunyuan-video: &hunyuan_video
   - vllm_omni/diffusion/models/hunyuan_video/
@@ -259,6 +271,7 @@ source_file_dependencies:
     - tests/entrypoints/test_duplex_omni.py
     - tests/entrypoints/openai_api/test_duplex_api_server.py
     - tests/model_executor/models/minicpmo_4_5/duplex/
+    - tests/protocol/
 
   omni_minicpmo_4_5_duplex_perf:
     - *minicpmo_4_5

.buildkite/cuda/test-nightly.yml

@@ -223,6 +223,17 @@ steps:
             -m "full_model and diffusion and H100 and B200 and cards_1"
             --run-level "full_model"
 
+      - label: ":robot_face: Robot Policy OpenPI · π0.5 · H100 · Single-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - pip install --no-deps "openpi-client @ git+https://github.com/Physical-Intelligence/openpi.git@215abfb217dbac7d5f1273282331b9b1866c0479#subdirectory=packages/openpi-client"
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_pi05_expansion.py
+            -m "full_model and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
       - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU"
         source_file_dependencies: diffusion_qwen_image_function
         timeout_in_minutes: 120

.buildkite/cuda/test-ready.yml

@@ -38,6 +38,12 @@ steps:
     depends_on: upload-ready-pipeline
     steps:
 
+      - label: "Diffusion · FlashAttention Test"
+        timeout_in_minutes: 20
+        commands:
+          - timeout 15m pytest -s -v tests/diffusion/attention/ -m 'core_model and cuda and L4 and cards_1' --run-level "core_model"
+        mirror_hardwares: l4_1
+
       - label: "Diffusion · Offloader Test"
         commands:
           - timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
@@ -96,7 +102,7 @@ steps:
     commands:
       - |
         timeout 15m bash -c "
-                  pytest -sv tests/engine/ tests/model_executor/ tests/worker/test_batched_omni_output.py -m 'core_model and cuda' --run-level 'core_model'
+                  pytest -sv tests/engine/ tests/model_executor/ tests/worker/test_batched_omni_output.py tests/worker_v2/ -m 'core_model and cuda' --run-level 'core_model'
         "
     mirror_hardwares: l4_1
 

.buildkite/npu/test-npu-nightly.yml

@@ -235,6 +235,37 @@ steps:
       build.env("NIGHTLY") == "1" ||
       build.pull_request.labels includes "nightly-test"
     steps:
+      - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test"
+        source_file_dependencies: diffusion_hunyuan_video_function
+        timeout_in_minutes: 180
+        mirror_hardwares: a2b3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+          HF_HUB_OFFLINE: "1"
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "npu and A2" --run-level "full_model"
+      - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Perf Test"
+        source_file_dependencies: diffusion_hunyuan_video_perf
+        key: nightly-diffusion-x2v-performance-hunyuanvideo15
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_2
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+          HF_HUB_OFFLINE: "1"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
       - label: ":full_moon: Diffusion X2V · Wan 2.2 Function Test"
         source_file_dependencies: diffusion_wan22_function
         mirror_hardwares: a2b3_npu_4

.claude/skills/production-add-diffusion-model/references/production-validation.md

@@ -340,7 +340,7 @@ On supported GPU hosts, wrap a test or soak with the repository monitor:
 
 ```bash
 bash tests/dfx/stability/scripts/resource_monitor.sh run --backend gpu -- \
-  pytest -s -v tests/dfx/stability/scripts/test_stability_<model>.py -m slow
+  pytest -s -v tests/dfx/stability/scripts/run_stability_<model>.py -m slow
 ```
 
 Check the target revision's monitor backends. Do not claim CPU/NPU monitoring

.github/CODEOWNERS

@@ -67,6 +67,11 @@
 # File-level overrides within entrypoints/
 /vllm_omni/entrypoints/openai/errors.py @alex-jw-brooks @NickCao
 /vllm_omni/entrypoints/cli/benchmark/ @alex-jw-brooks @Bounty-hunter
+# Shared wire codec: protocol/realtime (OpenAI Realtime GA) and protocol/duplex
+# (vLLM-Omni's extension of it) — RFC #6592 P0a. Layer-neutral by design — the
+# engine depends on it — so it is owned by the serving-boundary document rather
+# than by any one consumer. — docs/design/module/entrypoints.md
+/vllm_omni/protocol/ @alex-jw-brooks @linyueqian @NickCao
 
 # Error contracts — docs/design/module/error_contracts.md
 /vllm_omni/errors.py @alex-jw-brooks @NickCao

README.md

@@ -58,7 +58,7 @@ vLLM-Omni is flexible and easy to use with:
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
 - **Omni-modality models** (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
-- **TTS models** (e.g. Qwen3-TTS, IndexTTS 2.5, dots.tts, CosyVoice3)
+- **TTS models** (e.g. Qwen3-TTS, IndexTTS 2.5, CosyVoice3)
 - **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
 - **Robot-policy and action models** (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
 

apps/ComfyUI-vLLM-Omni/README.md

@@ -49,6 +49,7 @@ This extension offers the following nodes based on the output modalities (at **C
 
 - **Generate Image** for text-to-image and image-to-image tasks
 - **Generate Video** for text-to-video, first-frame/image-to-video, and reference-conditioned video
+- **Latent Mask Editing** for MiniMax-H3 latent-mask editing (source media plus video/audio noise masks)
 - **FastH3 Deployment** for routing text-to-video requests to a MiniMax-H3 server with FastH3 fused at startup
 - **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks
 - **TTS** and **TTS Voice Clone** for TTS tasks
@@ -67,7 +68,7 @@ Every node carries the vLLM-Omni mark in its title bar and is tinted by what it
 | Amber | AR / Diffusion / Multi-Stage Sampling Params | Sampling parameters that apply to any model |
 | Purple | Qwen TTS Params, Wan Video Params, MiniMax-H3 Video Params | Parameters that only one model family accepts |
 | Red | LoRA, FastH3 Deployment | Which weights the server is expected to have loaded |
-| Teal | Video References | Reference media |
+| Teal | Video References, Latent Mask Editing | Reference media |
 
 Recolouring a node by hand (right click -> Colors) overrides its tint, and the choice is kept.
 
@@ -134,11 +135,13 @@ You can configure per-stage sampling parameters for multi-stage models.
 </p>
 
 > [!TIP]
-> Connect a **frame** image for first-frame / image-to-video (e.g., Wan I2V, MiniMax-H3 FL2VA).
+> Connect **frame** for backward-compatible first-frame / image-to-video behavior. For MiniMax-H3 FL2VA,
+> connect **first_frame**, **last_frame**, or both to condition the start frame, end frame, or both.
 >
 > For reference-conditioned generation (MiniMax-H3 Ref2VA), connect a **Video References** node instead.
 >
-> Do not use `frame` and `references` together. Task routing is automatic from which inputs you connect.
+> Do not combine `frame` with `first_frame` or `last_frame`, and do not combine any frame input with
+> `references`. Task routing is automatic from which inputs you connect.
 
 For MiniMax-H3 Ref2VA, **Video References** accepts up to 9 images (`image_1`–`image_9`),
 3 videos (`video_1`–`video_3`), and 3 audio clips (`audio_1`–`audio_3`), with at most
@@ -261,6 +264,18 @@ Open the **vLLM-Omni FastH3 Text to Video** template, then:
 
 The node records which server the workflow targets; it does not start one, nor switch adapters or attention backends on a running server.
 
+#### Latent-mask editing (MiniMax-H3)
+
+The [WF-05 template](example_workflows/vLLM-Omni%20MiniMax-H3%20Latent%20Mask%20Editing.json) contains inpainting, object removal, continuation, and extension examples in one graph. See the [workflow guide](docs/wf05-h3-latent-editing.md) for inputs, mask settings, dependencies, and preview limitations.
+
+Connect a **Latent Mask Editing** node to **Generate Video → latent_edit** to edit a source clip instead of generating from scratch. It uploads the source media and serializes the video/audio noise masks the MiniMax H3 API accepts:
+
+- `source_video` / `source_audio` — the media to edit.
+- `video_mask` — a ComfyUI mask image; `0` preserves a region, `1` regenerates it, fractional values blend. A 2D mask `[H, W]` is applied to every frame; a 3D mask `[T, H, W]` is treated as a temporal mask (one slice per frame) for continuation or extension. It is resized to the video latent grid.
+- `audio_mask` — a scalar in `[0, 1]`; `0` keeps the source audio, `1` regenerates it, fractional values blend.
+
+A non-trivial mask requires its matching source, and a source without a mask is rejected. This node only forwards inputs to the server; the served model must declare latent-mask editing support (MiniMax-H3).
+
 ### TTS (e.g., Qwen TTS series)
 
 (Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni TTS**)

apps/ComfyUI-vLLM-Omni/__init__.py

@@ -21,7 +21,9 @@ from .comfyui_vllm_omni.nodes import (
     VLLMOmniGenerateImage,
     VLLMOmniGenerateMusic,
     VLLMOmniGenerateVideo,
+    VLLMOmniLatentMaskEditing,
     VLLMOmniMiniMaxH3Params,
+    VLLMOmniMiniMaxH3TemporalMask,
     VLLMOmniQwenTTSParams,
     VLLMOmniRemoteLoRA,
     VLLMOmniSamplingParamsList,
@@ -34,6 +36,7 @@ from .comfyui_vllm_omni.nodes import (
 
 # A dictionary that contains all nodes you want to export with their names
 NODE_CLASS_MAPPINGS = {
+    "VLLMOmniMiniMaxH3TemporalMask": VLLMOmniMiniMaxH3TemporalMask,
     # === Generation ===
     "VLLMOmniGenerateImage": VLLMOmniGenerateImage,
     "VLLMOmniGenerateVideo": VLLMOmniGenerateVideo,
@@ -42,6 +45,7 @@ NODE_CLASS_MAPPINGS = {
     "VLLMOmniGenerateMusic": VLLMOmniGenerateMusic,
     "VLLMOmniVoiceClone": VLLMOmniVoiceClone,
     "VLLMOmniVideoReferences": VLLMOmniVideoReferences,
+    "VLLMOmniLatentMaskEditing": VLLMOmniLatentMaskEditing,
     # === Params ===
     "VLLMOmniARSampling": VLLMOmniARSampling,
     "VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling,
@@ -55,6 +59,7 @@ NODE_CLASS_MAPPINGS = {
 
 # A dictionary that contains the friendly/humanly readable titles for the nodes
 NODE_DISPLAY_NAME_MAPPINGS = {
+    "VLLMOmniMiniMaxH3TemporalMask": "MiniMax-H3 Temporal Mask",
     # === Generation ===
     "VLLMOmniGenerateImage": "Generate Image",
     "VLLMOmniGenerateVideo": "Generate Video",
@@ -63,6 +68,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "VLLMOmniGenerateMusic": "Generate Music",
     "VLLMOmniVoiceClone": "TTS Voice Cloning",
     "VLLMOmniVideoReferences": "Video References",
+    "VLLMOmniLatentMaskEditing": "Latent Mask Editing",
     # === Params ===
     "VLLMOmniARSampling": "AR Sampling Params",
     "VLLMOmniDiffusionSampling": "Diffusion Sampling Params",

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py

@@ -1,12 +1,14 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
+import math
 from typing import Literal
 
 import torch
 from comfy_api.input import AudioInput, VideoInput
 
 from .utils.api_client import VLLMOmniClient
+from .utils.latent_mask import _align_frame_count, _video_latent_t
 from .utils.logger import get_logger
 from .utils.models import lookup_model_spec
 from .utils.types import (
@@ -17,6 +19,7 @@ from .utils.types import (
     AutoregressionSamplingParams,
     DiffusionSamplingParams,
     FastH3Deployment,
+    LatentMaskEditing,
     MiniMaxH3ModelSpecificParams,
     QwenTTSModelSpecificParams,
     VideoReferences,
@@ -203,11 +206,14 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
             },
             "optional": {
                 "frame": ("IMAGE",),
+                "first_frame": ("IMAGE",),
+                "last_frame": ("IMAGE",),
                 "references": ("VIDEO_REFERENCES",),
                 "sampling_params": ("SAMPLING_PARAMS",),
                 "lora": ("REMOTE_LORA",),
                 "model_params": ("VIDEO_PARAMS",),
                 "fast_h3": ("FASTH3_DEPLOYMENT",),
+                "latent_edit": ("LATENT_MASK_EDITING",),
             },
         }
 
@@ -216,12 +222,30 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
     FUNCTION = "generate"
 
     @classmethod
-    def VALIDATE_INPUTS(cls, url, model, frame=None, references=None, **_kwargs) -> str | Literal[True]:
+    def VALIDATE_INPUTS(
+        cls,
+        url,
+        model,
+        frame=None,
+        first_frame=None,
+        last_frame=None,
+        references=None,
+        fast_h3=None,
+        **_kwargs,
+    ) -> str | Literal[True]:
         base = super().VALIDATE_INPUTS(url, model)
         if base is not True:
             return base
         if frame is not None and references is not None:
             return "Provide only one of frame or references, not both."
+        if frame is not None and (first_frame is not None or last_frame is not None):
+            return "Provide either frame or first_frame/last_frame, not both."
+        if references is not None and (first_frame is not None or last_frame is not None):
+            return "Provide either first_frame/last_frame or references, not both."
+        if fast_h3 is not None and any(value is not None for value in (frame, first_frame, last_frame, references)):
+            return (
+                "FastH3 Preview supports T2VA only; disconnect frame, first_frame, last_frame, and references inputs."
+            )
         return True
 
     async def generate(
@@ -235,11 +259,14 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
         duration: float,
         negative_prompt: str | None = None,
         frame: torch.Tensor | None = None,
+        first_frame: torch.Tensor | None = None,
+        last_frame: torch.Tensor | None = None,
         references: dict | None = None,
         sampling_params: dict | list[dict] | None = None,
         model_params: dict | None = None,
         lora: dict | None = None,
         fast_h3: dict | None = None,
+        latent_edit: dict | None = None,
         **kwargs,
     ):
         if kwargs:
@@ -252,8 +279,11 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
         spec_model = model
 
         if fast_h3 is not None:
-            if frame is not None or references is not None:
-                raise ValueError("FastH3 Preview supports T2VA only; disconnect frame and references inputs.")
+            if any(value is not None for value in (frame, first_frame, last_frame, references)):
+                raise ValueError(
+                    "FastH3 Preview supports T2VA only; disconnect frame, first_frame, "
+                    "last_frame, and references inputs."
+                )
             if lora is not None:
                 raise ValueError(
                     "FastH3 is already fused into the selected server; disconnect the request-level LoRA input."
@@ -314,6 +344,8 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
             spec_model=spec_model,
             prompt=prompt,
             frame=frame,  # frame present => fl2va / Wan I2V
+            first_frame=first_frame,
+            last_frame=last_frame,
             references=references,
             width=width,
             height=height,
@@ -323,6 +355,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
             sampling_params=sampling_params,
             lora=lora,
             model_params=model_params,
+            latent_edit=latent_edit,
         )
         return (output,)
 
@@ -1024,3 +1057,89 @@ class VLLMOmniVideoReferences:
                 if value is not None:
                     refs[f"{kind}_{index}"] = value
         return (refs,)
+
+
+class VLLMOmniLatentMaskEditing:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {},
+            "optional": {
+                "source_video": ("VIDEO",),
+                "source_audio": ("AUDIO",),
+                "video_mask": ("MASK",),
+                "audio_mask": ("FLOAT", {"default": -1.0, "min": -1.0, "max": 1.0, "step": 0.01}),
+            },
+        }
+
+    RETURN_TYPES = ("LATENT_MASK_EDITING",)
+    RETURN_NAMES = ("latent_edit",)
+    FUNCTION = "get_latent_edit"
+    CATEGORY = "vLLM-Omni"
+
+    def get_latent_edit(
+        self,
+        source_video: VideoInput | None = None,
+        source_audio: AudioInput | None = None,
+        video_mask: torch.Tensor | None = None,
+        audio_mask: float = -1.0,
+        **kwargs,
+    ):
+        if kwargs:
+            logger.info("Uncaught kwargs: %s", kwargs)
+        edit = LatentMaskEditing()
+        if source_video is not None:
+            edit["source_video"] = source_video
+        if source_audio is not None:
+            edit["source_audio"] = source_audio
+        if video_mask is not None:
+            edit["video_mask"] = video_mask
+        if audio_mask >= 0.0:
+            edit["audio_mask"] = audio_mask
+        return (edit,)
+
+
+class VLLMOmniMiniMaxH3TemporalMask:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "images": ("IMAGE",),
+                "source_fps": ("FLOAT", {"default": 24.0, "min": 0.01}),
+                "duration": ("FLOAT", {"default": 5.0, "min": 0.01, "max": 15.0}),
+                "mode": (["continuation", "extension"],),
+                "preserve_fraction": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0}),
+            }
+        }
+
+    RETURN_TYPES = ("MASK", "FLOAT", "IMAGE", "MASK")
+    RETURN_NAMES = ("mask", "preview_fps", "preview_images", "preview_mask")
+    FUNCTION = "build"
+    CATEGORY = "vLLM-Omni"
+
+    def build(self, images, source_fps, duration, mode, preserve_fraction=0.5):
+        if images.shape[0] <= 0 or not math.isfinite(source_fps) or source_fps <= 0:
+            raise ValueError("Source must contain frames and have a positive finite FPS.")
+        if not math.isfinite(duration) or duration <= 0 or not 0 <= preserve_fraction <= 1:
+            raise ValueError("Invalid duration or preserve_fraction.")
+        frames = _align_frame_count(max(1, round(duration * 24)))
+        source_seconds = images.shape[0] / source_fps
+        if mode == "extension":
+            if frames / 24 <= source_seconds:
+                raise ValueError("Extension output must be longer than the source; increase duration.")
+            boundary = source_seconds
+        elif mode == "continuation":
+            boundary = min(source_seconds, frames / 24) * preserve_fraction
+        else:
+            raise ValueError(f"Unknown temporal mask mode: {mode}")
+        available = min(frames, math.floor(boundary * 24 + 1e-8))
+        prefix = 0 if available < 5 else 5 + 17 * ((available - 5) // 17)
+        preserved = _video_latent_t(prefix) if prefix else 0
+        total = _video_latent_t(frames)
+        mask = torch.ones(total, 1, 1)
+        mask[:preserved] = 0
+        indices = torch.arange(frames, device=images.device)
+        source_indices = (indices * (source_fps / 24)).floor().long().clamp(max=images.shape[0] - 1)
+        preview_images = images.index_select(0, source_indices)
+        preview_mask = mask.index_select(0, torch.arange(frames) * total // frames)
+        return mask, 24.0, preview_images, preview_mask

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py

@@ -30,6 +30,7 @@ from .format import (
     video_to_base64,
     video_to_bytes,
 )
+from .latent_mask import scalar_mask_to_json, video_mask_to_grid_json
 from .logger import get_logger, pretty_printer
 from .models import lookup_model_spec
 from .types import (
@@ -277,10 +278,13 @@ class VLLMOmniClient:
         fps: int,
         negative_prompt: str | None = None,
         frame: torch.Tensor | None = None,
+        first_frame: torch.Tensor | None = None,
+        last_frame: torch.Tensor | None = None,
         references: dict | None = None,
         sampling_params: dict | None = None,
         model_params: dict | None = None,
         lora: dict | None = None,
+        latent_edit: dict | None = None,
         spec_model: str | None = None,
         **extra_params,
     ) -> VideoInput:
@@ -292,6 +296,16 @@ class VLLMOmniClient:
         """
         if frame is not None and references is not None:
             raise ValueError("Provide only one of frame or references, not both.")
+        if frame is not None and (first_frame is not None or last_frame is not None):
+            raise ValueError("Provide either frame or first_frame/last_frame, not both.")
+        if references is not None and (first_frame is not None or last_frame is not None):
+            raise ValueError("Provide either first_frame/last_frame or references, not both.")
+
+        spec, matched_pattern = lookup_model_spec(spec_model or model)
+        if (first_frame is not None or last_frame is not None) and (
+            matched_pattern is None or "MiniMax-H3" not in matched_pattern
+        ):
+            raise ValueError("first_frame and last_frame are supported only for MiniMax-H3; use frame for this model.")
 
         # === regular payload fields ===
         form = aiohttp.FormData()
@@ -311,11 +325,24 @@ class VLLMOmniClient:
 
         # === multimodal inputs (first-last-frames, references, etc.) ===
         input_reference_image: torch.Tensor | None = None
+        keyframe_images: list[tuple[str, torch.Tensor]] = []
         video_task: str | None = None
 
         if frame is not None:
             input_reference_image = frame
             video_task = "fl2va"
+        elif first_frame is not None or last_frame is not None:
+            frame_indices: list[int] = []
+            if first_frame is not None:
+                keyframe_images.append(("first_frame.png", first_frame))
+                frame_indices.append(0)
+            if last_frame is not None:
+                keyframe_images.append(("last_frame.png", last_frame))
+                frame_indices.append(-1)
+            if len(keyframe_images) == 1:
+                input_reference_image = keyframe_images[0][1]
+            extra_params["frame_indices"] = frame_indices
+            video_task = "fl2va"
         elif references is not None:
             reference_formats = (
                 ("image", MAX_REFERENCE_IMAGES, "png", "image/png", image_tensor_to_png_bytes),
@@ -353,7 +380,7 @@ class VLLMOmniClient:
             video_task = "t2va"
 
         if input_reference_image is not None:
-            image_filename = "image.png"  # Required for multipart form
+            image_filename = keyframe_images[0][0] if keyframe_images else "image.png"
             form.add_field(
                 "input_reference",
                 image_tensor_to_png_bytes(input_reference_image, image_filename),
@@ -361,12 +388,67 @@ class VLLMOmniClient:
                 content_type="image/png",
             )
 
+        # === latent-mask editing (MiniMax H3) ===
+        if latent_edit is not None:
+            source_video = latent_edit.get("source_video")
+            source_audio = latent_edit.get("source_audio")
+            video_mask = latent_edit.get("video_mask")
+            audio_mask = latent_edit.get("audio_mask")
+
+            if video_mask is None and audio_mask is None:
+                raise ValueError("Latent-mask editing requires at least one mask.")
+
+            video_mask_trivial = video_mask is None or bool((video_mask == 1.0).all().item())
+            audio_mask_trivial = audio_mask is None or audio_mask == 1.0
+            if not video_mask_trivial and source_video is None:
+                raise ValueError("A non-trivial video mask requires a source video.")
+            if not audio_mask_trivial and source_audio is None and source_video is None:
+                raise ValueError("A non-trivial audio mask requires a source audio or a source video with audio.")
+
+            if source_video is not None:
+                form.add_field(
+                    "source_video",
+                    video_to_bytes(source_video, "source.mp4"),
+                    filename="source.mp4",
+                    content_type="video/mp4",
+                )
+            if source_audio is not None:
+                form.add_field(
+                    "source_audio",
+                    audio_to_bytes(source_audio, "source_audio.mp3"),
+                    filename="source_audio.mp3",
+                    content_type="audio/mpeg",
+                )
+            if video_mask is not None:
+                mask_json = video_mask_to_grid_json(video_mask, width=width, height=height, num_frames=num_frames)
+                form.add_field(
+                    "video_noise_mask",
+                    mask_json.encode("utf-8"),
+                    filename="video-mask.json",
+                    content_type="application/json",
+                )
+            if audio_mask is not None:
+                form.add_field(
+                    "audio_noise_mask",
+                    scalar_mask_to_json(audio_mask).encode("utf-8"),
+                    filename="audio-mask.json",
+                    content_type="application/json",
+                )
+
+        if len(keyframe_images) == 2:
+            for image_filename, image in keyframe_images:
+                form.add_field(
+                    "input_references",
+                    image_tensor_to_png_bytes(image, image_filename),
+                    filename=image_filename,
+                    content_type="image/png",
+                )
+
         # === model specific params. Either use a specialized builder, or add flattened fields as-is ===
         if model_params is not None:
             model_params = dict(model_params)
             model_params.pop("type", None)
 
-        spec, _ = lookup_model_spec(spec_model or model)
         params_builder = spec.get("params_builder") if spec else None
         if params_builder is not None:
             form_fields = params_builder(

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py

@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+import json
+
+import torch
+import torch.nn.functional as F
+
+
+def scalar_mask_to_json(value: float) -> str:
+    if not 0.0 <= value <= 1.0:
+        raise ValueError(f"mask value must be in [0, 1], got {value}")
+    return str(value)
+
+
+def _align_frame_count(frame_count: int) -> int:
+    if frame_count <= 0:
+        return 1
+    current = int(frame_count)
+    while current % 17 != 5:
+        current += 1
+    return current
+
+
+def _video_latent_t(frame_count: int) -> int:
+    if frame_count <= 5:
+        return 2
+    return ((int(frame_count) - 5) // 17) * 5 + 2
+
+
+def video_mask_to_grid(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> torch.Tensor:
+    tv = _video_latent_t(_align_frame_count(num_frames))
+    height = int(height) // 32 * 32
+    width = int(width) // 32 * 32
+    gh, gw = height // 16, width // 16
+
+    if mask.ndim == 2:
+        mask = mask.unsqueeze(0)
+    elif mask.ndim != 3:
+        raise ValueError(f"expected a 2D or 3D mask tensor, got {mask.ndim}D")
+
+    grid = F.interpolate(mask.unsqueeze(1).float(), size=(gh, gw), mode="area").squeeze(1)
+    if grid.shape[0] == 1:
+        grid = grid.expand(tv, gh, gw)
+    elif grid.shape[0] != tv:
+        grid = F.interpolate(grid.unsqueeze(0).unsqueeze(0), size=(tv, gh, gw), mode="nearest").squeeze(0).squeeze(0)
+    return grid
+
+
+def video_mask_to_grid_json(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> str:
+    return json.dumps(video_mask_to_grid(mask, width=width, height=height, num_frames=num_frames).tolist())

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py

@@ -40,6 +40,10 @@ class VideoReferences(dict):
     pass
 
 
+class LatentMaskEditing(dict):
+    pass
+
+
 class FastH3Deployment(dict):
     """Descriptor for a server that fused FastH3 at startup."""
 

apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md

@@ -0,0 +1,17 @@
+# WF-05: MiniMax-H3 latent-mask editing
+
+Edit a source video with MiniMax-H3 using spatial or temporal masks. The workflow includes object removal, inpainting, continuation, and extension, with a mask preview for each case.
+
+## Requirements
+
+- ComfyUI with ComfyUI-vLLM-Omni installed.
+- An H3 service using the FL2VA checkpoint partition with latent-mask editing support. Follow the [H3 recipe](../../../recipes/MiniMaxAI/MiniMax-H3.md) for server setup.
+
+## Use
+
+Open **vLLM-Omni MiniMax-H3 Latent Mask Editing.json** from the example workflows and set the Generate Video URL/model for the case you want to run.
+
+1. Upload your source video in Load Video.
+2. Adjust the case's mask. For removal and inpainting, set Mask Canvas to the video dimensions, Mask Rectangle to the region size, and Combine Masks x/y to its position. For continuation and extension, set MiniMax-H3 Temporal Mask's mode and duration; continuation also uses `preserve_fraction`. Match the duration in Generate Video, and use a duration longer than the source for extension. Mask values are `0` to preserve and `1` to regenerate.
+3. Adjust the prompt to describe the desired content. Set the output dimensions and duration in Generate Video. If editing audio, update the sound description and audio mask (`0` preserves, `1` regenerates the whole clip).
+4. Select the case's Mask Preview Save Video node and choose **Execute to selected output nodes** to preview the mask as a red overlay without inference, or select its Result Save Video node to generate the edited video. Running the entire workflow generates all four cases.

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json

@@ -0,0 +1,499 @@
+{
+  "id": "latent-mask-editing-7380",
+  "revision": 0,
+  "last_node_id": 7,
+  "last_link_id": 5,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "MarkdownNote",
+      "pos": [
+        40,
+        20
+      ],
+      "size": [
+        420,
+        210
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Latent-mask editing server prerequisite",
+      "properties": {},
+      "widgets_values": [
+        "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends."
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 2,
+      "type": "LoadVideo",
+      "pos": [
+        40,
+        280
+      ],
+      "size": [
+        420,
+        120
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "file",
+          "name": "file",
+          "type": "COMBO",
+          "widget": {
+            "name": "file"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        ""
+      ]
+    },
+    {
+      "id": 3,
+      "type": "LoadImageMask",
+      "pos": [
+        40,
+        460
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image",
+          "type": "STRING",
+          "widget": {
+            "name": "image"
+          },
+          "link": null
+        },
+        {
+          "name": "channel",
+          "type": "STRING",
+          "widget": {
+            "name": "channel"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "mask",
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadImageMask"
+      },
+      "widgets_values": [
+        "",
+        "red"
+      ]
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniLatentMaskEditing",
+      "pos": [
+        560,
+        260
+      ],
+      "size": [
+        420,
+        300
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "source_video",
+          "name": "source_video",
+          "type": "VIDEO",
+          "link": 1,
+          "shape": 7
+        },
+        {
+          "localized_name": "source_audio",
+          "name": "source_audio",
+          "type": "AUDIO",
+          "link": null
+        },
+        {
+          "localized_name": "video_mask",
+          "name": "video_mask",
+          "type": "MASK",
+          "link": 2,
+          "shape": 7
+        },
+        {
+          "localized_name": "audio_mask",
+          "name": "audio_mask",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_mask"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniLatentMaskEditing"
+      },
+      "widgets_values": [
+        0.5
+      ]
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        40,
+        680
+      ],
+      "size": [
+        420,
+        140
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "audio_flow_shift",
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "flow_shift",
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video params",
+          "name": "video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ]
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        1080,
+        200
+      ],
+      "size": [
+        540,
+        680
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "frame",
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "references",
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "sampling_params",
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": null
+        },
+        {
+          "localized_name": "lora",
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "localized_name": "model_params",
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 4,
+          "shape": 7
+        },
+        {
+          "localized_name": "fast_h3",
+          "name": "fast_h3",
+          "type": "FASTH3_DEPLOYMENT",
+          "link": null
+        },
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "link": 3,
+          "shape": 7
+        },
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "prompt",
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "negative_prompt",
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "fps",
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "duration",
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "Partially restyle the clip and soundtrack",
+        "",
+        160,
+        120,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 7,
+      "type": "SaveVideo",
+      "pos": [
+        1700,
+        300
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "link": 5,
+          "shape": 7
+        },
+        {
+          "localized_name": "filename_prefix",
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "format",
+          "name": "format",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "vllm-omni/latent-edit",
+        "mp4"
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      2,
+      0,
+      4,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      3,
+      0,
+      4,
+      2,
+      "MASK"
+    ],
+    [
+      3,
+      4,
+      0,
+      6,
+      6,
+      "LATENT_MASK_EDITING"
+    ],
+    [
+      4,
+      5,
+      0,
+      6,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      5,
+      6,
+      0,
+      7,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json

@@ -0,0 +1,560 @@
+{
+  "last_node_id": 9,
+  "last_link_id": 7,
+  "nodes": [
+    {
+      "id": 2,
+      "type": "LoadVideo",
+      "pos": [
+        40,
+        100
+      ],
+      "size": [
+        420,
+        120
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "file",
+          "type": "STRING",
+          "widget": {
+            "name": "file"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        ""
+      ]
+    },
+    {
+      "id": 3,
+      "type": "SolidMask",
+      "pos": [
+        40,
+        300
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "value",
+          "type": "FLOAT",
+          "widget": {
+            "name": "value"
+          },
+          "link": null
+        },
+        {
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SolidMask"
+      },
+      "widgets_values": [
+        0.0,
+        160,
+        120
+      ]
+    },
+    {
+      "id": 4,
+      "type": "SolidMask",
+      "pos": [
+        40,
+        500
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "value",
+          "type": "FLOAT",
+          "widget": {
+            "name": "value"
+          },
+          "link": null
+        },
+        {
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SolidMask"
+      },
+      "widgets_values": [
+        1.0,
+        160,
+        120
+      ]
+    },
+    {
+      "id": 5,
+      "type": "BatchMasksNode",
+      "pos": [
+        250,
+        380
+      ],
+      "size": [
+        200,
+        100
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "masks.mask0",
+          "type": "MASK",
+          "link": 2,
+          "slot_index": 0
+        },
+        {
+          "name": "masks.mask1",
+          "type": "MASK",
+          "link": 3,
+          "slot_index": 1
+        }
+      ],
+      "outputs": [
+        {
+          "name": "MASK",
+          "type": "MASK",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "BatchMasksNode"
+      },
+      "widgets_values": []
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniLatentMaskEditing",
+      "pos": [
+        450,
+        180
+      ],
+      "size": [
+        400,
+        120
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "source_video",
+          "type": "VIDEO",
+          "link": 1
+        },
+        {
+          "name": "source_audio",
+          "type": "AUDIO",
+          "link": null
+        },
+        {
+          "name": "video_mask",
+          "type": "MASK",
+          "link": 4
+        },
+        {
+          "name": "audio_mask",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_mask"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniLatentMaskEditing"
+      },
+      "widgets_values": [
+        -1.0
+      ]
+    },
+    {
+      "id": 7,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        40,
+        700
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            6
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ]
+    },
+    {
+      "id": 8,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        700,
+        180
+      ],
+      "size": [
+        450,
+        380
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null
+        },
+        {
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null
+        },
+        {
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": null
+        },
+        {
+          "name": "lora",
+          "type": "LORA",
+          "link": null
+        },
+        {
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 6
+        },
+        {
+          "name": "fast_h3",
+          "type": "FAST_H3",
+          "link": null
+        },
+        {
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "link": 5
+        },
+        {
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            7
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "temporal mask: regenerate the ending",
+        "",
+        160,
+        120,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 9,
+      "type": "SaveVideo",
+      "pos": [
+        900,
+        180
+      ],
+      "size": [
+        420,
+        200
+      ],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "link": 7
+        },
+        {
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "name": "format",
+          "type": "STRING",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "temporal_mask",
+        "mp4"
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      2,
+      0,
+      6,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      3,
+      0,
+      5,
+      0,
+      "MASK"
+    ],
+    [
+      3,
+      4,
+      0,
+      5,
+      1,
+      "MASK"
+    ],
+    [
+      4,
+      5,
+      0,
+      6,
+      2,
+      "MASK"
+    ],
+    [
+      5,
+      6,
+      0,
+      8,
+      6,
+      "LATENT_MASK_EDITING"
+    ],
+    [
+      6,
+      7,
+      0,
+      8,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      7,
+      8,
+      0,
+      9,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json

@@ -0,0 +1,511 @@
+{
+  "id": "latent-mask-editing-7380",
+  "revision": 0,
+  "last_node_id": 7,
+  "last_link_id": 5,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "MarkdownNote",
+      "pos": [
+        40,
+        20
+      ],
+      "size": [
+        420,
+        210
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Latent-mask editing server prerequisite",
+      "properties": {},
+      "widgets_values": [
+        "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends."
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 2,
+      "type": "LoadVideo",
+      "pos": [
+        40,
+        280
+      ],
+      "size": [
+        420,
+        120
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "file",
+          "name": "file",
+          "type": "COMBO",
+          "widget": {
+            "name": "file"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        ""
+      ]
+    },
+    {
+      "id": 3,
+      "type": "SolidMask",
+      "pos": [
+        40,
+        460
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "value",
+          "name": "value",
+          "type": "FLOAT",
+          "widget": {
+            "name": "value"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "mask",
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SolidMask"
+      },
+      "widgets_values": [
+        1.0,
+        160,
+        120
+      ]
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniLatentMaskEditing",
+      "pos": [
+        560,
+        260
+      ],
+      "size": [
+        420,
+        300
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "source_video",
+          "name": "source_video",
+          "type": "VIDEO",
+          "link": 1,
+          "shape": 7
+        },
+        {
+          "localized_name": "source_audio",
+          "name": "source_audio",
+          "type": "AUDIO",
+          "link": null
+        },
+        {
+          "localized_name": "video_mask",
+          "name": "video_mask",
+          "type": "MASK",
+          "link": 2,
+          "shape": 7
+        },
+        {
+          "localized_name": "audio_mask",
+          "name": "audio_mask",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_mask"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniLatentMaskEditing"
+      },
+      "widgets_values": [
+        0.5
+      ]
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        40,
+        680
+      ],
+      "size": [
+        420,
+        140
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "audio_flow_shift",
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "flow_shift",
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video params",
+          "name": "video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ]
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        1080,
+        200
+      ],
+      "size": [
+        540,
+        680
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "frame",
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "references",
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "sampling_params",
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": null
+        },
+        {
+          "localized_name": "lora",
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "localized_name": "model_params",
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 4,
+          "shape": 7
+        },
+        {
+          "localized_name": "fast_h3",
+          "name": "fast_h3",
+          "type": "FASTH3_DEPLOYMENT",
+          "link": null
+        },
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "link": 3,
+          "shape": 7
+        },
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "prompt",
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "negative_prompt",
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "fps",
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "duration",
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "Partially restyle the clip and soundtrack",
+        "",
+        160,
+        120,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 7,
+      "type": "SaveVideo",
+      "pos": [
+        1700,
+        300
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "link": 5,
+          "shape": 7
+        },
+        {
+          "localized_name": "filename_prefix",
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "format",
+          "name": "format",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "vllm-omni/latent-edit",
+        "mp4"
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      2,
+      0,
+      4,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      3,
+      0,
+      4,
+      2,
+      "MASK"
+    ],
+    [
+      3,
+      4,
+      0,
+      6,
+      6,
+      "LATENT_MASK_EDITING"
+    ],
+    [
+      4,
+      5,
+      0,
+      6,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      5,
+      6,
+      0,
+      7,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax H3 Image to Video.json

@@ -0,0 +1,219 @@
+{
+  "id": "6676f19a-7f41-4c80-8cb7-f80adcb4ac44",
+  "revision": 0,
+  "last_node_id": 9,
+  "last_link_id": 4,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "LoadImage",
+      "pos": [40, 20],
+      "size": [270, 314],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "image", "name": "image", "type": "COMBO", "widget": {"name": "image"}, "link": null},
+        {"localized_name": "choose file to upload", "name": "upload", "type": "IMAGEUPLOAD", "widget": {"name": "upload"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "IMAGE", "name": "IMAGE", "type": "IMAGE", "links": [1]},
+        {"localized_name": "MASK", "name": "MASK", "type": "MASK", "links": null}
+      ],
+      "title": "First Frame",
+      "properties": {"Node name for S&R": "LoadImage"},
+      "widgets_values": ["first_frame.png", "image"]
+    },
+    {
+      "id": 2,
+      "type": "LoadImage",
+      "pos": [40, 390],
+      "size": [270, 314],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "image", "name": "image", "type": "COMBO", "widget": {"name": "image"}, "link": null},
+        {"localized_name": "choose file to upload", "name": "upload", "type": "IMAGEUPLOAD", "widget": {"name": "upload"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "IMAGE", "name": "IMAGE", "type": "IMAGE", "links": null},
+        {"localized_name": "MASK", "name": "MASK", "type": "MASK", "links": null}
+      ],
+      "title": "Last Frame (optional)",
+      "properties": {"Node name for S&R": "LoadImage"},
+      "widgets_values": ["last_frame.png", "image"]
+    },
+    {
+      "id": 3,
+      "type": "MarkdownNote",
+      "pos": [350, 20],
+      "size": [390, 240],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "MiniMax H3 I2V modes",
+      "properties": {},
+      "widgets_values": ["This workflow uses the remote vLLM-Omni Generate Video node.\n\n- First-frame only (default): keep First Frame connected.\n- Last-frame only: disconnect First Frame and connect Last Frame to last_frame.\n- First+last-frame: connect both image loaders.\n\nLeave the legacy frame and references inputs disconnected. Select images from ComfyUI's input folder. H3 runs at 24 FPS; the 5.167-second duration resolves to 124 frames, satisfying 17k+5."],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniDiffusionSampling",
+      "pos": [350, 300],
+      "size": [300, 226],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "n", "name": "n", "type": "INT", "widget": {"name": "n"}, "link": null},
+        {"localized_name": "num_inference_steps", "name": "num_inference_steps", "type": "INT", "widget": {"name": "num_inference_steps"}, "link": null},
+        {"localized_name": "guidance_scale", "name": "guidance_scale", "type": "FLOAT", "widget": {"name": "guidance_scale"}, "link": null},
+        {"localized_name": "true_cfg_scale", "name": "true_cfg_scale", "type": "FLOAT", "widget": {"name": "true_cfg_scale"}, "link": null},
+        {"localized_name": "vae_use_slicing", "name": "vae_use_slicing", "type": "BOOLEAN", "widget": {"name": "vae_use_slicing"}, "link": null},
+        {"localized_name": "vae_use_tiling", "name": "vae_use_tiling", "type": "BOOLEAN", "widget": {"name": "vae_use_tiling"}, "link": null},
+        {"localized_name": "seed", "name": "seed", "type": "INT", "widget": {"name": "seed"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "diffusion sampling params", "name": "diffusion sampling params", "type": "SAMPLING_PARAMS", "links": [2]}
+      ],
+      "title": "Base H3 Sampling (50 steps)",
+      "properties": {"Node name for S&R": "VLLMOmniDiffusionSampling"},
+      "widgets_values": [1, 50, 1, 1, false, false, 757358688076805, "fixed"]
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [350, 570],
+      "size": [300, 82],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "audio_flow_shift", "name": "audio_flow_shift", "type": "FLOAT", "widget": {"name": "audio_flow_shift"}, "link": null},
+        {"localized_name": "flow_shift", "name": "flow_shift", "type": "FLOAT", "widget": {"name": "flow_shift"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "MiniMax-H3 video params", "name": "MiniMax-H3 video params", "type": "VIDEO_PARAMS", "links": [3]}
+      ],
+      "title": "Base H3 Flow Shifts",
+      "properties": {"Node name for S&R": "VLLMOmniMiniMaxH3Params"},
+      "widgets_values": [3, 12]
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniRemoteLoRA",
+      "pos": [700, 550],
+      "size": [360, 150],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "local_path", "name": "local_path", "type": "STRING", "widget": {"name": "local_path"}, "link": null},
+        {"localized_name": "name", "name": "name", "type": "STRING", "widget": {"name": "name"}, "link": null},
+        {"localized_name": "scale", "name": "scale", "type": "FLOAT", "widget": {"name": "scale"}, "link": null},
+        {"localized_name": "int_id", "name": "int_id", "type": "INT", "widget": {"name": "int_id"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "lora", "name": "lora", "type": "REMOTE_LORA", "links": null}
+      ],
+      "title": "Optional H3 Turbo Remote LoRA",
+      "properties": {"Node name for S&R": "VLLMOmniRemoteLoRA"},
+      "widgets_values": ["minimax_h3_fl2v_turbo_8step_v1.0_768p_bf16.safetensors", "h3-turbo-8step", 1, 0]
+    },
+    {
+      "id": 7,
+      "type": "MarkdownNote",
+      "pos": [700, 300],
+      "size": [420, 210],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Optional Turbo settings",
+      "properties": {},
+      "widgets_values": ["Base H3 is configured by default: 50 steps, flow_shift 12, audio_flow_shift 3.\n\nFor the shown 8-step Diffusers Turbo artifact, set its full local_path on the server, start the server with the matching preloaded LoRA, connect this Remote LoRA to Generate Video, set num_inference_steps to 9, and set flow_shift to 6. Keep audio_flow_shift at 3 and scale at 1.\n\nDo not use the _comfyui_ repack or FastH3 here."],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 8,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [1160, 80],
+      "size": [440, 610],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "frame", "name": "frame", "shape": 7, "type": "IMAGE", "link": null},
+        {"localized_name": "first_frame", "name": "first_frame", "shape": 7, "type": "IMAGE", "link": 1},
+        {"localized_name": "last_frame", "name": "last_frame", "shape": 7, "type": "IMAGE", "link": null},
+        {"localized_name": "references", "name": "references", "shape": 7, "type": "VIDEO_REFERENCES", "link": null},
+        {"localized_name": "sampling_params", "name": "sampling_params", "shape": 7, "type": "SAMPLING_PARAMS", "link": 2},
+        {"localized_name": "lora", "name": "lora", "shape": 7, "type": "REMOTE_LORA", "link": null},
+        {"localized_name": "model_params", "name": "model_params", "shape": 7, "type": "VIDEO_PARAMS", "link": 3},
+        {"localized_name": "fast_h3", "name": "fast_h3", "shape": 7, "type": "FASTH3_DEPLOYMENT", "link": null},
+        {"localized_name": "url", "name": "url", "type": "STRING", "widget": {"name": "url"}, "link": null},
+        {"localized_name": "model", "name": "model", "type": "STRING", "widget": {"name": "model"}, "link": null},
+        {"localized_name": "prompt", "name": "prompt", "type": "STRING", "widget": {"name": "prompt"}, "link": null},
+        {"localized_name": "negative_prompt", "name": "negative_prompt", "type": "STRING", "widget": {"name": "negative_prompt"}, "link": null},
+        {"localized_name": "width", "name": "width", "type": "INT", "widget": {"name": "width"}, "link": null},
+        {"localized_name": "height", "name": "height", "type": "INT", "widget": {"name": "height"}, "link": null},
+        {"localized_name": "fps", "name": "fps", "type": "INT", "widget": {"name": "fps"}, "link": null},
+        {"localized_name": "duration", "name": "duration", "type": "FLOAT", "widget": {"name": "duration"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "video", "name": "video", "type": "VIDEO", "links": [4]}
+      ],
+      "properties": {"Node name for S&R": "VLLMOmniGenerateVideo"},
+      "widgets_values": [
+        "http://localhost:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "A cinematic product shot begins exactly from the supplied keyframe. The subject moves naturally through a dark studio while blue and amber rim lights shift across reflective surfaces. The camera makes a slow controlled orbit. Audio: subtle room tone, precise mechanical details, and a restrained electronic score.",
+        "",
+        1344,
+        768,
+        24,
+        5.167
+      ]
+    },
+    {
+      "id": 9,
+      "type": "SaveVideo",
+      "pos": [1650, 170],
+      "size": [374, 314],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "video", "name": "video", "type": "VIDEO", "link": 4},
+        {"localized_name": "filename_prefix", "name": "filename_prefix", "type": "STRING", "widget": {"name": "filename_prefix"}, "link": null},
+        {"localized_name": "format", "name": "format", "type": "COMBO", "widget": {"name": "format"}, "link": null},
+        {"localized_name": "codec", "name": "codec", "type": "COMFY_DYNAMICCOMBO_V3", "widget": {"name": "codec"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "video", "name": "video", "type": "VIDEO", "links": null}
+      ],
+      "properties": {},
+      "widgets_values": ["video/MiniMax_H3_I2V", "auto", "auto"]
+    }
+  ],
+  "links": [
+    [1, 1, 0, 8, 1, "IMAGE"],
+    [2, 4, 0, 8, 4, "SAMPLING_PARAMS"],
+    [3, 5, 0, 8, 6, "VIDEO_PARAMS"],
+    [4, 8, 0, 9, 0, "VIDEO"]
+  ],
+  "groups": [
+    {"id": 1, "title": "Portable first/last-frame inputs", "bounding": [10, -30, 760, 770], "color": "#3f789e", "flags": {}},
+    {"id": 2, "title": "H3 sampling and optional Turbo", "bounding": [320, 270, 830, 460], "color": "#8a6d3b", "flags": {}},
+    {"id": 3, "title": "Remote generation and audio-preserving save", "bounding": [1130, 30, 930, 700], "color": "#3f789e", "flags": {}}
+  ],
+  "config": {},
+  "extra": {"workflowRendererVersion": "LG", "ds": {"scale": 0.75, "offset": [180, 120]}},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/web/main.js

@@ -52,6 +52,7 @@ const FAMILY_BY_OUTPUT = {
     REMOTE_LORA: "deployment",
     FASTH3_DEPLOYMENT: "deployment",
     VIDEO_REFERENCES: "media",
+    LATENT_MASK_EDITING: "media",
 };
 
 function familyOf(nodeData) {

benchmarks/README.md

@@ -41,6 +41,17 @@ LingBot-Video MoE transformer.
 - **Dense pipeline**: decoded-video MAE, MSE, PSNR, latency, and optional steady-state timings
 - **MoE transformer**: bitwise router, sparse-block, shared-expert, and full-transformer parity
 
+### [LingBot-World](lingbot_world/README.md) — Realtime Streaming Cadence
+
+Client-side benchmark for a `WS /v1/realtime/video` world-model session, where one
+request is a whole rollout and one AR block streams out as one video chunk. It
+measures a cadence rather than a throughput: there is exactly one session, and what
+matters is whether the next chunk arrives before the viewer finishes the last one.
+
+- **Layout**: `lingbot_world/benchmark_lingbot_world_realtime.py` (WS client), `lingbot_world/workload.py` (rollout and metric math), `lingbot_world/configs/` (single-GPU eager and Ulysses-4 compiled deploy configs)
+- **Workload**: built-in image-conditioned rollout with a generated camera script, or a JSON spec with per-chunk actions and mid-rollout prompt updates
+- **Key metrics**: TTFC (time to first chunk), steady-state chunk inter-arrival percentiles, VIDEO_RTF (wall seconds per video second; lower is better), chunk-deadline attainment, and simulated playback underruns
+
 ### [Distributed](distributed/omni_connectors/README.md) — RDMA Connector Testing
 
 RDMA environment setup and transfer tests for `MooncakeTransferEngineConnector`, including pytest-based single-node checks and manual cross-node benchmarks.

benchmarks/kernels/mot_linear_benchmarks.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 # ruff: noqa: N803
 
 """MoT (Mixture-of-Tokens) GEMM kernel benchmark and auto-tuning.
@@ -10,7 +13,7 @@ Usage:
     python benchmarks/kernels/mot_linear_benchmarks.py \
         --model ByteDance-Seed/BAGEL-7B-MoT \
         --tp-size 1 --dtype w16a16 --tune \
-        --save-dir vllm_omni/diffusion/layers/mot/configs/
+        --save-dir vllm_omni/diffusion/models/bagel/mot/configs/
 
     # Auto-tune with local model path (offline clusters):
     python benchmarks/kernels/mot_linear_benchmarks.py \
@@ -43,7 +46,7 @@ from vllm.triton_utils import triton
 from vllm.utils.torch_utils import set_random_seed
 
 # NOTE: you should use the same naming syetem for the kernel to load properly
-from vllm_omni.diffusion.layers.mot.ops.mot_gemm import build_config_filename, get_device_name
+from vllm_omni.diffusion.models.bagel.mot.ops.mot_gemm import build_config_filename, get_device_name
 
 # clear the triton cache from time to time, usaully no need to change
 _CACHE_CLEAR_INTERVAL_ENV = "VLLM_MOT_TUNE_CACHE_CLEAR_INTERVAL"
@@ -428,7 +431,7 @@ def benchmark_config(
     cache_flusher: torch.Tensor | None = None,
 ) -> float:
     """Run a MoT GEMM with the given tile config and return avg latency (us)."""
-    from vllm_omni.diffusion.layers.mot.ops.mot_gemm import invoke_mot_gemm
+    from vllm_omni.diffusion.models.bagel.mot.ops.mot_gemm import invoke_mot_gemm
 
     text_indices, vae_indices, M = build_regular_indices(
         image_num=image_num,
@@ -553,7 +556,7 @@ class BenchmarkWorker:
         use_int8_w8a16: bool,
     ) -> tuple[dict[str, int], float]:
         set_random_seed(self.seed)
-        from vllm_omni.diffusion.layers.mot.ops.mot_gemm import (
+        from vllm_omni.diffusion.models.bagel.mot.ops.mot_gemm import (
             get_best_mot_config,
         )
 

benchmarks/lingbot_world/README.md

@@ -0,0 +1,224 @@
+# LingBot-World Benchmarks — Realtime Streaming Cadence
+
+`benchmark_lingbot_world_realtime.py` measures LingBot-World 2.0 as it is
+actually deployed: one `WS /v1/realtime/video` session that streams one video
+chunk per AR block, for as long as the world keeps running.
+
+## Why this is not the diffusion serving benchmark
+
+[`benchmarks/diffusion`](../diffusion/README.md) measures "one request, one
+finished video" and reports request throughput. LingBot-World has neither shape:
+
+- **One request is the whole rollout.** `prepare_encode` runs once, then every AR
+  block is four DMD steps plus one clean-KV commit forward, and each block
+  streams out as a chunk. Throughput in requests per second is meaningless.
+- **There is exactly one session.** The stepwise deploy config pins
+  `max_num_seqs: 1`, and the AR-Diffusion consumer rejects a stage with more than
+  one replica, because session-affine routing does not exist yet. There is no
+  concurrency to sweep.
+- **What matters is cadence.** A deployment is healthy when chunk *N + 1* arrives
+  before a viewer finishes watching chunk *N*. That is a latency distribution
+  against a wall clock, not a throughput number.
+
+## Quick start
+
+Serve the model, then point the benchmark at it. The benchmark never starts a
+server.
+
+```bash
+# Terminal 1 — one GPU, eager (portable smoke configuration)
+vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+  --omni \
+  --deploy-config benchmarks/lingbot_world/configs/single_gpu_eager.yaml \
+  --port 8000
+
+# Terminal 2
+pip install websockets
+python benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py \
+  --host 127.0.0.1 --port 8000 \
+  --num-chunks 16 \
+  --output-json /tmp/lingbot_world_realtime.json
+```
+
+With no `--image`, the first frame comes from the shared vLLM image asset, so the
+default workload needs no setup and this repository carries no benchmark binary.
+Pass `--image /path/to/frame.png` to condition the rollout on your own frame.
+
+### Realtime topology
+
+The single-GPU config is a smoke configuration, not a realtime one: one card runs
+all five forwards of every block serially and nothing is compiled. Use the
+four-GPU config for numbers that mean anything about realtime behavior.
+
+```bash
+vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+  --omni \
+  --deploy-config benchmarks/lingbot_world/configs/usp4_compiled.yaml \
+  --port 8000
+
+python benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py \
+  --port 8000 --num-chunks 40 --warmup-sessions 1 --sessions 3 \
+  --target-fps 12 --output-json /tmp/lingbot_world_repeated.json
+```
+
+`usp4_compiled.yaml` sets pure Ulysses sequence parallelism over four ranks and
+`enforce_eager: false`. The pipeline validates the parallel shape itself:
+`sequence_parallel_size` must equal `ulysses_degree`, `ring_degree` and
+`allgather_degree` must be 1, `ulysses_mode` must be `strict`, and pipeline,
+CFG, and VAE parallel sizes above 1 are rejected. This benchmark config sets
+`tensor_parallel_size: 1`; the pipeline validator does not reject larger TP sizes.
+
+Compiled mode can pay compilation and capture costs on the first rollout.
+Use `--warmup-sessions 1` before measured repetitions. Excluding the first six
+chunks addresses the attention-window ramp; it does not guarantee that
+compilation and capture have finished.
+
+## Metrics
+
+| Metric | Meaning |
+| --- | --- |
+| `ttfc_ms` | **Time to first chunk**: `session.start` sent until the first chunk's bytes arrive. Pays for prompt encode, first-frame VAE encode, and any compile or capture. |
+| `interval_all` | Inter-arrival latency across every chunk: mean, median, std, min, p90, p95, p99, max. |
+| `interval_steady` | The same summary over steady-state chunks only. |
+| `video_rtf` | Wall seconds spent per second of video, matching vLLM-Omni's own RTF (`generation_s / video_duration`). **Lower is better; below 1.0 is real time.** |
+| `video_rtfx` | The reciprocal, under the name this repo uses for it: video seconds per wall second, where higher is better. |
+| `steady_video_rtf` | `video_rtf` restricted to steady-state chunks. |
+| `chunk_deadline_ms` | Wall-clock budget one steady chunk must meet to hold real time: 12 frames ÷ `--target-fps`. Override directly with `--chunk-slo-ms`. |
+| `slo_attainment` | Fraction of steady chunks that met the deadline. |
+| `playback` | Underrun count, total stall, and stall ratio from a simulated viewer. |
+
+### Which frame rate counts as real time
+
+**The checkpoint declares no frame rate.** There is none in `model_index.json`, the
+scheduler config, or the transformer config, and the pipeline never reads one. A
+chunk is twelve pixel frames; how long those frames *last* is a property of how the
+output is played, not of the model.
+
+That choice moves the verdict without moving anything measured:
+
+| basis | video per chunk | deadline | RTF at a measured 1357 ms/chunk |
+| --- | --- | --- | --- |
+| 16 fps | 0.750 s | 750 ms | 1.809 |
+| 12 fps | 1.000 s | 1000 ms | 1.357 |
+
+So `--fps` (sent to the server, which uses it to label the muxed MP4) and
+`--target-fps` (the real-time basis for RTF, the deadline, and the playback
+simulation) are separate flags. `--target-fps` defaults to `--fps`; set it when the
+two differ, and the report prints both. State the basis whenever you quote a number
+— an RTF without its fps is not a measurement. (RTF is a cost here, so a bigger number is a
+worse result; a steady chunk's RTF is simply its interval divided by its video duration.)
+An RTF above 1 means the measured deployment is slower than real time at the
+stated target frame rate. The benchmark measures this failure as well as success.
+
+### Why warmup chunks are not optional
+
+The DiT attends over a sliding window of `sliding_window_num_frames = 18` latent
+frames with `sink_size = 9`, so the key count a block attends over keeps growing
+until eighteen latent frames of history exist — six chunks at three frames per
+block. **Chunk cost is not stationary before chunk six.** A mean over a short
+rollout is therefore lower than anything a running session will sustain, which is
+why `--warmup-chunks` defaults to 6 and steady-state metrics are reported
+separately. Keep `--num-chunks` comfortably above the warmup, or the benchmark
+tells you no chunk reached steady state. The terminal chunk skips next-chunk
+preparation, so it is excluded from steady metrics too; it remains in overall
+RTF, all-chunk intervals, and playback simulation.
+
+The default 16-chunk rollout provides only nine steady intervals. Percentiles
+are interpolated sample summaries; the report warns when fewer than 100
+intervals contribute, and even 100 samples do not guarantee a stable p99.
+Use longer rollouts for tail analysis and at least `--sessions 3` after
+`--warmup-sessions 1` for reported comparisons. Session warmup pays compilation
+cost; chunk warmup separately excludes the attention-window ramp. The command
+above yields 33 steady intervals per session (indices 6 through 38), or 99 across
+three sessions. Report each session's mean and the spread across sessions; this
+sample still does not establish a reliable p99.
+
+### Why playback is simulated
+
+A p99 interval alone cannot say whether a viewer saw a stall: the same p99 is
+invisible behind a three-chunk buffer and a visible freeze behind none. The
+benchmark replays chunk arrivals against a wall clock — playback starts once
+`--playback-buffer-chunks` chunks have landed and then consumes video at `--target-fps`
+— and reports every moment the player ran dry.
+
+### What the client-side number is worth
+
+On a four-GPU Ulysses run the client's mean inter-arrival over all fifteen
+post-first-chunk intervals was **1326.743 ms**, against the server's own
+`StageRequestStats.inter_output_latency_ms` of **1327.479 ms** for the same
+request — a difference of **0.74 ms**. This cross-check shows agreement in
+average output cadence for that run, including warmup and the terminal chunk.
+It does not measure encoding or transport latency: a fixed delivery delay can
+cancel out of inter-arrival differences. Attributing that latency requires
+per-chunk timestamps at both boundaries or server-side profiling.
+
+Compare matching interval populations when investigating discrepancies; both
+measurement bugs and delivery buffering can change the observed cadence.
+
+### What this benchmark cannot see
+
+Chunk metadata carries no server-side timestamps, so the benchmark cannot split
+DiT time from VAE decode time; use `--enable-diffusion-pipeline-profiler` on the
+server for that. It also reports no GPU memory, which is a server-side quantity.
+
+## Custom rollouts
+
+`--workload rollout.json` replaces the built-in workload, including mid-rollout
+prompt changes, which the CLI flags cannot express:
+
+```json
+{
+  "prompt": "The camera moves slowly forward through the scene.",
+  "image": "/path/to/first_frame.png",
+  "num_chunks": 24,
+  "camera_pattern": "orbit",
+  "fps": 16,
+  "seed": 42,
+  "prompt_updates": [
+    {"after_chunk": 8, "prompt": "Rain begins to fall", "transition_chunks": 2}
+  ]
+}
+```
+
+Supply `camera_action_script` directly instead of `num_chunks` to control every
+frame's action. It must hold exactly one three-entry action list per chunk; one
+AR block is three latent frames, and the server rejects any other shape.
+Explicit `--image`, `--prompt`, `--negative-prompt`, `--width`, `--height`,
+`--fps`, `--seed`, and `--flow-shift` flags override the matching workload-file
+values. `--num-chunks` and `--camera-pattern` apply only to the built-in
+workload; when using `--workload`, set those fields in the file instead.
+An explicit `camera_action_script` takes precedence over the file's
+`num_chunks` and `camera_pattern`. Each prompt update must use a distinct
+`after_chunk` boundary so its interaction event ID is unique.
+
+Camera actions change what the world does, not what it costs, so `--camera-pattern`
+(`forward`, `orbit`, `hold`) keeps a rollout representative rather than sweeping a
+cost dimension.
+
+## Options worth knowing
+
+| Flag | Purpose |
+| --- | --- |
+| `--target-fps` | Playback rate the real-time verdict is measured against, independent of the mux label. |
+| `--sessions N` | N sequential measured rollouts; the report adds spread across them. |
+| `--warmup-sessions N` | Unmeasured rollouts first, to pay compile and capture cost. |
+| `--print-chunks` | One line per chunk as it arrives. |
+| `--save-video PATH` | Remux the last measured session for eyeballing (needs vLLM-Omni). |
+| `--ping-interval` | `session.ping` cadence keeping the server's stall clock fresh during a slow first chunk. `0` disables. |
+| `--first-chunk-timeout` | Raise it when the server is still compiling or capturing graphs. |
+
+## Output
+
+`--output-json` writes the run's configuration, every chunk's arrival time and
+size, per-session metrics, and the aggregate, along with the git revision and
+hostname so a number can be traced back to what produced it.
+
+## Tests
+
+`tests/benchmarks/test_lingbot_world_realtime.py` covers the chunk and frame
+arithmetic against the server's own formula, the metric math, and the protocol
+handling against a scripted WebSocket server. It needs no GPU:
+
+```bash
+pytest tests/benchmarks/test_lingbot_world_realtime.py
+```

benchmarks/lingbot_world/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py

@@ -0,0 +1,642 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Realtime streaming benchmark for LingBot-World 2.0 over ``WS /v1/realtime/video``.
+
+Every other diffusion benchmark in this repository measures "one request, one
+finished video". LingBot-World is not that: one request is a whole rollout, and
+the thing a deployment lives or dies by is whether chunk *N + 1* keeps arriving
+before a viewer has finished watching chunk *N*. This benchmark therefore
+measures a **cadence**, not a throughput:
+
+* ``TTFC`` -- session.start to the first chunk's bytes, which pays for prompt
+  encode, the first-frame VAE encode, and any compile or capture the deploy
+  config asks for;
+* **inter-arrival latency** over all chunks and, separately, over the steady
+  state after the attention window saturates;
+* ``VIDEO_RTF`` -- wall seconds spent per second of video generated, matching how
+  vLLM-Omni computes RTF everywhere else (``generation_s / video_duration``).
+  **Lower is better and below 1.0 is real time**; ``VIDEO_RTFX`` is reported
+  alongside it as the reciprocal, under the repo's own name for that;
+* **playback continuity** -- a simulated viewer with a start buffer, because a
+  p99 that is harmless behind a three-chunk buffer is a visible stall behind
+  none.
+
+The server is not started here. Bring one up first (see the README), then point
+this script at it. Requires ``websockets``; ``--save-video`` additionally needs
+a vLLM-Omni installation for the remux helper.
+
+Example:
+
+    python benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py \\
+      --host 127.0.0.1 --port 8000 \\
+      --num-chunks 16 \\
+      --output-json /tmp/lingbot_world_realtime.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import base64
+import json
+import mimetypes
+import os
+import platform
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from benchmarks.lingbot_world.workload import (  # noqa: E402
+    CAMERA_PATTERNS,
+    DEFAULT_FLOW_SHIFT,
+    DEFAULT_FPS,
+    DEFAULT_HEIGHT,
+    DEFAULT_MODEL,
+    DEFAULT_PROMPT,
+    DEFAULT_SEED,
+    DEFAULT_WARMUP_CHUNKS,
+    DEFAULT_WIDTH,
+    ChunkRecord,
+    Workload,
+    aggregate_metrics,
+    compute_metrics,
+    default_workload,
+    load_workload,
+    steady_chunk_deadline_ms,
+)
+
+# The shared vLLM asset keeps the default workload runnable with no setup and
+# adds no binary to this repository. Any real photograph conditions a rollout.
+_DEFAULT_IMAGE_ASSET = "2560px-Gfp-wisconsin-madison-the-nature-boardwalk"
+_WS_PATH = "/v1/realtime/video"
+
+
+class BenchmarkError(RuntimeError):
+    """A session failed in a way that invalidates its numbers."""
+
+
+# --------------------------------------------------------------------------- #
+# Input resolution
+# --------------------------------------------------------------------------- #
+
+
+def resolve_image_reference(value: str | None) -> str:
+    """Return an ``http(s)`` or ``data:`` URL for the session's first frame."""
+    if value is None:
+        from vllm.assets.image import ImageAsset
+
+        value = str(ImageAsset(_DEFAULT_IMAGE_ASSET).get_path("jpg"))
+    if value.startswith(("http://", "https://", "data:")):
+        return value
+    path = Path(value).expanduser()
+    if not path.is_file():
+        raise BenchmarkError(f"--image is neither a URL nor a readable file: {value}")
+    media_type = mimetypes.guess_type(path.name)[0] or "image/png"
+    return f"data:{media_type};base64,{base64.b64encode(path.read_bytes()).decode()}"
+
+
+def build_workload(args: argparse.Namespace) -> Workload:
+    if args.workload is not None:
+        # The file names its own first frame, so resolving happens inside
+        # load_workload -- never eagerly, which would fetch the default asset a
+        # --workload run has no use for.
+        return load_workload(
+            args.workload,
+            image_resolver=lambda value: resolve_image_reference(value if args.image is None else args.image),
+            overrides={
+                "width": args.width_override,
+                "height": args.height_override,
+                "fps": args.fps_override,
+                "seed": args.seed_override,
+                "prompt": args.prompt_override,
+                "flow_shift": args.flow_shift,
+                "negative_prompt": args.negative_prompt,
+            },
+        )
+    return default_workload(
+        image_reference=resolve_image_reference(args.image),
+        num_chunks=args.num_chunks,
+        camera_pattern=args.camera_pattern,
+        prompt=args.prompt_override if args.prompt_override is not None else DEFAULT_PROMPT,
+        width=args.width_override if args.width_override is not None else DEFAULT_WIDTH,
+        height=args.height_override if args.height_override is not None else DEFAULT_HEIGHT,
+        fps=args.fps_override if args.fps_override is not None else DEFAULT_FPS,
+        seed=args.seed_override if args.seed_override is not None else DEFAULT_SEED,
+        flow_shift=args.flow_shift if args.flow_shift is not None else DEFAULT_FLOW_SHIFT,
+        negative_prompt=args.negative_prompt,
+    )
+
+
+# --------------------------------------------------------------------------- #
+# One streaming session
+# --------------------------------------------------------------------------- #
+
+
+@dataclass
+class SessionResult:
+    request_id: str | None
+    records: list[ChunkRecord]
+    media: bytes
+    ttfc_s: float | None
+    video_start_s: float | None
+    wall_s: float
+    done_event: dict[str, Any]
+
+
+async def _keepalive(websocket: Any, send_lock: asyncio.Lock, interval: float) -> None:
+    """Refresh the server's stall clock while a slow first chunk is generating.
+
+    A compiled deploy config records CUDA graphs on the first block, which can
+    outlast the server's ~60 s stall timeout. ``session.ping`` is the documented
+    way to say the client is still there.
+    """
+    while True:
+        await asyncio.sleep(interval)
+        async with send_lock:
+            await websocket.send(json.dumps({"type": "session.ping"}))
+
+
+async def run_session(
+    url: str,
+    model: str,
+    workload: Workload,
+    *,
+    first_chunk_timeout: float,
+    chunk_timeout: float,
+    ping_interval: float,
+    collect_media: bool,
+    print_chunks: bool,
+) -> SessionResult:
+    """Drive one rollout and timestamp every chunk as its bytes land."""
+    try:
+        from websockets.asyncio.client import connect
+    except ImportError:  # pragma: no cover - exercised only without websockets
+        try:
+            from websockets import connect  # type: ignore[attr-defined]
+        except ImportError as exc:
+            raise BenchmarkError("This benchmark needs the websockets package: pip install websockets") from exc
+
+    payload = workload.session_start_payload(model)
+    records: list[ChunkRecord] = []
+    media = bytearray()
+    pending: dict[str, Any] | None = None
+    request_id: str | None = None
+    ttfc_s: float | None = None
+    video_start_s: float | None = None
+    last_arrival: float | None = None
+    done_event: dict[str, Any] = {}
+    send_lock = asyncio.Lock()
+    keepalive: asyncio.Task[None] | None = None
+    # Fire remaining prompt updates in chunk order as their boundary passes.
+    updates = sorted(workload.prompt_updates, key=lambda update: update.after_chunk)
+    update_index = 0
+
+    async with connect(url, max_size=None, ping_interval=None) as websocket:
+        async with send_lock:
+            await websocket.send(json.dumps(payload, ensure_ascii=False))
+        started = time.perf_counter()
+        if ping_interval > 0:
+            keepalive = asyncio.create_task(_keepalive(websocket, send_lock, ping_interval))
+
+        try:
+            while True:
+                timeout = first_chunk_timeout if not records else chunk_timeout
+                try:
+                    deadline = started + (last_arrival if last_arrival is not None else 0.0) + timeout
+                    remaining = deadline - time.perf_counter()
+                    if remaining <= 0:
+                        raise asyncio.TimeoutError
+                    message = await asyncio.wait_for(websocket.recv(), timeout=remaining)
+                except asyncio.TimeoutError as exc:
+                    raise BenchmarkError(
+                        f"No media chunk for {timeout:g} s after {len(records)} chunk(s); "
+                        "the server may still be loading, compiling, or capturing graphs."
+                    ) from exc
+                now = time.perf_counter()
+
+                if isinstance(message, (bytes, bytearray)):
+                    if pending is None:
+                        raise BenchmarkError("Received a binary frame with no preceding video.chunk_metadata event.")
+                    if collect_media:
+                        media.extend(message)
+                    if pending.get("kind") == "media":
+                        index = int(pending["generation_chunk_index"])
+                        arrival = now - started
+                        if ttfc_s is None:
+                            ttfc_s = arrival
+                        records.append(
+                            ChunkRecord(
+                                index=index,
+                                arrival_s=arrival,
+                                inter_arrival_s=arrival - (last_arrival if last_arrival is not None else 0.0),
+                                byte_length=len(message),
+                                num_frames=int(pending["num_frames"]),
+                            )
+                        )
+                        last_arrival = arrival
+                        if print_chunks:
+                            print(
+                                f"  chunk {index:3d}  t={arrival:8.3f}s  "
+                                f"interval={records[-1].inter_arrival_s * 1000:8.1f}ms  "
+                                f"frames={records[-1].num_frames:3d}  bytes={len(message)}",
+                                flush=True,
+                            )
+                        while update_index < len(updates) and updates[update_index].after_chunk <= index:
+                            async with send_lock:
+                                await websocket.send(json.dumps(updates[update_index].to_payload(), ensure_ascii=False))
+                            update_index += 1
+                    pending = None
+                    continue
+
+                event = json.loads(message)
+                event_type = event.get("type")
+                if event_type == "video.chunk_metadata":
+                    pending = event
+                elif event_type == "video.start":
+                    request_id = event.get("request_id")
+                    video_start_s = now - started
+                elif event_type == "error":
+                    # Say what was asked for alongside what the server refused:
+                    # some builds cap a rollout's num_frames, and the operator
+                    # needs to know which --num-chunks would fit.
+                    raise BenchmarkError(
+                        f"Server rejected the session after {len(records)} chunk(s): "
+                        f"{event.get('message', event)} "
+                        f"(requested {workload.num_chunks} chunks = num_frames {workload.num_frames})"
+                    )
+                elif event_type == "session.done":
+                    done_event = event
+                    break
+        finally:
+            if keepalive is not None:
+                keepalive.cancel()
+                try:
+                    await keepalive
+                except asyncio.CancelledError:
+                    pass
+
+        wall_s = time.perf_counter() - started
+
+    if not records:
+        raise BenchmarkError("The session finished without delivering a single media chunk.")
+    if done_event.get("stopped"):
+        raise BenchmarkError(f"The session was stopped before it finished: {done_event}")
+    observed = [record.index for record in records]
+    if observed != list(range(len(records))):
+        raise BenchmarkError(f"Chunk indices are not contiguous from zero: {observed}")
+    if len(records) != workload.num_chunks:
+        raise BenchmarkError(
+            f"Expected {workload.num_chunks} chunks for num_frames={workload.num_frames}, received {len(records)}."
+        )
+
+    return SessionResult(
+        request_id=request_id,
+        records=records,
+        media=bytes(media),
+        ttfc_s=ttfc_s,
+        video_start_s=video_start_s,
+        wall_s=wall_s,
+        done_event=done_event,
+    )
+
+
+# --------------------------------------------------------------------------- #
+# Reporting
+# --------------------------------------------------------------------------- #
+
+
+def _git_revision() -> str | None:
+    try:
+        return subprocess.check_output(
+            ["git", "-C", str(REPO_ROOT), "rev-parse", "HEAD"],
+            text=True,
+            stderr=subprocess.DEVNULL,
+        ).strip()
+    except (OSError, subprocess.CalledProcessError):
+        return None
+
+
+def _print_row(label: str, value: str) -> None:
+    print(f"{label:<44}{value:>16}")
+
+
+def print_report(
+    metrics: dict[str, Any],
+    *,
+    workload: Workload,
+    endpoint: str,
+    sessions: int,
+    target_fps: float,
+) -> None:
+    header = " LingBot-World realtime streaming benchmark "
+    print("\n{s:{c}^{n}}".format(s=header, n=60, c="="))
+    _print_row("Endpoint:", endpoint)
+    _print_row("Geometry:", f"{workload.width}x{workload.height}")
+    _print_row("Chunks per session:", str(workload.num_chunks))
+    _print_row("Sessions:", str(sessions))
+    frames = workload.num_frames
+    _print_row("Real-time basis:", f"{target_fps:g} fps")
+    _print_row("Video seconds per session:", f"{frames / target_fps:.2f} ({frames} frames)")
+    if abs(target_fps - workload.fps) > 1e-9:
+        _print_row("Mux label sent to server:", f"{workload.fps} fps")
+
+    if sessions == 1:
+        print("{s:{c}^{n}}".format(s=" Cadence ", n=60, c="-"))
+        if metrics.get("ttfc_ms") is not None:
+            _print_row("TTFC (session.start -> first chunk):", f"{metrics['ttfc_ms']:.1f} ms")
+        interval_all = metrics.get("interval_all") or {}
+        interval_steady = metrics.get("interval_steady") or {}
+        for title, summary in (("All chunks", interval_all), ("Steady state", interval_steady)):
+            if not summary:
+                continue
+            print(f"  {title} (n={int(summary['count'])}):")
+            _print_row("    mean / median:", f"{summary['mean_ms']:.1f} / {summary['median_ms']:.1f} ms")
+            _print_row("    std / max:", f"{summary['std_ms']:.1f} / {summary['max_ms']:.1f} ms")
+            tail = f"{summary['p90_ms']:.0f} / {summary['p95_ms']:.0f} / {summary['p99_ms']:.0f} ms"
+            _print_row("    p90 / p95 / p99:", tail)
+            if summary["count"] < 100:
+                print(
+                    "    NOTE: fewer than 100 intervals; tail percentiles are descriptive, not reliable tail estimates."
+                )
+
+        print("{s:{c}^{n}}".format(s=" Real time ", n=60, c="-"))
+        _print_row("VIDEO_RTF (wall s / video s, <1 real time):", f"{metrics['video_rtf']:.3f}")
+        _print_row("VIDEO_RTFX (video s / wall s):", f"{metrics['video_rtfx']:.3f}")
+        if metrics.get("steady_video_rtf") is not None:
+            _print_row("VIDEO_RTF steady state:", f"{metrics['steady_video_rtf']:.3f}")
+        _print_row("Frames per second:", f"{metrics['frames_per_second']:.2f}")
+        _print_row("Chunk deadline:", f"{metrics['chunk_deadline_ms']:.1f} ms")
+        if metrics.get("slo_attainment") is not None:
+            _print_row(
+                "Steady chunks within deadline:",
+                f"{metrics['slo_attainment'] * 100:.1f}% ({metrics['slo_violations']} late)",
+            )
+        if not metrics.get("steady_state_reached", False):
+            print(
+                f"  NOTE: no chunk reached the steady state; the attention window needs "
+                f"{metrics['warmup_chunks']} chunks. Raise --num-chunks."
+            )
+
+        playback = metrics.get("playback") or {}
+        print("{s:{c}^{n}}".format(s=" Playback continuity ", n=60, c="-"))
+        _print_row("Start buffer:", f"{playback.get('buffer_chunks')} chunk(s)")
+        _print_row("Underruns:", str(playback.get("underrun_count")))
+        _print_row("Total stall:", f"{playback.get('total_stall_ms', 0.0):.1f} ms")
+        _print_row("Stall ratio:", f"{playback.get('stall_ratio', 0.0) * 100:.2f}%")
+        if playback.get("first_underrun_chunk") is not None:
+            _print_row("First underrun at chunk:", str(playback["first_underrun_chunk"]))
+    else:
+        print("{s:{c}^{n}}".format(s=" Across sessions ", n=60, c="-"))
+        if metrics.get("ttfc_ms_mean") is not None:
+            _print_row("TTFC mean / max:", f"{metrics['ttfc_ms_mean']:.1f} / {metrics['ttfc_ms_max']:.1f} ms")
+        if metrics.get("steady_interval_ms_mean") is not None:
+            _print_row("Steady interval mean:", f"{metrics['steady_interval_ms_mean']:.1f} ms")
+            _print_row("Steady interval spread:", f"{metrics['steady_interval_ms_spread']:.1f} ms")
+        _print_row("VIDEO_RTF mean (lower is better):", f"{metrics['video_rtf_mean']:.3f}")
+        _print_row(
+            "VIDEO_RTF best / worst:",
+            f"{metrics['video_rtf_best']:.3f} / {metrics['video_rtf_worst']:.3f}",
+        )
+        _print_row("Chunk deadline:", f"{metrics['chunk_deadline_ms']:.1f} ms")
+        _print_row("Underruns (all sessions):", str(metrics["underrun_count"]))
+    print("=" * 60)
+
+
+def save_video(media: bytes, path: Path, *, fps: int) -> None:
+    from vllm_omni.diffusion.utils.media_utils import finalize_streaming_video_bytes
+
+    playback_bytes = finalize_streaming_video_bytes(media, input_format="m4s", fps=fps)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp_path = path.with_suffix(path.suffix + ".tmp")
+    tmp_path.write_bytes(playback_bytes)
+    os.replace(tmp_path, path)
+    print(f"Saved {len(media)} streamed bytes -> {len(playback_bytes)} playback bytes at {path}")
+
+
+# --------------------------------------------------------------------------- #
+# CLI
+# --------------------------------------------------------------------------- #
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Benchmark LingBot-World 2.0 realtime streaming over WS /v1/realtime/video.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    endpoint = parser.add_argument_group("endpoint")
+    endpoint.add_argument("--host", default="127.0.0.1")
+    endpoint.add_argument("--port", type=int, default=8000)
+    endpoint.add_argument("--base-url", default=None, help="Full ws:// URL; overrides --host/--port.")
+    endpoint.add_argument("--model", default=DEFAULT_MODEL)
+
+    workload = parser.add_argument_group("workload")
+    workload.add_argument("--workload", default=None, help="JSON rollout spec; overrides the built-in workload.")
+    workload.add_argument("--image", default=None, help="First frame: path or URL. Defaults to the shared vLLM asset.")
+    workload.add_argument("--prompt", dest="prompt_override", default=None, help=f"Default: {DEFAULT_PROMPT!r}")
+    workload.add_argument("--negative-prompt", default=None)
+    workload.add_argument("--num-chunks", type=int, default=16, help="AR blocks per session; ignored with --workload.")
+    workload.add_argument(
+        "--camera-pattern",
+        choices=CAMERA_PATTERNS,
+        default="forward",
+        help="Camera pattern for the built-in workload only; ignored with --workload.",
+    )
+    workload.add_argument("--width", dest="width_override", type=int, default=None, help=f"Default: {DEFAULT_WIDTH}")
+    workload.add_argument("--height", dest="height_override", type=int, default=None, help=f"Default: {DEFAULT_HEIGHT}")
+    workload.add_argument("--fps", dest="fps_override", type=int, default=None, help=f"Default: {DEFAULT_FPS}")
+    workload.add_argument("--seed", dest="seed_override", type=int, default=None, help=f"Default: {DEFAULT_SEED}")
+    workload.add_argument("--flow-shift", type=float, default=None, help=f"Default: {DEFAULT_FLOW_SHIFT}")
+
+    run = parser.add_argument_group("run")
+    run.add_argument("--sessions", type=int, default=1, help="Sequential measured rollouts.")
+    run.add_argument("--warmup-sessions", type=int, default=0, help="Unmeasured rollouts first (pays compile cost).")
+    run.add_argument(
+        "--warmup-chunks",
+        type=int,
+        default=DEFAULT_WARMUP_CHUNKS,
+        help="Chunks excluded from steady state; the attention window saturates after 6.",
+    )
+    run.add_argument(
+        "--chunk-slo-ms",
+        type=float,
+        default=None,
+        help="Deadline a steady chunk must meet. Default: one chunk of video at --target-fps.",
+    )
+    run.add_argument(
+        "--target-fps",
+        type=float,
+        default=None,
+        help=(
+            "Playback rate used for real-time factor, the chunk deadline, and the playback "
+            "simulation. The checkpoint declares no frame rate, so this is a consumption "
+            "choice, not a model property. Default: the --fps sent to the server."
+        ),
+    )
+    run.add_argument("--playback-buffer-chunks", type=int, default=1, help="Chunks buffered before playback starts.")
+    run.add_argument("--first-chunk-timeout", type=float, default=900.0)
+    run.add_argument("--chunk-timeout", type=float, default=300.0)
+    run.add_argument("--ping-interval", type=float, default=20.0, help="session.ping cadence; 0 disables.")
+
+    output = parser.add_argument_group("output")
+    output.add_argument("--output-json", default=None)
+    output.add_argument("--save-video", default=None, help="Write the last measured session's video here.")
+    output.add_argument("--print-chunks", action="store_true")
+
+    args = parser.parse_args(argv)
+    if args.sessions < 1:
+        parser.error("--sessions must be at least 1.")
+    if args.warmup_sessions < 0:
+        parser.error("--warmup-sessions must be non-negative.")
+    if args.warmup_chunks < 0:
+        parser.error("--warmup-chunks must be non-negative.")
+    if args.playback_buffer_chunks < 1:
+        parser.error("--playback-buffer-chunks must be at least 1.")
+    if args.target_fps is not None and args.target_fps <= 0:
+        parser.error("--target-fps must be positive.")
+    if args.workload is None and args.num_chunks < 1:
+        parser.error("--num-chunks must be at least 1.")
+    return args
+
+
+def endpoint_url(args: argparse.Namespace) -> str:
+    if args.base_url is not None:
+        base = args.base_url.rstrip("/")
+        for prefix, replacement in (("http://", "ws://"), ("https://", "wss://")):
+            if base.startswith(prefix):
+                base = replacement + base[len(prefix) :]
+                break
+        return base if base.endswith(_WS_PATH) else base + _WS_PATH
+    return f"ws://{args.host}:{args.port}{_WS_PATH}"
+
+
+async def run_benchmark(args: argparse.Namespace) -> dict[str, Any]:
+    workload = build_workload(args)
+    url = endpoint_url(args)
+    # The mux label and the real-time basis are different questions; only the
+    # latter decides whether a run passes.
+    target_fps = args.target_fps if args.target_fps is not None else float(workload.fps)
+    if max(1, args.warmup_chunks) >= workload.num_chunks - 1:
+        print(
+            f"WARNING: --warmup-chunks={args.warmup_chunks} leaves no nonterminal steady-state chunk in a "
+            f"{workload.num_chunks}-chunk rollout; steady-state metrics will be empty.",
+            file=sys.stderr,
+        )
+
+    print(f"Connecting to {url}")
+    print(
+        f"Rollout: {workload.num_chunks} chunks, num_frames={workload.num_frames}, "
+        f"{workload.video_seconds:.2f}s of video"
+    )
+    print(
+        f"Real-time basis: {target_fps:g} fps -> a steady 12-frame chunk is "
+        f"{steady_chunk_deadline_ms(target_fps):.1f} ms of video"
+        + ("" if args.target_fps is None else f" (mux label stays {workload.fps} fps)")
+    )
+
+    for index in range(args.warmup_sessions):
+        print(f"Warmup session {index + 1}/{args.warmup_sessions} (not measured)...")
+        await run_session(
+            url,
+            args.model,
+            workload,
+            first_chunk_timeout=args.first_chunk_timeout,
+            chunk_timeout=args.chunk_timeout,
+            ping_interval=args.ping_interval,
+            collect_media=False,
+            print_chunks=False,
+        )
+
+    session_metrics: list[dict[str, Any]] = []
+    session_payloads: list[dict[str, Any]] = []
+    last_result: SessionResult | None = None
+    for index in range(args.sessions):
+        if args.sessions > 1:
+            print(f"Session {index + 1}/{args.sessions}...")
+        result = await run_session(
+            url,
+            args.model,
+            workload,
+            first_chunk_timeout=args.first_chunk_timeout,
+            chunk_timeout=args.chunk_timeout,
+            ping_interval=args.ping_interval,
+            collect_media=args.save_video is not None,
+            print_chunks=args.print_chunks,
+        )
+        last_result = result
+        metrics = compute_metrics(
+            result.records,
+            fps=target_fps,
+            warmup_chunks=args.warmup_chunks,
+            slo_ms=args.chunk_slo_ms,
+            playback_buffer_chunks=args.playback_buffer_chunks,
+            ttfc_s=result.ttfc_s,
+            session_wall_s=result.wall_s,
+        )
+        session_metrics.append(metrics)
+        session_payloads.append(
+            {
+                "session_index": index,
+                "request_id": result.request_id,
+                "video_start_ms": (result.video_start_s * 1000.0) if result.video_start_s is not None else None,
+                "chunks": [record.to_dict() for record in result.records],
+                "metrics": metrics,
+            }
+        )
+
+    aggregate = aggregate_metrics(session_metrics, fps=target_fps)
+    print_report(aggregate, workload=workload, endpoint=url, sessions=args.sessions, target_fps=target_fps)
+
+    if args.save_video is not None and last_result is not None:
+        save_video(last_result.media, Path(args.save_video), fps=workload.fps)
+
+    document = {
+        "benchmark": "lingbot_world_realtime",
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "endpoint": url,
+        "model": args.model,
+        "git_revision": _git_revision(),
+        "hostname": platform.node(),
+        "workload": workload.describe(),
+        "config": {
+            "sessions": args.sessions,
+            "warmup_sessions": args.warmup_sessions,
+            "warmup_chunks": args.warmup_chunks,
+            "camera_pattern": args.camera_pattern,
+            "chunk_slo_ms": args.chunk_slo_ms,
+            "target_fps": target_fps,
+            "mux_fps": workload.fps,
+            "playback_buffer_chunks": args.playback_buffer_chunks,
+            "workload_file": args.workload,
+        },
+        "sessions": session_payloads,
+        "metrics": aggregate,
+    }
+    if args.output_json is not None:
+        path = Path(args.output_json)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n")
+        print(f"Wrote {path}")
+    return document
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv)
+    try:
+        asyncio.run(run_benchmark(args))
+    except BenchmarkError as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmarks/lingbot_world/configs/single_gpu_eager.yaml

@@ -0,0 +1,45 @@
+# Benchmark deploy config: LingBot-World 2.0 on one GPU, eager.
+#
+# This is the portable smoke configuration -- it matches the shipped
+# vllm_omni/deploy/lingbot_world_v2_stepwise.yaml so a number measured here is
+# comparable to the default deployment. It is NOT a realtime configuration: one
+# GPU runs all five forwards of every AR block serially and nothing is compiled,
+# so expect a chunk interval several times the real-time deadline.
+#
+# Use configs/usp4_compiled.yaml for the realtime topology.
+#
+#   vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+#     --omni --deploy-config benchmarks/lingbot_world/configs/single_gpu_eager.yaml \
+#     --port 8000
+
+pipeline: lingbot_world
+async_chunk: false
+distributed_executor_backend: mp
+dtype: bfloat16
+
+stages:
+  - stage_id: 0
+    devices: "0"
+    # AR-Diffusion binds one session per runner invocation; the runner rejects
+    # anything larger.
+    max_num_seqs: 1
+    enforce_eager: true
+    model_class_name: LingBotWorldCausalDMDPipeline
+    engine_backend: vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine
+    step_execution: true
+    streaming_output: true
+    default_sampling_params:
+      # The causal-fast checkpoint pins a four-step DMD schedule; this sizes the
+      # step budget before the pipeline is reached.
+      num_inference_steps: 4
+      output_type: pil
+      extra_args:
+        flow_shift: 5.0
+    model_config:
+      # The AR cache geometry is fixed at load time, so a benchmark request must
+      # ask for exactly this resolution.
+      ar_diffusion_height: 480
+      ar_diffusion_width: 832
+      ar_diffusion_kv_config:
+        gpu_memory_fraction: 0.6
+        warmup_cudagraph: true

benchmarks/lingbot_world/configs/usp4_compiled.yaml

@@ -0,0 +1,60 @@
+# Benchmark deploy config: LingBot-World 2.0 on four GPUs, Ulysses SP-4, compiled.
+#
+# This is the realtime topology. Two things differ from the single-GPU config,
+# and both matter to the number the benchmark reports:
+#
+#   parallel_config   Pure Ulysses sequence parallelism over four ranks. The
+#                     pipeline validates this: LingBot requires
+#                     sequence_parallel_size == ulysses_degree, ring_degree and
+#                     allgather_degree of 1, ulysses_mode "strict", and
+#                     ulysses_a2a_permute disabled. Pipeline, CFG, and VAE
+#                     parallel sizes above 1 are rejected. This config uses
+#                     TP=1; the validator does not reject larger TP sizes.
+#   enforce_eager     false puts LingBotDMDBlockRunner on the compiled path
+#                     (regional torch.compile with cudagraph step marking). The
+#                     first rollout therefore pays compile and capture cost --
+#                     use --warmup-sessions 1 before measured repetitions.
+#                     Chunk warmup only excludes the attention-window ramp.
+#
+# Serve it under a GPU reservation, since it needs four cards:
+#
+#   gpu run --gpus 4 -- vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+#     --omni --deploy-config benchmarks/lingbot_world/configs/usp4_compiled.yaml \
+#     --port 8000
+#
+# `devices` is the rank->card mapping inside the reservation, so it stays
+# "0,1,2,3" no matter which physical cards CUDA_VISIBLE_DEVICES selected.
+
+pipeline: lingbot_world
+async_chunk: false
+distributed_executor_backend: mp
+dtype: bfloat16
+
+stages:
+  - stage_id: 0
+    devices: "0,1,2,3"
+    max_num_seqs: 1
+    enforce_eager: false
+    model_class_name: LingBotWorldCausalDMDPipeline
+    engine_backend: vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine
+    step_execution: true
+    streaming_output: true
+    parallel_config:
+      tensor_parallel_size: 1
+      sequence_parallel_size: 4
+      ulysses_degree: 4
+      ring_degree: 1
+    default_sampling_params:
+      num_inference_steps: 4
+      output_type: pil
+      extra_args:
+        flow_shift: 5.0
+    model_config:
+      ar_diffusion_height: 480
+      ar_diffusion_width: 832
+      ar_diffusion_kv_config:
+        # Each rank holds its own shard of the paged KV cache, so the per-card
+        # fraction can stay at the single-GPU value; lower it if the four-rank
+        # start OOMs on smaller cards.
+        gpu_memory_fraction: 0.6
+        warmup_cudagraph: true

benchmarks/lingbot_world/workload.py

@@ -0,0 +1,631 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Workload construction and metric math for the LingBot-World realtime benchmark.
+
+Nothing here imports torch, vllm, or websockets, so the arithmetic that decides
+what a benchmark number *means* is unit-testable without a GPU or a server.
+
+Two model properties drive every formula in this module, both read from the
+checkpoint's ``transformer/config.json`` rather than assumed:
+
+``num_frames_per_block = 3``
+    One AR block commits three latent frames, so one streamed chunk is three
+    latent frames and a request's ``camera_action_script`` carries exactly one
+    three-entry action list per chunk.
+
+``sliding_window_num_frames = 18`` with ``sink_size = 9``
+    Self-attention sees nine sink frames plus a nine-frame recent window, so the
+    key count a block attends over keeps growing until eighteen latent frames of
+    history exist -- six chunks. Chunk cost is therefore *not* stationary before
+    chunk six, and a mean taken over a short rollout reports a number no steady
+    session will ever produce. :data:`DEFAULT_WARMUP_CHUNKS` exists for that
+    reason and is not a round-number guess.
+
+The causal VAE expands the first latent frame to one pixel frame and every later
+latent frame to four, so chunk zero carries nine pixel frames and every chunk
+after it carries twelve. Video-duration math has to respect that asymmetry or
+real-time factor comes out wrong on short rollouts.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+# transformer.config.num_frames_per_block: latent frames committed per AR block.
+FRAMES_PER_BLOCK = 3
+# vae_scale_factor_temporal: latent frames -> pixel frames, after the first.
+TEMPORAL_COMPRESSION = 4
+# (sink_size 9 + recent window 9) / FRAMES_PER_BLOCK: chunks until the attention
+# window saturates and per-chunk cost becomes stationary.
+DEFAULT_WARMUP_CHUNKS = 6
+
+DEFAULT_MODEL = "robbyant/lingbot-world-v2-14b-causal-fast-diffusers"
+DEFAULT_PROMPT = "The camera moves slowly forward through the scene."
+# The AR KV cache geometry is fixed at load time, so a request must ask for
+# exactly the resolution the deploy config declares.
+DEFAULT_WIDTH = 832
+DEFAULT_HEIGHT = 480
+DEFAULT_FPS = 16
+DEFAULT_FLOW_SHIFT = 5.0
+DEFAULT_SEED = 42
+
+_HOLD = "hold"
+_ACTION_KEYS = ("w", "a", "s", "d")
+
+
+def num_frames_for_chunks(num_chunks: int) -> int:
+    """Pixel-frame count whose rollout is exactly ``num_chunks`` chunks.
+
+    Inverse of :func:`chunks_for_num_frames`, which is the server's own
+    ``((num_frames - 1) // 4 + 1) // 3``.
+    """
+    if num_chunks < 1:
+        raise ValueError("num_chunks must be at least 1.")
+    return (num_chunks * FRAMES_PER_BLOCK - 1) * TEMPORAL_COMPRESSION + 1
+
+
+def chunks_for_num_frames(num_frames: int) -> int:
+    """Chunks the server generates for ``num_frames``, mirroring its own formula."""
+    if num_frames < 1:
+        raise ValueError("num_frames must be at least 1.")
+    return ((num_frames - 1) // TEMPORAL_COMPRESSION + 1) // FRAMES_PER_BLOCK
+
+
+def pixel_frames_in_chunk(chunk_index: int) -> int:
+    """Pixel frames carried by one chunk.
+
+    The causal decoder expands the session's opening latent frame once and every
+    later latent frame four times, so chunk zero is short by three frames.
+    """
+    if chunk_index < 0:
+        raise ValueError("chunk_index must be non-negative.")
+    if chunk_index == 0:
+        return (FRAMES_PER_BLOCK - 1) * TEMPORAL_COMPRESSION + 1
+    return FRAMES_PER_BLOCK * TEMPORAL_COMPRESSION
+
+
+def camera_actions_for_chunk(pattern: str, chunk_index: int) -> list[list[str]]:
+    """One three-entry action list for ``chunk_index``.
+
+    Camera actions select camera embeddings; they change what the world does,
+    not how much arithmetic it costs. The pattern therefore exists to keep a
+    benchmark rollout representative rather than to sweep a cost dimension.
+    """
+    if chunk_index < 0:
+        raise ValueError("chunk_index must be non-negative.")
+    if pattern == "forward":
+        return [["w"] for _ in range(FRAMES_PER_BLOCK)]
+    if pattern == _HOLD:
+        return [[] for _ in range(FRAMES_PER_BLOCK)]
+    if pattern == "orbit":
+        key = _ACTION_KEYS[chunk_index % len(_ACTION_KEYS)]
+        return [[key] for _ in range(FRAMES_PER_BLOCK)]
+    raise ValueError(f"Unknown camera pattern {pattern!r}; expected one of {sorted(CAMERA_PATTERNS)}.")
+
+
+CAMERA_PATTERNS = ("forward", "orbit", _HOLD)
+
+
+def build_camera_script(num_chunks: int, pattern: str) -> list[list[list[str]]]:
+    """Full ``camera_action_script``: one action list per generated chunk."""
+    if num_chunks < 1:
+        raise ValueError("num_chunks must be at least 1.")
+    return [camera_actions_for_chunk(pattern, index) for index in range(num_chunks)]
+
+
+def _validate_camera_script(script: Sequence[Any]) -> list[list[list[str]]]:
+    validated: list[list[list[str]]] = []
+    for chunk_index, actions in enumerate(script):
+        if not isinstance(actions, Sequence) or isinstance(actions, (str, bytes)):
+            raise ValueError(f"camera_action_script[{chunk_index}] must be a list of three action lists.")
+        if len(actions) != FRAMES_PER_BLOCK:
+            raise ValueError(
+                f"camera_action_script[{chunk_index}] must hold exactly {FRAMES_PER_BLOCK} action lists "
+                f"(got {len(actions)}); one AR block is {FRAMES_PER_BLOCK} latent frames."
+            )
+        frames: list[list[str]] = []
+        for frame_index, frame in enumerate(actions):
+            if not isinstance(frame, Sequence) or isinstance(frame, (str, bytes)):
+                raise ValueError(f"camera_action_script[{chunk_index}][{frame_index}] must be a list of strings.")
+            if any(not isinstance(action, str) for action in frame):
+                raise ValueError(f"camera_action_script[{chunk_index}][{frame_index}] must contain only strings.")
+            frames.append([str(action) for action in frame])
+        validated.append(frames)
+    if not validated:
+        raise ValueError("camera_action_script must not be empty.")
+    return validated
+
+
+@dataclass(frozen=True)
+class PromptUpdate:
+    """A mid-rollout ``session.interaction`` scheduled on a chunk boundary."""
+
+    after_chunk: int
+    prompt: str
+    transition_chunks: int = 3
+
+    def __post_init__(self) -> None:
+        if self.after_chunk < 0:
+            raise ValueError("PromptUpdate.after_chunk must be non-negative.")
+        if not self.prompt.strip():
+            raise ValueError("PromptUpdate.prompt must contain non-whitespace text.")
+        if self.transition_chunks < 0:
+            raise ValueError("PromptUpdate.transition_chunks must be non-negative.")
+
+    def to_payload(self) -> dict[str, Any]:
+        return {
+            "type": "session.interaction",
+            "interaction": {
+                "event_id": f"chunk-{self.after_chunk}",
+                "event": {"prompt": self.prompt},
+                "transition_chunks": self.transition_chunks,
+            },
+        }
+
+
+@dataclass(frozen=True)
+class Workload:
+    """One rollout: what the benchmark asks the server to generate."""
+
+    prompt: str
+    image_reference: str
+    camera_script: list[list[list[str]]]
+    width: int = DEFAULT_WIDTH
+    height: int = DEFAULT_HEIGHT
+    fps: int = DEFAULT_FPS
+    seed: int = DEFAULT_SEED
+    flow_shift: float = DEFAULT_FLOW_SHIFT
+    negative_prompt: str | None = None
+    prompt_updates: tuple[PromptUpdate, ...] = ()
+    extra_params: Mapping[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if not self.prompt.strip():
+            raise ValueError("Workload.prompt must contain non-whitespace text.")
+        if not self.image_reference:
+            raise ValueError("Workload.image_reference is required; LingBot-World is image-conditioned.")
+        if self.width <= 0 or self.height <= 0 or self.width % 16 or self.height % 16:
+            raise ValueError("Workload width/height must be positive multiples of 16.")
+        if self.fps <= 0:
+            raise ValueError("Workload.fps must be positive.")
+        object.__setattr__(self, "camera_script", _validate_camera_script(self.camera_script))
+        seen_update_chunks: set[int] = set()
+        for update in self.prompt_updates:
+            if update.after_chunk in seen_update_chunks:
+                raise ValueError(f"Duplicate prompt update after chunk {update.after_chunk}; event IDs must be unique.")
+            seen_update_chunks.add(update.after_chunk)
+            if update.after_chunk >= self.num_chunks:
+                raise ValueError(
+                    f"Prompt update after chunk {update.after_chunk} never fires in a {self.num_chunks}-chunk rollout."
+                )
+
+    @property
+    def num_chunks(self) -> int:
+        return len(self.camera_script)
+
+    @property
+    def num_frames(self) -> int:
+        return num_frames_for_chunks(self.num_chunks)
+
+    @property
+    def video_seconds(self) -> float:
+        return self.num_frames / self.fps
+
+    def session_start_payload(self, model: str) -> dict[str, Any]:
+        """The ``session.start`` event for ``WS /v1/realtime/video``."""
+        extra_params: dict[str, Any] = {
+            "flow_shift": self.flow_shift,
+            "camera_action_script": self.camera_script,
+        }
+        extra_params.update(self.extra_params)
+        payload: dict[str, Any] = {
+            "type": "session.start",
+            "model": model,
+            "prompt": self.prompt,
+            "image_reference": {"image_url": self.image_reference},
+            "width": self.width,
+            "height": self.height,
+            "num_frames": self.num_frames,
+            "fps": self.fps,
+            "seed": self.seed,
+            "extra_params": extra_params,
+        }
+        if self.negative_prompt is not None:
+            payload["negative_prompt"] = self.negative_prompt
+        return payload
+
+    def describe(self) -> dict[str, Any]:
+        return {
+            "num_chunks": self.num_chunks,
+            "num_frames": self.num_frames,
+            "width": self.width,
+            "height": self.height,
+            "fps": self.fps,
+            "seed": self.seed,
+            "flow_shift": self.flow_shift,
+            "video_seconds": self.video_seconds,
+            "prompt_updates": [
+                {
+                    "after_chunk": update.after_chunk,
+                    "transition_chunks": update.transition_chunks,
+                }
+                for update in self.prompt_updates
+            ],
+        }
+
+
+def default_workload(
+    *,
+    image_reference: str,
+    num_chunks: int,
+    camera_pattern: str = "forward",
+    prompt: str = DEFAULT_PROMPT,
+    width: int = DEFAULT_WIDTH,
+    height: int = DEFAULT_HEIGHT,
+    fps: int = DEFAULT_FPS,
+    seed: int = DEFAULT_SEED,
+    flow_shift: float = DEFAULT_FLOW_SHIFT,
+    negative_prompt: str | None = None,
+) -> Workload:
+    """Built-in rollout: a fixed prompt and a generated camera script."""
+    return Workload(
+        prompt=prompt,
+        image_reference=image_reference,
+        camera_script=build_camera_script(num_chunks, camera_pattern),
+        width=width,
+        height=height,
+        fps=fps,
+        seed=seed,
+        flow_shift=flow_shift,
+        negative_prompt=negative_prompt,
+    )
+
+
+def load_workload(
+    path: str | Path,
+    *,
+    image_resolver: Any = None,
+    overrides: Mapping[str, Any] | None = None,
+) -> Workload:
+    """Read a rollout spec from JSON.
+
+    ``image_resolver`` turns the spec's ``image`` field into an ``http(s)`` or
+    ``data:`` URL; the caller owns that because inlining a local file is I/O.
+    Keys in ``overrides`` that are not ``None`` win over the file, so CLI flags
+    stay meaningful next to ``--workload``.
+    """
+    document = json.loads(Path(path).read_text())
+    if not isinstance(document, dict):
+        raise ValueError("A workload file must contain a JSON object.")
+
+    merged = dict(document)
+    for key, value in (overrides or {}).items():
+        if value is not None:
+            merged[key] = value
+
+    camera_script = merged.get("camera_action_script")
+    if camera_script is None:
+        num_chunks = merged.get("num_chunks")
+        if num_chunks is None:
+            raise ValueError("A workload file needs either camera_action_script or num_chunks.")
+        camera_script = build_camera_script(int(num_chunks), str(merged.get("camera_pattern", "forward")))
+
+    image = merged.get("image") or merged.get("image_reference")
+    if image is None:
+        raise ValueError("A workload file needs an image or image_reference field.")
+    image_reference = image_resolver(str(image)) if image_resolver is not None else str(image)
+
+    updates_field = merged.get("prompt_updates") or ()
+    prompt_updates = tuple(
+        PromptUpdate(
+            after_chunk=int(entry["after_chunk"]),
+            prompt=str(entry["prompt"]),
+            transition_chunks=int(entry.get("transition_chunks", 3)),
+        )
+        for entry in updates_field
+    )
+
+    extra_params = merged.get("extra_params") or {}
+    if not isinstance(extra_params, Mapping):
+        raise ValueError("Workload extra_params must be a JSON object.")
+
+    return Workload(
+        prompt=str(merged.get("prompt", DEFAULT_PROMPT)),
+        image_reference=image_reference,
+        camera_script=camera_script,
+        width=int(merged.get("width", DEFAULT_WIDTH)),
+        height=int(merged.get("height", DEFAULT_HEIGHT)),
+        fps=int(merged.get("fps", DEFAULT_FPS)),
+        seed=int(merged.get("seed", DEFAULT_SEED)),
+        flow_shift=float(merged.get("flow_shift", DEFAULT_FLOW_SHIFT)),
+        negative_prompt=merged.get("negative_prompt"),
+        prompt_updates=prompt_updates,
+        extra_params=dict(extra_params),
+    )
+
+
+# --------------------------------------------------------------------------- #
+# Metrics
+# --------------------------------------------------------------------------- #
+
+
+@dataclass
+class ChunkRecord:
+    """One media chunk as the client observed it."""
+
+    index: int
+    arrival_s: float
+    """Seconds from ``session.start`` being sent to this chunk's bytes arriving."""
+    inter_arrival_s: float
+    """Seconds since the previous chunk; for chunk zero this equals ``arrival_s``."""
+    byte_length: int
+    num_frames: int
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "index": self.index,
+            "arrival_s": self.arrival_s,
+            "inter_arrival_ms": self.inter_arrival_s * 1000.0,
+            "byte_length": self.byte_length,
+            "num_frames": self.num_frames,
+        }
+
+
+def percentile(values: Sequence[float], fraction: float) -> float:
+    """Linearly interpolated percentile, so a 10-sample run still reports a p95.
+
+    Implemented here rather than pulled from numpy to keep this module importable
+    in a bare environment; the benchmark's own numbers must not depend on which
+    optional package happens to be installed.
+    """
+    if not values:
+        raise ValueError("percentile() needs at least one value.")
+    if not 0.0 <= fraction <= 1.0:
+        raise ValueError("percentile fraction must be within [0, 1].")
+    ordered = sorted(values)
+    if len(ordered) == 1:
+        return ordered[0]
+    position = fraction * (len(ordered) - 1)
+    lower = int(position)
+    upper = min(lower + 1, len(ordered) - 1)
+    weight = position - lower
+    return ordered[lower] * (1.0 - weight) + ordered[upper] * weight
+
+
+def _stddev(values: Sequence[float]) -> float:
+    if len(values) < 2:
+        return 0.0
+    mean = sum(values) / len(values)
+    return (sum((value - mean) ** 2 for value in values) / (len(values) - 1)) ** 0.5
+
+
+def _latency_summary(values: Sequence[float]) -> dict[str, float]:
+    """Millisecond summary of a set of inter-arrival intervals."""
+    if not values:
+        return {}
+    milliseconds = [value * 1000.0 for value in values]
+    return {
+        "count": float(len(milliseconds)),
+        "mean_ms": sum(milliseconds) / len(milliseconds),
+        "median_ms": percentile(milliseconds, 0.5),
+        "std_ms": _stddev(milliseconds),
+        "min_ms": min(milliseconds),
+        "p90_ms": percentile(milliseconds, 0.90),
+        "p95_ms": percentile(milliseconds, 0.95),
+        "p99_ms": percentile(milliseconds, 0.99),
+        "max_ms": max(milliseconds),
+    }
+
+
+@dataclass(frozen=True)
+class PlaybackResult:
+    """What a viewer would have seen, given a start buffer and a playback rate."""
+
+    underrun_count: int
+    total_stall_s: float
+    playback_span_s: float
+    first_underrun_chunk: int | None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "underrun_count": self.underrun_count,
+            "total_stall_ms": self.total_stall_s * 1000.0,
+            "playback_span_s": self.playback_span_s,
+            "stall_ratio": (self.total_stall_s / self.playback_span_s) if self.playback_span_s > 0 else 0.0,
+            "first_underrun_chunk": self.first_underrun_chunk,
+        }
+
+
+def simulate_playback(
+    records: Sequence[ChunkRecord],
+    *,
+    fps: float,
+    buffer_chunks: int = 1,
+) -> PlaybackResult:
+    """Replay arrivals against a wall clock and count stalls.
+
+    Playback starts when ``buffer_chunks`` chunks have arrived and then consumes
+    video at ``fps``. A chunk that has not arrived by the moment the player needs
+    it is an underrun, and the player resumes only when the bytes land -- which
+    is what a viewer experiences, and what a single p99 number cannot tell you:
+    the same p99 is harmless behind a three-chunk buffer and fatal behind none.
+    """
+    if fps <= 0:
+        raise ValueError("fps must be positive.")
+    if buffer_chunks < 1:
+        raise ValueError("buffer_chunks must be at least 1.")
+    if not records:
+        return PlaybackResult(0, 0.0, 0.0, None)
+    if buffer_chunks > len(records):
+        raise ValueError(f"buffer_chunks={buffer_chunks} exceeds the {len(records)} chunks received.")
+
+    ordered = sorted(records, key=lambda record: record.index)
+    start = ordered[buffer_chunks - 1].arrival_s
+    cursor = start
+    underruns = 0
+    total_stall = 0.0
+    first_underrun: int | None = None
+
+    for record in ordered:
+        if record.arrival_s > cursor:
+            stall = record.arrival_s - cursor
+            underruns += 1
+            total_stall += stall
+            if first_underrun is None:
+                first_underrun = record.index
+            cursor = record.arrival_s
+        cursor += record.num_frames / fps
+
+    return PlaybackResult(
+        underrun_count=underruns,
+        total_stall_s=total_stall,
+        playback_span_s=cursor - start,
+        first_underrun_chunk=first_underrun,
+    )
+
+
+def steady_chunk_deadline_ms(fps: float) -> float:
+    """Wall-clock budget a steady chunk must meet to hold real time.
+
+    ``fps`` here is the rate the frames are *intended to be played at*, which the
+    checkpoint does not declare: nothing in ``model_index.json``, the scheduler
+    config, or the transformer config carries a frame rate, and the pipeline
+    never reads one. It is a property of how the output is consumed, so both
+    this deadline and real-time factor scale linearly with the choice and the
+    caller has to state it rather than inherit it from a muxing default.
+    """
+    if fps <= 0:
+        raise ValueError("fps must be positive.")
+    return pixel_frames_in_chunk(1) / fps * 1000.0
+
+
+def compute_metrics(
+    records: Sequence[ChunkRecord],
+    *,
+    fps: float,
+    warmup_chunks: int = DEFAULT_WARMUP_CHUNKS,
+    slo_ms: float | None = None,
+    playback_buffer_chunks: int = 1,
+    ttfc_s: float | None = None,
+    session_wall_s: float | None = None,
+) -> dict[str, Any]:
+    """Summarize one session.
+
+    ``warmup_chunks`` are excluded from the steady-state block because the
+    attention window has not saturated before then (see the module docstring).
+    The terminal chunk also stays out of steady metrics because it skips
+    next-chunk preparation. All chunks stay in the playback simulation, which is
+    what a session really costs end to end.
+    """
+    if not records:
+        raise ValueError("compute_metrics() needs at least one chunk record.")
+    if warmup_chunks < 0:
+        raise ValueError("warmup_chunks must be non-negative.")
+
+    ordered = sorted(records, key=lambda record: record.index)
+    deadline_ms = slo_ms if slo_ms is not None else steady_chunk_deadline_ms(fps)
+
+    all_intervals = [record.inter_arrival_s for record in ordered[1:]]
+    # The terminal chunk skips next-chunk preparation and is not representative
+    # of sustained generation. Keep it in end-to-end and playback metrics.
+    steady = [record for record in ordered[:-1] if record.index >= warmup_chunks]
+    steady_intervals = [record.inter_arrival_s for record in steady if record.index > 0]
+
+    total_frames = sum(record.num_frames for record in ordered)
+    video_seconds = total_frames / fps
+    wall_seconds = session_wall_s if session_wall_s is not None else ordered[-1].arrival_s
+
+    metrics: dict[str, Any] = {
+        "chunks_received": len(ordered),
+        "warmup_chunks": warmup_chunks,
+        "steady_chunks": len(steady_intervals),
+        "steady_state_reached": bool(steady_intervals),
+        "total_frames": total_frames,
+        "total_bytes": sum(record.byte_length for record in ordered),
+        "video_seconds": video_seconds,
+        "wall_seconds": wall_seconds,
+        # RTF is processing time over content duration, as vLLM-Omni computes it
+        # (`patch.py`: generation_s / video_duration; `compute_audio_rtf`: latency
+        # / audio_duration, "SLO red line < 1"). LOWER IS BETTER and below 1.0 is
+        # real time. RTFX is the reciprocal the repo reports under its own name,
+        # kept here because "fraction of real time achieved" reads better on a
+        # dashboard -- but the two must never be confused for one another.
+        "video_rtf": (wall_seconds / video_seconds) if video_seconds > 0 else 0.0,
+        "video_rtfx": (video_seconds / wall_seconds) if wall_seconds > 0 else 0.0,
+        "frames_per_second": (total_frames / wall_seconds) if wall_seconds > 0 else 0.0,
+        "chunk_deadline_ms": deadline_ms,
+        "interval_all": _latency_summary(all_intervals),
+        "interval_steady": _latency_summary(steady_intervals),
+    }
+    if ttfc_s is not None:
+        metrics["ttfc_ms"] = ttfc_s * 1000.0
+
+    if steady_intervals:
+        steady_frames = sum(record.num_frames for record in steady if record.index > 0)
+        steady_wall = sum(steady_intervals)
+        steady_video = steady_frames / fps
+        metrics["steady_video_rtf"] = (steady_wall / steady_video) if steady_video > 0 else 0.0
+        metrics["steady_video_rtfx"] = (steady_video / steady_wall) if steady_wall > 0 else 0.0
+        within = sum(1 for value in steady_intervals if value * 1000.0 <= deadline_ms)
+        metrics["slo_attainment"] = within / len(steady_intervals)
+        metrics["slo_violations"] = len(steady_intervals) - within
+
+    metrics["playback"] = simulate_playback(
+        ordered,
+        fps=fps,
+        buffer_chunks=playback_buffer_chunks,
+    ).to_dict()
+    metrics["playback"]["buffer_chunks"] = playback_buffer_chunks
+    return metrics
+
+
+def aggregate_metrics(sessions: Iterable[Mapping[str, Any]], *, fps: float) -> dict[str, Any]:
+    """Combine per-session metrics across sequential repeats.
+
+    Steady intervals are pooled rather than averaging per-session means, so a
+    session that happened to produce fewer steady chunks does not carry the same
+    weight as a full one.
+    """
+    entries = list(sessions)
+    if not entries:
+        raise ValueError("aggregate_metrics() needs at least one session.")
+    if len(entries) == 1:
+        single = dict(entries[0])
+        single["sessions"] = 1
+        return single
+
+    deadline_ms = entries[0]["chunk_deadline_ms"]
+    if any(entry["chunk_deadline_ms"] != deadline_ms for entry in entries):
+        raise ValueError("Cannot aggregate sessions with different chunk deadlines.")
+    ttfcs = [float(entry["ttfc_ms"]) for entry in entries if entry.get("ttfc_ms") is not None]
+    # Lower is better, so "max" is the worst session, not the best.
+    rtfs = [float(entry["video_rtf"]) for entry in entries]
+    steady_means = [
+        float(entry["interval_steady"]["mean_ms"]) for entry in entries if entry.get("interval_steady", {}).get("count")
+    ]
+    steady_count = sum(entry.get("interval_steady", {}).get("count", 0) for entry in entries)
+    steady_total_ms = sum(
+        entry["interval_steady"]["mean_ms"] * entry["interval_steady"]["count"]
+        for entry in entries
+        if entry.get("interval_steady", {}).get("count")
+    )
+    return {
+        "sessions": len(entries),
+        "chunks_received": sum(int(entry["chunks_received"]) for entry in entries),
+        "total_frames": sum(int(entry["total_frames"]) for entry in entries),
+        "video_seconds": sum(float(entry["video_seconds"]) for entry in entries),
+        "wall_seconds": sum(float(entry["wall_seconds"]) for entry in entries),
+        "video_rtf_mean": sum(rtfs) / len(rtfs),
+        "video_rtf_best": min(rtfs),
+        "video_rtf_worst": max(rtfs),
+        "ttfc_ms_mean": (sum(ttfcs) / len(ttfcs)) if ttfcs else None,
+        "ttfc_ms_max": max(ttfcs) if ttfcs else None,
+        "steady_interval_ms_mean": (steady_total_ms / steady_count) if steady_count else None,
+        "steady_interval_ms_spread": (max(steady_means) - min(steady_means)) if steady_means else None,
+        "underrun_count": sum(int(entry["playback"]["underrun_count"]) for entry in entries),
+        "chunk_deadline_ms": deadline_ms,
+    }

benchmarks/qwen3_omni/embedding_harness.py

@@ -0,0 +1,226 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Small fixtures for the real Qwen3 thinker embedding method and its oracle."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import asdict, dataclass
+from types import SimpleNamespace
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+SCENARIOS = ("text", "empty", "audio", "image", "video", "mixed", "interleaved", "vision_no_deepstack")
+
+
+@dataclass(frozen=True)
+class ModelShape:
+    vocab_size: int = 152064
+    hidden_size: int = 2048
+    deepstack_levels: int = 3
+    image_token_id: int = 151655
+    video_token_id: int = 151656
+    audio_token_id: int = 151675
+
+    def __post_init__(self):
+        if min(self.vocab_size, self.hidden_size, self.deepstack_levels) <= 0:
+            raise ValueError("vocab_size, hidden_size and deepstack_levels must be positive")
+        ids = (self.image_token_id, self.video_token_id, self.audio_token_id)
+        if len(set(ids)) != 3 or not all(0 < token < self.vocab_size for token in ids):
+            raise ValueError("Provide three distinct, positive, in-vocabulary modality IDs; use oov=True to test OOV")
+
+    def token_ids(self, oov=False):
+        if oov:
+            return dict(zip(("image", "video", "audio"), range(self.vocab_size, self.vocab_size + 3)))
+        return {name: getattr(self, f"{name}_token_id") for name in ("image", "video", "audio")}
+
+
+@dataclass
+class Feature:
+    modality: str
+    positions: torch.Tensor
+    main: torch.Tensor
+    deepstack: torch.Tensor | None
+    combined: torch.Tensor
+
+
+@dataclass
+class EmbeddingCase:
+    name: str
+    input_ids: torch.Tensor
+    is_multimodal: torch.Tensor | None
+    features: tuple[Feature, ...]
+    fingerprint: str
+
+    def fresh_embeddings(self):
+        # The thinker replaces list entries with main-scale views.
+        return None if self.name == "text" else [feature.combined for feature in self.features]
+
+
+def _tensor_bytes(tensor):
+    return tensor.detach().contiguous().view(torch.uint8).cpu().numpy().tobytes()
+
+
+def build_case(
+    name,
+    num_tokens,
+    num_mm_tokens,
+    shape,
+    *,
+    device="cuda",
+    dtype=torch.bfloat16,
+    mask_device="cpu",
+    seed=42,
+    oov=False,
+):
+    """Build known feature placements; no model merge is used to define them."""
+    if name not in SCENARIOS:
+        raise ValueError(f"Unknown scenario: {name}")
+    if num_tokens <= 0 or not 0 <= num_mm_tokens < num_tokens:
+        raise ValueError("Require N > 0 and 0 <= M < N")
+    if (name in {"text", "empty"}) != (num_mm_tokens == 0):
+        raise ValueError("Only text/empty scenarios accept M=0")
+    if name in {"mixed", "interleaved"} and (num_mm_tokens < 4 or num_mm_tokens % 2):
+        raise ValueError("Mixed/interleaved scenarios require an even M >= 4")
+    token_ids = shape.token_ids(oov)
+    generator = torch.Generator().manual_seed(seed)
+    ids = torch.randint(0, min(shape.vocab_size, *token_ids.values()), (num_tokens,), generator=generator)
+    mask = torch.zeros(num_tokens, dtype=torch.bool)
+    positions = torch.arange((num_tokens - num_mm_tokens) // 2, (num_tokens - num_mm_tokens) // 2 + num_mm_tokens)
+    groups = []
+    if name == "mixed":
+        groups = [("audio", positions[: num_mm_tokens // 2]), ("image", positions[num_mm_tokens // 2 :])]
+    elif name == "interleaved":
+        groups = [("video", positions[::2]), ("audio", positions[1::2])]
+    elif num_mm_tokens:
+        groups = [("image" if name == "vision_no_deepstack" else name, positions)]
+    features = []
+    for modality, indexes in groups:
+        ids[indexes] = token_ids[modality]
+        mask[indexes] = True
+        rows = torch.arange(len(indexes), dtype=torch.float32)[:, None].remainder(31) / 32
+        cols = torch.arange(shape.hidden_size, dtype=torch.float32)[None, :].remainder(7) / 16
+        main_cpu = rows + cols + {"audio": 2, "image": 4, "video": 6}[modality]
+        main = main_cpu.to(device=device, dtype=dtype)
+        deepstack = None
+        parts = [main]
+        if modality != "audio" and name != "vision_no_deepstack":
+            deepstack = torch.stack(
+                [(main_cpu + 16 + 4 * level).to(device=device, dtype=dtype) for level in range(shape.deepstack_levels)]
+            )
+            parts.extend(deepstack.unbind(0))
+        combined = torch.cat(parts, dim=-1)
+        # set_mm_embedding_modality simply attaches this attribute. Tag only
+        # after cat/to: views and copies do not preserve arbitrary attributes.
+        combined.modality = modality
+        features.append(Feature(modality, indexes, main, deepstack, combined))
+    digest = hashlib.sha256(json.dumps({"name": name, "shape": asdict(shape), "seed": seed, "oov": oov}).encode())
+    digest.update(_tensor_bytes(ids))
+    digest.update(_tensor_bytes(mask))
+    for feature in features:
+        digest.update(feature.modality.encode())
+        digest.update(_tensor_bytes(feature.combined))
+    return EmbeddingCase(
+        name,
+        ids.to(device=device),
+        None if name == "text" else mask.to(device=mask_device),
+        tuple(features),
+        digest.hexdigest(),
+    )
+
+
+class LanguageModelAdapter(nn.Module):
+    def __init__(self, embedding):
+        super().__init__()
+        self.embedding = embedding
+
+    def embed_input_ids(self, input_ids):
+        return self.embedding(input_ids)
+
+    def forward(self, input_ids):
+        return self.embed_input_ids(input_ids)
+
+
+def make_thinker(
+    shape,
+    *,
+    device="cuda",
+    dtype=torch.bfloat16,
+    seed=42,
+    backend="torch",
+    deepstack=True,
+    buffer_capacity=32,
+    oov=False,
+):
+    """Skip checkpoint construction, preserving the real class and its MRO."""
+    from tests.model_executor.helpers import bootstrap_vllm_layer_custom_op_modules
+
+    bootstrap_vllm_layer_custom_op_modules()
+    from vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker import (
+        Qwen3OmniMoeThinkerForConditionalGeneration,
+    )
+
+    if buffer_capacity <= 0:
+        raise ValueError("buffer_capacity must be positive")
+    device = torch.device(device)
+    if backend == "torch":
+        embedding = nn.Embedding(shape.vocab_size, shape.hidden_size, device=device, dtype=dtype)
+    elif backend == "vllm":
+        from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
+
+        with torch.device(device):
+            embedding = VocabParallelEmbedding(shape.vocab_size, shape.hidden_size, params_dtype=dtype, disable_tp=True)
+    else:
+        raise ValueError(f"Unknown embedding backend: {backend}")
+    with torch.no_grad():
+        embedding.weight.uniform_(-0.5, 0.5, generator=torch.Generator(device=device).manual_seed(seed))
+    thinker = object.__new__(Qwen3OmniMoeThinkerForConditionalGeneration)
+    nn.Module.__init__(thinker)
+    thinker.config = SimpleNamespace(text_config=SimpleNamespace(hidden_size=shape.hidden_size))
+    for modality, token_id in shape.token_ids(oov).items():
+        setattr(thinker.config, f"{modality}_token_id", token_id)
+    thinker.visual = SimpleNamespace(
+        deepstack_visual_indexes=list(range(shape.deepstack_levels)) if deepstack else None
+    )
+    thinker.language_model = LanguageModelAdapter(embedding)
+    thinker._has_oov_mm_tokens = oov
+    thinker.use_deepstack = deepstack
+    thinker.deepstack_num_level = shape.deepstack_levels if deepstack else 0
+    thinker.deepstack_input_embeds = [
+        torch.zeros(buffer_capacity, shape.hidden_size, device=device, dtype=dtype)
+        for _ in range(thinker.deepstack_num_level)
+    ]
+    thinker.deepstack_input_embeds_num_tokens = 0
+    return thinker.eval()
+
+
+@torch.inference_mode()
+def expected_outputs(model, case):
+    """Independent oracle: direct lookup and explicit positions, not merge helpers."""
+    ids = case.input_ids
+    if model._has_oov_mm_tokens and case.is_multimodal is not None:
+        ids = ids.masked_fill(case.is_multimodal.to(ids.device), 0)
+    main = F.embedding(ids, model.language_model.embedding.weight)
+    has_deepstack = any(feature.deepstack is not None for feature in case.features)
+    deepstack = main.new_zeros((model.deepstack_num_level, *main.shape)) if has_deepstack else None
+    for feature in case.features:
+        indexes = feature.positions.to(main.device)
+        main[indexes] = feature.main
+        if feature.deepstack is not None:
+            deepstack[:, indexes, :] = feature.deepstack
+    return main, deepstack
+
+
+@torch.inference_mode()
+def assert_case_output(model, case, result):
+    expected, deepstack = expected_outputs(model, case)
+    torch.testing.assert_close(result, expected, rtol=0, atol=0)
+    if deepstack is None:
+        assert model.deepstack_input_embeds_num_tokens == 0, "Unexpected stale DeepStack buffer"
+    else:
+        assert model.deepstack_input_embeds_num_tokens == case.input_ids.numel()
+        actual = torch.stack([buffer[: case.input_ids.numel()] for buffer in model.deepstack_input_embeds])
+        torch.testing.assert_close(actual, deepstack, rtol=0, atol=0)

docker/Dockerfile.rocm

@@ -2,7 +2,11 @@
 # docker/Dockerfile.ci. A stale base image can build successfully while failing
 # at runtime when vLLM-Omni imports APIs added by the newer vLLM release.
 ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.29.0
-FROM ${BASE_IMAGE} AS base
+
+# Keep the dependency-heavy portion independent from the source tree. The
+# registry cache is keyed on these inputs, while the final source copy can be
+# linked onto the cached image without materializing its large ROCm layers.
+FROM ${BASE_IMAGE} AS deps
 
 # NOTE: REMINDER to vLLM-Omni rebase maintainer
 # BASE_IMAGE pins the default prebuilt runtime when USE_NIGHTLY_BUILD=0.
@@ -64,9 +68,24 @@ RUN --mount=type=cache,target=/root/.cache/pip \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-# Step 3: Copy vllm-omni code and install without uv
-RUN mkdir -p ${COMMON_WORKDIR}/vllm-omni
-COPY . ${COMMON_WORKDIR}/vllm-omni
+# Step 3: Install project dependencies against a placeholder package tree.
+# The editable install keeps resolving imports from this directory after the
+# final COPY --link replaces the placeholder with the real source tree.
+WORKDIR ${COMMON_WORKDIR}/vllm-omni
+RUN mkdir -p vllm_omni && touch vllm_omni/__init__.py
+COPY pyproject.toml setup.py ./
+COPY requirements/ requirements/
+ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0
+
+RUN uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" \
+        --no-cache-dir -e ".[dev]" --no-build-isolation && \
+    python3 -m pip uninstall -y onnxruntime && \
+    python3 -m pip install --no-cache-dir --force-reinstall --no-deps \
+        "onnxruntime-rocm>=1.22.2" && \
+    python3 -c "import importlib.metadata as m; import onnxruntime as ort; assert 'ROCMExecutionProvider' in ort.get_available_providers(), ort.get_available_providers(); assert m.version('onnxruntime-rocm'); assert not any(d.metadata['Name'] == 'onnxruntime' for d in m.distributions())" && \
+    python3 -c "from torchcodec.decoders import VideoDecoder"
+
+RUN ln -sf /usr/bin/python3 /usr/bin/python
 
 # This is a workaround to ensure pytest exits with the correct status code in CI tests.
 RUN printf '%s\n' \
@@ -85,19 +104,13 @@ RUN printf '%s\n' \
     '    os._exit(_exit_code)' \
     > ${COMMON_WORKDIR}/vllm-omni/conftest.py
 
-# Some optional dependencies pull in the CPU ONNX Runtime distribution. The
-# CPU and ROCm wheels own the same import path and cannot coexist, so remove
-# the CPU wheel and reinstall the ROCm wheel after dependency resolution.
-RUN cd ${COMMON_WORKDIR}/vllm-omni && \
-    uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" \
-        --no-cache-dir ".[dev]" --no-build-isolation && \
-    python3 -m pip uninstall -y onnxruntime && \
-    python3 -m pip install --no-cache-dir --force-reinstall --no-deps \
-        "onnxruntime-rocm>=1.22.2" && \
-    python3 -c "import importlib.metadata as m; import onnxruntime as ort; assert 'ROCMExecutionProvider' in ort.get_available_providers(), ort.get_available_providers(); assert m.version('onnxruntime-rocm'); assert not any(d.metadata['Name'] == 'onnxruntime' for d in m.distributions())" && \
-    python3 -c "from torchcodec.decoders import VideoDecoder"
-
-RUN ln -sf /usr/bin/python3 /usr/bin/python
+# Keep source changes in one lightweight linked layer. Do not add RUN, COPY, or
+# ADD instructions that depend on the resulting filesystem after this point;
+# doing so would force BuildKit to download and extract the cached ROCm layers.
+FROM deps AS base
+ARG COMMON_WORKDIR=/app
+WORKDIR ${COMMON_WORKDIR}/vllm-omni
+COPY --link . .
 
 FROM base AS test
 

docker/Dockerfile.rocm.dockerignore

@@ -0,0 +1,35 @@
+# Keep the linked source layer small and deterministic.
+.git
+.venv
+/build
+dist
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+.mypy_cache
+
+# Distribution / packaging
+.Python
+/build/
+cmake-build-*/
+CMakeUserPresets.json
+develop-eggs/
+/dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+rust/target/

docker/Dockerfile.xpu

@@ -40,7 +40,6 @@ ENV VLLM_OMNI_TARGET_DEVICE=xpu
 # Keep the upstream oneCCL packages paired with the XPU torch runtime.
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install --no-build-isolation \
-    --extra-index-url https://mirrors.ustc.edu.cn/pypi/simple/ \
     ".[dev]" && \
     uv pip uninstall triton && \
     uv pip install triton-xpu==3.7.2 --reinstall

docs/.nav.yml

@@ -161,6 +161,7 @@ nav:
         - Cache-DiT: design/feature/cache_dit.md
         - TeaCache: design/feature/teacache.md
         - Diffusion Continuous Batching: design/feature/diffusion_continuous_batching.md
+        - Mooncake KV Transfer: design/feature/mooncake-kv-transfer.md
     - Infrastructure and Performance:
       - Prometheus Metrics: design/metrics.md
       - Speech Generation Performance Optimizations: design/qwen3_omni_tts_performance_optimization.md

docs/README.md

@@ -56,7 +56,7 @@ vLLM-Omni is flexible and easy to use with:
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
 - Omni-modality models (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
-- TTS models (e.g. Qwen3-TTS, IndexTTS 2.5, dots.tts, CosyVoice3)
+- TTS models (e.g. Qwen3-TTS, IndexTTS 2.5, CosyVoice3)
 - Diffusion models — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
 - Robot-policy and action models (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
 

docs/cli/bench/serve.md

@@ -23,7 +23,9 @@ You can use `vllm bench serve --omni --help=all` to get descriptions of all para
   The API endpoint exposed externally, to which clients send their requests.
 
 - `--dataset-name`
-  The name of the dataset used; random-mm indicates generating random multimodal inputs (images, videos, audio), while `omniinteract` replays official OmniInteract videos.
+  The name of the dataset used; `random-mm` generates random multimodal inputs (images, videos, audio),
+  `omniinteract` replays official OmniInteract videos, and `videomme` loads Video-MME MCQ videos
+  (`lmms-eval/Video-MME`, MiniCPM OmniEvalKit packing).
 
 - `--num-prompts`
   The total number of requests to send, an integer.
@@ -343,6 +345,57 @@ transcript, event, and result artifacts without errors. A valid LISTEN-only case
 chunks. Official-manifest eligibility is reported separately because clipped or cancelled output is a benchmark-quality
 signal, not a transport failure. This local performance test does not score answer accuracy.
 
+### Video-MME Benchmark
+
+Video-MME (`--dataset-name videomme`) scores multiple-choice video QA. Default packing is
+OmniEvalKit MiniCPM `minicpm-frames` (up to 96 sampled frames as `image_url`). Pass a local
+mirror with `--dataset-path` / `--videomme-parquet` + `--videomme-video-dir`, or a Hugging
+Face dataset id (`lmms-eval/Video-MME` by default; any `org/name` is accepted when
+`--dataset-name videomme` is explicit). Relative `--videomme-video-dir` values are resolved
+to absolute `file://` URLs. For those URLs, start the server with
+`--allowed-local-media-path` covering the video root; otherwise use `--videomme-inline-local-video`.
+
+For MiniCPM-o 4.5, the default deployment allows 64 images per request. To run the
+96-frame recipe, start the server with a stage 0 override (replace `/path/to/Video-MME`
+with the local video root shared by the benchmark client and server):
+
+```bash
+vllm serve openbmb/MiniCPM-o-4_5 --omni \
+  --allowed-local-media-path /path/to/Video-MME \
+  --stage-overrides '{"0":{"limit_mm_per_prompt":{"image":96,"audio":64,"video":1}}}'
+```
+
+The override applies to this server invocation only. Alternatively, use
+`--videomme-max-frames 64` in the benchmark to stay within the default image limit;
+this evaluates a different frame-sampling setting from the 96-frame recipe.
+
+Run the benchmark against that server:
+
+```bash
+vllm bench serve --omni \
+  --backend openai-chat-omni \
+  --dataset-name videomme \
+  --dataset-path /path/to/Video-MME \
+  --videomme-pack-mode minicpm-frames \
+  --videomme-max-frames 96 \
+  --model openbmb/MiniCPM-o-4_5 \
+  --endpoint /v1/chat/completions \
+  --num-prompts 8 \
+  --max-concurrency 1 \
+  --save-result
+```
+
+Accuracy keys (`videomme_accuracy`, per-duration / domain / task breakdowns) are written into
+the saved JSON. `videomme_accuracy` excludes HTTP failures; `videomme_accuracy_incl_http_fail`
+counts them as wrong. `videomme_submitted` / `videomme_unique_question_ids` show when
+oversampling made the request count differ from unique-question coverage.
+`videomme_skipped_rows` counts rows skipped during sampling because of missing or unreadable
+media or invalid question fields. The opt-in accuracy runner rejects skipped rows, HTTP
+failures, duplicate question IDs, and missing gold answers. This checks the sampled subset;
+it does not establish that a supplied local mirror contains the entire official dataset.
+Use `--videomme-save-eval-items` (or `VIDEOMME_SAVE_EVAL_ITEMS=1`) for per-request
+`videomme_eval_items` rows.
+
 ### Multi-Modal Benchmark
 
 <details class="admonition abstract" markdown="1">

docs/configuration/environment_variables.md

@@ -70,6 +70,7 @@ The lifecycle labels used below are:
 | `OMNI_DIFFUSION_PROMPT_EMBED_CACHE_SIZE` | Positive integer; default `32` entries | Each diffusion runner; resolved during model setup | A valid environment value overrides the explicit cache size. A non-integer logs a warning; a non-positive value is ignored. | Experimental |
 | `OMNI_DIFFUSION_SESSION_STATE_MANAGER` | Boolean with the same accepted spellings as the prompt cache; default disabled | Experimental diffusion session manager; model setup | A recognized environment value overrides the explicit enable setting. An unrecognized value is ignored. | Experimental |
 | `OMNI_DIFFUSION_SESSION_STATE_MANAGER_MAX_SESSIONS` | Positive integer; default `64` | Experimental diffusion session manager; model setup | A valid environment value overrides the explicit maximum. A non-integer logs a warning; a non-positive value is ignored. | Experimental |
+| `VLLM_OMNI_AR_DIFFUSION_KV_GATHER` | `1` enables; default `0` (off) | AR-diffusion KV manager at allocation and attention at dispatch; set before worker startup | Only the exact value `1` enables contiguous KV gathering. History staging additionally requires `ARDiffusionKVConfig.reuse_history_staging`. | Experimental |
 
 Backend names for `DIFFUSION_ATTENTION_BACKEND` are the members of
 `DiffusionAttentionBackendEnum`, such as `FLASH_ATTN`, `TORCH_SDPA`,
@@ -84,7 +85,7 @@ depends on the installed kernels and model path.
 | `SPEAKER_MAX_UPLOADED` | Integer; default `1000` | Speech server; read when speaker storage initializes | Environment-only setting. A non-integer logs a warning and uses `1000`; range is not otherwise validated. | Stable |
 | `VLLM_OMNI_ABORT_TIMEOUT` | Float seconds; default `2` | Engine abort wait for Videos DELETE and `generate()` cancel/error cleanup; read when `async_omni` imports | Environment-only setting. A non-float raises `ValueError` during import. | Experimental |
 | `VLLM_OMNI_ASYNC_OUTPUT_TIMEOUT` | Float seconds; default `600` | Diffusion engine async-output wait in `step_streaming`; resolved per call on the request path, not at import | Environment-only setting. A non-float or `<=0` value warns once and uses the default. | Experimental |
-| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `1`, `true`, `yes` or `on` enables; default `0` (off) | Orchestration loop and the serving-side final-output drain; read once when the `Orchestrator` is constructed | Environment-only setting. Values are stripped and case-normalized; any unrecognized value leaves the legacy poll loop selected. | Experimental |
+| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `1`, `true`, `yes` or `on` enables; defaults to on for Qwen3-TTS and off for other pipelines | Pipeline default computed from `pipeline_config.model_type` at engine initialization; env override resolved at `Orchestrator` construction and separately when the serving-side final-output drain starts | An explicit env value wins; otherwise both consumers use the engine's pipeline default. Values are stripped and case-normalized; any unrecognized value selects the legacy poll loop. Set before server startup. | Experimental |
 | `VLLM_OMNI_INPUT_WAIT_TIMEOUT_S` | Float seconds; default `600`; `<=0` disables | Full-payload input coordinator, not async-chunk transfer; read when the scheduler module imports in each worker | Environment-only setting. A non-float logs a warning and uses `600`. | Stable operational control |
 | `VLLM_OMNI_ORCH_MONITOR_PATH` | Filesystem path; default `<current-working-directory>/vllm_omni_orch_monitor_<timestamp>.json` | Orchestrator monitor enabled by `--enable-orch-monitor`; read when the monitor is created | Environment-only path override. Parent directories are created; write errors are logged. | Diagnostic |
 | `VLLM_OMNI_VIDEO_SYNC_TIMEOUT` | Float seconds; default `600` | Synchronous Videos API; read when the API server module imports | Environment-only setting. A non-float raises `ValueError` during import. | Experimental |

docs/configuration/stage_configs.md

@@ -290,3 +290,90 @@ vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/depl
 
 !!! important
     We are actively iterating on the definition of deployment configurations, and we welcome feedback from users and developers.
+
+## Qwen3-TTS with Model Runner V2
+
+Qwen3-TTS can opt into the native CUDA Model Runner V2 pipeline on vLLM
+0.29.0. Select one of these deployment profiles:
+
+| Profile | Runner | Code2Wav graph batches | Intended use |
+| --- | --- | --- | --- |
+| `qwen3_tts.yaml` | V1 | Existing defaults | Existing deployment / regression control |
+| `qwen3_tts_mrv2.yaml` | V2 | B1 | Native runner validation |
+| `qwen3_tts_high_concurrency_mrv2.yaml` | V2 | B1, B2 | Opt-in throughput tuning |
+| `qwen3_tts_high_concurrency_mrv2_b4.yaml` | V2 | B1, B2, B3, B4 | Experimental throughput / buffered playback |
+
+```bash
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base --omni \
+  --deploy-config vllm_omni/deploy/qwen3_tts_mrv2.yaml
+```
+
+The V2 profiles bound Talker prefill to 512 tokens per step and select the
+Talker AR runner and the Code2Wav generation
+runner together. Native inter-stage delivery carries codec payloads directly;
+request-owned snapshots preserve buffers through asynchronous completion and
+CUDA graph reuse. Terminal completion waits for upstream stage metrics before
+releasing request state. Platform sections retain V1 on NPU, XPU, ROCm and MUSA;
+this change does not qualify MRV2 on those backends or enable other model families.
+Selecting V2 for a stage without `supports_native_mrv2_data_plane` emits a warning
+when pipeline and deployment settings are merged. That stage retains the legacy
+transport path; the warning does not establish support for that combination.
+
+MRV2 model hooks use capability declarations rather than architecture or stage
+names. Omni lifecycle flags select the model state, and `_returns_tuple` declares
+the capture output contract. The optional batched predictor hook is `mtp`, with
+an explicit `mtp_output_key` (a string or two-part payload key), optional
+`mtp_validity_key`, and `mtp_graph_safe`/`mtp_disable_graph` capture controls.
+Its inputs are token IDs, embeddings, previous hidden states and per-row
+conditioning; it returns updated embeddings and prediction codes. Models may
+supply `mtp_sampling_params` and `get_mtp_seed(sampling_params)` for model-local explicit seeds, and declare
+`mtp_accepts_per_row_generators`, `mtp_accepts_req_infos`, or `mtp_sample_uniforms`
+(with `mtp_sample_steps` and `mtp_sample_vocab_size`) as needed. Qwen3-TTS retains
+its existing `talker_mtp` entry point for V1.
+
+### Included performance work
+
+- Request snapshots have a fast path for immutable scalar leaves, including
+  waveform lists. Mutable containers, aliases and cycles keep deep-copy
+  semantics. This is not the historical shallow `dict(prompt)` experiment.
+- The API reference cache holds owned float32 arrays with a default capacity
+  of 1024 entries and a 512 MiB waveform-payload budget. Cache hits return
+  independent lists. Qwen3-TTS model artifact caches also default to 1024
+  entries. These caches have different owners and eviction policies; equal
+  entry limits do not make them a single coherent cache.
+- Talker state stays on GPU where its lifecycle allows it. BOS/EOS projections
+  are cached in their projection dtype and invalidated on weight loading.
+- Code2Wav packs CPU codec inputs before device transfer. Optional B2/B4 graph
+  buckets batch compatible requests without sharing their per-request state.
+- Native output materialization runs in a bounded worker and drains before
+  closing the data plane. The existing control-only shortcut avoids launching
+  model work for a step that has only lifecycle events.
+
+MTP prefix re-prefill remains disabled in the high-concurrency V2 profile.
+Adding more graph shapes increases compilation cost and has not established a
+stable end-to-end gain for this extracted version.
+
+### Decoder batches and playback
+
+Changing `decode_cudagraph_batch_sizes` selects the captured batch buckets.
+Keep `decode_batch_max_size` consistent with the intended maximum too; the
+stateful graph path currently groups according to the captured buckets, while
+the stateless path also uses the explicit maximum.
+
+B4 remains experimental. First-packet latency alone does not establish
+uninterrupted playback. Validate inter-chunk arrival times, buffering, WER and
+speaker similarity before adopting either batching preset for a production
+workload. Floating-point decoder outputs can differ across batch sizes; this PR
+does not claim bitwise or quality equivalence.
+
+### Optional MPS deployment
+
+NVIDIA MPS is an optional operator setting for colocated CUDA processes, not a
+YAML option or a library default. This PR does not establish a throughput or
+first-packet latency benefit from MPS. Measure the exact deployment with and
+without MPS before enabling it.
+
+Use only assigned GPUs and an independent MPS pipe directory. A private MPS
+server does not provide exclusive GPU ownership or MIG isolation. For a
+single-GPU deployment, explicitly place both stages on that GPU; the supplied
+high-concurrency profile places its two stages on different GPUs by default.

docs/contributing/ci/test_execution_guide.md

@@ -177,10 +177,10 @@ Failed jobs: 1/2
     cd tests
 
     # Stability: Qwen3-Omni
-    pytest -s -v dfx/stability/scripts/test_stability_qwen3_omni.py -m slow
+    pytest -s -v dfx/stability/scripts/run_stability_qwen3_omni.py -m slow
 
     # Stability: Wan2.2 (v1/videos diffusion benchmark loop)
-    pytest -s -v dfx/stability/scripts/test_stability_wan22.py -m slow
+    pytest -s -v dfx/stability/scripts/run_stability_wan22.py -m slow
 
     # Reliability: Qwen3-Omni (H100 × 2)
     pytest -s -v dfx/reliability/test_reliability_qwen3_omni.py -m slow

docs/contributing/ci/test_writing_guide.md

@@ -555,7 +555,7 @@ L5 level testing focuses on the performance of model services under ***long-runn
 
 - ***Trigger Timing***: **`Weekly`** (weekly) or **`Days before Release`** (several days before a major release). Due to long execution times, the frequency is lower.
 - ***Run Command***:
-    - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_<model>.py` alongside a matching JSON config)
+    - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/run_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/run_stability_wan22.py` (or add `run_stability_<model>.py` alongside a matching JSON config)
     - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_<model>.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`).
 - ***Script Example***:
 
@@ -564,7 +564,7 @@ L5 level testing focuses on the performance of model services under ***long-runn
 
 ##### Stability
 
-When you want to add L5-level stability test cases, add or extend the appropriate JSON file under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` for Omni bench traffic, or `test_wan22.json` for diffusion `/v1/videos` workloads). Pair the JSON with `tests/dfx/stability/scripts/test_stability_<model>.py`. The following illustrates the Qwen3-Omni shape:
+When you want to add L5-level stability test cases, add or extend the appropriate JSON file under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` for Omni bench traffic, or `test_wan22.json` for diffusion `/v1/videos` workloads). Pair the JSON with `tests/dfx/stability/scripts/run_stability_<model>.py`. The following illustrates the Qwen3-Omni shape:
 
 ```json
 {

docs/contributing/profiling.md

@@ -22,7 +22,7 @@ profiler_config:
 Supported fields:
 
 | Field | Description |
-|---|---|
+| --- | --- |
 | `profiler` | Profiler backend. Supported values: `torch`, `cuda`. Use `torch` for `trace.json`, Excel operator tables, and optional memory snapshots. Use `cuda` for Nsight Systems only. |
 | `torch_profiler_dir` | Output directory for torch-profiler artifacts. Required when `profiler: torch`. |
 | `torch_profiler_use_gzip` | Compress `trace_rank*.json` into `trace_rank*.json.gz`. |
@@ -287,6 +287,52 @@ Recommended viewers:
 
 For upstream background on the underlying vLLM profiling infrastructure, see the [vLLM profiling guide](https://docs.vllm.ai/en/stable/contributing/profiling/).
 
+## 6. Profiling Omni AR/TTS stages
+
+For Omni AR and TTS stages, put `profiler_config` under each stage that should
+be captured. The serving API then controls a narrow window with
+`POST /start_profile` and `POST /stop_profile`; model initialization and
+warmup stay outside the trace.
+
+Example stage override for a host-stack torch trace:
+
+```bash
+PROFILE='{"profiler":"torch","torch_profiler_dir":"/tmp/qwen3_tts_profile",'\
+'"torch_profiler_with_stack":true,"torch_profiler_record_shapes":false,'\
+'"torch_profiler_use_gzip":true,"torch_profiler_dump_cuda_time_total":true}'
+
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni \
+  --deploy-config vllm_omni/deploy/qwen3_tts_high_concurrency.yaml \
+  --stage-overrides "{\"0\":{\"profiler_config\":$PROFILE},\"1\":{\"profiler_config\":$PROFILE}}"
+```
+
+After one or more warmup requests, capture one short measured request:
+
+```bash
+curl -sS -X POST http://127.0.0.1:8000/start_profile \
+  -H 'Content-Type: application/json' -d '{"stages":[0,1]}'
+# send one fixed /v1/audio/speech or /v1/chat/completions request here
+curl -sS -X POST http://127.0.0.1:8000/stop_profile \
+  -H 'Content-Type: application/json' -d '{"stages":[0,1]}'
+```
+
+For an Nsight Systems CUDA timeline, use the same stage override with
+`{"profiler":"cuda"}` and wrap the server:
+
+```bash
+nsys profile --trace=cuda,nvtx,osrt --sample=process-tree \
+  --cpuctxsw=process-tree --trace-fork-before-exec=true \
+  --capture-range=cudaProfilerApi --capture-range-end=stop-shutdown \
+  --cuda-graph-trace=node -o /tmp/qwen3_tts_nsys \
+  vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni ...
+```
+
+For Omni AR workers, `profiler="cuda"` requires the native CUDA profiler
+wrapper and uses `cudaProfilerStart/Stop`; the report is only valid if the
+`/start_profile` and `/stop_profile` calls both succeed. Use `nsys stats` for
+CUDA API, GPU kernel, OS runtime, and NVTX summaries. Treat profiler latency as
+diagnostic only; use a separate no-profiler run for performance numbers.
+
 ## 7. Orchestrator Monitor
 
 > **Warning:** Diagnostic only. Adds lightweight counters on the orchestrator poll loop and per-replica queue depth. Disable in production unless you are actively debugging orchestrator saturation.
@@ -314,9 +360,11 @@ If unset, the monitor writes `./vllm_omni_orch_monitor_<MMDDHHMM>.json` at shutd
 Each 1-second window records:
 
 | Series | Description |
-|---|---|
+| --- | --- |
 | `windows.duration_s` | Wall time covered by the window |
 | `windows.loop_idle` / `windows.loop_active` | Orchestrator poll-loop iterations with no work vs. active forwarding |
+| `windows.dispatch_queue_size` | Ready-queue size sampled at each monitor window boundary |
+| `windows.dispatch_queue_high_water` | Maximum Event Driven ready-queue size observed in each window |
 | `replicas.<stage,replica>.outputs_queue_size` | MP client `outputs_queue` backlog for that replica |
 | `replicas.<stage,replica>.inflight` | Requests currently bound/routed to the replica |
 
@@ -331,12 +379,16 @@ orchestrator wakes only once per 0.5 s reconcile timeout, while a busy one still
 records one iteration per routed output. Window counts and the `loop_active_pct`
 summary are therefore not comparable across the two modes.
 
+For Event Driven runs, use `dispatch_queue_high_water` rather than only the
+window-boundary `dispatch_queue_size`: a serial dispatcher can drain a burst
+before the window closes, hiding a transient ready-queue backlog in the latter.
+
 ### Relationship to other diagnostics
 
 This monitor is intentionally separate from the existing profiling tools:
 
 | Tool | Scope | Output |
-|---|---|---|
+| --- | --- | --- |
 | `--enable-diffusion-pipeline-profiler` | Diffusion stage functions (`vae.decode`, `diffuse`, …) | Per-request `stage_durations` / logs |
 | `--enable-ar-profiler` | AR stage generation time | Per-request `stage_durations` |
 | `profiler_config` (`torch` / `cuda`) | GPU/CPU kernels inside a stage worker | `trace.json`, Nsight, operator tables |

docs/design/feature/mooncake-kv-transfer.md

@@ -0,0 +1,89 @@
+# Mooncake KV Transfer
+
+## PR summary
+
+This change adds an opt-in AR-to-DiT KV handoff using vLLM's native
+`KVTransferConfig`, Scheduler-owned diffusion KV pages, and
+`MooncakeConnector`. The existing `OmniKVTransferManager` path remains the
+fallback when `kv_transfer_config` is absent.
+
+The orchestrator creates one transfer ticket per request and resolves the
+bound AR replica's Mooncake endpoint after generation. DiT allocates the
+destination pages, waits for every worker rank to finish loading, and releases
+the source and destination pages only after the connector lifecycle completes.
+Request registration failures are rolled back before Worker dispatch. Receive
+timeouts and cancellation terminate the request without recycling in-flight
+pages: the scheduler retains them until every CFG row has completed on every
+rank. Late completion is polled only while such requests remain. If a peer
+never completes, its pages remain reserved until shutdown; other requests can
+continue using the remaining capacity. Invalid pages, missing Worker ranks,
+and failures without a proven rollback still stop the replica before teardown.
+The example sets `transfer_timeout: 60.0` seconds explicitly. Cancellation
+queued during a synchronous receive is processed when that receive completes
+or times out; it does not interrupt the remote write.
+
+## Reference configuration
+
+| Stage | Replicas | Devices | Parallelism | KV cache |
+| --- | ---: | --- | --- | ---: |
+| AR | 2 | 0-3 | TP=2 per replica | 2 GiB per worker |
+| DiT | 2 | 4-7 | TP=2 per replica, EP enabled | 1 GiB per worker |
+
+The acceptance workload uses 512x512 images, two denoising steps, guidance
+scale 2.0, and `max_num_seqs=1`.
+
+For a four-GPU debug run, set both stages to one replica and assign AR TP=2 to
+devices `0,1` and DiT TP=2 to devices `2,3`.
+
+## Experiment
+
+The final comparison used one node with eight NVIDIA RTX PRO 6000 Blackwell
+Server Edition GPUs, TCP Mooncake transport, the same 50-prompt/seed T2I
+dataset, concurrency 8, and two independent 100-request runs per
+implementation. Each side completed 200/200 requests without an OOM or request
+failure.
+
+| Implementation | Run 1 (req/s) | Run 2 (req/s) | Mean (req/s) |
+| --- | ---: | ---: | ---: |
+| `OmniKVTransferManager` | 4.4103 | 4.4497 | 4.4300 |
+| Native `MooncakeConnector` | 4.4457 | 4.4699 | 4.4578 |
+
+Native throughput is **0.63% higher** than the legacy mean. The absolute
+performance difference is below the 1% acceptance threshold.
+
+For accuracy, a non-zero transferred prefix and the resulting attention output
+were compared bit-for-bit with the legacy manager path:
+
+```text
+prefix_bits_equal    True
+attention_bits_equal True
+max_abs_diff         0.0
+```
+
+The one-AR/one-DiT TP=2 debug topology also completed 20/20 sequential
+requests. A separate real-TCP transport check transferred both CFG rows from a
+TP=1 producer to a strict-Ulysses SP=2 consumer with exact values on both
+target ranks.
+
+## Scope and limitations
+
+- The fully validated end-to-end topology is 2x AR TP=2 plus 2x DiT TP=2.
+- Heterogeneous TP=1 to SP=2 reshaping is transport-tested, but arbitrary mixed
+  TP/CFG/SP end-to-end combinations have not been exhaustively validated.
+- CFG-parallel end-to-end deployment is not part of this acceptance run.
+- Native AR-to-DiT transfer currently requires `async_chunk: false`.
+- TCP is covered by the acceptance run. The supplementary single-node RDMA
+  measurements use a separate short workload; they do not establish RDMA
+  support for multi-node deployment, which remains out of scope.
+- The native path currently validates the two-stage `AR(0) -> DiT(1)` graph. Other stage graphs are rejected during startup until
+  their producer/consumer edge mapping is implemented.
+- Sleep mode is disabled for the native paged path because registered KV pages
+  must remain mapped for the connector lifetime.
+
+- Ascend NPU adaptation and its performance validation are maintained in a
+  separate follow-up PR; the GPU acceptance results above do not validate NPU.
+- The homogeneous piecewise CUDA batching optimization is retained. A native
+  paged-attention CUDA graph capture/replay regression covers two CFG rows,
+  strided QKV, mixed causal/image spans, changed inputs, and output stability.
+  It must be run on CUDA hardware; CPU unit tests do not establish graph or
+  end-to-end performance acceptance.

docs/design/feature/offloader/README.md

@@ -84,10 +84,22 @@ Block topology comes from the pipeline `OffloadPlan` first: `block_attrs` maps
 each DiT path to its ordered block containers, `encoder_block_attrs` declares
 streamable encoder stacks, `on_demand_component_paths` marks pipeline-managed
 residency, `resident_dit_paths` marks the DiTs that may hold resident layers,
-and `encoder_dlo_weight_replication` marks the encoders whose loader-produced
-weights are safe for AllGather. DiTs absent from the plan fall back to
-`_layerwise_offload_blocks_attrs` (including the deprecated singular-name
-compatibility path).
+`encoder_dlo_weight_replication` marks the encoders whose loader-produced
+weights are safe for AllGather, and `offload_submodules` maps a DiT submodule
+that owns its residency to its own block container. DiTs absent from the plan
+fall back to `_layerwise_offload_blocks_attrs` (including the deprecated
+singular-name compatibility path).
+
+An undeclared DiT submodule joins the plan only when it is large enough that
+keeping it resident would defeat streaming; its block container is then found
+by scanning well-known attribute names, which warns once per submodule class.
+Size is computed from parameter shapes and dtypes, including meta parameters,
+so resolving before mmap loading preserves the same residency decision.
+Block ownership validation includes nested components: a block shared with a
+parent, sibling, or encoder is rejected before any hooks or storage are changed.
+A submodule with no block container must implement the `load_to_device` /
+`offload_to_cpu` lifecycle, and the resolver rejects it before any component of
+the pipeline is placed or hooked.
 
 The resolver is pure: it moves no tensor, installs no hook, writes no module
 attribute, and reads no process group — multi-rank facts come from

docs/design/feature/omni_async_output_materialization.md

@@ -318,18 +318,18 @@ The same behavior applies to the Base and VoiceDesign Qwen3-TTS checkpoints
 because they use the same Talker implementation.
 
 !!! warning
-    Do not pass `--no-async-chunk` or enable prefix caching when you want this
-    optimization. Either change causes the runner to fall back to synchronous
-    output construction. Enabling async chunk alone cannot activate the feature
-    for a model that has not opted into async Omni output.
+    Do not pass `--no-async-chunk` when you want this optimization. Disabling
+    async chunk causes the runner to fall back to synchronous output
+    construction. Enabling async chunk alone cannot activate the feature for a
+    model that has not opted into async Omni output.
 
 !!! note "Prefix cache compatibility"
-    Async Omni output materialization and Omni prefix caching cannot currently
-    run together. `_should_use_async_omni_output()` returns `False` whenever
-    `self.omni_prefix_cache` is present, so prefix caching selects synchronous
-    output materialization. Supporting both features requires snapshotting or
-    otherwise synchronizing prefix-cache merge and update state before the
-    background output path can consume it safely.
+    Prefix cache and async Omni output can run together. `save_outputs` returns
+    a `step_id`; the output builder later calls `materialize(sid, req_ids)` with
+    the save-time request list, which may be a step late relative to the live
+    `input_batch`. `_should_use_async_omni_output()` does not disable itself
+    when the cache is present. See
+    [Automatic Prefix Caching](prefix_caching.md#implementation).
 
 ## Compatibility and Fallbacks
 
@@ -341,10 +341,9 @@ runtime conditions hold:
 | AR async scheduling is enabled | The optimization relies on the scheduler advancing while the prior output is materialized |
 | `async_chunk` is enabled | The feature targets incremental downstream Omni payloads |
 | The model stage opts in with `use_async_omni_output` | Models must declare that their output lifecycle is safe to defer |
-| Omni prefix cache is disabled | Prefix-cache merge and update ordering currently requires synchronous materialization |
 | Speculative decoding is disabled | Speculative output state is not included in this deferred path |
 | Routed-expert output is disabled | Routed-expert extraction currently requires the synchronous path |
-| Postprocess is absent or explicitly runs eagerly | State needed by the next decode step must be updated before the runner returns |
+| Postprocess is absent or explicitly runs eagerly | State needed by the next decode step must be updated before the runner returns. Eager postprocess sees the scheduled slice (`None, None` combined tensors), not the prefix-cache-merged full prompt; it must only consume the tail |
 
 These checks are evaluated per stage on runners based on `GPUARModelRunner`.
 An unsupported combination does not prevent serving; it only falls back to
@@ -380,6 +379,8 @@ so Omni payload materialization remains synchronous.
 - `tests/worker/test_gpu_ar_model_runner.py`: Snapshot, guard, connector
   ordering, and background error-propagation tests.
 - [Async Chunk](async_chunk.md): Inter-stage chunking and scheduling design.
+- [Automatic Prefix Caching](prefix_caching.md#implementation): Prefix-cache
+  `step_id` / `materialize` contract used by the deferred output builder.
 - [Qwen3-Omni optimization blog](https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization):
   Optimization context and controlled performance results.
 - [PR #4476](https://github.com/vllm-project/vllm-omni/pull/4476): Feature

docs/design/feature/prefix_caching.md

@@ -1,6 +1,5 @@
 # Automatic Prefix Caching in Omni Models
 
-
 ---
 
 ## Table of Contents
@@ -9,6 +8,8 @@
 - [High-Level Approach](#high-level-approach)
 - [Example](#example)
 - [What About Multimodal Inputs?](#what-about-multimodal-inputs)
+- [Implementation](#implementation)
+- [Related Files](#related-files)
 
 ---
 
@@ -27,6 +28,7 @@ vLLM implements automatic prefix caching for managing its kv-cache, which is bes
     This document describes vLLM-Omni's mechanism for caching tensor outputs that are meant to be passed between stages, when requests have common prefixes, similar to the way in which vLLM has prefix caching for the kv-cache. This works in conjunction with vLLM's multimodal encoder caching, but is distinct. See the final section for a concrete example for how they tie together in practice.
 
 ### High-Level Approach
+
 !!! note "Note 2"
     Prior to reading this section, it's recommended to take a look at the design documents in vLLM for [Automatic Prefix Caching](https://docs.vllm.ai/en/latest/features/automatic_prefix_caching/), which will make some of the concepts more clear.
 
@@ -38,8 +40,10 @@ The main focus of vLLM-Omni's approach to prefix caching stage outputs is to bui
 
 With this in mind, consider the set of blocks in a 2D layout, where the row represents the index of blocks being considered, and the columns represent the slots corresponding to tokens within each block. Since we know the `num_blocks` and `block_size` from our kv cache config, if we want to cache a tensor with feature size `D`, we can preallocate a CPU tensor of size `(num_blocks, block_size, D)`, and use the same block index and slot mapping to retrieve the corresponding feature vector.
 
+Host footprint: each cached key costs `num_blocks × block_size × D × dtype_bytes` of **pinned** CPU memory (pinned so device→host can overlap compute), allocated on the first `save_outputs` that sees the key — the first real request pays the `cudaHostAlloc`. Measured on a Qwen3-Omni deployment: thinker `__hidden_states__` `[15092, 16, 2048]` bf16 ≈ 0.92 GiB (plus the same again for each `hidden_states.layer_*` key a model exposes), Qwen3-TTS talker `codes.audio` `[16180, 16, 16]` ≈ 33 MiB. Budget host RAM for the stage accordingly.
 
 ### Example
+
 !!! note "Note 3"
     Prefix caching in vLLM-Omni currently is only supported on AutoRegressive stages with one kv-cache group. Configure it with the pipeline-wide `enable_prefix_caching` field in the deploy config.
 
@@ -56,7 +60,7 @@ The prefix cache flow is then outlined below.
 
 2. Say we process the request `The quick brown fox was tired and slept beneath the shady tree`, which is 12 tokens and evenly divides into 3 blocks as shown below.
 
-```
+```text
          [  The quick brown fox  ] [  was tired and slept ] [beneath the shady tree ]
 Block 1: |<--- block tokens ---->|
 Block 2: |<------- prefix ------>| |<--- block tokens --->|
@@ -65,10 +69,9 @@ Block 3: |<------------------ prefix -------------------->| |<--- block tokens -
 
 When the request processes, we inspect the multimodal outputs and identify the `mm_feature` tensor, which will be of shape `(seq_len, feature_dim)`, i.e., `(12, 16)` in this example. We note that the first axis is dependent on the `seq_len` and add a new cache_tensor of shape `(num_blocks, block_size, feature_dim)` to our multimodal cache for tensors.
 
+1. If we lay out the cache as a 2D tensor of shape (`num_blocks`, `block_size`), we'll have something like the following:
 
-3. If we lay out the cache as a 2D tensor of shape (`num_blocks`, `block_size`), we'll have something like the following:
-
-```
+```text
 0: [  The quick brown fox  ]
 1: [  was tired and slept  ]
 2: [beneath the shady tree ]
@@ -78,7 +81,8 @@ When the request processes, we inspect the multimodal outputs and identify the `
 ```
 
 Or, if we flatten it down to 1D,
-```
+
+```text
 0: The
 1: quick
 2: brown
@@ -90,7 +94,8 @@ Or, if we flatten it down to 1D,
 ```
 
 which we can think of as row indices into the hidden states tensor if we view it as the 2D shape `(num_blocks x block_size, feature_dim)`. That is, the analogous flattened (from 3D -> 2D) mapping of the cache for hidden states becomes the following.
-```
+
+```text
 0: <hidden states vector of len 2 corresponding to 'The'>
 1: <hidden states vector of len 2 corresponding to 'quick'>
 2: <hidden states vector of len 2 corresponding to 'brown'>
@@ -103,10 +108,9 @@ which we can think of as row indices into the hidden states tensor if we view it
 
 Similarly, for the multimodal outputs cache, the flattened coordinates are the same, but the `mm_feature` maps to vectors of length `16` instead of the hidden size of `2`. Note that in practice, we may have multiple  multimodal output tensors per forward pass, which may have different names and different feature dimensions.
 
+1. Now, say that we receive a new request `The quick brown fox jumped over the dog`.
 
-4. Now, say that we receive a new request `The quick brown fox jumped over the dog`.
-
-```
+```text
          [  The quick brown fox  ] [  jumped over the dog ]
 Block 1: |<--- block tokens ---->|
 Block 2: |<------- prefix ------>| |<--- block tokens --->|
@@ -116,7 +120,7 @@ Here, we will have a cache hit for `Block 1` which will be detected by vLLM base
 
 Since we have the block indices / slot mappings from the kv cache manager, we can simply mirror the mappings and leverage the same indices for the cached hidden states and multimodal outputs. This allows us to look up the correct tensors from our externally maintained 3D caches.
 
-```
+```text
 0: [  The quick brown fox  ] < already in the cache
 1: [  was tired and slept  ]
 2: [beneath the shady tree ]
@@ -129,11 +133,11 @@ Since we have the block indices / slot mappings from the kv cache manager, we ca
 
 Finally, to pass the full hidden states and multimodal outputs to the next stage, we simply concatenate the cached contents with the corresponding new tensors computed from the current forward call.
 
-
 ### What About Multimodal Inputs?
+
 It's also useful to consider the case about how Omni prefix caching is handled when we have multimodal inputs that don't cleanly end on block boundaries, as well as how this works with multimodal encoder caching in vLLM. For example:
 
-```
+```text
          [   Im0  Im1  Im2  Im3  ] [ Im4  Im5 foo <empty> ]
 Block 1: |<--- block tokens ---->|
 Block 2: |<------- prefix ------>| |<--- block tokens --->|
@@ -147,9 +151,9 @@ In reality, this isn't a big problem for correctness, because vLLM also maintain
 - The hash describing the image data starting at position 0 and with length 6
 - In vLLM's encoder cache, a mapping from the image hash above to the encoder output
 
-
 To understand what happens, say we get the following input as a second request:
-```
+
+```text
          [   Im0  Im1  Im2  Im3  ] [  Im4  Im5 bar  baz  ]
 Block 1: |<--- block tokens ---->|
 Block 2: |<------- prefix ------>| |<--- block tokens --->|
@@ -162,3 +166,162 @@ Because we have multimodal data in a scheduled span that isn't fully precomputed
 When we pass our multimodal tensors to the language model component in the same stage, we'll then expect the same outputs, because the prefix caching behaviors in vLLM-Omni / vLLM match, so the LLM will use vLLM's KV cache manager's prefix caching to correctly handle the attention information for `Block 1` while calculating the outputs for `Block 2`, giving us the correct results for processing `Block 2` with the context of `Block 1`.
 
 Finally, we look up the output hidden states/multimodal tensors corresponding to the prefix cache hit `Block 1` and concatenate it with the forward pass result to get the final result, which is expected to be identical to the full hidden states when prefix caching is disabled.
+
+### Implementation
+
+The block/slot model is `vllm_omni/core/prefix_cache/`.
+`OmniPrefixCacheManager` owns slot occupancy, the request-task table,
+hit spans, and merge.
+`OmniPrefixCacheController` moves data: a reusable `StagingBufferPool` for
+this step's device→host copy, and writes into the durable `PrefixBlockPool`.
+The state lock covers those tables only.
+
+Miss is not an error (this step's forward slice only). A hit span whose
+hidden rows are absent is fatal. For mm keys the rule is looser: a model may
+emit a key only for some requests, so a hit span with no rows behind an mm key
+reads zeros from the pool for those positions rather than raising. Abort still
+writes: once a hash entered this step's batch it must land in the cache.
+
+`enable_prefix_caching` is refused on KV-consumer stages (`kv_role` of
+`kv_consumer` or `kv_both`). KV received from a producer is reported as
+`num_computed_tokens` too, and the manager cannot tell it from a local hit.
+Producer-only stages are unaffected. Pooling stages never save, so they get
+no cache. This gate is one function, `stage_prefix_cache_config`, called by
+both the GPU and the NPU model runner at kv-cache init.
+
+Which stages may set `enable_prefix_caching: true`:
+
+| Stage | `enable_prefix_caching` | Why |
+| --- | --- | --- |
+| AR stage with one full-attention kv group whose hidden states / per-token mm feed the next stage (Qwen3-Omni thinker and talker) | supported | The case the cache is built for: full-prompt hidden states are merged from the pool on a hit. |
+| AR stage that sets `requires_full_prefix_cached_hidden_states = False`, optionally with `deferred_prefix_cache_mm_keys` (Qwen3-TTS talker, Higgs v3 talker) | supported | Hidden is not cached; deferred codec rows are written once on finish. |
+| Pooling stage | ignored | Never saves; the gate returns no config. |
+| `kv_role: kv_consumer` / `kv_both` | refused at kv-cache init (`OmniPrefixCacheUnmatchError`) | Producer KV shows up as `num_computed_tokens` and is indistinguishable from a local hit. |
+| Speculative decoding on the stage | refused at kv-cache init | Under async scheduling vLLM keeps `num_computed_tokens_cpu` optimistic (all drafts accepted) during the forward and corrects it afterwards; `step_slots_cpu` would mirror rows at the wrong slots. Not verified; refused as a whole. |
+| `prefix_match_unit` smaller than `block_size` | refused at kv-cache init | Sub-block hits make `num_computed_tokens` unaligned; the hit registry only mirrors whole blocks. |
+| Hybrid / sliding-window / multi-group kv cache (e.g. a talker with `attention_type: sliding_recompute`) | refused at kv-cache init | The cache mirrors exactly one full-attention block table. |
+| Attention backend whose kernel block size differs from `--block-size` (FlashInfer / FlashMLA / CutlassMLA with a block size they do not list natively), or decode context parallel | refused at first step (`FullAttentionGroupView`) | `step_slots_cpu` computes `table[req, pos // block_size] * block_size + pos % block_size` over allocator block ids; hybrid kernel blocks and DCP token striping change that row layout. FlashAttention / Triton accept any multiple of 16 and never split blocks. |
+| Codec decoder / Code2Wav stages (Qwen3-Omni stage 2, Qwen3-TTS stage 1) | keep `false` | Nothing downstream consumes their hidden states; the cache would only add device→host copies. Not validated. |
+| Diffusion stages | n/a | No vLLM KV cache to mirror. |
+
+Hit spans come from `scheduled_new_reqs` only, as in the pre-refactor cache:
+
+- A new request with a (partial) prefix hit is the normal path: the hit
+  blocks are read from the pool, the rest is this step's rows, and the
+  divergent tail is written under its own blocks. The gather starts on the
+  prefetch thread at `new_step_starts`; a same-step hit (vLLM hashes blocks
+  at schedule time, so `b` can hit blocks `a` computes in the same forward)
+  cannot plan until `a`'s write is registered and starts at `save_outputs`
+  instead. Either way it runs before the next step can hand those blocks
+  to a new tenant. vLLM frees blocks one step before `finished_req_ids`
+  arrives, so a finished request whose last step materializes late may
+  still find a hit slot reassigned. Each `(slot, key)` carries a write
+  version, bumped whenever a new write claims it. A planned read captures
+  that version and is registered in `_pending_reads`; a later write that
+  reclaims those slots copy-on-writes the still-`COMMITTED` rows into the
+  ref before it overwrites the pool, so a delayed fetch serves the original
+  tenant. A version mismatch with no preserved copy raises for live and
+  finished alike (the pool rows are a newer tenant's). Production defaults
+  stay opt-in until preempt/resume hit spans are reconstructed.
+- `async_chunk` continuation: when the next upstream chunk arrives, the same
+  request id re-enters `scheduled_new_reqs` with `num_computed_tokens` equal
+  to what it already computed itself. Ids already in `live_reqs` are skipped
+  for hit marking: those rows were delivered in earlier steps and re-emitting
+  them would duplicate output. A `delivered_upto` span for this case is
+  Phase 2.
+- Preemption + reschedule: vLLM resets `num_computed_tokens` to 0 on
+  preemption and re-runs prefix matching on resume, so the resumed request
+  can come back with a fresh hit. With the V1 model runner it arrives
+  through `scheduled_cached_reqs` (id in `resumed_req_ids`, `new_block_ids`
+  replaces the table); with the V2 runner it re-enters `scheduled_new_reqs`
+  while still in `live_reqs`. Neither path marks an omni hit span: the
+  resumed request gets only the rows it recomputes, and its still-open
+  deferred write keeps appending (a slot written twice keeps the later
+  chunk). Cache integrity holds either way — the hit blocks already have
+  rows, from this request or the one it hit. Stages that need full prompt
+  hidden states should be sized so preemption does not occur while prefix
+  caching is on. Same as before this refactor; tracked for Phase 2.
+
+Two write paths, split by `ModelCachePolicy.deferred_keys`:
+
+- Immediate (`JOIN_NEXT_STEP`): save launches a whole-step device→host into
+  a staging slot; the committer waits that event and writes the CPU pool.
+  The next save waits `done` (the pool write): a reused slot must never
+  leave a pending pool write behind, or a delayed hit read of the old rows
+  would find nothing recoverable.
+- Deferred (`JOIN_ON_FINISH`): mm whose first dim is this step's token count
+  stays on a per-request GPU clone; `_WriteChunk`s append across steps;
+  finish/abort (or GPU-byte-budget pressure) forces the copy. One open
+  WriteTask per request; a budget flush may close it mid-request, in which
+  case the next save opens a new one (`write_n` + 1) and a hit reads both,
+  so one long request cannot pin the whole budget.
+
+A `WriteTask` moves through `TaskState` only via `transition()`, one step at
+a time along a strict chain:
+`PENDING` (registered, not queued) → `QUEUED` (on a copy queue) →
+`COPYING` (one thread owns the copy stage) → `HOST_READY` (host rows ready) →
+`WRITTEN` (in the CPU pool); `FAILED` is reachable from any non-terminal
+state. Skipping a step raises. The worker claims `COPYING` in the same `_wake`
+critical section as the queue pop, so `escalate` never re-queues a task it can
+see is already claimed. `host_ready` / `done` are wait primitives set by the
+transitions into `HOST_READY` / `WRITTEN` / `FAILED`. GPU-clone bytes are
+charged once per clone on a `_BudgetTicket` pinned by every task that views
+it, and uncharged when the last holder releases (idempotent per tid).
+
+Mm whose first dim equals the unpadded scheduled length *or* the CUDA-graph
+padded length is registered on first sighting. Talker `codes.audio` is a
+cat of scheduled rows and stays unpadded while hidden is padded; both must
+open a pool key, whether immediate or deferred.
+Leftover mm (lists, `codes.ref`, any tensor whose first dim is not this
+step's token count) is copied to CPU at save without truncating that first
+dim. That leftover copy is the async-builder read replica for this step; it
+does not write the pool or carry abort/preempt occupancy.
+
+```python
+cache.register_policy(ModelCachePolicy.from_model(model))   # load_model
+cache.new_step_starts(scheduler_output)   # before _update_states
+sid = cache.save_outputs(hidden, mm_outputs, num_tokens_unpadded=n,
+                         num_tokens_padded=n_pad)
+outs = cache.materialize(sid, req_ids)    # or discard_step(sid)
+```
+
+Each step id is consumed exactly once. `req_ids` must be a subset of the save
+snapshot. At most `staging_depth` unused step ids may exist at once: every
+`save_outputs` claims one staging slot, including saves with only leftover
+mm that copy no device→host page. A slot is also held by each immediate
+write that views it, until the committer's pool write. A later save waits
+for `materialize`/`discard_step` or that pool write to free a slot;
+`staging_claim_timeout_s` then errors with the unused ids and the task count.
+`join`/`join_host_ready` use the same bound and raise with the stuck task's
+id and state instead of hanging the caller.
+`materialize` may run on the async output builder after the engine has
+entered the next step; leftover mm (not written to the pool) is copied
+to CPU at `save_outputs` so the builder never reads live CUDA-graph
+buffers. See
+[Async Omni Output Materialization](omni_async_output_materialization.md).
+
+The cache is constructed only on the last pipeline-parallel rank
+(`_ensure_omni_prefix_cache`). Other ranks skip it: they never call
+`save_outputs`, so a hit table there would resolve to absent slots.
+
+Threads, locks, and what each may block on:
+
+| Thread | Role | May block on | Must not hold while blocked |
+| --- | --- | --- | --- |
+| Engine | `new_step_starts`, `save_outputs` | previous-step `join` (`done`) / finished-write `join_host_ready`; `reserve()` GPU-byte flush; staging-slot claim; `dispatch()` / finish-abort `escalate()` (eager mode: the copy + pool write run inline) | `_state_lock` |
+| Async output builder | `materialize` (may overlap the next engine step) | this step's `step_d2h_event`; `join` (`done`); deferred `fetch_host` | `_state_lock` |
+| Committer | `_worker_loop`: wait device→host / deferred copy / pool write | `_wake.wait`; `step_d2h_event` or copy-stream sync | never takes `_state_lock` |
+| Prefetch pool | hit-span gather during forward | `join` (`done`); deferred `fetch_host` | `_state_lock` |
+
+| Lock | Covers | Does not cover |
+| --- | --- | --- |
+| manager `_state_lock` | occupancy tables, step contexts, hit spans, task registration, pool-key publish (`install_key`) | join, GPU-byte flush, copy, `step_d2h_event` wait, pool-key allocation, eager `dispatch()` / `escalate()` |
+| controller `_lock` / `_wake` | task registry, queues, GPU-clone byte budget | device→host / pool-write body (released before `synchronize`) |
+| `WriteTask.lock` | `state`, `reassigned`, `append_chunk` | waiting on `host_ready` / `done` (those are events) |
+
+### Related Files
+
+- `vllm_omni/core/prefix_cache/`
+- `vllm_omni/worker/gpu_model_runner.py` (`_ensure_omni_prefix_cache`)
+- `tests/core/test_prefix_cache.py`
+- [Async Omni Output Materialization](omni_async_output_materialization.md)

docs/design/fullduplex-personaplex.md

@@ -271,3 +271,111 @@ The required evidence is:
 
 Audio that is empty, all zero, non-finite, or only a protocol `listen` event is
 not a successful end-to-end result.
+
+### Paced multi-session load driver
+
+The existing driver keeps its two-session admission and slot-reuse validation
+when `--sessions` is omitted. Passing `--sessions N` selects a separate,
+bounded load mode; it does not replace the lifecycle test or change server
+admission capacity. Start an existing PersonaPlex server configured to admit
+the requested number of sessions, then run from a source checkout with the
+vLLM-Omni client dependencies installed:
+
+```bash
+python -m tests.e2e.online_serving.personaplex_realtime_duplex \
+  --model nvidia/personaplex-7b-v1 \
+  --input-wav /path/to/input.wav \
+  --sessions 2 --load-frames 1000 --drain-s 2 \
+  --server-revision SERVER_COMMIT --server-hardware A100-80GB \
+  --output-dir /tmp/personaplex-load-n2
+```
+
+Use a new output directory for every run. The driver refuses to overwrite an
+existing `load-result.json`. The server labels are supplied by the operator,
+not detected or verified by the client. Record the server command, package
+versions, checkpoint revision and hardware alongside a published measurement.
+
+All attempted sessions finish admission before admitted sessions share a
+common input start. A rejected session remains a failed row in the report;
+it is not removed from the denominator. Each session receives at most
+`--load-frames` input frames, without looping the WAV, and uses 80 ms pacing.
+A stalled send cannot cause a catch-up burst. The timeline records send
+schedule lateness so an overloaded client is visible rather than mistaken
+for a faster server. This mode measures steady concurrency, not join/leave
+interference; the original lifecycle mode still checks slot reuse.
+Load mode does not probe an additional N+1 session for `resource_exhausted`;
+the lifecycle mode owns that admission-limit check. A per-session
+`client_pacing_warning` and the summary's `client_pacing_warning_sessions`
+identify send-lateness p99 above one nominal frame (80 ms). This is a warning,
+not an extra acceptance gate: read it alongside server-facing measurements
+so a slow client is not mistaken for a slow server.
+
+The fixed drain window is part of the measurement configuration. Audio
+arriving during cleanup is excluded, so closing cannot rescue a session that
+missed its observation deadline. Cleanup is attempted for admitted sessions
+even when audio validation fails. Protocol errors, missing/invalid audio,
+shared response IDs and cleanup failures make the report fail and the CLI
+exit nonzero. Cancellation closes transports and propagates to the caller.
+`--cleanup-timeout-s` (default 5 s) bounds session-close send and acknowledgement
+together, independently of `--timeout-s`. The subsequent WebSocket close
+handshake has the same separate bound; an unresponsive close acknowledgement
+cannot hold cancellation for the full inference timeout.
+
+`load-result.json` retains per-session send and audio-packet timelines using
+the client's monotonic clock. It does not export handshake credentials or
+raw audio payloads. Metric definitions are explicit:
+
+Both modes validate each nonempty audio packet's strict base64 encoding, PCM16
+sample alignment, 24 kHz sample rate and response identity. Valid aggregate
+audio cannot hide a malformed packet. In load mode, rejected packets make the session fail
+and appear in `invalid_audio_packets` with their event index, receive time and
+fixed diagnostic code; they do not contribute samples or timing intervals.
+Empty audio deltas remain legal and contribute no samples. This check does
+not change the public client's decoding. The lifecycle scenario and dispatch
+remain the same, but malformed packets that aggregate checks once tolerated
+now fail that mode as well.
+
+The probe accepts both the current OpenAI Realtime audio event name
+`response.output_audio.delta` and the legacy `response.audio.delta`. Validation
+uses the wire payload directly rather than relying on a particular client
+library's alias table, so a server-side event-name migration cannot silently
+turn real audio into a zero-output measurement.
+
+| Field | Meaning |
+| --- | --- |
+| `client_audio_packet_interval_ms` | Consecutive nonempty packet arrivals: median, linear p99, max, and zero-based index of the first maximal interval. Interval index 0 ends at packet 1. One packet can contain multiple codec frames; this is not server tick latency. |
+| `client_first_audio_after_stream_start_ms` | First audio receipt minus first input send start; admission is excluded. |
+| `client_stream_rtf` | First input send start to last audio receipt, divided by received audio duration. This includes real-time input pacing and is not an inference-only RTF. |
+| `client_send_lateness_ms` | Send start minus its original shared schedule, clipped at zero. |
+| `frame_deficit` | Integer input-frame minus output-frame count from the shared session validation; missing when valid whole-frame accounting is unavailable. |
+| `output_frames`, `voiced_frames`, `silent_frames`, `frame_coverage_ratio`, `audio_rms`, `audio_chunks` | Session-validation statistics retained on success and on acceptance failure once computed. |
+| `acceptance_check` | Failed local check, with numeric diagnostics in `error`; other exceptions retain phase/type only, without remote handshake or metadata text. |
+| `audio_underrun_s`, `audio_underrun_event_count`, `audio_continuity_ok` | Shared `compute_continuity_stats` on valid PCM16 packet arrivals, with its default 100 ms underrun threshold; absent audio gives missing values rather than a continuity pass. |
+
+No output gives missing latency/RTF values, not zero. By default RTF is
+diagnostic; `--max-client-rtf` adds an explicit ceiling. The existing frame
+coverage, audible-frame and scheduler-data-plane checks still apply. A longer
+`--drain-s` can demonstrate eventual completion under backlog, but it must not
+be reported as real-time capacity when the fixed-window run failed or client
+RTF exceeded the stated target. Likewise, the first live session can pay
+one-time model/codec/JIT costs; either use the server's configured duplex
+warmup path or label that run as cold-start evidence before comparing steady
+capacity. Passing client-only metrics does not certify model quality or a
+real-time capacity target on another GPU. Begin with N=1 and N=2; higher N is a
+capacity experiment, not a promised property of an A100 or of this patch.
+Continuity is a client-arrival diagnostic, not proof of speaker playback and
+not an additional pass/fail gate. Packet-interval and send-lateness summaries
+retain their documented linear p99; the older client interval helper reports
+rounded nearest-rank p95 instead and is not substituted for that schema.
+
+Deterministic driver and localhost WebSocket checks need no model or GPU:
+
+```bash
+python -m pytest tests/e2e/online_serving/test_personaplex_load_driver.py \
+  -m 'core_model and cpu' --run-level core_model -q
+```
+
+These checks test the driver against controlled protocol responses. Run the
+load-driver command against real weights separately before reporting model-serving
+performance. Server tick timestamps and generic TTFT/TPOT aggregation remain
+the responsibility of the existing duplex metrics paths, not this driver.

docs/design/fullduplex.md

@@ -9,6 +9,12 @@ the owner document for `vllm_omni/engine/duplex/**`,
 `vllm_omni/entrypoints/duplex_omni.py`, `vllm_omni/entrypoints/duplex/**`,
 `vllm_omni/clients/**` and the per-model `duplex/` packages.
 
+`vllm_omni/protocol/realtime/**` (the OpenAI Realtime API) and
+`vllm_omni/protocol/duplex/**` (vLLM-Omni's extension of it) are the shared
+wire codec duplex is built on. Duplex is its first consumer but not its owner: the codec
+is deliberately runtime-agnostic so a second Realtime surface can reuse it
+(RFC #6592 P0a).
+
 The public wire contract lives in
 [`docs/serving/realtime_duplex_api.md`](../serving/realtime_duplex_api.md);
 this page describes how it is produced.
@@ -124,6 +130,40 @@ plugin and the session runtime config to `DuplexOrchestrator` directly.
    attachment/resume/replay bookkeeping; it holds no session state.
 6. **No `typing.Protocol`** in the duplex surfaces: the plugin, data plane,
    session state, PCM buffer, stage port and client transport seams are ABCs.
+7. **The wire codec is not duplex, and a wire type is not a mailbox message.**
+   The event and command *vocabulary* plus its wire rendering, the audio format
+   negotiation, the conversation-item rules and the error envelope are model-
+   and runtime-agnostic, and split along the three tiers
+   [`docs/serving/realtime_duplex_api.md`](../serving/realtime_duplex_api.md)
+   already defines: **Tier 1** (identical to OpenAI) is
+   `vllm_omni/protocol/realtime/**`; **Tier 2** (OpenAI names carrying our
+   extensions) and **Tier 3** (ours alone) are `vllm_omni/protocol/duplex/**`.
+   A Tier 2 class *subclasses* its Tier 1 twin and declares only what it adds,
+   so the OpenAI surface stays honestly pure --- `session.created` without
+   `resume_token`, `input_audio_buffer.append` without `video_frames`. Our
+   error **codes** are Tier 3 (`protocol/duplex/errors.py`); only the envelope
+   shape and OpenAI's three `error.type` classes are Tier 1.
+   What stays engine-side is everything that is *internal representation*
+   rather than contract: `DuplexCommand.payload()` renders the session runner's
+   mailbox dictionary, whose channel genuinely differs from the client event
+   (`session.update` and the three `conversation.item.*` commands all travel on
+   `turn.signal`); `engine/duplex/realtime_commands.py` decides which command a
+   decoded event becomes; `engine/duplex/realtime_events.py` holds the
+   session's projection state. The codec may not import the engine, the
+   entrypoints, `model_executor` or the clients
+   (`tests/protocol/realtime/test_protocol_import_boundary.py`), and there is
+   exactly one implementation of each codec behaviour
+   (`tests/protocol/realtime/test_realtime_codec_single_source.py`).
+   The dependency chain is `protocol/realtime` <- `protocol/duplex` <- engine /
+   entrypoints / clients, and a duplex consumer uses **only** the middle link:
+   `protocol/duplex/**` re-exports the Tier 1 names it does not extend, so a
+   helper that later needs a duplex-specific version (`convert_input_audio_with_rate`
+   resamples to MiniCPM-o's 16 kHz rather than the client's rate) is overridden
+   in one file instead of at every call site
+   (`tests/protocol/duplex/test_duplex_protocol_facade.py`).
+   `DuplexEvent` is therefore a plain alias of `RealtimeEvent` --- an event has
+   no engine-internal half --- while `DuplexCommand` is a real class, because a
+   command does.
 
 ## Package layout
 
@@ -142,6 +182,31 @@ vllm_omni/
 │   │   ├── chat_completions.py      DuplexChatCompletionsAdapter (/v1/chat/completions on a session per request)
 │   │   └── websocket.py             websocket send/close/receive helpers
 │   └── openai/api_server.py         builds DuplexOmni for duplex models; session-backed app state
+├── protocol/                        SHARED WIRE CODEC (no engine / no model / no transport)
+│   ├── realtime/                    the OpenAI Realtime API itself
+│   │   ├── events.py                RealtimeEvent + OpenAI's 30 server events (22 ship as Tier 1;
+│   │   │                            8 have a Tier 2 twin in protocol/duplex)
+│   │   ├── commands.py              RealtimeCommand + OpenAI's 10 client commands (8 Tier 1,
+│   │   │                            2 with a Tier 2 twin) --- wire_type + fields, no payload()
+│   │   ├── formats.py               audio format spellings, what is supported, per-session validation
+│   │   ├── session.py               session-object readers; RealtimeInputDefaults (append defaults)
+│   │   ├── items.py                 conversation item shape, truncation, transcripts, camera frames
+│   │   ├── audio_input.py           decode_audio_append -> RealtimeAudioAppend; client speech hints
+│   │   ├── audio.py                 PCM / G.711 / WAV decode, resample, re-encode
+│   │   ├── errors.py                RealtimeProtocolError (the envelope's exception type)
+│   │   └── capabilities.py          RealtimeProtocolCapabilities: what one consumer can serve
+│   └── duplex/                      Tier 2 + Tier 3, and the one door a duplex consumer uses
+│       ├── __init__.py              re-exports the Tier 1 helper functions (audio, formats,
+│       │                            items, session, capabilities) so the engine never
+│       │                            imports protocol/realtime directly
+│       ├── events.py                the whole 42-event vocabulary: 22 Tier-1 re-exports,
+│       │                            8 Tier-2 subclasses (session.created + resume_token, ...),
+│       │                            12 Tier-3 events (listen/speak, playback, resume, ...);
+│       │                            DuplexEvent = RealtimeEvent
+│       ├── commands.py              the whole 17-command vocabulary: 8 Tier-1 re-exports,
+│       │                            2 Tier-2 subclasses (append + hints, commit + final),
+│       │                            7 Tier-3 commands (barge_in, playback.ack, ...)
+│       └── errors.py                our error-code vocabulary (Tier 3) + RealtimeProtocolError
 ├── engine/
 │   ├── omni_engine_base.py          OmniEngineBase (stage processes, orchestrator thread, queues, RPC)
 │   ├── async_omni_engine.py         AsyncOmniEngine (turn-based request building)
@@ -149,16 +214,20 @@ vllm_omni/
 │   ├── orchestrator.py              OrchestratorBase + Orchestrator
 │   ├── duplex_orchestrator.py       DuplexOrchestrator (+ DuplexOrchestratorRequestState; implements DuplexStagePort)
 │   └── duplex/
-│       ├── commands.py              DuplexCommand dataclasses, command_from_realtime
-│       ├── realtime_commands.py     Realtime client event -> DuplexCommand translation
-│       ├── events.py                DuplexEvent dataclasses (+ to_realtime), REALTIME_ERROR_TYPES_BY_CODE
+│       ├── commands.py              mailbox half: DuplexCommand.payload() + the mailbox `type`,
+│       │                            paired with each protocol command; command_from_realtime
+│       ├── realtime_commands.py     duplex binding of the codec: decoded event -> DuplexCommand,
+│       │                            DUPLEX_REALTIME_CAPABILITIES, duplex_response_format
+│       ├── events.py                re-export shim (DuplexEvent = RealtimeEvent) + the runner's
+│       │                            epoch-filter sets DOMAIN_TERMINAL_EVENTS / MODEL_OUTPUT_EVENTS
 │       ├── realtime_events.py       RealtimeProjectionState: internal event -> typed events
 │       ├── messages.py              queue envelopes (Open/Close/Resume/Touch/Command/Result/Event), DuplexSessionError
 │       ├── config.py                DuplexSessionConfig, DuplexCapabilities, ResponseCreateOptions
 │       ├── contracts.py             DuplexFence (session_id, epoch, turn_id), stage request records, DuplexStagePort
 │       ├── plugin.py                DuplexModelPlugin, DuplexModelSessionState, DuplexDataPlane, PcmAppendBuffer ABCs
 │       ├── turn_detection.py        server-side VAD turn detector used by the session
-│       ├── audio.py / vad.py / intermediate.py
+│       ├── audio.py                 compatibility re-export, via protocol/duplex
+│       ├── vad.py / intermediate.py
 │       └── session/                 one engine-resident session and everything that runs it
 │           ├── engine_session.py    DuplexEngineSession: ledgers, lease, fence, stage resources, append sequencing
 │           ├── runner.py            DuplexSessionRunner (per-session mailbox on the orchestrator loop)

docs/design/module/engine_orchestration.md

@@ -79,7 +79,7 @@ the boundary affected by the in-flight stage client/process refactor in
 responsibilities proposed only by that PR are not current contracts.
 
 The orchestration loop also has an opt-in event-driven mode
-(`VLLM_OMNI_EVENT_DRIVEN_ORCH=1`, default off) proposed in
+(`VLLM_OMNI_EVENT_DRIVEN_ORCH=1`, default off except for Qwen3-TTS) proposed in
 [#5221](https://github.com/vllm-project/vllm-omni/pull/5221). It changes poll
 cadence only: the routing, ordering, and terminal-state contracts below hold
 identically on both loops.

docs/design/module/entrypoints.md

@@ -30,6 +30,7 @@ primary_code_paths:
   - vllm_omni/entrypoints/utils.py
   - vllm_omni/entrypoints/duplex/**
   - vllm_omni/clients/**
+  - vllm_omni/protocol/**
 primary_path_exceptions:
   - path: vllm_omni/entrypoints/openai/errors.py
     owner: error_contracts.md
@@ -53,6 +54,7 @@ depends_on:
   - error_contracts.md
   - engine_orchestration.md
 validation_paths:
+  - tests/protocol/**
   - tests/entrypoints/test_omni_entrypoints.py
   - tests/entrypoints/test_async_omni.py
   - tests/entrypoints/test_async_omni_pause_sleep_routing.py
@@ -106,6 +108,31 @@ placement, payload implementation, or semantic error classification.
 `entrypoints/openai/errors.py` is an explicit primary-path exception owned by
 `error_contracts.md`.
 
+It also owns `vllm_omni/protocol/**`, the shared wire codec promoted by RFC
+[#6592](https://github.com/vllm-project/vllm-omni/issues/6592) P0a. It follows the three tiers
+`docs/serving/realtime_duplex_api.md` already defines. `protocol/realtime/**`
+is Tier 1, identical to OpenAI: 30 server events, 10 client commands, their
+typed fields and pure wire rendering, audio-format negotiation,
+conversation-item rules and the error-envelope shape. `protocol/duplex/**` is
+Tier 2 (OpenAI names carrying vLLM-Omni extensions, each a subclass of its
+Tier 1 twin declaring only what it adds) and Tier 3 (ours alone, including our
+error-code vocabulary), kept separate so a GA-only consumer is not handed a
+vocabulary its clients never send. That package sits
+outside `entrypoints/` on purpose --- the engine depends on it, so it belongs
+to neither layer --- but deciding what a Realtime client may put on the wire is
+this document's subject, and the same reasoning already places
+`vllm_omni/errors.py`, `inputs/` and `outputs/` under contract documents rather
+than under a consumer.
+
+It is deliberately **not** owned by `../fullduplex.md`, even though duplex is
+the codec's only consumer today. RFC #6592 asks for a named owner precisely so
+a second Realtime consumer does not need duplex review to change the codec
+(#6592 open question 8). Ownership here is review scope, not a dependency: the
+codec may not import the engine, the entrypoints or the model code, which
+`tests/protocol/realtime/test_protocol_import_boundary.py` enforces. When a
+second consumer lands (#6592 P0b) the package is expected to graduate to its
+own module document with owners drawn from both consumers.
+
 ## Candidate invariants
 
 These identifiers are proposals while the document is `draft`.
@@ -115,6 +142,22 @@ These identifiers are proposals while the document is `draft`.
 **Rule:** Entrypoints MUST NOT implement cross-stage routing or stage lifecycle
 policy.
 
+### ENTRY-INV-002: The shared Realtime codec does not depend on its consumers
+
+**Rule:** `vllm_omni/protocol/**` MUST NOT import `vllm_omni.engine`,
+`vllm_omni.entrypoints`, `vllm_omni.model_executor`, `vllm_omni.worker` or
+`vllm_omni.clients`; `protocol/duplex/**` MAY depend on `protocol/realtime/**`
+but never the reverse, and a Tier 1 class MUST NOT carry a vLLM-Omni extension
+field; and each protocol/codec behavior MUST have exactly one
+implementation that every consumer uses. A wire type MUST NOT carry a
+consumer's internal representation: the duplex mailbox rendering
+(`DuplexCommand.payload()`, whose channel differs from the client event for
+`session.update` and the `conversation.item.*` commands) stays engine-side.
+A duplex consumer (`engine/duplex/**`, `entrypoints/duplex/**`, the duplex
+clients) MUST import `vllm_omni.protocol.duplex` and MUST NOT import
+`vllm_omni.protocol.realtime` directly, so the tier boundary has exactly one
+extension point.
+
 ### ENTRY-INV-100: Public requests are normalized once
 
 **Rule:** Public protocol values MUST be validated and converted to internal

docs/features/custom_pipeline.md

@@ -1,7 +1,7 @@
 # Custom Pipeline Extension Guide
 
 Transformer already support Custom Pipeline via
-https://github.com/huggingface/diffusers/blob/main/docs/source/en/using-diffusers/custom_pipeline_overview.md
+<https://github.com/huggingface/diffusers/blob/main/docs/source/en/using-diffusers/custom_pipeline_overview.md>
 
 This guide demonstrates how to use the newly added features for extending vLLM-Omni's diffusion pipeline with custom functionality.
 
@@ -20,8 +20,9 @@ Three main features enable custom pipeline extension:
 `WorkerWrapperBase` is a wrapper class that creates `DiffusionWorker` instances with optional extension support. It enables dynamic inheritance, allowing you to add custom methods and functionality to workers without modifying the base worker class.
 
 **Key capabilities:**
+
 - Dynamic worker class extension via `worker_extension_cls`
-- Support for custom pipeline initialization via `custom_pipeline_args`
+- Support for custom pipeline initialization via `custom_pipeline_args["pipeline_class"]`
 - Method delegation to underlying worker
 - Attribute access forwarding
 
@@ -42,6 +43,7 @@ The `diffusion_load_format` parameter controls the initial diffusion model load.
 `CustomPipelineWorkerExtension` is a mixin class that extends `DiffusionWorker` with the ability to re-initialize the pipeline with a custom implementation.
 
 **Key method:**
+
 - `re_init_pipeline(custom_pipeline_args)`: Re-initializes the pipeline with custom arguments, properly cleaning up the old pipeline first
 
 **Location:** `vllm_omni/diffusion/worker/diffusion_worker.py`
@@ -150,6 +152,7 @@ omni = Omni(
     },
     worker_extension_cls=MyCustomExtension,
     # Note: worker_extension_cls is an internal parameter
-    # CustomPipelineWorkerExtension will automatically init pipeline when custom_pipeline_args is provided
+    # CustomPipelineWorkerExtension will automatically init pipeline when
+    # custom_pipeline_args includes "pipeline_class".
 )
 ```

docs/models/supported_models.md

@@ -58,6 +58,7 @@ th {
 | `Magi2Pipeline` | MAGI-2 native Preview-stage T2VA and I2VA (272p/540p) | `sand-ai/MAGI-2-preview` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SandAI/MAGI-2-preview-L20X.md) |
 | `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Pi0Pipeline` | π0 (Pi-Zero) VLA | `lerobot/pi0_base` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/lerobot/Pi0.md) |
+| `Pi05Pipeline` | π0.5 VLA | `lerobot/pi05_base` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/lerobot/Pi05.md) |
 | `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | | — |
 | `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
@@ -72,7 +73,6 @@ th {
 | `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | ✅︎ | — |
 | `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | ✅︎ | — |
 | `BreezeForConditionalGeneration` | Breeze-TTS-2 (TTS, voice clone, voice direction) | `BreezeBlue/Breeze-TTS-2` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/BreezeBlue/Breeze-TTS-2.md) |
-| `DotsTTSForConditionalGeneration` | dots.tts | `dots-studio/dots.tts-soar` | ✅︎ | | | | — |
 | `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | | — |
 | `MammothModa2ForConditionalGeneration` | MammothModa2-Dev (AR-only image understanding) | `bytedance-research/MammothModa2-Dev` | ✅︎ | | | | — |
 | `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
@@ -106,12 +106,12 @@ th {
 | `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | | — |
 | `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | | — |
 | `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | | — |
-| `DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | ✅︎ | — |
 | `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/OpenBMB/MiniCPM-o-4_5.md) |
 | `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `GepardTalkerForConditionalGeneration` | Gepard-1.0 | `nineninesix/gepard-1.0` | ✅︎ | | | ✅︎ | — |
 | `HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | | — |
 | `HiDreamO1ImagePipeline` | HiDream-O1-Image | `HiDream-ai/HiDream-O1-Image` | ✅︎ | | | | — |
+| `AnimaPipeline` | Anima | `circlestone-labs/Anima` | ✅︎ | ✅︎ | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/circlestone-labs/Anima.md) |
 
 The `Recipe` column links the corresponding deployment recipe. `Published` is
 preferred when a corresponding `recipes.vllm.ai` page exists; otherwise the

docs/serving/realtime_duplex_api.md

@@ -8,12 +8,41 @@ history stays honest, and a dropped connection can resume the same session.
 This page covers how to run a duplex deployment, how to drive it from Python
 with `vllm_omni.clients.duplex.DuplexClient`, and the complete wire contract.
 
-The endpoint is served for models that ship a duplex plugin — currently
-MiniCPM-o 4.5 only — and just for deploy configurations that declare
+The endpoint is served for models that ship a duplex plugin — including
+MiniCPM-o 4.5 and Qwen3-Omni — and just for deploy configurations that declare
 `session_mode: duplex`. PersonaPlex and Nemotron VoiceChat are ported to the
 plugin contract in follow-up PRs and are not served over this endpoint yet. The runtime architecture
 is described in [Full-Duplex Runtime (MiniCPM-o 4.5)](../design/fullduplex.md).
 
+## Qwen3-Omni conversation history
+
+Qwen3-Omni keeps incoming conversation items independently addressable by item
+ID. Before applying the model's chat template, it merges consecutive user
+items into one multimodal user message. An assistant message, including an
+empty message reserved for an interrupted or unheard answer, separates turns.
+The final user message places media before text instructions, following the
+[official Qwen3-Omni demo's `format_history()`](https://github.com/QwenLM/Qwen3-Omni/blob/e4235853125589c789f06a2dd83e9f4126df5e9d/web_demo.py#L84-L179).
+
+A camera image or text item may arrive after an audio commit and before
+`response.create`. Those items belong to the same pending user turn: the
+committed audio remains associated with its history item even when that item
+is no longer last. Prompt preparation snapshots the selected inputs; later
+items do not modify an already prepared request.
+
+The duplex adapter retains its existing budgets: at most four audio payloads
+(including the current input), an 8 MiB encoded-audio history budget (keeping
+at least the newest payload), and at most eight prompt images. The recent
+16-message window is rounded outwards to a complete turn. If any audio in an
+older turn is no longer retained, the entire turn's model context, including
+its images/text and assistant replies, is omitted. Images from that turn are
+therefore **not** permanent session-wide model context. These limits differ
+from the demo's one image-bearing turn and five audio-bearing turns; the
+shared rule is grouping before pruning, rather than copying its UI budgets.
+
+Grouping and pruning affect only the model-input view. They do not merge or
+delete the source conversation items exposed to clients, and do not change
+playback acknowledgement or interruption handling.
+
 ## Quick Start
 
 ### Start the Server
@@ -768,7 +797,10 @@ events with `server_event_seq > 40` are replayed in order)
 ```
 
 with camera frames (omni video; rides the append that closes a 1 s model
-unit; each entry is a **bare** base64 JPEG/PNG — no `data:` URL prefix)
+unit; each entry is a **bare** base64 JPEG/PNG — no `data:` URL prefix).
+Only a model whose Stage 0 interleaves a frame track at unit boundaries takes
+this field; a turn model has no such boundary and rejects it, expecting
+`conversation.item.create` with `input_image` content instead.
 
 ```json
 {
@@ -945,6 +977,10 @@ deferred commit during an active response (`event.response_create_deferred`)
 {"type": "conversation.item.truncated", "item_id": "item_resp_01", "content_index": 0, "audio_end_ms": 1850, "event": {"…": "…"}}
 ```
 
+After this acknowledgement, `conversation.item.retrieve` returns the truncated
+transcript. Without text/audio alignment marks, its prefix is estimated from
+the requested position and audio duration; this is not exact word alignment.
+
 #### Response lifecycle
 
 `C→S response.create`

docs/serving/speech_api.md

@@ -521,6 +521,15 @@ are cached in-process with a shared LRU so repeated requests with the same
 all TTS model types; deleting a voice invalidates every model-type slot at
 once.
 
+Decoded reference waveforms use a separate LRU cache of owned, contiguous
+float32 arrays at the source sampling rate. Its byte budget counts numeric
+buffers (four bytes per mono sample), excluding cache metadata. Numeric storage
+avoids retaining a Python float object for every sample during garbage
+collection. MOSS reference encoding consumes these arrays directly; other
+list-based interfaces, including MOSS Nano, receive temporary lists that are
+not retained in the resolve cache. The MOSS reference encoder also releases
+completed batch inputs before waiting for more work.
+
 ### Precomputed Custom Voices
 
 Qwen3-TTS Base and VoxCPM2 can load offline-precomputed voices at startup.
@@ -948,22 +957,24 @@ Use `/v1/audio/voices` to list available voices for the loaded model.
 ## Orchestration Loop (experimental)
 
 Multi-stage omni deployments route stage outputs through a single orchestrator
-loop. By default that loop polls every stage replica on a 1 ms cadence. An
-opt-in event-driven mode replaces the poll with one reader task per live stage
+loop. The legacy loop polls every stage replica on a 1 ms cadence. The
+event-driven mode replaces the poll with one reader task per live stage
 replica awaiting its client directly, and switches the serving-side
-final-output drain to a condition-variable wakeup at the same time.
+final-output drain to a condition-variable wakeup at the same time. Qwen3-TTS
+uses this mode by default; other pipelines keep the legacy poll unless enabled.
 
 **Configuration (environment variables):**
 
 | Variable | Default | Description |
 | --- | --- | --- |
-| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `0` (off) | Switches the orchestration loop and the final-output drain from the legacy 1 ms poll to event-driven wakeups. Enabled by `1`, `true`, `yes`, or `on`, matched case-insensitively after surrounding whitespace is stripped; any other value leaves it off. |
+| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | On for Qwen3-TTS; off for other pipelines | Switches the orchestration loop and the final-output drain from the legacy 1 ms poll to event-driven wakeups. An explicit value wins; otherwise the pipeline default computed at engine initialization is used. The override is resolved at orchestrator construction and separately when the final-output drain starts. `1`, `true`, `yes`, or `on` enables it, ignoring case and surrounding whitespace; other values select the legacy poll loop. |
 
-Set it on the process that runs the orchestrator (stage 0 of an omni
-deployment) before starting the server:
+Set it on the process that runs the orchestrator (stage 0 of an omni deployment)
+before starting the server when overriding the pipeline default. For example,
+explicitly disable event-driven orchestration for Qwen3-TTS:
 
 ```bash
-export VLLM_OMNI_EVENT_DRIVEN_ORCH=1
+VLLM_OMNI_EVENT_DRIVEN_ORCH=0 \
 vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base \
     --omni \
     --port 8091
@@ -973,8 +984,9 @@ The server logs the selected loop mode and its reader/poller counts once at
 startup, so you can confirm which loop is live.
 
 Routing, output ordering, and terminal-state behavior are identical on both
-loops; only the poll cadence changes. Leaving the variable unset keeps the
-legacy poll loop, which is the supported default.
+loops; only the poll cadence changes. Leaving the variable unset selects the
+pipeline default: event-driven for Qwen3-TTS, and legacy polling for other
+pipelines, including Qwen3-Omni.
 
 **Known limitations:**
 

docs/serving/streaming_video_output_api.md

@@ -41,7 +41,7 @@ bytes.
 | Server to client | `video.start` | Confirms the request ID, format, and accepted configuration |
 | Server to client | `video.chunk_metadata` | Describes the next binary media frame |
 | Server to client | Binary frame | Carries one fragmented MP4 chunk |
-| Client to server | `session.interaction` | Queues an optional prompt update during generation |
+| Client to server | `session.interaction` | Queues an optional interaction (e.g., prompt. camera action) during generation |
 | Server to client | `session.interaction.queued` | Confirms that an interaction was queued |
 | Client to server | `session.ping` | Refreshes the stall clock |
 | Server to client | `session.pong` | Acknowledges a ping |
@@ -83,10 +83,39 @@ Models that implement interaction updates can change the active prompt:
 }
 ```
 
+Models that register the camera modality (for example LingBot-World) also
+accept structural SE3 payloads on `event.multi_modal_data.camera`. Coordinates
+follow Unity semantics (`+X` right, `+Y` up, `+Z` forward; same as LingBot's
+native camera frame). WASD key tokens are not admitted by the engine:
+
+```json
+{
+  "type": "session.interaction",
+  "interaction": {
+    "event_id": "forward-1",
+    "event": {
+      "multi_modal_data": {
+        "camera": {
+          "mode": "velocity",
+          "data": {
+            "translation": [0.0, 0.0, 0.05],
+            "rotation": [0.0, 0.0, 0.0, 1.0]
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+`mode` is `target` (lerp to an absolute pose over `transition_chunks`) or
+`velocity` (hold a per-frame SE3 delta until replaced). Prompt and camera may
+be combined in one interaction event.
+
 This feature is pipeline-dependent. An enabled endpoint does not imply that
-the loaded model supports midway prompt updates.
+the loaded model supports midway prompt or camera updates.
 
 The default start timeout is 10 seconds and the server reports a stall after
 about 60 seconds without generation progress or a `session.ping`. See the
 [streaming video generation example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/streaming_video_generation)
-for prompt-update scheduling and browser playback.
+for prompt/camera-update scheduling and browser playback.

docs/serving/videos_api.md

@@ -73,6 +73,10 @@ curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4
 | `image_reference` | string | null | JSON-encoded reference image payload; do not combine with `input_reference` or `video_reference` |
 | `video_reference` | string | null | JSON-encoded reference video payload; do not combine with `input_reference` or `image_reference` |
 | `audio_reference` | string | null | JSON-encoded audio reference for speech-to-video: `{"audio_url": "..."}` — supports HTTP(s) URLs or base64 data URLs |
+| `source_video` | file | null | MiniMax H3 latent-edit source video (`.mp4` or `.mov`, up to 512 MiB) |
+| `source_audio` | file | null | Optional MiniMax H3 latent-edit source audio (`.wav` or `.mp3`, up to 512 MiB) |
+| `video_noise_mask` | file | null | UTF-8 JSON MiniMax H3 video mask; `0` preserves and `1` regenerates a token |
+| `audio_noise_mask` | file | null | UTF-8 JSON MiniMax H3 audio mask; `0` preserves and `1` regenerates a token |
 | `width` | integer | model default | Output video width |
 | `height` | integer | model default | Output video height |
 | `num_frames` | integer | 1 | Number of generated frames |
@@ -214,6 +218,49 @@ upstream vLLM's media URL allowlist protection. Deployments should therefore
 treat remote media URLs as untrusted and choose this setting as part of their
 URL access policy.
 
+### MiniMax H3 Latent-Mask Editing
+
+MiniMax H3 accepts request-scoped source media and video/audio noise masks.
+At least one mask is required. A nontrivial `video_noise_mask` requires
+`source_video`, while a nontrivial `audio_noise_mask` requires either
+`source_audio` or a `source_video` with an audio stream. Mask values are in
+`[0, 1]`: `0` preserves the source, `1` regenerates it, and fractional values
+blend the two behaviors. Exact all-one masks are no-ops and do not require a
+source. Source uploads without a mask are rejected. Masks may be a JSON scalar
+or arrays matching the H3 latent/token grid; pixel-resolution masks must be
+resized or pooled by the client before upload.
+
+For an aligned output of `F` frames at `W x H`, the video latent grid is
+`[Tv, H/16, W/16]`, where `Tv = 2 + 5 * ((F - 5) / 17)`. The video mask may be
+a scalar, a flat token vector, `[Tv, H/32, W/32]`, or the full latent grid. For
+the model input, timestep, and velocity, a full-grid mask is max-pooled over
+each 2x2 spatial token and fractional values are rounded upward to 1/256
+levels. The final x0 restore uses the original, unquantized mask, so full-grid
+masks retain cell-level preservation inside a model token. The audio length is
+`Ta = round(F * 40 / 24)`, and its mask may be a scalar, `[Ta]`, `[2, Ta]`, or
+a flat `2 * Ta` vector. If source audio is short, its missing latent tail is
+forced to mask value `1` so H3 generates that portion. Each mask must be sent
+as a UTF-8 JSON file part and is limited to 8 MiB. A file may contain either a
+JSON scalar or an array.
+
+```bash
+curl -s http://localhost:8091/v1/videos/sync \
+  -F "prompt=Partially restyle the complete clip and soundtrack" \
+  -F 'extra_params={"task":"t2va","duration":4.0,"aspect_ratio":"16:9"}' \
+  -F "source_video=@source.mp4;type=video/mp4" \
+  -F "source_audio=@source.wav;type=audio/wav" \
+  -F "video_noise_mask=@video-mask.json;type=application/json" \
+  -F "audio_noise_mask=@audio-mask.json;type=application/json" \
+  -o edited.mp4
+```
+
+For example, each mask file in the request above may contain the JSON scalar
+`0.5`.
+
+The server streams source files to temporary request-scoped storage and removes
+them after synchronous or asynchronous generation finishes. Models that do not
+declare latent-mask editing support reject these fields.
+
 ### Speech-to-Video
 
 For models that support audio-driven generation (e.g., Wan2.2-S2V), pass both
@@ -251,6 +298,7 @@ These `extra_params` control how the server turns decoded frames into MP4 bytes.
 | Field | Type | Default | Description |
 | --- | --- | --- | --- |
 | `preencode_mp4` | boolean | false | Encode the MP4 on the worker while the VAE is still decoding, instead of after the full video is materialized |
+| `preencode_batch_frames` | positive integer | 17 (H3, Wan T2V/I2V); 1 (Wan S2V) | Minimum accumulated frames per worker transfer/encoding batch; used only with `preencode_mp4=true` |
 | `video_codec_options` | object | null | Encoder options passed through to the H.264 encoder, such as `{"preset": "ultrafast", "threads": "0"}` |
 
 With `preencode_mp4` enabled, each committed VAE chunk leaves the accelerator and
@@ -261,16 +309,29 @@ unchanged: the same complete MP4, byte-for-byte equivalent frames.
 ```bash
 curl -X POST http://localhost:8091/v1/videos/sync \
   -F "prompt=A small robot walking through a neon city" \
-  -F 'extra_params={"preencode_mp4": true, "video_codec_options": {"preset": "ultrafast"}}' \
+  -F 'extra_params={"preencode_mp4": true, "preencode_batch_frames": 33, "video_codec_options": {"preset": "ultrafast"}}' \
   -o output.mp4
 ```
 
+Set `preencode_batch_frames` in `extra_params` (or `extra_args` for offline
+sampling) to tune batching. The worker accumulates complete VAE chunks until
+it has at least this many frames, then transfers and encodes them together.
+It always flushes the final partial batch. This is a threshold, not an exact
+chunk length: a value of 1 submits every native chunk immediately, and a value
+smaller than a native chunk does not split it. Larger values reduce transfers
+but retain more frames on the accelerator and delay encoding. The VAE decode
+window and output frame count stay unchanged. Wan S2V keeps its existing
+per-clip behavior by default. Zero, negative, fractional, boolean, string, and
+null values are rejected when pre-encoding is enabled.
+
 `preencode_mp4` applies to the complete-MP4 response paths only. The
 `/v1/realtime/video` WebSocket endpoint rejects it, because that path already
-overlaps encoding through its own incremental fragmented-MP4 encoder.
+overlaps encoding through its own incremental fragmented-MP4 encoder. Wan also
+rejects it together with `enable_frame_interpolation`, which needs the decoded
+frames the pre-encoded path no longer materializes.
 
-Support is per model: MiniMax-H3 implements it, and other models ignore the flag
-and take the full-decode path.
+Support is per model: MiniMax-H3 and Wan 2.2 (T2V, I2V, and S2V) implement it,
+and other models ignore the flag and take the full-decode path.
 
 ## Storage
 

docs/user_guide/diffusion/attention_backends/rainfusion.md

@@ -78,3 +78,35 @@ Protected video tails require a compatible MindIE-SD release.
 For common configuration and selector behavior, see the
 [attention backend overview](../attention_backends.md) and the
 [backend selection design](../../../design/feature/attention_backend_selection.md).
+
+## Wan2.2 T2V quantization on Ascend
+
+`RAINFUSION_ATTN` accepts `diffusion_kv_cache_dtype: fp8` or `mxfp4` for
+Wan2.2 T2V A14B.
+MindIE-SD must expose `sparse_attention` with `rf_v3` and an explicit `precision`
+argument. Omni passes the selected precision directly to this API; the installed
+native operators must support that precision. Unsupported inputs or an
+incompatible Python API raise an actionable error; Omni does not change BSA
+precision automatically. Update `diffusion_kv_cache_dtype` to a precision
+supported by the installed MindIE-SD, or unset it. Native execution errors also
+propagate without retry.
+BSA FP8 and MXFP4 retain MindIE-SD's default rotation policy.
+
+`diffusion_kv_cache_skip_layers`/`diffusion_kv_cache_skip_steps` explicitly
+select floating-point BSA and keep attention sparse. The existing `block_sparse`
+warmup/layer exclusions and short-sequence threshold instead select Dense
+attention, using the configured Dense precision. Leave `block_sparse.precision`
+at its default when using `diffusion_kv_cache_dtype` as the shared Dense/BSA
+precision setting.
+
+Configure BSA quantization in the model's existing deployment YAML with
+`backend: RAINFUSION_ATTN` and stage-level `diffusion_kv_cache_dtype: fp8` or
+`mxfp4`. The precision and skip selectors are siblings of
+`diffusion_attention_config`; they are not per-role `quant` fields.
+For 40 denoising steps, `diffusion_kv_cache_skip_steps: "0,1,38,39"` and
+`diffusion_kv_cache_skip_layers: "0,39"` are recommended starting points. These
+fixed step indices do not automatically follow a changed step count. Selected
+forwards remain floating-point BSA; unsupported BSA precision is an error rather
+than an automatic fallback. See
+[quantized attention](../../quantization/quantized_kvcache.md#wan22-t2v-quantized-attention-on-ascend)
+for the shared configuration structure and index semantics.

docs/user_guide/diffusion/mot_config.md

@@ -28,7 +28,7 @@ and their optimal Triton tile configurations:
 
 Config loading order (3-tier, see ops/mot_gemm.py):
     1. $VLLM_TUNED_CONFIG_FOLDER/<filename>  (env override)
-    2. This directory: vllm_omni/diffusion/layers/mot/configs/<filename>
+    2. This directory: vllm_omni/diffusion/models/bagel/mot/configs/<filename>
     3. Conservative default config (compiles everywhere, sub-optimal perf)
 
 If no config file matches the current device, a warning is printed with
@@ -39,7 +39,7 @@ To generate configs for your hardware:
     python benchmarks/kernels/mot_linear_benchmarks.py \
         --model ByteDance-Seed/BAGEL-7B-MoT \
         --tp-size 1 --dtype w16a16 --tune \
-        --save-dir vllm_omni/diffusion/layers/mot/configs/
+        --save-dir vllm_omni/diffusion/models/bagel/mot/configs/
 
 For multi-GPU tuning (uses Ray for parallel search):
 

docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md

@@ -353,9 +353,14 @@ modes can resolve the same `None` value to different schedules.
   Cache-DiT on any selected component. Prompt-embedding cache is also
   incompatible when the text encoder uses `allgather`. Rank-local cache hits
   could otherwise make ranks enter different weight collectives.
-- Per-tensor online FP8 linears use the ordinary loader and can run with either
-  DiT transfer path. With DiT `allgather`, every rank temporarily materializes
-  the complete FP8 model in host memory before DLO retains only its shard.
+- Per-tensor online FP8, INT8, and MXFP8 linears use the ordinary loader and
+  can run with either DiT transfer path. With DiT `allgather`, every rank
+  temporarily materializes the complete quantized model in host memory before
+  DLO retains only its shard. Layers too wide for the NPU quantization kernel
+  (`npu_quant_matmul` rejects an output dimension past 65535) stay
+  unquantized and load straight into host memory under offload-after-quant —
+  their runtime layout is a plain contiguous bf16 weight, the same layout DLO
+  shards on the ordinary path, so they are allowed under `allgather` too.
   Other online quantization methods require rank-local transfer for the
   affected component until their runtime layouts are validated.
 - Resident leading layers require DiT `rank-local` transfer and a model

docs/user_guide/examples/online_serving/bagel.md

@@ -247,12 +247,18 @@ curl http://localhost:8091/v1/chat/completions \
   -d @payload.json
 ```
 
-BAGEL img2img derives its generated image size from the input image. It
-preserves the input aspect ratio, aligns dimensions to the latent stride, and
-applies the checkpoint size limit; explicit `height` and `width` values do not
-override this model-specific resize policy. In step mode, the effective size is
-resolved before scheduler admission so requests with different resulting
-shapes are not combined.
+When no output size is requested (`size=auto` on `/v1/images/edits`, or no
+`height`/`width` in a chat request), BAGEL img2img derives the canvas from the
+input image: it preserves the input aspect ratio, aligns dimensions to the
+latent stride (16 px), and applies the checkpoint size limit. An explicit
+`size`, `height`, or `width` is honored instead, in both the single-stage and
+the two-stage deployment; the input image is still resized to a stride-aligned
+size for the VAE/ViT prefill. The latent grid needs multiples of 16, so a
+requested side that is not one is floored to the next multiple (1000x700
+generates 992x688) and the server logs a warning; the response reports the
+generated size. A side above the checkpoint limit is rejected with HTTP 400.
+In step mode, the effective size is resolved before scheduler admission so
+requests with different resulting shapes are not combined.
 
 ### Image to Text (img2text)
 
@@ -318,8 +324,8 @@ curl http://localhost:8091/v1/chat/completions \
 | `--server` / `-s` | `http://localhost:8091` | Server URL |
 | `--image-url` / `-i` | `None` | Input image URL or local path (img2img/img2text) |
 | `--modality` / `-m` | `text2img` | `text2img`, `img2img`, `img2text`, `text2text` |
-| `--height` | `512` | Text-to-image output height; BAGEL img2img derives its size from the input image |
-| `--width` | `512` | Text-to-image output width; BAGEL img2img derives its size from the input image |
+| `--height` | `512` | Output height, a multiple of 16; omit both `--height` and `--width` to let img2img derive the size from the input image |
+| `--width` | `512` | Output width, a multiple of 16; omit both `--height` and `--width` to let img2img derive the size from the input image |
 | `--steps` | `25` | Number of inference steps |
 | `--seed` | `42` | Random seed |
 | `--negative` | `None` | Negative prompt for CFG |

docs/user_guide/quantization/overview.md

@@ -13,7 +13,7 @@ For the internal architecture and backend extension points, see the
 | Mode | Guide | Description | Methods |
 | ------ | ------- | ------------- | --------- |
 | Online quantization | [Online Quantization](online.md) | vLLM-Omni computes quantized weights and scales while loading the model. | FP8 W8A8, Int8 W8A8, BitsAndBytes W4, MXFP8 W8A8, MXFP4 W4A4 |
-| Runtime attention quantization | [Quantized KV Cache](quantized_kvcache.md) | vLLM-Omni dynamically quantizes eligible diffusion Flash Attention tensors during inference. | FP8 FA |
+| Runtime attention quantization | [Quantized KV Cache](quantized_kvcache.md) | vLLM-Omni dynamically quantizes eligible diffusion attention tensors during inference. | FP8, MXFP8, MXFP4 |
 | Pre-quantized checkpoints | Method-specific guides | The checkpoint or an offline quantizer provides quantized weights and scales before serving. | ModelOpt, AutoRound, TorchAO, msModelSlim, serialized Int8, offline MXFP8, offline MXFP4 DualScale |
 
 ## Hardware Support

docs/user_guide/quantization/quantized_kvcache.md

@@ -2,41 +2,36 @@
 
 ## Overview
 
-In DiT-based image and video generation, Flash Attention can take a large share
-of denoising time, especially for high-resolution or long-frame workloads.
-vLLM-Omni supports online FP8 quantization for eligible diffusion Flash
-Attention (FA) to reduce FA latency while keeping model weights in their
-original dtype.
+In DiT-based image and video generation, attention can take a large share of
+denoising time, especially for high-resolution or long-frame workloads.
+vLLM-Omni supports online Q/K/V quantization for eligible diffusion attention
+paths while keeping model weights in their original dtype.
 
-This feature is configured through `diffusion_kv_cache_dtype` on
-`OmniDiffusionConfig` (CLI: `--diffusion-kv-cache-dtype`). It is intentionally
-**not** the same as vLLM's `--kv-cache-dtype`, which controls autoregressive
-language-model KV cache storage and defaults to `"auto"`. Diffusion FA
-quantization uses the dedicated diffusion flags so omni serve does not inherit
-that default.
+Runtime attention precision is configured through `diffusion_kv_cache_dtype`
+on `OmniDiffusionConfig` (CLI: `--diffusion-kv-cache-dtype`). This setting is
+separate from vLLM's `--kv-cache-dtype`, which controls autoregressive
+language-model KV cache storage.
 
 In vLLM-Omni diffusion pipelines, this is a runtime FA path: Q/K/V tensors are
 dynamically quantized before the attention operator. It does not quantize model
 weights and is separate from [FP8 W8A8](fp8.md), [Int8 W8A8](int8.md), or
 pre-quantized checkpoint formats.
 
-If `diffusion_kv_cache_dtype` is not set, behavior is unchanged and attention
-runs in the native dtype.
+If `diffusion_kv_cache_dtype` is not set, attention runs in the native dtype.
 
 ## Hardware Support
 
-| Device | FP8 FA |
-|--------|--------|
-| Ascend NPU | ✅ |
-| NVIDIA GPU | ❌ |
-| AMD ROCm | ❌ |
-| Intel XPU | ❌ |
+| Device | Dense FP8 | Dense MXFP8/MXFP4 | BSA FP8/MXFP4 |
+|--------|------------|-------------------|----------------|
+| Ascend NPU | ✅ | ✅ | ✅ |
+| NVIDIA GPU | ❌ | ❌ | ❌ |
+| AMD ROCm | ❌ | ❌ | ❌ |
+| Intel XPU | ❌ | ❌ | ❌ |
 
 Legend: `✅` supported, `❌` unsupported.
 
-FP8 FA is currently implemented only for the NPU Flash Attention backend. Other
-backends do not support `diffusion_kv_cache_dtype="fp8"` for diffusion attention
-and fall back to native dtype execution.
+These runtime methods are implemented only by the NPU Flash Attention and BSA
+backends. Other backends reject an incompatible explicit configuration.
 
 ## Model Type Support
 
@@ -44,7 +39,7 @@ and fall back to native dtype execution.
 
 | Model | Scope | Status | Notes |
 |-------|-------|--------|-------|
-| Wan2.2 | Eligible DiT full-attention FA on Ascend NPU | Tested | Compare quality and latency against a BF16 baseline before production use |
+| Wan2.2 | Self-attention on Ascend NPU | NPU interface and T2V smoke tested | Compare quality and latency against a BF16 baseline before production use |
 | Other diffusion models | Eligible DiT full-attention FA on Ascend NPU | Not tested | You can try `diffusion_kv_cache_dtype="fp8"`; tune `diffusion_kv_cache_skip_steps` and `diffusion_kv_cache_skip_layers` when higher precision is needed |
 
 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
@@ -105,12 +100,13 @@ The legacy keyword aliases `kv_cache_dtype`, `kv_cache_skip_steps`, and
 
 | Parameter | Type | Default | Description |
 |-----------|------|---------|-------------|
-| `diffusion_kv_cache_dtype` | str \| None | `None` | Set to `"fp8"` to enable dynamic FP8 FA on supported attention backends |
+| `diffusion_kv_cache_dtype` | str \| None | `None` | Runtime attention method. Dense FA supports `fp8`, `mxfp8`, and `mxfp4`; BSA supports `fp8` and `mxfp4` |
 | `diffusion_kv_cache_skip_steps` | str \| None | `None` | Denoising step selector to keep in native dtype, for example `"0,1,4-6"` |
 | `diffusion_kv_cache_skip_layers` | str \| None | `None` | Transformer layer selector to keep in native dtype, for example `"0-2,10"` |
 
 Selectors use comma-separated integers and inclusive ranges. Listed steps or
-layers skip FP8 FA; all other eligible full-attention forwards use the FP8 path.
+layers run floating-point attention through the selected Dense or BSA backend;
+all other eligible forwards use `diffusion_kv_cache_dtype`.
 
 ## Validation and Notes
 
@@ -123,3 +119,60 @@ layers skip FP8 FA; all other eligible full-attention forwards use the FP8 path.
 4. Report both latency and quality results when enabling this option for a new
    model. For image or video models, include visual comparison and quantitative
    metrics when available, such as PSNR or SSIM.
+
+## Wan2.2 T2V quantized attention on Ascend
+
+Wan2.2 T2V A14B supports quantized self-attention through MindIE-SD. Model
+weights are unchanged, and model-level opt-out keeps cross-attention in its
+native dtype.
+
+| Attention path | self-attention `backend` | Supported `diffusion_kv_cache_dtype` |
+| --- | --- | --- |
+| Dense FA | `FLASH_ATTN` | `fp8`, `mxfp8`, `mxfp4` |
+| BSA | `RAINFUSION_ATTN` | `fp8`, `mxfp4` |
+
+Configure the method and fallback selectors once at the diffusion stage. The
+per-role attention config only selects the self-attention backend:
+
+```yaml
+stages:
+  - stage_id: 0
+    diffusion_kv_cache_dtype: "mxfp8"
+    diffusion_kv_cache_skip_layers: "0,39"
+    diffusion_kv_cache_skip_steps: "0,1,38,39"
+    diffusion_attention_config:
+      per_role:
+        self:
+          backend: FLASH_ATTN  # Use RAINFUSION_ATTN with fp8 or mxfp4 for BSA.
+```
+
+For 40 denoising steps, `diffusion_kv_cache_skip_steps: "0,1,38,39"` and
+`diffusion_kv_cache_skip_layers: "0,39"` are recommended starting points. Steps
+are zero-based across the complete request and do not reset when Wan switches
+transformers; layer indices are zero-based and local to each transformer.
+Selectors accept comma-separated indices and inclusive ranges such as
+`"0,3-5"`. A selected forward uses floating-point attention while preserving
+the Dense or BSA path.
+
+Unsupported precision/input combinations raise an error and must be corrected
+in the configuration; operator errors are not retried with another precision.
+
+For BSA, change the backend to `RAINFUSION_ATTN` and set
+`diffusion_kv_cache_dtype` to `fp8` or `mxfp4`.
+See [RainFusion attention](../diffusion/attention_backends/rainfusion.md) for
+sparse-path configuration and behavior.
+
+### Requirement and interface test
+
+Until these APIs are available in a MindIE-SD release, use revision
+[`8637b5333b0225381b215390fd09a8732e671cc4`](https://gitcode.com/zqxu/MindIE-SD/commit/8637b5333b0225381b215390fd09a8732e671cc4).
+Build the Python package, PyTorch plugin, and custom operators from that same
+checkout. With CANN sourced, run the NPU interface tests from the Omni checkout:
+
+```bash
+python -m pytest tests/platforms/npu/quant/test_kv_quant_npu.py \
+    -k real_npu -vv -s -o addopts=''
+```
+
+A skipped NPU test is not a pass. Validate generated video quality against a
+floating-point run with the same prompt, seed, resolution, and step count.

examples/model_display_names.yml

@@ -2,7 +2,6 @@
 # The docs generator combines these names with the serving mode for each page H1.
 aura_omni: AURA Omni
 covo_audio: Covo-Audio-Chat
-dynin_omni: Dynin-Omni
 lance: Lance
 mimo_audio: MiMo-Audio
 ming_flash_omni: Ming-flash-omni 2.0

examples/offline_inference/dynin_omni/README.md

@@ -1,110 +0,0 @@
-# Dynin-Omni: Offline inference
-
-This folder contains a unified offline inference entrypoint:
-
-- `end2end.py`
-
-## 1. Environment Setup
-
-Run from repository root:
-
-```bash
-cd <REPO_ROOT>
-```
-
-If needed, install this repo in editable mode:
-
-```bash
-pip install -e .
-```
-
-## 2. Extra Dependencies (EMOVA)
-
-Install the following packages for EMOVA-related components:
-
-```bash
-pip install \
-  "phonemizer==3.3.0" \
-  "Unidecode==1.4.0" \
-  "hydra-core==1.3.2" \
-  "pytorch-lightning==1.1.0" \
-  "wget==3.2" \
-  "wrapt==2.1.1" \
-  "onnx==1.20.1" \
-  "frozendict==2.4.7" \
-  "inflect==7.5.0" \
-  "braceexpand==0.1.7" \
-  "webdataset==1.0.2" \
-  "torch-stft==0.1.4" \
-  "editdistance==0.8.1"
-```
-
-## 3. Hardware and VRAM Requirements
-
-This example uses a 3-stage pipeline on one GPU by default
-([`dynin_omni.yaml`](../../../vllm_omni/deploy/dynin_omni.yaml)):
-
-- Stage-0 (`token2text`): `gpu_memory_utilization: 0.5`
-- Stage-1 (`token2image`): `gpu_memory_utilization: 0.1`
-- Stage-2 (`token2audio`): `gpu_memory_utilization: 0.1`
-
-### Requested GPU Memory Budget from `gpu_memory_utilization`
-
-| Stage | Utilization | A100 80GB | H200 141GB |
-| :-- | :-- | :-- | :-- |
-| Stage-0 (token2text) | 0.5 | ~40.0 GB | ~70.5 GB |
-| Stage-1 (token2image) | 0.1 | ~8.0 GB | ~14.1 GB |
-| Stage-2 (token2audio) | 0.1 | ~8.0 GB | ~14.1 GB |
-| Total requested budget | 0.7 | ~56.0 GB | ~98.7 GB |
-
-### Observed Runtime Signal (from your log)
-
-- Stage-0 reported: `Model loading took 15.12 GiB memory` (weights footprint signal).
-- Stages 1/2 can still add runtime memory depending on task path and backend allocations.
-- Keep extra headroom for CUDA/PyTorch overhead and temporary allocations.
-
-### GPU Compatibility
-
-- Confirmed target GPUs for this setup: **NVIDIA H200**, **NVIDIA A100**.
-- CI/e2e coverage in this repo also includes CUDA **L4** markers for Dynin tests.
-
-## 4. End2End Run Examples
-
-```bash
-# t2t
-python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
-  --task t2t --model snu-aidas/Dynin-Omni --text <INSTRUCTION_TEXT>
-
-# i2t
-python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
-  --task i2t --model snu-aidas/Dynin-Omni --image <IMAGE_PATH> --text "Please describe this image in detail."
-
-# s2t
-python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
-  --task s2t --model snu-aidas/Dynin-Omni --audio <AUDIO_PATH> --text "Transcribe the given audio."
-
-# t2i
-python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
-  --task t2i --model snu-aidas/Dynin-Omni --text <INSTRUCTION_TEXT>
-
-# v2t
-python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
-  --task v2t --model snu-aidas/Dynin-Omni --video <VIDEO_PATH> --text "Describe this video in detail."
-
-# i2i
-python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
-  --task i2i --model snu-aidas/Dynin-Omni --image <IMAGE_PATH> --text <INSTRUCTION_TEXT>
-
-# t2s
-python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
-  --task t2s --model snu-aidas/Dynin-Omni --text <INSTRUCTION_TEXT>
-```
-
-## 5. Notes
-
-- Outputs are saved under task-specific directories in `/tmp` by default.
-- You can override output path with `--output-dir`.
-- If you want to force local config resolution, pass `--dynin-config-path <PATH_TO_DYNIN_OMNI_YAML>`.
-- If you see the warning
-  `max_num_batched_tokens (32768) exceeds max_num_seqs * max_model_len (4096)`,
-  reduce `max_num_batched_tokens` in the deploy config (for example, `4096` in the CI config).

examples/offline_inference/text_to_image/README.md

@@ -19,23 +19,23 @@ This folder provides several entrypoints for experimenting with text-to-image di
 
 ### Supported Models
 
-| Model | Image Shape  | Peak VRAM (GiB) * | Model Weights (GiB) |
+| Model | Image Shape | Peak VRAM (GiB) * | Model Weights (GiB) |
 | ----- | ----------- | ----------- | ----------------- |
 | `Qwen/Qwen-Image` | 1024 x 1024 | 60.0 | 53.7 |
-| `Qwen/Qwen-Image-2512` |1024 x 1024 | 60.0 | 53.7 |
+| `Qwen/Qwen-Image-2512` | 1024 x 1024 | 60.0 | 53.7 |
 | `Tongyi-MAI/Z-Image-Turbo` | 1024 x 1024 | 24.8 | 19.2 |
 | `stepfun-ai/NextStep-1.1` | 512 x 512 | 71.8 | 28.1 |
 | `meituan-longcat/LongCat-Image` | 1024 x 1024 | 71.2 | 27.3 |
 | `AIDC-AI/Ovis-Image-7B` | 1024 x 1024 | 71.8 | 17.1 |
-| `OmniGen2/OmniGen2` |  1024 x 1024 | 20.1 | 14.7 |
+| `OmniGen2/OmniGen2` | 1024 x 1024 | 20.1 | 14.7 |
 | `stabilityai/stable-diffusion-3.5-medium` | 1024 x 1024 | 20.1 | 15.6 |
 | `black-forest-labs/FLUX.1-dev` | 1024 x 1024 | 33.9 | 31.4 |
 | `black-forest-labs/FLUX.1-schnell` | 1024 x 1024 | 33.9 | 31.4 |
 | `black-forest-labs/FLUX.2-klein-4B` | 1024 x 1024 | 72.7 | 14.9 |
 | `black-forest-labs/FLUX.2-klein-9B` | 1024 x 1024 | 37.1 | 32.3 |
 | `black-forest-labs/FLUX.2-dev` | 1024 x 1024 | 65.7 | >80 (CPU offload required) |
-| `HunyuanImage-3.0` | 1024 x 1024 | 80.0 (TP≥3)  | 160 |
-| `HiDream-I1-Full` | 1024 x 1024 | 63.7  | 57.7 |
+| `HunyuanImage-3.0` | 1024 x 1024 | 80.0 (TP≥3) | 160 |
+| `HiDream-I1-Full` | 1024 x 1024 | 63.7 | 57.7 |
 | `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | 1024 (Raw) / 2048 (Turbo) | — | ~30 |
 
 !!! info
@@ -112,9 +112,9 @@ python text_to_image.py \
 | `--enable-cpu-offload` | flag | off | Enable CPU offloading for diffusion models |
 | `--lora-path` | str | — | Path to PEFT LoRA adapter folder or checkpoint file |
 | `--lora-scale` | float | `1.0` | Scale factor for LoRA weights |
-| `--lora-backend` | str |`"peft"`| LoRA backend for loading LoRA adapters. Default: peft. Available options: peft, distill |
-| `--use-system-prompt` | str | `None` | System prompt preset: `en_unified`, `en_vanilla`, `en_recaption`, `en_think_recaption`, `dynamic`, `None`, or custom text. Recommended: `en_unified`. Only for HunyuanImage-3.0.|
-| `--system-prompt` | str | `None` | Custom system prompt text. Only used when `--use-system-prompt` is set to `custom`. Only for HunyuanImage-3.0.|
+| `--lora-backend` | str | `"peft"` | LoRA backend for loading LoRA adapters. Default: peft. Available options: peft, distill |
+| `--use-system-prompt` | str | `None` | System prompt preset: `en_unified`, `en_vanilla`, `en_recaption`, `en_think_recaption`, `dynamic`, `None`, or custom text. Recommended: `en_unified`. Only for HunyuanImage-3.0. |
+| `--system-prompt` | str | `None` | Custom system prompt text. Only used when `--use-system-prompt` is set to `custom`. Only for HunyuanImage-3.0. |
 | `--auxiliary-text-encoder` | str | `None` | Supplementary auxiliary text encoder parameters model name or path (especially for Hidream-l1-full). |
 
 **NextStep-1.1 specific arguments:**
@@ -127,7 +127,7 @@ python text_to_image.py \
 | `--use-norm` | flag | off | Apply layer normalization to sampled tokens |
 
 > If you encounter OOM errors, try using `--vae-use-slicing` and `--vae-use-tiling` to reduce memory usage.
-
+>
 > Qwen-Image currently publishes best-effort presets at `1328x1328`, `1664x928`, `928x1664`, `1472x1140`, `1140x1472`, `1584x1056`, and `1056x1584`. Adjust `--height/--width` accordingly for the most reliable outcomes.
 
 ## More CLI Examples
@@ -216,6 +216,26 @@ python examples/offline_inference/text_to_image/text_to_image.py \
   --output /output.png
 ```
 
+### Anima Single-File Checkpoints
+
+Anima uses the official model dimensions by default. A checkpoint may provide a JSON file with the same stem (for example, `anima.json` beside `anima.safetensors`) containing `transformer` and `text_conditioner` configuration objects. These override the default dimensions and must match the checkpoint weights. The shared tiny-model tests use this to load small random-weight models. Components in `text_encoder/`, `vae/`, `tokenizer/`, `t5_tokenizer/`, and `scheduler/` beside the checkpoint are discovered automatically.
+
+To load Anima, point `--model` to the single-file checkpoint path, pass the native pipeline class name using `--model-class-name`, and supply the converted components directory using `--custom-pipeline-args`:
+
+```bash
+python examples/offline_inference/text_to_image/text_to_image.py \
+  --model /path/to/models/anima-official/split_files/diffusion_models/anima-base-v1.0.safetensors \
+  --model-class-name AnimaPipeline \
+  --custom-pipeline-args '{"components_path": "/path/to/models/anima-components"}' \
+  --prompt "A cinematic close-up of a glass teapot on a wooden table." \
+  --seed 42 \
+  --guidance-scale 4.0 \
+  --num-inference-steps 50 \
+  --height 1024 \
+  --width 1024 \
+  --output anima_output.png
+```
+
 ### Krea 2
 
 Krea 2 is published as diffusers-format checkpoints: `krea/Krea-2-Turbo` (few-step distilled) and `krea/Krea-2-Raw`.
@@ -336,7 +356,7 @@ python text_to_image.py \
 
 LoRA adapters must be in PEFT format. A typical adapter directory structure:
 
-```
+```text
 lora_adapter/
 ├── adapter_config.json
 └── adapter_model.safetensors

examples/offline_inference/text_to_image/text_to_image.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
 import argparse
 import functools
@@ -58,6 +58,7 @@ def parse_json_object(value: str, flag_name: str = "argument") -> dict[str, Any]
 
 
 parse_profiler_config = functools.partial(parse_json_object, flag_name="--profiler-config")
+parse_custom_pipeline_args = functools.partial(parse_json_object, flag_name="--custom-pipeline-args")
 
 
 def build_text_to_image_prompt(prompt: str, negative_prompt: str | None) -> dict[str, Any]:
@@ -379,6 +380,18 @@ def parse_args() -> argparse.Namespace:
         default=None,
         help="Supplementary auxiliary text encoder parameters model name or path (especially for Hidream-l1-full).",
     )
+    parser.add_argument(
+        "--model-class-name",
+        type=str,
+        default=None,
+        help="Override the diffusion pipeline class name (e.g. AnimaPipeline).",
+    )
+    parser.add_argument(
+        "--custom-pipeline-args",
+        type=parse_custom_pipeline_args,
+        default=None,
+        help='JSON object passed to native/custom pipelines (e.g. \'{"components_path": "/path"}\').',
+    )
     parser.add_argument(
         "--trust-remote-code",
         action="store_true",
@@ -558,7 +571,6 @@ def main():
         "vae_patch_parallel_size": args.vae_patch_parallel_size,
         "enable_expert_parallel": args.enable_expert_parallel,
         "enable_cpu_offload": args.enable_cpu_offload,
-        "mode": "text-to-image",
         "log_stats": args.log_stats,
         "enable_diffusion_pipeline_profiler": args.enable_diffusion_pipeline_profiler,
         "profiler_config": args.profiler_config,
@@ -576,9 +588,13 @@ def main():
         omni_kwargs["trust_remote_code"] = True
     if args.deploy_config:
         omni_kwargs["deploy_config"] = args.deploy_config
-    if use_nextstep:
+    if args.model_class_name:
+        omni_kwargs["model_class_name"] = args.model_class_name
+    elif use_nextstep:
         # NextStep-1.1 requires explicit pipeline class
         omni_kwargs["model_class_name"] = "NextStep11Pipeline"
+    if args.custom_pipeline_args is not None:
+        omni_kwargs["custom_pipeline_args"] = args.custom_pipeline_args
     # Cosmos3 loads its (gated) guardrail models at build time, so the guardrails
     # gate is an engine-level config (offline analog of the server's --no-guardrails).
     if args.extra_body and "guardrails" in args.extra_body:
@@ -614,6 +630,10 @@ def main():
         print(f"  LoRA: scale={args.lora_scale}")
     if args.deploy_config:
         print(f"  deploy-config: {args.deploy_config}")
+    if args.model_class_name:
+        print(f"  Model class name: {args.model_class_name}")
+    if args.custom_pipeline_args is not None:
+        print(f"  Custom pipeline args: {args.custom_pipeline_args}")
     print(f"{'=' * 60}\n")
 
     # Build LoRA request when --lora-path is set
@@ -723,7 +743,10 @@ def main():
     # stop-token-ids declaratively from the plain prompt + extra_body, so this
     # example stays model-agnostic. Models without one are untouched.
     ar_input_builder = get_ar_input_builder(model_class_name)
-    if ar_input_builder is not None:
+    # A model can also be deployed with only its diffusion stage. Keep those
+    # requests on the string-prompt path, as in the single-stage images API.
+    has_ar_stage = any(not isinstance(params, OmniDiffusionSamplingParams) for params in sampling_params_list)
+    if ar_input_builder is not None and has_ar_stage:
         _apply_ar_stage_inputs(
             ar_input_builder,
             model=args.model,

examples/offline_inference/text_to_speech/README.md

@@ -13,7 +13,7 @@ list of supported architectures across all modalities, see
 ## Supported Models
 
 | Model | HuggingFace repo | Stages | Voice cloning | Streaming | Special modes | Sample rate |
-|---|---|---|---|---|---|---|
+| --- | --- | --- | --- | --- | --- | --- |
 | AuK | `tencent/AuK`, `tencent/AuK-Flash` (assembled with `tools/prepare_auk_checkpoint.py`) | 2 (thinker encoder + DiT) | ✓ | — | instruction-driven editing, enhancement, separation; no example script (use `vllm_omni.model_extras.auk`) | 24 kHz |
 | Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | dual-AR | ✓ | ✓ | 11 languages | 44.1 kHz |
 | Breeze-TTS-2 | `BreezeBlue/Breeze-TTS-2` | 2 (talker + codec) | ✓ | ✓ (async chunk) | voice design / voice direction (`--instruction`) | 24 kHz |
@@ -27,7 +27,6 @@ list of supported architectures across all modalities, see
 | OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design, language hint | 24 kHz |
 | Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | 2 (talker + code2wav) | ✓ (Base) | ✓ | 3 task variants | 24 kHz |
 | VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | ✓ (online) | continuation | 48 kHz |
-| dots.tts | `dots-studio/dots.tts-soar` | single (native AR) | — (text-only) | ✓ (online) | no-reference synthesis | 48 kHz |
 | IndexTTS-2 | `IndexTeam/IndexTTS-2` | 2 (AR talker + S2Mel DiT + BigVGAN) | ✓ (required) | — | emotion control (`--emo-audio`, `--emo-text`, `--emo-vector`) | 22.05 kHz |
 | IndexTTS-2.5 | native `checkpoints/` bundle | 2 (AR talker + EnhancedCodec + S2Mel DiT + BigVGAN) | ✓ (required) | — | multilingual (`--lang`) + emotion control | 22.05 kHz |
 | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | varies | ✓ | ✓ | voice presets | 24 kHz |
@@ -54,32 +53,38 @@ codec: a slow AR transformer predicts one semantic token per codec frame, and a
 4-layer fast AR predicts that frame's remaining 9 codebooks.
 
 ### Prerequisites
+
 No extra packages: the codec architecture ships in tree and its weights
 (`codec.pth`) come with the checkpoint.
 
 ### Quick start
+
 ```bash
 python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
     --text "Welcome to Audio8 TTS, a compact multilingual text to speech model."
 ```
 
 ### Voice cloning
+
 ```bash
 python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
     --text "Welcome to Audio8 TTS." \
     --ref-audio /path/to/reference.wav \
     --ref-text  "The exact transcript of the reference recording."
 ```
+
 The reference transcript must match what is actually spoken in the clip;
 a mismatch degrades both stability and speaker similarity.
 
 ### Streaming
+
 ```bash
 python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
     --text "Welcome to Audio8 TTS." --streaming
 ```
 
 ### Notes
+
 - Output: 44.1 kHz mono WAV; the codec emits ~21.5 frames/s (2048 samples per frame).
 - Context is 2048 packed text+audio positions, so `max_model_len` is 2048.
 - Greedy decoding (`temperature: 0`) is degenerate for this checkpoint — it
@@ -92,6 +97,7 @@ python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
 - `--num-prompts 4` smoke-tests concurrency (stage 0 runs `max_num_seqs: 4`).
 
 ### Measured behaviour (1x H20, shared GPU)
+
 `--streaming` on a 58-character English sentence: TTFA ~0.74 s (4-frame first
 chunk), 7 chunks, 1.55 s wall for 3.16 s of audio (RTF ~0.49). Treat these as
 smoke-test magnitudes, not a benchmark.
@@ -103,12 +109,14 @@ smoke-test magnitudes, not a benchmark.
 2-stage TTS pipeline (`talker` + `code2wav`) at 24 kHz.
 
 ### Prerequisites
+
 ```bash
 uv pip install -e .
 # Includes soundfile, onnxruntime, x-transformers, einops via requirements.
 ```
 
 Download the model snapshot:
+
 ```python
 from huggingface_hub import snapshot_download
 snapshot_download('FunAudioLLM/Fun-CosyVoice3-0.5B-2512',
@@ -116,15 +124,18 @@ snapshot_download('FunAudioLLM/Fun-CosyVoice3-0.5B-2512',
 ```
 
 If your downloaded checkpoint lacks `config.json`, add it:
+
 ```json
 {
     "model_type": "cosyvoice3",
     "architectures": ["CosyVoice3Model"]
 }
 ```
+
 This is required because `AutoConfig.register("cosyvoice3", CosyVoice3Config)` only registers the class mapping; the loader still reads `model_type` from `config.json` to select the class.
 
 ### Quick start
+
 ```bash
 python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
     --model pretrained_models/Fun-CosyVoice3-0.5B \
@@ -132,9 +143,11 @@ python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
 ```
 
 ### Voice cloning
+
 If `--ref-audio` is omitted, the script downloads the upstream
 [`zero_shot_prompt.wav`](https://github.com/FunAudioLLM/CosyVoice/blob/main/asset/zero_shot_prompt.wav) from the CosyVoice repo into the current directory.
 To use your own clip, pass `--ref-audio /path/to/reference.wav`, and modify `--prompt-text` correspondingly.
+
 ```bash
 python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
     --model pretrained_models/Fun-CosyVoice3-0.5B \
@@ -144,9 +157,11 @@ python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
 ```
 
 ### Streaming
+
 Streaming is enabled by default via `async_chunk: true` in `vllm_omni/deploy/cosyvoice3.yaml`. Pass `--no-async-chunk` on `vllm serve` to switch to the legacy synchronous path.
 
 ### Notes
+
 - Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform.
 - Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config <path>` to override.
 
@@ -157,6 +172,7 @@ Streaming is enabled by default via `async_chunk: true` in `vllm_omni/deploy/cos
 2-stage TTS pipeline (AR + DiT flow-matching) at 24 kHz. Every request requires reference audio and its transcript for zero-shot voice cloning.
 
 ### Quick start
+
 ```bash
 python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
     --model zai-org/GLM-TTS \
@@ -167,12 +183,14 @@ python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
 ```
 
 ### Architecture
+
 ```text
 Text → [Stage 0: AR] → Speech Tokens → [Stage 1: DiT + HiFT] → Audio (24 kHz)
         (Llama-based)    (32k vocab)      (Flow Matching)
 ```
 
 ### Notes
+
 - `--ref-audio` and `--ref-text` are **required** together; GLM-TTS does not support text-only synthesis.
 - Reference audio should be 3-10 seconds.
 - First run may be slow due to lazy loading of WhisperVQ tokenizer and CampPlus ONNX speaker embedder.
@@ -187,12 +205,14 @@ Text → [Stage 0: AR] → Speech Tokens → [Stage 1: DiT + HiFT] → Audio (24
 No extra packages are required; the DAC codec is vendored in vLLM-Omni.
 
 ### Quick start
+
 ```bash
 python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
     --text "Hello, this is a test of the Fish Speech text to speech system."
 ```
 
 ### Voice cloning
+
 ```bash
 python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
     --text "Hello, this is a cloned voice." \
@@ -201,14 +221,17 @@ python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
 ```
 
 ### Streaming
+
 ```bash
 python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
     --text "Hello, this is a streaming test." \
     --streaming
 ```
+
 Streaming requires `async_chunk: true` in the deploy config.
 
 ### Notes
+
 - Output: 44.1 kHz mono WAV.
 - DAC codec weights (`codec.pth`) are loaded lazily from the model directory.
 
@@ -219,6 +242,7 @@ Streaming requires `async_chunk: true` in the deploy config.
 Dense 0.5B two-stage TTS pipeline (`AR + flow` + audio VAE) at 44.1 kHz. The example covers style, IP voice, music-only generation, text-to-audio events, emotion, dialect, zero-shot cloning, podcast, speech+BGM, and speech+environment-sound cases.
 
 ### Quick start
+
 ```bash
 python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
     --case style \
@@ -227,6 +251,7 @@ python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
 ```
 
 ### Voice cloning
+
 ```bash
 python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
     --case zero_shot \
@@ -237,6 +262,7 @@ python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
 ```
 
 ### Streaming
+
 ```bash
 python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
     --case basic \
@@ -247,6 +273,7 @@ python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
 ```
 
 ### Notes
+
 - `style`, `ip`, `bgm`, and `tta` do not require reference audio.
 - Reference-audio cases use `--ref-audio`; `zero_shot` also requires `--ref-text`.
 - `podcast` uses multiple references via `--ref-audio-paths`.
@@ -259,14 +286,17 @@ python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
 Standalone talker-only deployment of Ming-flash-omni-2.0 at 44.1 kHz. Voice is controlled through caption fields (`风格` / `IP` / `语速`/`基频`/`音量`) rather than reference audio.
 
 ### Prerequisites
+
 The example calls into `vllm_omni.model_executor.models.ming_flash_omni.prompt_utils` for the default prompt and instruction builder; no extra pip install on top of the base vLLM-Omni install.
 
 ### Quick start
+
 ```bash
 python examples/offline_inference/text_to_speech/ming_flash_omni_tts/end2end.py --case style
 ```
 
 ### Cases
+
 ```bash
 # ASMR-style whisper (caption-driven)
 python examples/offline_inference/text_to_speech/ming_flash_omni_tts/end2end.py --case style
@@ -281,6 +311,7 @@ python examples/offline_inference/text_to_speech/ming_flash_omni_tts/end2end.py
 Override the default text per case with `--text`, write to a custom path with `--output`.
 
 ### Notes
+
 - Talker-only deployment — for the multimodal Ming-flash-omni example, see [`examples/offline_inference/ming_flash_omni/`](../../ming_flash_omni/).
 - Deploy config: `vllm_omni/deploy/ming_flash_omni_tts.yaml` (single GPU, `enforce_eager`, `max_num_seqs: 1`).
 - Decode defaults from the Ming cookbook: `max_decode_steps=200`, `cfg=2.0`, `sigma=0.25`, `temperature=0.0`, `use_zero_spk_emb=True`.
@@ -294,6 +325,7 @@ Single-stage 0.1B AR LM + MOSS-Audio-Tokenizer-Nano codec at 48 kHz mono (mixed
 > **No built-in speaker presets.** `--ref-audio` is required on every call. Default `--mode voice_clone` matches upstream's recommended workflow; `--mode continuation` is exposed for completeness but upstream's continuation-with-prompt path emits very short / near-silent output, so it is rarely useful in practice. Sample reference clips ship in the upstream repo under [`assets/audio/`](https://github.com/OpenMOSS/MOSS-TTS-Nano/tree/main/assets/audio) (e.g. `zh_1.wav`, `en_2.wav`, `jp_2.wav`).
 
 ### Quick start
+
 ```bash
 # Fetch a sample reference clip (one-off, user-scoped cache).
 REF_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/moss-tts-nano"
@@ -305,9 +337,11 @@ python examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py \
     --text "你好，这是MOSS-TTS-Nano的语音合成演示。" \
     --ref-audio "$REF_DIR/zh_1.wav"
 ```
+
 The first run downloads `OpenMOSS-Team/MOSS-TTS-Nano` and `OpenMOSS-Team/MOSS-Audio-Tokenizer-Nano` from Hugging Face.
 
 ### Reproducible runs
+
 ```bash
 python examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py \
     --text "Deterministic test." \
@@ -316,6 +350,7 @@ python examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py \
 ```
 
 ### Notes
+
 - Output: 48 kHz mono WAV (the tokenizer is internally stereo at 48 kHz; the wrapper averages to mono before reaching the engine).
 - Deploy config: `vllm_omni/deploy/moss_tts_nano.yaml` (auto-loaded; override with `--deploy-config`).
 - Default `--max-new-frames 375` ≈ 14 s of audio; raise for longer outputs.
@@ -329,12 +364,15 @@ python examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py \
 Zero-shot multilingual TTS supporting 600+ languages, with three modes (auto / clone / design).
 
 ### Prerequisites
+
 ```bash
 huggingface-cli download k2-fsa/OmniVoice
 ```
+
 Voice cloning requires `transformers>=5.3.0`. Auto and design modes work with `transformers>=4.57.0`.
 
 ### Quick start (auto voice)
+
 ```bash
 python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
     --model k2-fsa/OmniVoice \
@@ -342,6 +380,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
 ```
 
 ### Voice cloning
+
 ```bash
 python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
     --model k2-fsa/OmniVoice \
@@ -351,6 +390,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
 ```
 
 ### Voice design
+
 ```bash
 python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
     --model k2-fsa/OmniVoice \
@@ -359,6 +399,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
 ```
 
 ### Language hint
+
 ```bash
 python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
     --model k2-fsa/OmniVoice \
@@ -367,6 +408,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
 ```
 
 ### Seed for Reproducibility
+
 ```bash
 python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
     --model k2-fsa/OmniVoice \
@@ -375,6 +417,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
 ```
 
 ### Notes
+
 - Stage 0 (Generator): Qwen3-0.6B with 32-step iterative unmasking.
 - Stage 1 (Decoder): HiggsAudioV2 RVQ + DAC at 24 kHz.
 
@@ -385,13 +428,16 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
 3-task-variant TTS with 24 kHz output. Has its own argparse surface (this script does not follow the common `--text` / `--ref-audio` shape).
 
 ### Prerequisites
+
 For ROCm builds, replace `onnxruntime` with `onnxruntime-rocm`:
+
 ```bash
 pip uninstall onnxruntime
 pip install onnxruntime-rocm
 ```
 
 ### Task variants
+
 - `CustomVoice`: predefined speaker (speaker ID) with optional style instruction.
 - `VoiceDesign`: text + descriptive instruction designs a new voice.
 - `Base`: voice cloning from reference audio + transcript.
@@ -415,22 +461,27 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-ty
 ```
 
 ### Streaming
+
 ```bash
 python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
     --query-type CustomVoice \
     --streaming \
     --output-dir /tmp/out_stream
 ```
+
 Streaming requires `async_chunk: true` in the deploy config.
 
 ### Word Timestamps
+
 Word-level timestamps are currently a serving-path feature: launch
 `vllm-omni serve` with `--forced-aligner` and request `word_timestamps`
 (see `examples/online_serving/text_to_speech/README.md`). An offline
 example is not provided in this release.
 
 ### Batched decoding
+
 The Code2Wav stage supports batched decoding through the SpeechTokenizer. Pass multiple prompts via `--txt-prompts` and set `--batch-size` accordingly. To raise `max_num_seqs` on either stage, create an overlay based on `vllm_omni/deploy/qwen3_tts.yaml` and pass it with `--deploy-config`:
+
 ```bash
 python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
     --query-type CustomVoice \
@@ -438,9 +489,11 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
     --batch-size 4 \
     --deploy-config /path/to/qwen3_tts_batched.yaml
 ```
+
 `--batch-size` must match a CUDA-graph capture size (1, 2, 4, 8, 16…).
 
 ### Notes
+
 - Run `--help` for the full argument surface.
 - See `qwen3_tts/end2end.py` for the prompt-length-estimation logic the Talker uses.
 
@@ -451,6 +504,7 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
 Single-stage native AR TTS at 22.05 kHz. Pipeline: `Qwen3.5 backbone → 32 FSQ codebook heads (one frame/step) → NeMo NanoCodec`. Backbone runs under vLLM paged attention; the 32-head sampling + learned embedding feedback ride the native-AR runner hooks. `enforce_eager` (CUDA graph is a perf follow-up).
 
 ### Prerequisites
+
 The NanoCodec decoder needs NeMo (NVIDIA Open Model License), installed separately:
 
 ```bash
@@ -480,15 +534,18 @@ failure rather than an environment one. The PyTorch sampler it falls back to cos
 here.
 
 ### Quick start (zero-shot, default voice)
+
 ```bash
 python examples/offline_inference/text_to_speech/gepard/end2end.py \
     --text "Hello, this is a Gepard demo."
 ```
 
 ### Voice cloning
+
 Not yet — PR1 is zero-shot only (the learned `null_prefix` default voice). Reference-audio cloning (`ref_compressor` speaker prefix) is a follow-up PR.
 
 ### Notes
+
 - Output: 22.05 kHz mono WAV.
 - First run downloads two checkpoints, not one: the model itself, and the NanoCodec decoder it names in `codec_id` (`nvidia/nemo-nano-codec-22khz-1.89kbps-21.5fps` by default). Both are fetched automatically; expect the first startup to be correspondingly slower.
 - Offline and online: `examples/offline_inference/text_to_speech/gepard/end2end.py` and `/v1/audio/speech` (see the [online Gepard section](../../online_serving/text_to_speech/README.md#gepard-10)).
@@ -502,6 +559,7 @@ Not yet — PR1 is zero-shot only (the learned `null_prefix` default voice). Ref
 Single-stage native AR TTS at 48 kHz. Pipeline: `feat_encoder → MiniCPM4 → FSQ → residual_lm → LocDiT → AudioVAE`.
 
 ### Prerequisites
+
 ```bash
 pip install voxcpm
 # or, for a local source checkout:
@@ -509,6 +567,7 @@ export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/voxcpm
 ```
 
 ### Quick start
+
 ```bash
 python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
     --model openbmb/VoxCPM2 \
@@ -516,7 +575,9 @@ python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
 ```
 
 ### Voice cloning
+
 Pass a reference audio for isolated cloning, or both `--ref-audio` + `--ref-text` for prompt continuation:
+
 ```bash
 python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
     --text "Hello, this is a voice clone demo." \
@@ -525,39 +586,16 @@ python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
 ```
 
 ### Streaming
+
 Streaming is exposed through the online OpenAI Speech API (`stream=true`). See [`examples/online_serving/text_to_speech/voxcpm2/gradio_demo.py`](../../online_serving/text_to_speech/voxcpm2/gradio_demo.py) for an AudioWorklet-based gapless streaming player; the offline `end2end.py` script does not expose a streaming path.
 
 ### Notes
+
 - Output: 48 kHz mono WAV.
 - Deploy config: `vllm_omni/deploy/voxcpm2.yaml` (auto-loaded by HF `model_type`).
 
 ---
 
-## dots.tts
-
-Single-stage native AR TTS at 48 kHz (rednote-hilab). Pipeline: `Qwen2.5-1.5B base LM → DiT (10-step Euler flow matching) → patch_encoder AR loopback → AudioVAE (streaming decode)`. Same "vLLM-native base LM + side-path computation" pattern as VoxCPM2, with a plain Qwen2 backbone instead of MiniCPM4 and no FSQ / residual LM stage.
-
-### Quick start
-```bash
-python examples/offline_inference/text_to_speech/dots_tts/end2end.py \
-    --model dots-studio/dots.tts-soar \
-    --text "Hello, this is a test of dots TTS running on vLLM Omni."
-```
-
-### Voice cloning
-Not wired in this release. Generation is text-only and does not consume reference audio, a named voice, or a precomputed speaker embedding. The CAM++ x-vector speaker encoder weights load, but `end2end.py` has no `--ref-audio`/`--ref-text` flags yet.
-
-### Streaming
-The AudioVAE decoder emits incremental audio through its internal streaming path (`init_stream_state` / `stream_step` / `stream_flush`). Online streaming is exposed through the OpenAI-compatible `/v1/audio/speech` endpoint with `stream=true`; see the [online serving guide](../../online_serving/text_to_speech/README.md#dotstts). The offline `end2end.py` script returns the consolidated waveform.
-
-### Notes
-- Output: 48 kHz mono WAV.
-- Deploy config: `vllm_omni/deploy/dots_tts.yaml` (auto-loaded by HF `model_type`).
-- Checkpoints: `dots-studio/dots.tts-soar` is the validated default. `dots.tts-base` shares the same architecture but is unvalidated in this repo. `dots.tts-mf` (MeanFlow, 2-4 step) is not supported yet.
-- Known limitation: no CUDA graph capture and no batched side-path computation yet, so concurrent requests do not currently scale (each request's DiT Euler steps run serially). See `recipes/rednote-hilab/dots.tts.md` for details and the roadmap.
-
----
-
 ## IndexTTS-2 and IndexTTS-2.5
 
 Both versions use a 2-stage TTS pipeline and produce 22.05 kHz mono audio. Stage 0 is a GPT AR talker. IndexTTS-2 Stage 1 uses RepCodec semantic embeddings, GPT latent, S2Mel CFM/DiT, and BigVGAN. IndexTTS-2.5 replaces RepCodec with EnhancedCodec and uses the official code-only path (`use_gpt_latent=false`). Every request requires reference audio for zero-shot voice cloning.
@@ -592,6 +630,7 @@ python examples/offline_inference/text_to_speech/indextts2/end2end.py \
 ```
 
 ### Emotion control
+
 ```bash
 # Emotion from reference audio
 python examples/offline_inference/text_to_speech/indextts2/end2end.py \
@@ -616,6 +655,7 @@ python examples/offline_inference/text_to_speech/indextts2/end2end.py \
 ```
 
 ### Notes
+
 - `--ref-audio` is **required** — neither version provides text-only synthesis.
 - IndexTTS-2.5 Stage 0 (AR Talker) generates mel codes from text + reference audio using plain vLLM sampling. It does not reproduce the official default `num_beams=3` beam search. For parity comparisons, run upstream with `num_beams=1`; output quality can differ from the official beam-search result.
 - IndexTTS-2.5 accepts language codes such as `zh`, `en`, `zhen` (mixed Chinese/English), `ja`, and `yue`. `Mandarin` is a vLLM-Omni convenience alias for `zh`. Japanese (`ja`) uses `fugashi` tokenization and produces audio, but it does not automatically expand numbers, dates, or percentages; write those inputs as readable Japanese text before calling the model. Use `--no-text-normalization` only when the input is already normalized.
@@ -631,12 +671,15 @@ python examples/offline_inference/text_to_speech/indextts2/end2end.py \
 Voxtral-4B-TTS (Mistral). Has its own argparse surface; uses voice presets and the `mistral_common` `SpeechRequest` protocol.
 
 ### Prerequisites
+
 Latest `mistral_common` with `SpeechRequest` support:
+
 ```bash
 pip install -e /path/to/mistral-common  # or upgrade from PyPI when available
 ```
 
 ### Quick start (voice preset)
+
 ```bash
 python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
     --write-audio --voice cheerful_female \
@@ -645,6 +688,7 @@ python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
 ```
 
 ### Voice cloning (capability gated upstream)
+
 ```bash
 python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
     --write-audio \
@@ -654,12 +698,14 @@ python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
 ```
 
 ### Streaming + concurrency
+
 ```bash
 python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
     --num-prompts 32 --concurrency 8 --streaming --write-audio --voice neutral_female \
     --model mistralai/Voxtral-4B-TTS-2603 \
     --text "..."
 ```
+
 Available voice presets are listed on the HF model card (`mistralai/Voxtral-4B-TTS-2603`).
 
 ### Notes

examples/offline_inference/text_to_speech/dots_tts/end2end.py

@@ -1,125 +0,0 @@
-"""Offline dots.tts inference example (single-stage native AR pipeline).
-
-Uses the single-stage native AR config (vllm_omni/deploy/dots_tts.yaml).
-Reference checkpoint: dots-studio/dots.tts-soar.
-"""
-
-from __future__ import annotations
-
-import os
-import time
-from pathlib import Path
-
-import soundfile as sf
-import torch
-
-from vllm_omni import Omni
-from vllm_omni.utils.tracking_parser import TrackingArgumentParser
-
-REPO_ROOT = Path(__file__).resolve().parents[4]
-SAMPLE_RATE = 48_000
-
-
-def parse_args():
-    parser = TrackingArgumentParser(description="Offline dots.tts native AR inference")
-    parser.add_argument(
-        "--model",
-        type=str,
-        default="dots-studio/dots.tts-soar",
-        help="dots.tts model path or HuggingFace repo ID.",
-    )
-    parser.add_argument(
-        "--text",
-        type=str,
-        default="Hello, this is a test of dots TTS running on vLLM Omni.",
-        help="Text to synthesize.",
-    )
-    parser.add_argument(
-        "--output-dir",
-        type=str,
-        default="output_audio",
-        help="Directory for output WAV files.",
-    )
-    parser.add_argument(
-        "--deploy-config",
-        type=str,
-        default=None,
-        help="Override the deploy config path. If unset, auto-loads "
-        "vllm_omni/deploy/dots_tts.yaml based on the HF model_type.",
-    )
-    return parser.parse_args()
-
-
-def extract_audio(multimodal_output: dict) -> torch.Tensor:
-    """Extract the final complete audio tensor from multimodal output.
-
-    The output processor concatenates per-step delta tensors under
-    ``model_outputs``.  Falls back to ``audio`` for backwards compat.
-    """
-    audio = multimodal_output.get("model_outputs")
-    if audio is None:
-        audio = multimodal_output.get("audio")
-    if audio is None:
-        raise ValueError(f"No audio key in multimodal_output: {list(multimodal_output.keys())}")
-
-    if isinstance(audio, list):
-        valid = [torch.as_tensor(a).float().cpu().reshape(-1) for a in audio if a is not None]
-        if not valid:
-            raise ValueError("Audio list is empty or all elements are None.")
-        return torch.cat(valid, dim=0) if len(valid) > 1 else valid[0]
-
-    return torch.as_tensor(audio).float().cpu().reshape(-1)
-
-
-def main():
-    args = parse_args()
-
-    output_dir = Path(args.output_dir)
-    output_dir.mkdir(parents=True, exist_ok=True)
-
-    engine = Omni(
-        model=args.model,
-        deploy_config=args.deploy_config,
-    )
-
-    from transformers import AutoTokenizer
-
-    from vllm_omni.model_executor.models.dots_tts.dots_tts_prompt import (
-        build_dots_tts_prompt,
-    )
-
-    tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
-
-    prompt = build_dots_tts_prompt(
-        tokenizer=tokenizer,
-        text=args.text,
-    )
-
-    print(f"Model       : {args.model}")
-    print(f"Text        : {args.text}")
-    print(f"Output dir  : {output_dir}")
-    print(f"Prompt len  : {len(prompt['prompt_token_ids'])} tokens")
-
-    t_start = time.perf_counter()
-    outputs = engine.generate([prompt])
-    elapsed = time.perf_counter() - t_start
-
-    request_output = outputs[0]
-    mm = request_output.outputs[0].multimodal_output
-    audio = extract_audio(mm)
-
-    duration = audio.numel() / SAMPLE_RATE
-    rtf = elapsed / duration if duration > 0 else float("inf")
-
-    output_path = output_dir / "output.wav"
-    sf.write(str(output_path), audio.numpy(), SAMPLE_RATE, format="WAV")
-
-    print(f"Saved       : {output_path}")
-    print(f"Duration    : {duration:.2f}s")
-    print(f"Inference   : {elapsed:.2f}s")
-    print(f"RTF         : {rtf:.3f}")
-
-
-if __name__ == "__main__":
-    os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-    main()

examples/online_serving/dynin_omni/README.md

@@ -1,97 +0,0 @@
-# Dynin-Omni: Online serving
-
-## Installation
-
-Please refer to [README.md](../../../README.md).
-
-## Launch the Server
-
-First, find the `transformers_modules` path:
-
-```bash
-python - <<'PY'
-from transformers.utils.hub import HF_MODULES_CACHE
-print(HF_MODULES_CACHE)
-PY
-```
-
-Then export it for both `PYTHONPATH` and `HF_MODULES_CACHE`:
-
-```bash
-export PYTHONPATH=<transformers_modules_path>:$PYTHONPATH
-export HF_MODULES_CACHE=<transformers_modules_path>
-```
-
-Run from repository root. The 3-stage pipeline is resolved automatically from
-the model type (`vllm_omni/deploy/dynin_omni.yaml`), so no config path is
-required. Pass `--deploy-config` only to override the defaults:
-
-```bash
-vllm-omni serve snu-aidas/Dynin-Omni \
-  --omni \
-  --port 8091
-```
-
-If `vllm-omni` is not in PATH, run:
-
-```bash
-PYTHONPATH="$(pwd)" python -m vllm_omni.entrypoints.cli.main serve snu-aidas/Dynin-Omni \
-  --omni \
-  --port 8091
-```
-
-Wait until the server logs show both `All stages initialized successfully` and
-`Application startup complete.` before sending requests.
-
-## Send Requests via Python Client
-
-Move to the example directory:
-
-```bash
-cd examples/online_serving/dynin_omni
-```
-
-### Text -> Image
-
-```bash
-python openai_chat_completion_client_for_multimodal_generation.py \
-  --query-type t2i \
-  --prompt "A realistic indoor living room with natural daylight."
-```
-
-### Image -> Image
-
-```bash
-python openai_chat_completion_client_for_multimodal_generation.py \
-  --query-type i2i \
-  --image-path ../../offline_inference/dynin_omni/data/image/sofa_under_water.jpg \
-  --prompt "Transform this surreal underwater setting into a realistic indoor living room while preserving the sofa layout."
-```
-
-### Text -> Speech
-
-```bash
-python openai_chat_completion_client_for_multimodal_generation.py \
-  --query-type t2s \
-  --prompt "Hello. This is Dynin-omni."
-```
-
-## CLI Arguments
-
-- `--query-type` (`t2i|t2s|i2i`)
-- `--model` (default: `snu-aidas/Dynin-Omni`)
-- `--host` / `--port` (OpenAI-compatible vLLM endpoint)
-- `--prompt` (custom text)
-- `--image-path` (required for `i2i`)
-- `--modalities` (optional output modalities override)
-- `--output-dir` (default: `/tmp/dynin_online_outputs`)
-
-## Notes
-
-- This client currently supports only `t2i`, `t2s`, and `i2i`.
-- `t2t` is intentionally not exposed in this online example.
-- This example intentionally uses the OpenAI-compatible chat completion endpoint.
-- Task routing for non-text outputs relies on Dynin task trigger tokens (`<|t2i|>`, `<|i2i|>`, `<|t2s|>`) injected by the client.
-- Outputs are saved under `/tmp/dynin_online_outputs` by default.
-- Dynin stage-0 warmup can take a while on first startup; do not send requests before startup completes.
-- Dynin itself can execute text-returning tasks such as `t2t`, `s2t`, `i2t`, and `v2t`, but this online serving example currently runs stage-0 in `generation` mode. In that path, the generation worker does not surface the final text as `output.text`, so OpenAI chat responses for those text-output tasks may complete internally but still return empty text.

examples/online_serving/dynin_omni/openai_chat_completion_client_for_multimodal_generation.py

@@ -1,342 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import annotations
-
-import argparse
-import base64
-import json
-import mimetypes
-import os
-import time
-from pathlib import Path
-from typing import Any
-
-DEFAULT_MODEL = "snu-aidas/Dynin-Omni"
-DEFAULT_OUTPUT_DIR = "/tmp/dynin_online_outputs"
-
-QUERY_CHOICES = ("t2i", "t2s", "i2i")
-DEFAULT_PROMPT_BY_QUERY = {
-    "t2i": "A high quality detailed living room interior photo.",
-    "t2s": "Please read this sentence naturally: Hello from Dynin-Omni online serving.",
-    "i2i": "Transform this image into a realistic indoor living room while preserving layout.",
-}
-DEFAULT_MODALITIES_BY_QUERY = {
-    "t2i": ["image"],
-    "t2s": ["audio"],
-    "i2i": ["image"],
-}
-OFFLINE_PARITY_STAGE_COUNT = 3
-OFFLINE_PARITY_STAGE_SAMPLING = {
-    "max_tokens": 1,
-    "temperature": 0.0,
-    "top_p": 1.0,
-    "detokenize": False,
-}
-
-
-def _infer_mime_type(path: Path) -> str:
-    mime_type, _ = mimetypes.guess_type(str(path))
-    return mime_type or "application/octet-stream"
-
-
-def _encode_file_as_data_url(path: Path) -> str:
-    mime_type = _infer_mime_type(path)
-    raw = path.read_bytes()
-    encoded = base64.b64encode(raw).decode("utf-8")
-    return f"data:{mime_type};base64,{encoded}"
-
-
-def _to_image_url(path_or_url: str) -> str:
-    value = str(path_or_url)
-    if value.startswith(("http://", "https://", "data:image/")):
-        return value
-    path = Path(value).expanduser().resolve()
-    if not path.exists():
-        raise FileNotFoundError(f"Image file not found: {path}")
-    return _encode_file_as_data_url(path)
-
-
-def _build_user_content(query_type: str, prompt: str, image_path: str | None) -> list[dict[str, Any]]:
-    if query_type == "t2i":
-        return [{"type": "text", "text": f"<|t2i|> {prompt}"}]
-
-    if query_type == "t2s":
-        return [{"type": "text", "text": f"<|t2s|> {prompt}"}]
-
-    if query_type == "i2i":
-        if not image_path:
-            raise ValueError("--image-path is required for query type i2i")
-        return [
-            {"type": "text", "text": f"<|i2i|> {prompt}"},
-            {"type": "image_url", "image_url": {"url": _to_image_url(image_path)}},
-        ]
-
-    raise ValueError(f"Unsupported query_type: {query_type}")
-
-
-def _collect_text_from_content(content: Any) -> list[str]:
-    texts: list[str] = []
-    if isinstance(content, str):
-        stripped = content.strip()
-        if stripped:
-            texts.append(stripped)
-        return texts
-
-    if isinstance(content, dict):
-        for key in ("text", "content", "value", "output_text"):
-            text_value = content.get(key)
-            if isinstance(text_value, str) and text_value.strip():
-                texts.append(text_value.strip())
-        return texts
-
-    if isinstance(content, list):
-        for item in content:
-            texts.extend(_collect_text_from_content(item))
-        return texts
-
-    content_text = getattr(content, "text", None)
-    if isinstance(content_text, str) and content_text.strip():
-        texts.append(content_text.strip())
-    content_value = getattr(content, "content", None)
-    if isinstance(content_value, str) and content_value.strip():
-        texts.append(content_value.strip())
-    output_text = getattr(content, "output_text", None)
-    if isinstance(output_text, str) and output_text.strip():
-        texts.append(output_text.strip())
-    return texts
-
-
-def _extract_text_outputs(chat_completion: Any) -> list[str]:
-    texts: list[str] = []
-    for choice in getattr(chat_completion, "choices", []) or []:
-        message = getattr(choice, "message", None)
-        if message is None:
-            continue
-        content = getattr(message, "content", None)
-        texts.extend(_collect_text_from_content(content))
-        reasoning_content = getattr(message, "reasoning_content", None)
-        if isinstance(reasoning_content, str) and reasoning_content.strip():
-            texts.append(reasoning_content.strip())
-        choice_text = getattr(choice, "text", None)
-        if isinstance(choice_text, str) and choice_text.strip():
-            texts.append(choice_text.strip())
-    top_level_output_text = getattr(chat_completion, "output_text", None)
-    if isinstance(top_level_output_text, str) and top_level_output_text.strip():
-        texts.append(top_level_output_text.strip())
-    return texts
-
-
-def _extract_image_data_urls(chat_completion: Any) -> list[str]:
-    urls: list[str] = []
-    for choice in getattr(chat_completion, "choices", []) or []:
-        message = getattr(choice, "message", None)
-        if message is None:
-            continue
-        content = getattr(message, "content", None)
-        if not isinstance(content, list):
-            continue
-        for item in content:
-            if not isinstance(item, dict):
-                continue
-            if item.get("type") != "image_url":
-                continue
-            image_url = (item.get("image_url") or {}).get("url")
-            if isinstance(image_url, str) and image_url.startswith("data:image"):
-                urls.append(image_url)
-    return urls
-
-
-def _extract_audio_payloads(chat_completion: Any) -> list[bytes]:
-    payloads: list[bytes] = []
-    for choice in getattr(chat_completion, "choices", []) or []:
-        message = getattr(choice, "message", None)
-        if message is None:
-            continue
-        message_audio = getattr(message, "audio", None)
-        if message_audio is None:
-            continue
-        data_b64 = getattr(message_audio, "data", None)
-        if isinstance(data_b64, str) and data_b64:
-            try:
-                payloads.append(base64.b64decode(data_b64))
-            except Exception:
-                continue
-    return payloads
-
-
-def _decode_data_url(data_url: str) -> tuple[bytes, str]:
-    header, data = data_url.split(",", 1)
-    mime_type = "image/png"
-    if ";" in header and ":" in header:
-        mime_type = header.split(":", 1)[1].split(";", 1)[0]
-    return base64.b64decode(data), mime_type
-
-
-def _image_extension_from_mime(mime_type: str) -> str:
-    if mime_type == "image/jpeg":
-        return ".jpg"
-    if mime_type == "image/webp":
-        return ".webp"
-    if mime_type == "image/gif":
-        return ".gif"
-    return ".png"
-
-
-def _save_outputs(
-    *,
-    query_type: str,
-    chat_completion: Any,
-    output_dir: Path,
-) -> None:
-    output_dir.mkdir(parents=True, exist_ok=True)
-    stamp = time.strftime("%Y%m%d_%H%M%S")
-
-    text_outputs = _extract_text_outputs(chat_completion)
-    image_data_urls = _extract_image_data_urls(chat_completion)
-    audio_payloads = _extract_audio_payloads(chat_completion)
-
-    if text_outputs:
-        text_path = output_dir / f"{query_type}_{stamp}.txt"
-        text_path.write_text("\n\n".join(text_outputs) + "\n", encoding="utf-8")
-        print(f"[dynin-online] text saved: {text_path}")
-        print(text_outputs[0])
-
-    for idx, image_url in enumerate(image_data_urls):
-        image_bytes, mime_type = _decode_data_url(image_url)
-        ext = _image_extension_from_mime(mime_type)
-        image_path = output_dir / f"{query_type}_{stamp}_{idx}{ext}"
-        image_path.write_bytes(image_bytes)
-        print(f"[dynin-online] image saved: {image_path}")
-
-    for idx, audio_bytes in enumerate(audio_payloads):
-        audio_path = output_dir / f"{query_type}_{stamp}_{idx}.wav"
-        audio_path.write_bytes(audio_bytes)
-        print(f"[dynin-online] audio saved: {audio_path}")
-
-    if not text_outputs and not image_data_urls and not audio_payloads:
-        print("[dynin-online] no output extracted from response")
-        raw_path = output_dir / f"{query_type}_{stamp}_raw_response.json"
-        try:
-            if hasattr(chat_completion, "model_dump_json"):
-                serialized = chat_completion.model_dump_json(indent=2)
-            else:
-                if hasattr(chat_completion, "model_dump"):
-                    raw_payload: Any = chat_completion.model_dump(mode="json")
-                else:
-                    raw_payload = chat_completion
-                try:
-                    serialized = json.dumps(raw_payload, ensure_ascii=False, indent=2)
-                except Exception:
-                    serialized = json.dumps({"repr": repr(raw_payload)}, ensure_ascii=False, indent=2)
-            raw_path.write_text(serialized + "\n", encoding="utf-8")
-            print(f"[dynin-online] raw response saved: {raw_path}")
-        except Exception:
-            pass
-
-
-def _build_offline_parity_sampling_params_list() -> list[dict[str, Any]]:
-    return [dict(OFFLINE_PARITY_STAGE_SAMPLING) for _ in range(OFFLINE_PARITY_STAGE_COUNT)]
-
-
-def run_request(args: argparse.Namespace) -> None:
-    from openai import OpenAI
-
-    client = OpenAI(
-        api_key="EMPTY",
-        base_url=f"http://{args.host}:{args.port}/v1",
-    )
-    prompt = args.prompt.strip() if args.prompt else DEFAULT_PROMPT_BY_QUERY[args.query_type]
-    user_content = _build_user_content(
-        query_type=args.query_type,
-        prompt=prompt,
-        image_path=args.image_path,
-    )
-    if args.modalities:
-        modalities = [item.strip() for item in args.modalities.split(",") if item.strip()]
-    else:
-        modalities = DEFAULT_MODALITIES_BY_QUERY[args.query_type]
-
-    extra_body = {
-        "sampling_params_list": _build_offline_parity_sampling_params_list(),
-    }
-    chat_completion = client.chat.completions.create(
-        model=args.model,
-        messages=[{"role": "user", "content": user_content}],
-        modalities=modalities,
-        extra_body=extra_body,
-    )
-    _save_outputs(
-        query_type=args.query_type,
-        chat_completion=chat_completion,
-        output_dir=Path(args.output_dir).expanduser(),
-    )
-
-
-def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Dynin-Omni online chat completion client")
-    parser.add_argument(
-        "--query-type",
-        "-q",
-        type=str,
-        default="t2i",
-        choices=QUERY_CHOICES,
-        help="Dynin query type",
-    )
-    parser.add_argument(
-        "--model",
-        "-m",
-        type=str,
-        default=DEFAULT_MODEL,
-        help="Model name/path",
-    )
-    parser.add_argument(
-        "--host",
-        type=str,
-        default="localhost",
-        help="Host/IP of the vLLM Omni API server",
-    )
-    parser.add_argument(
-        "--port",
-        type=int,
-        default=8091,
-        help="Port of the vLLM Omni API server",
-    )
-    parser.add_argument(
-        "--prompt",
-        "-p",
-        type=str,
-        default="",
-        help="Custom prompt text",
-    )
-    parser.add_argument(
-        "--image-path",
-        "-i",
-        type=str,
-        default=None,
-        help="Image path/URL for i2i",
-    )
-    parser.add_argument(
-        "--modalities",
-        type=str,
-        default="",
-        help="Comma-separated output modalities override (e.g., text,image,audio)",
-    )
-    parser.add_argument(
-        "--output-dir",
-        "-o",
-        type=str,
-        default=DEFAULT_OUTPUT_DIR,
-        help="Directory to save outputs",
-    )
-    return parser.parse_args()
-
-
-def main() -> None:
-    args = parse_args()
-    os.environ.setdefault("HF_HUB_DISABLE_XET", "1")
-    run_request(args)
-
-
-if __name__ == "__main__":
-    main()

examples/online_serving/minicpmo/README.md

@@ -316,3 +316,12 @@ them unless `--allow-invalid-clock` is explicit.
   [`examples/offline_inference/minicpmo/`](../../offline_inference/minicpmo/)
 - Recipe:
   [`recipes/OpenBMB/MiniCPM-o-4_5.md`](../../../recipes/OpenBMB/MiniCPM-o-4_5.md)
+
+### Shared realtime UI implementation
+
+The `python -m examples.online_serving.minicpmo.realtime_web` command remains
+available with the same MiniCPM defaults and required `--ref-audio`. Its assets
+now live in [the shared realtime UI](../realtime_web/README.md), with a dedicated
+`minicpm-native` profile preserving native duplex input, playback ACKs and camera
+frames. Qwen3 uses a separate profile in the same shell, supporting manual STT
+turns or Server VAD turns with speech interruption.

examples/online_serving/minicpmo/realtime_web/server.py

@@ -1,185 +1,23 @@
-"""Static host and same-origin Realtime proxy for the full-duplex demo."""
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
-from __future__ import annotations
+"""Backward-compatible MiniCPM entry point for the shared realtime UI."""
 
-import argparse
-import asyncio
-import base64
-import contextlib
-import hashlib
-import json
-import logging
-from pathlib import Path
-from urllib.parse import urlencode
+from examples.online_serving.realtime_web.server import (
+    APP_DIR,
+    STATIC_DIR,
+    _join_ws_url,
+    build_app,
+)
+from examples.online_serving.realtime_web.server import (
+    main as _main,
+)
 
-import uvicorn
-import websockets
-from fastapi import FastAPI, WebSocket, WebSocketDisconnect
-from fastapi.responses import HTMLResponse, Response
-from fastapi.staticfiles import StaticFiles
-
-logger = logging.getLogger(__name__)
-APP_DIR = Path(__file__).parent / "app"
-STATIC_DIR = APP_DIR / "static"
-
-
-def _join_ws_url(base: str, path: str, query: str) -> str:
-    return base.rstrip("/") + path + (("?" + query) if query else "")
-
-
-async def _pump_client_to_backend(client: WebSocket, backend) -> None:
-    try:
-        while True:
-            message = await client.receive()
-            if message["type"] == "websocket.disconnect":
-                await backend.close()
-                return
-            if message.get("text") is not None:
-                await backend.send(message["text"])
-            elif message.get("bytes") is not None:
-                await backend.send(message["bytes"])
-    except (WebSocketDisconnect, websockets.ConnectionClosed):
-        with contextlib.suppress(Exception):
-            await backend.close()
-
-
-async def _pump_backend_to_client(client: WebSocket, backend) -> None:
-    try:
-        async for message in backend:
-            if isinstance(message, bytes):
-                await client.send_bytes(message)
-            else:
-                await client.send_text(message)
-    except (WebSocketDisconnect, websockets.ConnectionClosed):
-        return
-    except RuntimeError as exc:
-        if "websocket.send" in str(exc) and "websocket.close" in str(exc):
-            return
-        raise
-
-
-def _expected_proxy_close(exc: BaseException) -> bool:
-    if isinstance(exc, (WebSocketDisconnect, websockets.ConnectionClosed, asyncio.CancelledError)):
-        return True
-    return isinstance(exc, RuntimeError) and "websocket.send" in str(exc) and "websocket.close" in str(exc)
-
-
-def build_app(
-    *,
-    ws_backend: str = "ws://127.0.0.1:8099",
-    model: str = "openbmb/MiniCPM-o-4_5",
-    public_realtime_url: str | None = None,
-    ref_audio: str | None = None,
-) -> FastAPI:
-    app = FastAPI(title="Experimental Full-Duplex Web Demo")
-    index_path = APP_DIR / "index.html"
-    app_version_hash = hashlib.sha256()
-    for asset_path in (
-        STATIC_DIR / "app.js",
-        STATIC_DIR / "pcm_worklet.js",
-        STATIC_DIR / "playback_worklet.js",
-    ):
-        app_version_hash.update(asset_path.read_bytes())
-    app_version = app_version_hash.hexdigest()[:12]
-
-    ref_audio_uri: str | None = None
-    if ref_audio:
-        ref_path = Path(ref_audio)
-        if not ref_path.is_file():
-            raise SystemExit(f"--ref-audio not found: {ref_audio}")
-        encoded = base64.b64encode(ref_path.read_bytes()).decode("ascii")
-        ref_audio_uri = f"data:audio/wav;base64,{encoded}"
-
-    @app.get("/", response_class=HTMLResponse)
-    def index() -> HTMLResponse:
-        config = json.dumps(
-            {
-                "model": model,
-                "realtimePath": public_realtime_url or "v1/realtime",
-                "refAudio": ref_audio_uri,
-                "appVersion": app_version,
-            },
-            ensure_ascii=True,
-        )
-        html = (
-            index_path.read_text(encoding="utf-8")
-            .replace("__FULL_DUPLEX_CONFIG__", config)
-            .replace("__FULL_DUPLEX_APP_VERSION__", app_version)
-        )
-        return HTMLResponse(html, headers={"Cache-Control": "no-store"})
-
-    @app.get("/healthz")
-    def healthz() -> Response:
-        return Response(content="ok", media_type="text/plain")
-
-    @app.websocket("/v1/realtime")
-    async def realtime_proxy(websocket: WebSocket) -> None:
-        await websocket.accept()
-        query = urlencode(websocket.query_params.multi_items())
-        backend_url = _join_ws_url(ws_backend, "/v1/realtime", query)
-        logger.info("Proxying Realtime WebSocket to %s", backend_url)
-        try:
-            async with websockets.connect(
-                backend_url,
-                max_size=64 * 1024 * 1024,
-            ) as backend:
-                tasks = {
-                    asyncio.create_task(_pump_client_to_backend(websocket, backend)),
-                    asyncio.create_task(_pump_backend_to_client(websocket, backend)),
-                }
-                done, pending = await asyncio.wait(
-                    tasks,
-                    return_when=asyncio.FIRST_COMPLETED,
-                )
-                for task in pending:
-                    task.cancel()
-                await asyncio.gather(*pending, return_exceptions=True)
-                for result in await asyncio.gather(*done, return_exceptions=True):
-                    if isinstance(result, BaseException) and not _expected_proxy_close(result):
-                        raise result
-        except (WebSocketDisconnect, websockets.ConnectionClosed):
-            return
-        except Exception:
-            logger.exception("Realtime WebSocket proxy failed")
-            with contextlib.suppress(Exception):
-                await websocket.close(code=1011)
-
-    app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
-    return app
+__all__ = ["APP_DIR", "STATIC_DIR", "_join_ws_url", "build_app", "main"]
 
 
 def main() -> None:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--host", default="0.0.0.0")
-    parser.add_argument("--port", type=int, default=7862)
-    parser.add_argument("--ws-backend", default="ws://127.0.0.1:8099")
-    parser.add_argument(
-        "--public-realtime-url",
-        help="Browser-visible ws:// or wss:// Realtime URL; defaults to the same-origin proxy.",
-    )
-    parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
-    parser.add_argument(
-        "--ref-audio",
-        required=True,
-        help=(
-            "Reference voice wav for TTS voice cloning, e.g. the "
-            "official MiniCPM-o-Demo assets/ref_audio/ref_minicpm_signature.wav"
-        ),
-    )
-    args = parser.parse_args()
-
-    logging.basicConfig(level=logging.INFO)
-    uvicorn.run(
-        build_app(
-            ws_backend=args.ws_backend,
-            model=args.model,
-            public_realtime_url=args.public_realtime_url,
-            ref_audio=args.ref_audio,
-        ),
-        host=args.host,
-        port=args.port,
-        log_level="info",
-    )
+    _main(default_profile="minicpm-native")
 
 
 if __name__ == "__main__":

examples/online_serving/qwen3_omni/README.md

@@ -320,24 +320,17 @@ python openai_realtime_client.py \
 Server VAD uses the pinned Silero v6.2 ONNX artifact from `istupakov/silero-vad-onnx` and never downloads model files while processing a session. Make the artifact available in the Hugging Face cache before startup, or configure a local artifact in the deployment YAML:
 
 ```yaml
+base_config: /path/to/vllm-omni/vllm_omni/deploy/qwen3_omni_duplex.yaml
 duplex_session:
   server_vad_model_path: /models/silero_vad.onnx
 ```
 
-Start from the bundled Qwen3-Omni deployment YAML, add the fields above at the top level, and serve that complete configuration:
-
-```bash
-cp vllm_omni/deploy/qwen3_omni_moe.yaml /path/to/qwen3_omni_server_vad.yaml
-# Edit /path/to/qwen3_omni_server_vad.yaml and add duplex_session.
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \
-  --omni \
-  --port 8091 \
-  --deploy-config /path/to/qwen3_omni_server_vad.yaml
-```
-
-Keep Qwen's default `session_mode: turn`; `duplex_session` enables the Realtime handler without changing scheduler
-semantics. Bare `/v1/realtime` selects that handler. The client uses `?duplex=0` only for the existing non-Server-VAD
-wire flow; `?duplex=1` remains a supported compatibility alias.
+For the engine-owned plugin, use an overlay based on
+`vllm_omni/deploy/qwen3_omni_duplex.yaml`, with `duplex_session.server_vad_model_path`
+pointing to your local artifact. See the [complete duplex setup](../realtime_web/README.md#qwen3-with-server-vad-automatic-turns).
+The default `qwen3_omni_moe.yaml` remains a turn deployment. Duplex mode serves
+`/v1/realtime?duplex=1` and `/v1/chat/completions`; the legacy STT handler requires
+a turn deployment.
 
 The Python client supports the following command-line arguments:
 
@@ -584,3 +577,53 @@ The gradio script supports the following arguments:
 - `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
 - `--port`: Port for Gradio server (default: 7861)
 - `--share`: Share the Gradio demo publicly (creates a public link)
+
+## Browser voice call (shared realtime UI)
+
+Run these commands from the repository root with the vLLM-Omni environment activated.
+
+### Without VAD: manually submit each turn
+
+Start the backend:
+
+```bash
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
+```
+
+In another terminal, check backend readiness and start the UI:
+
+```bash
+curl --fail http://127.0.0.1:8091/health
+python -m examples.online_serving.qwen3_omni.realtime_web \
+    --backend ws://127.0.0.1:8091 --stt --port 7863
+```
+
+Open `http://localhost:7863`, start a session, speak, and press **Send turn**.
+`--stt` is the default. Each turn uses a separate connection without shared model
+history; the conversation shown in the browser is a local log.
+
+### With VAD: automatically submit after silence
+
+Start the backend with a Server VAD deployment overlay and a compatible Silero
+ONNX artifact, following the
+[complete VAD setup](../realtime_web/README.md#qwen3-with-server-vad-automatic-turns).
+After its health check succeeds, start the UI:
+
+```bash
+python -m examples.online_serving.qwen3_omni.realtime_web \
+    --backend ws://127.0.0.1:8091 --vad --port 7863
+```
+
+Speak and pause for 500 ms to submit automatically. `--vad` configures the client;
+it does not load a VAD model or change the backend deployment by itself.
+
+VAD mode keeps uploading audio during replies and supports speech interruption.
+The engine-owned Qwen duplex plugin maintains conversation history and tracks
+played replies through playback acknowledgements. Each committed utterance starts
+a new generation request; the model does not use native streaming-input KV decoding.
+STT mode pauses microphone upload while the reply is generated and played.
+In VAD mode, click **Camera** to upload sampled frames with your spoken question;
+see [camera setup and limits](../realtime_web/README.md#qwen-vad-camera-input).
+STT remains audio-only. To switch back to manual turns, use the default turn
+deployment, restart the UI with `--stt`, refresh the page, and reconnect. Stop the existing UI before reusing port 7863.
+See the [shared UI guide](../realtime_web/README.md) for HTTPS access and testing.

examples/online_serving/qwen3_omni/openai_realtime_client.py

@@ -130,7 +130,7 @@ async def run_client(
                                 "prefix_padding_ms": 300,
                                 "silence_duration_ms": 500,
                                 "create_response": True,
-                                "interrupt_response": False,
+                                "interrupt_response": True,
                             },
                         }
                     },
@@ -233,6 +233,7 @@ async def run_client(
                 continue
 
             if event_type == "response.done" and server_vad:
+                await ws.send(json.dumps({"type": "session.close"}))
                 break
 
             if event_type == "error":

examples/online_serving/qwen3_omni/realtime_web/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Qwen3-Omni turn-based browser call UI."""

examples/online_serving/qwen3_omni/realtime_web/__main__.py

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+from examples.online_serving.realtime_web.server import main
+
+if __name__ == "__main__":
+    main(default_profile="qwen3-turn")

examples/online_serving/realtime_web/README.md

@@ -0,0 +1,221 @@
+# Shared Realtime Web UI
+
+One browser shell provides microphone capture, speaker playback, a level meter,
+conversation text, and an event log. Model profiles own the wire protocol.
+This implements [RFC #7222](https://github.com/vllm-project/vllm-omni/issues/7222).
+
+| Profile | Turn control | Camera | Playback acknowledgement |
+| --- | --- | --- | --- |
+| `minicpm-native` | Model-controlled listen/speak, continuous audio input | Frames accompany audio | Yes |
+| `qwen3-turn --stt` (default) | User presses **Send turn** | No | No |
+| `qwen3-turn --vad` | Server detects trailing silence; speech interrupts replies | Sampled frames with each spoken turn | Yes |
+
+Qwen3 VAD uses an engine-owned duplex plugin: microphone upload continues
+while replies stream, and speech can interrupt generation and playback. The
+checkpoint still generates committed turns; this is not native streaming-input
+KV decoding. STT remains explicit-turn and audio-only.
+
+## MiniCPM: existing command stays valid
+
+Start your MiniCPM backend using the [existing deployment instructions](../minicpmo/README.md), then:
+
+```bash
+python -m examples.online_serving.minicpmo.realtime_web \
+    --ws-backend ws://127.0.0.1:8099 \
+    --ref-audio /path/to/ref_minicpm_signature.wav
+```
+
+The compatibility wrapper serves the shared assets with `minicpm-native`.
+The current native query, `extra_body`, reference voice, continuous microphone
+upload, camera frames and playback acknowledgements are retained.
+
+## Qwen3: explicit-turn STT adapter
+
+Start the backend, then the UI in a second terminal:
+
+```bash
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
+
+python -m examples.online_serving.qwen3_omni.realtime_web \
+    --backend ws://127.0.0.1:8091 --stt --port 7863
+```
+
+Open `http://localhost:7863`, start a session, speak, and press **Send turn**.
+Wait for the answer to finish playing before speaking again. The UI opens a
+fresh STT connection for each turn; the displayed conversation is a local log,
+and **previous turns are not provided as model history**. The STT endpoint
+accepts only model selection, so system-prompt controls are hidden in this mode.
+
+The adapter follows the shipped `qwen3_omni/openai_realtime_client.py` and
+`vllm_omni/entrypoints/openai/realtime_connection.py`:
+
+1. Explicit `duplex=0` selects the legacy STT handler on a turn deployment.
+   It does not enable STT on a duplex deployment.
+2. Send `{type: "session.update", model: ...}` and `commit(final=false)`.
+3. Stream mono PCM16 at 16 kHz; **Send turn** flushes buffered audio, then sends `commit(final=true)`.
+4. `response.output_audio.delta.audio` contains PCM; `response.output_audio.done`
+   terminates this adapter, including a response with no audio.
+5. The local implementation emits **model output** on `transcription.*`, so the
+   UI displays it as assistant text. This differs from the early RFC's suggested
+   user-transcript mapping. The VAD adapter also supports `response.output_text.*`
+   and audio-transcript events without duplicating the answer.
+
+## Qwen3 with Server VAD (automatic turns)
+
+Run commands from the repository root with the vLLM-Omni environment activated.
+Prepare the pinned Silero v6.2 ONNX artifact as described in the
+[Qwen3 Server VAD instructions](../qwen3_omni/README.md#realtime-websocket-client-openai_realtime_clientpy).
+The backend requires ONNX Runtime and a compatible local artifact; setting
+`--vad` on the UI alone does not enable VAD on the backend.
+
+Create an overlay named `qwen3_vad.yaml`, replacing both paths with absolute paths
+on the backend host:
+
+```yaml
+base_config: /path/to/vllm-omni/vllm_omni/deploy/qwen3_omni_duplex.yaml
+duplex_session:
+  server_vad_model_path: /path/to/silero_vad.onnx
+```
+
+The base configuration enables duplex mode and sets the per-prompt audio and image limits.
+
+Start the backend:
+
+```bash
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
+    --deploy-config qwen3_vad.yaml
+```
+
+In a second terminal, wait until the health check succeeds, then start the UI:
+
+```bash
+curl --fail http://127.0.0.1:8091/health
+python -m examples.online_serving.qwen3_omni.realtime_web \
+    --backend ws://127.0.0.1:8091 --vad --port 7863
+```
+
+Open `http://localhost:7863` and start a session. Speak and pause: the server
+commits the turn automatically after 500 ms of silence. No **Send turn** action
+is needed. Defaults are a speech threshold of 0.5 and 300 ms of prefix padding.
+Speak again to interrupt an answer. The engine owns input buffering, VAD,
+request cancellation, playback acknowledgements, and conversation history.
+Each committed utterance starts a normal Thinker → Talker → Code2Wav request;
+it does not append input to a live Qwen KV cache. The capability response reports
+`implementation_level: turn_based_duplex` and `supports_core_resumable_request: false`.
+
+The plugin retains up to four audio inputs (including the current turn), pruning
+older audio at an 8 MiB base64 budget; one current utterance is bounded by the
+engine's pending-input limit. The UI reports playback progress both on normal
+drain and on interruption. Qwen does not provide text/audio alignment, so assistant
+text enters history only after the complete audio response has been generated and
+acknowledged as played. A partially played answer contributes no guessed text
+prefix. Image conversation items remain available until deleted. Model context limits still apply.
+The bundled duplex deploy allows four audio inputs and eight images per prompt.
+Use headphones to avoid speaker audio triggering VAD interruption. Reference
+voices and tool calls are not supported by this plugin.
+
+### Switching between VAD and manual turns
+
+| Mode | Backend | UI flag | Submit a turn |
+| --- | --- | --- | --- |
+| Without VAD | Default Qwen turn deployment | `--stt` (default) | Press **Send turn** |
+| With VAD | Server VAD configuration and Silero artifact | `--vad` | Pause after speaking |
+
+Stop the existing UI process before starting another on the same port, then
+refresh the browser and reconnect. To use STT, start a backend with the default
+turn deployment and restart the UI with `--stt`. The duplex backend exposes
+Realtime duplex and ChatCompletion; it does not expose the legacy STT handler.
+To enable VAD on a default backend, first restart that backend with the overlay.
+Make `--backend` match the actual backend port; the UI and backend use separate
+ports. `Connection refused` means the target backend is unavailable: check its
+startup log and `/health` before connecting.
+
+VAD connects with `duplex=1` and sends nested `session.audio.input` configuration
+with `create_response: true` and `interrupt_response: true`. It sends playback
+ACKs when speaker audio drains. `response.audio.done` / `response.output_audio.done`
+only drain playback; `response.done` terminates a turn. Backpressure displays a
+notice without pausing the microphone. Failed handshakes time out after 15 seconds;
+Silero initialization failures display a specific error.
+
+After updating the backend Python code, restart the backend, refresh the UI, and
+start a new session so the plugin and browser assets use the updated code. A
+successful `/health` check confirms readiness, not which deployment mode is active.
+
+## Qwen VAD camera input
+
+After starting a session with `--vad`, click **Camera** and allow access. The UI
+sends one JPEG per second, resized to a longest side of 448 pixels, independently
+of microphone uploads (including while muted). Each frame is a user conversation
+item using the [OpenAI Realtime image input](https://developers.openai.com/api/docs/guides/realtime-conversations#image-inputs) shape:
+
+```json
+{"type":"conversation.item.create","item":{"id":"camera_1","type":"message","role":"user","content":[{"type":"input_image","image_url":"data:image/jpeg;base64,..."}]}}
+```
+
+Images become conversation context immediately; they do not start a response.
+Speak and let VAD commit your audio, or send `response.create` after adding an
+image/text item. A manual `input_audio_buffer.commit` only commits audio; follow
+it with `response.create` to generate. Images persist across audio commits,
+interruptions, and audio-buffer clears. Remove them with `conversation.item.delete`.
+The browser retains eight camera items and deletes the oldest before adding a
+ninth. Camera-off stops capture but leaves those items in the conversation.
+
+This deployment accepts JPEG/PNG Base64 data URLs with a total image-context
+budget of eight images and 4 MiB; external image URLs are not supported. Clients
+must delete old image items when reaching the limit. There is no precise
+video/audio timestamp alignment or native streaming-video KV update.
+
+### Protocol compatibility boundary
+
+Qwen VAD uses `session.update`, 24 kHz PCM16 `input_audio_buffer.append`,
+`conversation.item.create` with `input_image`/`input_text`, and `response.create`.
+The engine resamples input to Qwen's 16 kHz processing rate. Qwen rejects the
+custom `video_frames` field on an append: it answers whole turns, so it has no
+audio unit for a frame track to align to. MiniCPM keeps that field for its
+native camera protocol, where Stage 0 interleaves frames at unit boundaries.
+
+This is a supported subset, not full OpenAI Realtime API compatibility.
+`duplex=1`, `overlap_policy`, `playback.ack`, and `session.close` remain vLLM-Omni
+extensions. The legacy `--stt` mode is unchanged and is not the standard Realtime
+conversation API. This demo does not add WebRTC, semantic VAD, or tool calling.
+
+## Shared host options
+
+The shared entry point can select either profile explicitly:
+
+```bash
+python -m examples.online_serving.realtime_web --profile qwen3-turn \
+    --backend ws://127.0.0.1:8091 --stt --port 7863
+```
+
+- `--backend` / `--ws-backend`: backend WebSocket origin (not the full `/v1/realtime` URL).
+- `--model`: override the wrapper's model name.
+- `--public-realtime-url`: optional browser-visible WebSocket URL; otherwise the
+  static host proxies `/v1/realtime` on the same origin.
+- `--host`, `--port`: UI bind address and port (default port 7862).
+- `--ref-audio`: required for MiniCPM; rejected for Qwen3.
+
+Microphone and camera access require `localhost` or HTTPS. For a remote backend,
+use an SSH tunnel to the UI host or an HTTPS reverse proxy with WebSocket support.
+An explicit `wss://` public URL retains its scheme.
+
+## Validation
+
+```bash
+node --test tests/examples/test_realtime_web_profiles.cjs
+pytest tests/examples/test_minicpmo_realtime_web_server.py \
+       tests/examples/test_minicpmo_realtime_web_static.py \
+       tests/examples/test_realtime_web_profiles.py
+pytest -m cpu tests/model_executor/models/qwen3_omni/test_duplex_plugin.py \
+       tests/engine/duplex/test_session_runner.py \
+       tests/engine/test_duplex_orchestrator.py
+```
+
+These CPU tests cover profile messages, two STT turns, VAD terminal/drain ordering,
+backpressure, native capture during playback, server config injection, and shared
+worklets. Backend regressions cover current audio inclusion in history-based
+prompts, delayed playback ACK ordering, and no extra generation during silence
+after a completed VAD turn. Hardware acceptance still requires a MiniCPM call
+with camera/barge-in and two Qwen turns against each enabled backend, including
+speech interruption in VAD mode; simulated tests do not
+establish model quality or hardware latency.

examples/online_serving/realtime_web/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Shared browser audio UI and model-specific realtime profiles."""

examples/online_serving/realtime_web/__main__.py

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+from .server import main
+
+if __name__ == "__main__":
+    main()

examples/online_serving/realtime_web/app/index.html

@@ -4,37 +4,35 @@
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <meta name="color-scheme" content="light">
-    <title>MiniCPM-o 4.5 Voice</title>
+    <title>Omni Realtime Voice</title>
     <link rel="stylesheet" href="static/styles.css">
   </head>
   <body>
     <header class="app-header">
       <div>
-        <p class="eyebrow">Experimental full-duplex runtime</p>
-        <h1>MiniCPM-o 4.5 Voice</h1>
+        <p id="pageEyebrow" class="eyebrow"></p>
+        <h1 id="pageTitle">Omni Realtime Voice</h1>
       </div>
       <span id="connectionState" class="status status-offline">Offline</span>
     </header>
 
     <main>
+      <p id="profileDescription" role="note"></p>
       <section class="call-bar" aria-label="Session controls">
         <div class="call-actions">
           <button id="callButton" class="primary" type="button">Start session</button>
+          <button id="sendTurnButton" class="secondary" type="button" hidden disabled>Send turn</button>
           <button id="muteButton" class="secondary" type="button" disabled>Mute</button>
           <button id="cameraButton" class="secondary" type="button" disabled>Camera</button>
           <video id="cameraPreview" muted playsinline style="display:none;width:96px;height:72px;border-radius:8px;object-fit:cover;background:#000"></video>
         </div>
-        <div style="display:flex;gap:8px;align-items:flex-start;margin-top:8px;width:100%">
+        <div id="promptControls" style="display:flex;gap:8px;align-items:flex-start;margin-top:8px;width:100%">
           <label style="display:inline-flex;flex-direction:column;gap:4px;font-size:13px;opacity:.85;white-space:nowrap">
             Prompt preset
             <select id="promptPreset" style="font-size:13px;padding:4px 6px;border-radius:6px">
-              <option value="omni" selected>Omni (official omni preset)</option>
-              <option value="chinese_call">中文通话 (official audio preset)</option>
-              <option value="english_call">English Call (official audio preset)</option>
-              <option value="custom">Custom</option>
             </select>
           </label>
-          <textarea id="systemPrompt" rows="2" placeholder="System prompt" style="flex:1;font-size:13px;padding:6px 8px;border-radius:6px;resize:vertical">Streaming Omni Conversation.</textarea>
+          <textarea id="systemPrompt" rows="2" placeholder="System prompt" style="flex:1;font-size:13px;padding:6px 8px;border-radius:6px;resize:vertical"></textarea>
         </div>
         <div class="meter" aria-label="Microphone level">
           <span>Mic</span>
@@ -53,7 +51,7 @@
             <p class="eyebrow">Live transcript</p>
             <h2 id="conversationTitle">Conversation</h2>
           </div>
-          <span class="policy-label">Model-controlled listen / speak</span>
+          <span id="policyLabel" class="policy-label"></span>
         </div>
         <div id="conversation" class="conversation" aria-live="polite">
           <p id="emptyConversation" class="empty-state">Start a session and speak naturally.</p>
@@ -73,7 +71,10 @@
       </details>
     </main>
 
-    <script>window.FULL_DUPLEX_CONFIG = __FULL_DUPLEX_CONFIG__;</script>
+    <script>window.OMNI_REALTIME_CONFIG = __FULL_DUPLEX_CONFIG__; window.FULL_DUPLEX_CONFIG = window.OMNI_REALTIME_CONFIG;</script>
+    <script src="static/profiles/common.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
+    <script src="static/profiles/minicpm_native.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
+    <script src="static/profiles/qwen3_turn.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
     <script src="static/app.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
   </body>
 </html>

examples/online_serving/realtime_web/app/static/app.js

@@ -1,8 +1,10 @@
 (() => {
   'use strict';
 
-  const config = window.FULL_DUPLEX_CONFIG || {};
+  const config = window.OMNI_REALTIME_CONFIG || window.FULL_DUPLEX_CONFIG || {};
+  const profile = window.OmniRealtimeProfiles[config.profile || 'minicpm-native'](config);
   const callButton = document.getElementById('callButton');
+  const sendTurnButton = document.getElementById('sendTurnButton');
   const muteButton = document.getElementById('muteButton');
   const cameraButton = document.getElementById('cameraButton');
   const cameraPreview = document.getElementById('cameraPreview');
@@ -27,20 +29,26 @@
   const INITIAL_PLAYBACK_BUFFER_MS = 400;
   const SESSION_CLOSE_TIMEOUT_MS = 1000;
 
-  // Default prompts mirroring the official MiniCPM-o-Demo presets
-  // (assets/presets/{omni,audio_duplex}/*.yaml).
-  const PROMPT_PRESETS = {
-    omni: 'Streaming Omni Conversation.',
-    chinese_call: '扮演一个具有以上声音特征的助手。请认真、高质量地回复用户的问题。'
-      + '请用高自然度的方式和用户聊天。你处于双工模式，可以一边听、一边说。'
-      + '你是由面壁智能开发的人工智能助手：面壁小钢炮。',
-    english_call: 'Replicate the tone and style from the input audio. Your task is to be '
-      + 'a helpful assistant using this voice pattern. Please answer the user\'s questions '
-      + 'seriously and in a high quality. Please chat with the user in a high naturalness '
-      + 'style. You are in duplex mode, where you can listen and speak at the same time.',
-  };
+  const PROMPT_PRESETS = profile.presets;
+  document.title = profile.title;
+  document.getElementById('pageTitle').textContent = profile.title;
+  document.getElementById('pageEyebrow').textContent = profile.eyebrow;
+  document.getElementById('profileDescription').textContent = profile.description;
+  document.getElementById('policyLabel').textContent = profile.policy;
+  cameraButton.hidden = !profile.camera;
+  sendTurnButton.hidden = !profile.clientCommit;
+  promptPreset.replaceChildren();
+  for (const name of [...Object.keys(PROMPT_PRESETS), 'custom']) {
+    const option = document.createElement('option');
+    option.value = name;
+    option.textContent = name.replaceAll('_', ' ');
+    promptPreset.appendChild(option);
+  }
+  systemPromptInput.value = Object.values(PROMPT_PRESETS)[0];
+  document.getElementById('promptControls').hidden = !profile.instructions;
 
   let socket = null;
+  let closingSocket = null;
   let mediaStream = null;
   let captureContext = null;
   let captureNode = null;
@@ -65,6 +73,70 @@
   let liveUserTurn = null;
   let liveAssistantTurn = null;
   let sessionCloseResolver = null;
+  let responseComplete = false;
+  let playbackComplete = true;
+  let turnSubmitted = false;
+  let echoTimer = null;
+  let sessionGeneration = 0;
+  let audioChain = Promise.resolve();
+  // Retain interrupted IDs until startSession() resets the session. Queued
+  // events and pending audio decodes can outlive response.done, so removing
+  // IDs there could resume interrupted playback. This set grows with the
+  // interruptions in a call; sessionGeneration rejects old-session decodes
+  // after the set is cleared for a new session.
+  const interruptedResponses = new Set();
+  const pendingEvents = new Set();
+  let assistantTextChannel = null;
+  let connectionReady = false;
+  let turnCounter = 0;
+  let stopping = null;
+  let turnTimeout = null;
+
+  function markBusy() {
+    assistantActive = true;
+    if (profile.halfDuplex) pendingCapture = [];
+  }
+
+  function armTurnTimeout() {
+    clearTimeout(turnTimeout);
+    turnTimeout = window.setTimeout(() => {
+      failSession('Timed out waiting for the model response. Check the backend event log and restart the session.');
+    }, 120000);
+  }
+
+  async function failSession(message) {
+    appendLog(message, true);
+    await stopSession({ terminal: false });
+    setConnection('Error', 'error');
+    runtimeDetail.textContent = message;
+  }
+
+  function finishResponseIfReady() {
+    if (!responseComplete || !playbackComplete || echoTimer !== null) return;
+    const generation = sessionGeneration;
+    echoTimer = window.setTimeout(async () => {
+      echoTimer = null;
+      if (!running || generation !== sessionGeneration) return;
+      if (profile.reconnectEachTurn) {
+        connectionReady = false;
+        const previous = socket;
+        socket = null;
+        if (previous) { previous.onclose = null; previous.onmessage = null; previous.close(); }
+        try { await openSocket(); }
+        catch (error) { await failSession(error.message); return; }
+        if (!running || generation !== sessionGeneration) return;
+      }
+      assistantActive = false;
+      responseComplete = false;
+      turnSubmitted = false;
+      currentResponseId = null;
+      responseHasAudio = false;
+      assistantTextChannel = null;
+      if (profile.halfDuplex) pendingCapture = [];
+      sendTurnButton.disabled = !profile.clientCommit;
+      setModel(profile.waiting);
+    }, ECHO_GUARD_MS);
+  }
 
   function staticAssetUrl(path) {
     const version = String(config.appVersion || '').trim();
@@ -82,12 +154,7 @@
   }
 
   function realtimeUrl() {
-    const url = new URL(config.realtimePath, window.location.href);
-    url.protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
-    url.searchParams.set('duplex', '1');
-    url.searchParams.set('model', config.model || 'openbmb/MiniCPM-o-4_5');
-    url.searchParams.set('autostart', '0');
-    return url.toString();
+    return profile.url(config, window.location.href);
   }
 
   function setConnection(label, kind) {
@@ -250,7 +317,7 @@
   }
 
   function microphoneUploadEnabled() {
-    return running && !muted;
+    return running && connectionReady && !muted && (!profile.halfDuplex || !assistantActive);
   }
 
   function flushCapture() {
@@ -267,19 +334,9 @@
       offset += chunk.length;
     }
     pendingCapture = [];
-    const pcm = resampleInt16(merged, captureRate, INPUT_RATE);
-    const appendEvent = {
-      type: 'input_audio_buffer.append',
-      audio: int16ToBase64(pcm),
-      format: 'pcm16',
-      sample_rate_hz: INPUT_RATE,
-    };
-    // Omni duplex: ~1 fps camera frame rides the audio append (official
-    // MiniCPM-o-Demo contract: one base64 JPEG per ~1 s chunk).
-    if (cameraPendingFrame) {
-      appendEvent.video_frames = [cameraPendingFrame];
-      cameraPendingFrame = null;
-    }
+    const pcm = resampleInt16(merged, captureRate, profile.inputSampleRate || INPUT_RATE);
+    const appendEvent = profile.append(int16ToBase64(pcm), cameraPendingFrame);
+    cameraPendingFrame = null;
     socket.send(JSON.stringify(appendEvent));
   }
 
@@ -287,13 +344,17 @@
     currentResponseId = responseId || currentResponseId;
     responseHasAudio = false;
     assistantActive = true;
-    setModel('Speaking');
+    responseComplete = false;
+    playbackComplete = true;
+    if (profile.halfDuplex) pendingCapture = [];
+    setModel(profile.halfDuplex ? 'Thinking' : 'Speaking');
   }
 
   function feedPlayback(decoded, responseId) {
     if (!decoded || !decoded.pcm || decoded.pcm.length === 0 || !playbackNode) return;
     const pcm = resampleInt16(decoded.pcm, decoded.sourceRate, playbackRate);
     responseHasAudio = true;
+    playbackComplete = false;
     assistantActive = true;
     setPlayback('Buffering');
     playbackNode.port.postMessage({
@@ -305,6 +366,7 @@
   }
 
   function requestPlaybackDrain(responseId) {
+    if (!responseHasAudio) { playbackComplete = true; finishResponseIfReady(); return; }
     if (!playbackNode) return;
     playbackNode.port.postMessage({ type: 'drain', responseId: responseId || currentResponseId });
   }
@@ -314,90 +376,133 @@
       if (!responseId && playedMs > 0) appendLog('playback ack skipped: missing response id', true);
       return;
     }
-    socket.send(JSON.stringify({
-      type: 'playback.ack',
-      response_id: responseId,
-      item_id: `item_${responseId}`,
-      played_ms: playedMs,
-      committed_ms: playedMs,
-    }));
+    const ack = profile.ack(responseId, playedMs);
+    if (ack) socket.send(JSON.stringify(ack));
   }
 
   function playbackDrained(message) {
     const responseId = message.responseId || currentResponseId;
-    sendPlaybackAck(responseId, Number(message.playedMs) || 0);
+    if (currentResponseId && responseId !== currentResponseId) return;
+    if (profile.playbackAck) sendPlaybackAck(responseId, Number(message.playedMs) || 0);
     setPlayback('Idle');
-    if (message.underrunMs > 0) {
-      appendLog(`playback underrun ${message.underrunMs} ms`);
-    }
-    window.setTimeout(() => {
-      assistantActive = false;
-      currentResponseId = null;
-      responseHasAudio = false;
-      if (running) setModel('Listening');
-    }, ECHO_GUARD_MS);
+    playbackComplete = true;
+    if (message.underrunMs > 0) appendLog(`playback underrun ${message.underrunMs} ms`);
+    if (!profile.halfDuplex && !profile.waitForResponseDone) responseComplete = true;
+    finishResponseIfReady();
   }
 
-  function handleEvent(event) {
+  async function handleAudioEvent(action) {
+    markBusy();
+    currentResponseId = action.responseId || currentResponseId || `turn-${turnCounter}`;
+    setModel('Speaking');
+    const generation = sessionGeneration;
+    const responseId = currentResponseId;
+    const decoded = await decodeAudioDelta(action.event);
+    if (generation === sessionGeneration && !interruptedResponses.has(responseId)) feedPlayback(decoded, responseId);
+  }
+
+  function handleTranscriptEvent(action) {
+    if (action.role === 'assistant') {
+      // Qwen may emit both text and audio-transcript representations.
+      // Display one channel per response instead of duplicating the answer.
+      if (profile.deduplicateTranscript && assistantTextChannel && action.channel !== assistantTextChannel) return;
+      if (action.text) assistantTextChannel = action.channel || assistantTextChannel;
+    }
+    if (action.kind === 'text') addTranscript(action.role, action.text);
+    else if (profile.deduplicateTranscript && action.role === 'assistant') {
+      if (action.text) {
+        const turn = ensureTurn('assistant');
+        turn.value = action.text;
+        turn.text.textContent = action.text;
+      }
+    } else finishTranscript(action.role, action.text);
+  }
+
+  async function handleEvent(event, action = profile.mapEvent(event)) {
     appendEventLog(event);
-    const responseId = responseIdOf(event);
-    switch (event.type) {
-      case 'session.created':
-      case 'session.updated':
+    const responseId = action.responseId || responseIdOf(event);
+    if (responseId && interruptedResponses.has(responseId) && action.kind !== 'interrupt') return;
+    switch (action.kind) {
+      case 'connected':
         setConnection('Connected', 'online');
-        setModel('Listening');
+        if (!assistantActive) setModel(profile.waiting);
         break;
-      case 'response.listen':
+      case 'interrupt': {
+        if (responseId) interruptedResponses.add(responseId);
+        // A late cancellation belongs to its own response, not the newer one
+        // currently playing. Speech-start events interrupt the local playback.
+        if (responseId && responseId !== currentResponseId) break;
+        if (currentResponseId) interruptedResponses.add(currentResponseId);
+        sessionGeneration += 1;
+        audioChain = Promise.resolve();
+        // Move queued events off the interrupted decode, preserving any newer
+        // response already received. Its chunks must not be lost with the old one.
+        const queued = [...pendingEvents];
+        pendingEvents.clear();
+        for (const pending of queued) enqueueEvent(pending.event, pending.action, pending.source);
+        clearTimeout(echoTimer);
+        echoTimer = null;
+        // Local playback completes immediately on interruption, including
+        // when the server already completed generation.
+        clearTimeout(turnTimeout);
+        if (playbackNode) playbackNode.port.postMessage({ type: 'clear' });
+        responseComplete = true;
+        playbackComplete = true;
+        responseHasAudio = false;
         assistantActive = false;
-        setModel('Listening');
+        currentResponseId = null;
+        assistantTextChannel = null;
+        finishTranscript('assistant');
+        setPlayback('Idle');
+        setModel(profile.waiting);
         break;
-      case 'response.created':
-      case 'response.speak':
+      }
+      case 'listen':
+        assistantActive = false;
+        setModel(profile.waiting);
+        break;
+      case 'begin':
+        if (profile.halfDuplex) armTurnTimeout();
+        if (echoTimer !== null) { clearTimeout(echoTimer); echoTimer = null; }
         beginAssistant(responseId);
         break;
-      case 'response.output_audio.delta':
-        currentResponseId = responseId || currentResponseId;
-        assistantActive = true;
-        setModel('Speaking');
-        decodeAudioDelta(event)
-          .then((decoded) => feedPlayback(decoded, responseId))
-          .catch((error) => appendLog(`audio decode failed: ${error.message || error}`, true));
+      case 'audio':
+        await handleAudioEvent(action);
         break;
-      case 'response.output_audio.done':
+      case 'drain':
         requestPlaybackDrain(responseId);
         break;
-      case 'response.output_audio_transcript.delta':
-        addTranscript('assistant', event.delta || '');
+      case 'text': case 'text-final':
+        handleTranscriptEvent(action);
         break;
-      case 'response.output_audio_transcript.done':
-        finishTranscript('assistant', event.transcript || '');
-        break;
-      case 'conversation.item.input_audio_transcription.delta':
-        addTranscript('user', event.delta || '');
-        break;
-      case 'conversation.item.input_audio_transcription.completed':
-        finishTranscript('user', event.transcript || '');
-        break;
-      case 'response.done':
+      case 'done':
+        clearTimeout(turnTimeout);
+        responseComplete = true;
         finishTranscript('assistant');
-        if (!responseHasAudio) requestPlaybackDrain(responseId);
+        requestPlaybackDrain(responseId);
+        finishResponseIfReady();
         break;
-      case 'playback.acknowledged':
-        {
-          const acknowledgement = event.event || event;
-          runtimeDetail.textContent = `Playback committed ${acknowledgement.committed_ms || 0} ms`;
-        }
+      case 'backpressure':
+        armTurnTimeout();
+        markBusy();
+        setModel('Thinking / Speaking');
+        runtimeDetail.textContent = action.message;
         break;
-      case 'session.closed':
+      case 'ack':
+        runtimeDetail.textContent = `Playback committed ${action.committedMs} ms`;
+        break;
+      case 'closed':
         if (sessionCloseResolver) sessionCloseResolver();
         sessionCloseResolver = null;
         break;
       case 'error':
-        setConnection('Error', 'error');
-        runtimeDetail.textContent = String(event.error || event.code || 'Server error');
-        break;
-      default:
+        if (action.fatal) { await failSession(action.message); break; }
+        // A rejected frame or a refused update costs one operation, not the
+        // call: report it and keep listening.
+        appendLog(action.message, true);
+        runtimeDetail.textContent = action.message;
         break;
+      default: break;
     }
   }
 
@@ -406,8 +511,15 @@
     playbackRate = playbackContext.sampleRate;
     await playbackContext.audioWorklet.addModule(staticAssetUrl('static/playback_worklet.js'));
     playbackNode = new AudioWorkletNode(playbackContext, 'fullduplex-pcm-playback');
+    const currentPlaybackNode = playbackNode;
     playbackNode.port.onmessage = (message) => {
+      if (playbackNode !== currentPlaybackNode) return;
       if (message.data.type === 'playback-started') setPlayback('Playing');
+      else if (message.data.type === 'playback-stopped') {
+        // The old response may report after the next one has started. ACK its
+        // own cursor without changing the current response's UI state.
+        if (profile.playbackAck) sendPlaybackAck(message.data.responseId, Number(message.data.playedMs) || 0);
+      }
       else if (message.data.type === 'playback-drained') playbackDrained(message.data);
       else if (message.data.type === 'playback-underrun') {
         runtimeDetail.textContent = `Playback underrun ${message.data.underrunMs || 0} ms`;
@@ -448,51 +560,86 @@
     await captureContext.resume();
   }
 
+  function enqueueEvent(event, action, source) {
+    const pending = { event, action, source };
+    pendingEvents.add(pending);
+    const generation = sessionGeneration;
+    audioChain = audioChain.then(() => {
+      if (!pendingEvents.delete(pending)) return;
+      if (socket === source || action.kind === 'closed') return handleEvent(event, action);
+    }).catch((error) => {
+      if (generation === sessionGeneration) return failSession(`Server event failed: ${error.message}`);
+    });
+  }
+
   function openSocket() {
     return new Promise((resolve, reject) => {
       const url = realtimeUrl();
-      socket = new WebSocket(url);
+      const current = new WebSocket(url);
+      socket = current;
+      connectionReady = false;
+      turnCounter += 1;
       let settled = false;
-      socket.onopen = () => {
+      const timer = window.setTimeout(() => {
+        if (settled) return;
         settled = true;
-        const extraBody = {
-          auto_response: true,
-        };
-        const session = {
-          modalities: ['audio', 'text'],
-          voice: 'default',
-          extra_body: extraBody,
-        };
-        // Reference voice for TTS cloning, provided by the server via
-        // --ref-audio (mirrors the official demo's default ref audio).
-        if (config.refAudio) session.ref_audio = config.refAudio;
-        const instructions = systemPromptInput ? systemPromptInput.value.trim() : '';
-        if (instructions) session.instructions = instructions;
-        socket.send(JSON.stringify({ type: 'session.update', session }));
-        runtimeDetail.textContent = `${captureRate} Hz capture / ${playbackRate} Hz playback`;
+        current.onclose = null;
+        current.close();
+        reject(new Error(`Session handshake timed out. ${profile.connectionHint}`));
+      }, 15000);
+      const rejectOnce = (message) => {
+        clearTimeout(timer);
+        if (!settled) { settled = true; reject(new Error(message)); }
+      };
+      current.onopen = () => {
+        const instructions = systemPromptInput.value.trim();
+        for (const event of profile.initialMessages(config, instructions)) current.send(JSON.stringify(event));
         appendLog(`websocket open  ${url}`);
-        resolve();
       };
-      socket.onmessage = (message) => {
-        if (typeof message.data !== 'string') return;
-        try {
-          handleEvent(JSON.parse(message.data));
-        } catch (error) {
-          appendLog(`invalid server event: ${error.message || error}`, true);
+      current.onmessage = (message) => {
+        if (typeof message.data !== 'string' || (socket !== current && closingSocket !== current)) return;
+        let event;
+        try { event = JSON.parse(message.data); }
+        catch (error) {
+          if (socket !== current) return;
+          const detail = `Invalid server event: ${error.message}`;
+          if (!settled) rejectOnce(detail);
+          else void failSession(detail);
+          return;
         }
-      };
-      socket.onerror = () => {
-        if (!settled) {
+        // Shutdown has detached the active socket. Only its close acknowledgement
+        // may pass, directly: the event queue can itself be awaiting cleanup.
+        if (closingSocket === current) {
+          if (event?.type === 'session.closed' && sessionCloseResolver) sessionCloseResolver();
+          return;
+        }
+        const action = profile.mapEvent(event);
+        if (action.kind === 'error' && !settled) {
+          rejectOnce(action.message);
+          return;
+        }
+        if (event.type === profile.readyEvent && !settled) {
+          clearTimeout(timer);
           settled = true;
-          reject(new Error(`WebSocket connection failed: ${url}`));
+          connectionReady = true;
+          runtimeDetail.textContent = `${captureRate} Hz capture / ${playbackRate} Hz playback`;
+          resolve();
         }
+        // Stop the speaker immediately, even while an old WAV is decoding.
+        // Ordinary completion still waits for its preceding audio chunks.
+        if (action.kind === 'interrupt') {
+          void handleEvent(event, action);
+          return;
+        }
+        enqueueEvent(event, action, current);
       };
-      socket.onclose = (event) => {
+      current.onerror = () => rejectOnce(`WebSocket connection failed. ${profile.connectionHint}`);
+      current.onclose = (event) => {
+        clearTimeout(timer);
+        connectionReady = false;
         appendLog(`websocket closed  code=${event.code}`);
-        if (running) {
-          setConnection('Disconnected', 'error');
-          stopSession();
-        }
+        if (!settled) rejectOnce(`Connection closed before session ready. ${profile.connectionHint}`);
+        else if (running && socket === current) failSession(`Backend disconnected (${event.code}). ${profile.connectionHint}`);
       };
     });
   }
@@ -515,46 +662,61 @@
     setConnection('Connecting', 'connecting');
     runtimeDetail.textContent = 'Requesting microphone access';
     try {
+      sessionGeneration += 1;
+      audioChain = Promise.resolve();
+      interruptedResponses.clear();
+      pendingEvents.clear();
       await openPlayback();
       await openCapture();
       await openSocket();
       running = true;
       muted = false;
       assistantActive = false;
-      sendTimer = window.setInterval(flushCapture, SEND_INTERVAL_MS);
+      sendTimer = window.setInterval(flushCapture, profile.sendIntervalMs || SEND_INTERVAL_MS);
       startClock();
       callButton.textContent = 'End session';
       callButton.classList.add('is-active');
       muteButton.disabled = false;
-      cameraButton.disabled = false;
+      cameraButton.disabled = !profile.camera;
+      sendTurnButton.disabled = !profile.clientCommit;
       setConnection('Connected', 'online');
-      setModel('Listening');
+      setModel(profile.waiting);
       appendLog('session started');
     } catch (error) {
       appendLog(`start failed: ${error.message || error}`, true);
+      await stopSession({ terminal: false });
       setConnection('Error', 'error');
       runtimeDetail.textContent = String(error.message || error);
-      await stopSession();
     } finally {
       callButton.disabled = false;
     }
   }
 
   async function startCamera() {
-    if (cameraStream) return;
+    if (!profile.camera || cameraStream) return;
     cameraStream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false });
     cameraPreview.srcObject = cameraStream;
     cameraPreview.style.display = '';
     await cameraPreview.play().catch(() => {});
     // Official omni-duplex cadence: one JPEG (quality 0.7) per ~1 s chunk,
     // no client-side resize (the server normalizes at scale_resolution=448).
-    cameraTimer = window.setInterval(() => {
+    const captureCameraFrame = () => {
       if (!cameraStream || cameraPreview.videoWidth === 0) return;
-      cameraCanvas.width = cameraPreview.videoWidth;
-      cameraCanvas.height = cameraPreview.videoHeight;
-      cameraCanvas.getContext('2d').drawImage(cameraPreview, 0, 0);
-      cameraPendingFrame = cameraCanvas.toDataURL('image/jpeg', 0.7).split(',')[1];
-    }, 1000);
+      const scale = profile.cameraMaxDimension
+        ? Math.min(1, profile.cameraMaxDimension / Math.max(cameraPreview.videoWidth, cameraPreview.videoHeight)) : 1;
+      cameraCanvas.width = Math.max(1, Math.round(cameraPreview.videoWidth * scale));
+      cameraCanvas.height = Math.max(1, Math.round(cameraPreview.videoHeight * scale));
+      cameraCanvas.getContext('2d').drawImage(cameraPreview, 0, 0, cameraCanvas.width, cameraCanvas.height);
+      const frame = cameraCanvas.toDataURL('image/jpeg', 0.7).split(',')[1];
+      if (profile.imageMessages) {
+        if (connectionReady && socket?.readyState === WebSocket.OPEN) {
+          for (const event of profile.imageMessages(frame)) socket.send(JSON.stringify(event));
+        }
+      } else cameraPendingFrame = frame;
+    };
+    // Do not make the first spoken turn race a one-second timer.
+    captureCameraFrame();
+    cameraTimer = window.setInterval(captureCameraFrame, 1000);
     cameraButton.textContent = 'Camera off';
     cameraButton.classList.add('is-active');
     appendLog('camera on (1 fps omni frames)');
@@ -590,15 +752,33 @@
       const finish = () => {
         if (done) return;
         done = true;
+        clearTimeout(timer);
         if (sessionCloseResolver === finish) sessionCloseResolver = null;
         resolve();
       };
       sessionCloseResolver = finish;
-      window.setTimeout(finish, timeoutMs);
+      const timer = window.setTimeout(finish, timeoutMs);
     });
   }
 
-  async function stopSession({ terminal = true } = {}) {
+  function stopSession(options = {}) {
+    if (stopping) return stopping;
+    stopping = cleanupSession(options).finally(() => { stopping = null; });
+    return stopping;
+  }
+
+  async function cleanupSession({ terminal = true } = {}) {
+    sessionGeneration += 1;
+    pendingEvents.clear();
+    connectionReady = false;
+    clearTimeout(echoTimer);
+    clearTimeout(turnTimeout);
+    echoTimer = null;
+    responseComplete = false;
+    playbackComplete = true;
+    turnSubmitted = false;
+    assistantTextChannel = null;
+    sendTurnButton.disabled = true;
     running = false;
     assistantActive = false;
     pendingCapture = [];
@@ -607,15 +787,16 @@
     sendTimer = null;
     clockTimer = null;
     if (socket) {
-      const closingSocket = socket;
+      closingSocket = socket;
       socket = null;
       closingSocket.onclose = null;
-      if (terminal && closingSocket.readyState === WebSocket.OPEN) {
+      if (terminal && profile.closeSession && closingSocket.readyState === WebSocket.OPEN) {
         const closed = waitForSessionClosed(closingSocket, SESSION_CLOSE_TIMEOUT_MS);
         closingSocket.send(JSON.stringify({ type: 'session.close' }));
         await closed;
       }
       closingSocket.close(1000, 'client stop');
+      closingSocket = null;
     }
     if (playbackNode) playbackNode.port.postMessage({ type: 'clear' });
     if (mediaStream) {
@@ -632,6 +813,8 @@
     playbackNode = null;
     currentResponseId = null;
     responseHasAudio = false;
+    finishTranscript('user');
+    finishTranscript('assistant');
     meterFill.style.width = '0%';
     sessionTimer.textContent = '00:00';
     callButton.textContent = 'Start session';
@@ -659,6 +842,18 @@
     if (running) stopSession();
     else startSession();
   });
+  sendTurnButton.addEventListener('click', () => {
+    if (!running || !connectionReady || !profile.clientCommit || turnSubmitted) return;
+    flushCapture();
+    for (const event of profile.commitMessages()) socket.send(JSON.stringify(event));
+    turnSubmitted = true;
+    responseComplete = false;
+    markBusy();
+    armTurnTimeout();
+    sendTurnButton.disabled = true;
+    setModel('Thinking');
+    appendLog('turn submitted');
+  });
   muteButton.addEventListener('click', toggleMute);
   clearLogButton.addEventListener('click', () => {
     eventLog.textContent = '';

examples/online_serving/realtime_web/app/static/playback_worklet.js

@@ -39,6 +39,13 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
       }
       this.notifyIfDrained();
     } else if (message.type === 'clear') {
+      if (this.activeResponseId) {
+        this.port.postMessage({
+          type: 'playback-stopped',
+          responseId: this.activeResponseId,
+          playedMs: Math.round((this.playedFrames * 1000) / sampleRate),
+        });
+      }
       this.queue = [];
       this.offset = 0;
       this.playedFrames = 0;

examples/online_serving/realtime_web/app/static/profiles/common.js

@@ -0,0 +1,56 @@
+/* Shared wire helpers. Profiles translate server events into shell actions. */
+(function (root) {
+  'use strict';
+  const profiles = root.OmniRealtimeProfiles = root.OmniRealtimeProfiles || {};
+  profiles.url = (config, location) => {
+    const url = new URL(config.realtimePath || 'v1/realtime', location);
+    if (url.protocol === 'http:') url.protocol = 'ws:';
+    if (url.protocol === 'https:') url.protocol = 'wss:';
+    return url;
+  };
+  /* Errors that leave the session unusable, so the shell has to tear it down.
+     Everything else costs one operation --- a rejected image, a refused
+     session.update, a late playback ack --- and the call carries on.
+     'runtime_append_failed' is deliberately absent: the engine reports it and
+     keeps the session, unlike 'runtime_append_task_failed' which kills it. */
+  profiles.fatalErrorCodes = new Set([
+    'unknown_session',
+    'session_closed',
+    'resource_exhausted',
+    'unsupported',
+    'unsupported_turn_detection',
+    'server_vad_unavailable',
+    'runtime_append_task_failed',
+    'internal_error',
+  ]);
+  profiles.event = (event) => {
+    const responseId = event.response_id || event.response?.id || null;
+    const action = (kind, extra = {}) => ({ kind, responseId, ...extra });
+    switch (event.type) {
+      case 'session.created': case 'session.updated': return action('connected');
+      case 'response.created': return action('begin');
+      case 'response.audio.delta': case 'response.output_audio.delta':
+        return action('audio', { event: { ...event, delta: event.delta || event.audio || event.response?.audio } });
+      case 'response.audio.done': case 'response.output_audio.done': return action('drain');
+      case 'response.audio_transcript.delta': case 'response.output_audio_transcript.delta':
+        return action('text', { role: 'assistant', channel: 'audio', text: event.delta || '' });
+      case 'response.audio_transcript.done': case 'response.output_audio_transcript.done':
+        return action('text-final', { role: 'assistant', channel: 'audio', text: event.transcript || '' });
+      case 'conversation.item.input_audio_transcription.delta':
+        return action('text', { role: 'user', text: event.delta || '' });
+      case 'conversation.item.input_audio_transcription.completed':
+        return action('text-final', { role: 'user', text: event.transcript || '' });
+      case 'response.done': return action('done');
+      case 'session.closed': return action('closed');
+      case 'error': {
+        const code = event.code || event.error?.code;
+        return action('error', {
+          code,
+          fatal: profiles.fatalErrorCodes.has(code),
+          message: typeof event.error === 'string' ? event.error : event.error?.message || event.message || code || 'Server error',
+        });
+      }
+      default: return action('ignore');
+    }
+  };
+})(globalThis);

examples/online_serving/realtime_web/app/static/profiles/minicpm_native.js

@@ -0,0 +1,55 @@
+(function (root) {
+  'use strict';
+  const profiles = root.OmniRealtimeProfiles;
+  profiles['minicpm-native'] = () => ({
+    title: 'MiniCPM-o 4.5 Voice', eyebrow: 'Experimental full-duplex runtime',
+    policy: 'Model-controlled listen / speak',
+    description: 'Speak naturally. The model can listen and speak at the same time. Camera frames are sent with your audio.',
+    waiting: 'Listening', camera: true, playbackAck: true, clientCommit: false,
+    halfDuplex: false, closeSession: true, reconnectEachTurn: false,
+    readyEvent: 'session.updated', instructions: true, sendIntervalMs: 200,
+    presets: {
+      omni: 'Streaming Omni Conversation.',
+      chinese_call: '扮演一个具有以上声音特征的助手。请认真、高质量地回复用户的问题。'
+        + '请用高自然度的方式和用户聊天。你处于双工模式，可以一边听、一边说。'
+        + '你是由面壁智能开发的人工智能助手：面壁小钢炮。',
+      english_call: 'Replicate the tone and style from the input audio. Your task is to be '
+        + 'a helpful assistant using this voice pattern. Please answer the user\'s questions '
+        + 'seriously and in a high quality. Please chat with the user in a high naturalness '
+        + 'style. You are in duplex mode, where you can listen and speak at the same time.',
+    },
+    url(config, location) {
+      const url = profiles.url(config, location);
+      url.searchParams.set('duplex', '1');
+      url.searchParams.set('model', config.model || 'openbmb/MiniCPM-o-4_5');
+      url.searchParams.set('autostart', '0');
+      return url.toString();
+    },
+    initialMessages(config, instructions) {
+      const session = { modalities: ['audio', 'text'], voice: 'default',
+        extra_body: { auto_response: true } };
+      if (config.refAudio) session.ref_audio = config.refAudio;
+      if (instructions) session.instructions = instructions;
+      return [{ type: 'session.update', session }];
+    },
+    append(audio, frame) {
+      const event = { type: 'input_audio_buffer.append', audio, format: 'pcm16', sample_rate_hz: 16000 };
+      if (frame) event.video_frames = [frame];
+      return event;
+    },
+    commitMessages: () => [],
+    ack(responseId, playedMs) {
+      return { type: 'playback.ack', response_id: responseId, item_id: `item_${responseId}`,
+        played_ms: playedMs, committed_ms: playedMs };
+    },
+    mapEvent(event) {
+      if (event.type === 'response.listen') return { kind: 'listen' };
+      if (event.type === 'response.speak') return { kind: 'begin', responseId: event.response_id || event.response?.id };
+      if (event.type === 'playback.acknowledged') {
+        return { kind: 'ack', committedMs: (event.event || event).committed_ms || 0 };
+      }
+      return profiles.event(event);
+    },
+    connectionHint: 'Check the MiniCPM duplex deployment and reference voice configuration.',
+  });
+})(globalThis);

examples/online_serving/realtime_web/app/static/profiles/qwen3_turn.js

@@ -0,0 +1,116 @@
+(function (root) {
+  'use strict';
+  const profiles = root.OmniRealtimeProfiles;
+  profiles['qwen3-turn'] = (config) => {
+    const vad = config.adapter === 'vad';
+    let interruptOnSpeech = vad;
+    let imageSeq = 0;
+    let imageItems = [];
+    // The engine refuses a ninth image (or 4 MiB of them) for the whole
+    // session, so the camera track retires its own oldest frame instead.
+    const MAX_IMAGE_ITEMS = 8;
+    const hint = 'This Qwen deploy needs session_mode: duplex and Server VAD support; use --stt or enable Server VAD.';
+    return {
+      title: 'Qwen3-Omni Voice', eyebrow: vad ? 'Engine-owned duplex call' : 'Turn-based realtime call',
+      policy: vad ? 'Server VAD • interruptible responses' : 'Send turn • one response at a time',
+      description: vad
+        ? 'Pause to send your turn; speak again to interrupt. Microphone upload continues during playback. Requires the Qwen duplex plugin and Silero. Optional camera frames accompany each spoken turn.'
+        : 'Speak, then press Send turn. Microphone upload pauses while the model answers. Each turn uses a fresh connection; conversation history is not carried between turns. No barge-in or camera input.',
+      waiting: 'Waiting for you', camera: vad, cameraMaxDimension: 448, playbackAck: vad, clientCommit: !vad,
+      waitForResponseDone: true, halfDuplex: !vad, deduplicateTranscript: true, closeSession: vad, reconnectEachTurn: !vad,
+      inputSampleRate: vad ? 24000 : 16000,
+      readyEvent: vad ? 'session.updated' : 'session.created', instructions: vad, sendIntervalMs: 200,
+      presets: { assistant: 'You are a helpful assistant. Answer clearly and concisely.' },
+      url(config, location) {
+        const url = profiles.url(config, location);
+        for (const key of ['native_duplex', 'minicpmo45_native_duplex', 'autostart']) url.searchParams.delete(key);
+        // Explicit opt-out matches the local file client, including servers
+        // that auto-select the duplex endpoint when the query is omitted.
+        url.searchParams.set('duplex', vad ? '1' : '0');
+        url.searchParams.set('model', config.model || 'Qwen/Qwen3-Omni-30B-A3B-Instruct');
+        return url.toString();
+      },
+      initialMessages(config, instructions) {
+        imageItems = [];
+        interruptOnSpeech = vad;
+        if (!vad) return [
+          { type: 'session.update', model: config.model },
+          { type: 'input_audio_buffer.commit', final: false },
+        ];
+        const session = { model: config.model, overlap_policy: 'barge_in_on_speech', audio: { input: {
+          format: { type: 'audio/pcm', rate: 24000 },
+          turn_detection: { type: 'server_vad', threshold: 0.5, prefix_padding_ms: 300,
+            silence_duration_ms: 500, create_response: true, interrupt_response: true },
+        } } };
+        if (instructions) session.instructions = instructions;
+        return [{ type: 'session.update', session }];
+      },
+      append(audio) {
+        return vad ? { type: 'input_audio_buffer.append', audio }
+          : { type: 'input_audio_buffer.append', audio, format: 'pcm16', sample_rate_hz: 16000 };
+      },
+      // Qwen is a turn model: a frame is a picture added to the conversation,
+      // not a track interleaved into the audio. That is exactly the OpenAI
+      // Realtime image interface, so the camera sends conversation items
+      // rather than an extension field on the append.
+      imageMessages(frame) {
+        if (!vad) return [];
+        const messages = [];
+        if (imageItems.length >= MAX_IMAGE_ITEMS) {
+          // Retire before creating: the engine counts what is already stored.
+          messages.push({ type: 'conversation.item.delete', item_id: imageItems.shift() });
+        }
+        const id = `camera_${++imageSeq}`;
+        imageItems.push(id);
+        messages.push({ type: 'conversation.item.create', item: {
+          id, type: 'message', role: 'user',
+          content: [{ type: 'input_image', image_url: `data:image/jpeg;base64,${frame}` }],
+        } });
+        return messages;
+      },
+      commitMessages: () => vad ? [] : [{ type: 'input_audio_buffer.commit', final: true }],
+      ack: (responseId, playedMs) => vad ? { type: 'playback.ack', response_id: responseId,
+        item_id: `item_${responseId}`, played_ms: playedMs, committed_ms: playedMs } : null,
+      mapEvent(event) {
+        if (event.type === 'session.created' || event.type === 'session.updated') {
+          const input = event.session?.audio?.input;
+          const turnDetection = input && 'turn_detection' in input
+            ? input.turn_detection : event.session?.turn_detection;
+          if (turnDetection !== undefined) {
+            interruptOnSpeech = vad && turnDetection?.type === 'server_vad'
+              && turnDetection.interrupt_response !== false;
+          }
+        }
+        // Generation can finish long before the speaker drains its queue. In
+        // that case the server has no active response left to cancel for us.
+        if (interruptOnSpeech && event.type === 'input_audio_buffer.speech_started') {
+          return { kind: 'interrupt' };
+        }
+        if (event.type === 'playback.acknowledged') return { kind: 'ack', committedMs: (event.event || event).committed_ms || 0 };
+        if (event.type === 'response.output_text.delta' || event.type === 'transcription.delta') {
+          // On the shipped STT route transcription.* is model-generated text,
+          // not a separate ASR transcript of the user. See realtime_connection.py.
+          return { kind: 'text', role: 'assistant', channel: 'text', text: event.delta || '' };
+        }
+        if (event.type === 'response.output_text.done' || event.type === 'transcription.done') {
+          return { kind: 'text-final', role: 'assistant', channel: 'text', text: event.text || '' };
+        }
+        if (vad && (event.type === 'output_audio_buffer.cleared' ||
+            (event.type === 'response.done' && event.response?.status === 'cancelled'))) {
+          return { kind: 'interrupt', responseId: event.response_id || event.response?.id || null };
+        }
+        if (event.type === 'input_audio_buffer.committed') return { kind: 'begin' };
+        if (event.type === 'input_audio_buffer.cleared') return { kind: 'backpressure', message: 'Input cleared. Model is answering; please wait.' };
+        if (!vad && event.type === 'response.output_audio.done') return { kind: 'done' };
+        const action = profiles.event(event);
+        if (action.kind === 'error') {
+          if (action.code === 'input_backpressure') return { ...action, kind: 'backpressure', message: 'Input buffer is full; please repeat dropped speech after the response.' };
+          if (action.code === 'server_vad_initialization_failed') action.message = 'Server VAD could not load Silero. Configure the server artifact or restart this UI with --stt.';
+          else if (vad && action.code === 'unsupported') action.message += ` ${hint}`;
+        }
+        return action;
+      },
+      connectionHint: vad ? hint : 'Check the Qwen /v1/realtime STT endpoint and model name.',
+    };
+  };
+})(globalThis);

examples/online_serving/realtime_web/app/static/styles.css

@@ -125,3 +125,6 @@ button:disabled { opacity: .46; cursor: default; }
 @media (prefers-reduced-motion: reduce) {
   *, *::before, *::after { animation-duration: .01ms !important; animation-iteration-count: 1 !important; }
 }
+
+/* Profile-specific controls may have inline flex styles. */
+[hidden] { display: none !important; }

examples/online_serving/realtime_web/server.py

@@ -0,0 +1,229 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Static host and same-origin proxy for the shared realtime browser UI."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import base64
+import contextlib
+import hashlib
+import json
+import logging
+from pathlib import Path
+from urllib.parse import urlencode
+
+import uvicorn
+import websockets
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect
+from fastapi.responses import HTMLResponse, Response
+from fastapi.staticfiles import StaticFiles
+
+logger = logging.getLogger(__name__)
+APP_DIR = Path(__file__).parent / "app"
+STATIC_DIR = APP_DIR / "static"
+
+
+def _join_ws_url(base: str, path: str, query: str) -> str:
+    return base.rstrip("/") + path + (("?" + query) if query else "")
+
+
+async def _pump_client_to_backend(client: WebSocket, backend) -> None:
+    try:
+        while True:
+            message = await client.receive()
+            if message["type"] == "websocket.disconnect":
+                await backend.close()
+                return
+            if message.get("text") is not None:
+                await backend.send(message["text"])
+            elif message.get("bytes") is not None:
+                await backend.send(message["bytes"])
+    except (WebSocketDisconnect, websockets.ConnectionClosed):
+        with contextlib.suppress(Exception):
+            await backend.close()
+
+
+async def _pump_backend_to_client(client: WebSocket, backend) -> None:
+    try:
+        async for message in backend:
+            if isinstance(message, bytes):
+                await client.send_bytes(message)
+            else:
+                await client.send_text(message)
+    except (WebSocketDisconnect, websockets.ConnectionClosed):
+        return
+    except RuntimeError as exc:
+        if "websocket.send" in str(exc) and "websocket.close" in str(exc):
+            return
+        raise
+
+
+def _expected_proxy_close(exc: BaseException) -> bool:
+    if isinstance(exc, (WebSocketDisconnect, websockets.ConnectionClosed, asyncio.CancelledError)):
+        return True
+    return isinstance(exc, RuntimeError) and "websocket.send" in str(exc) and "websocket.close" in str(exc)
+
+
+def build_app(
+    *,
+    ws_backend: str | None = None,
+    model: str | None = None,
+    public_realtime_url: str | None = None,
+    ref_audio: str | None = None,
+    profile: str = "minicpm-native",
+    adapter: str = "stt",
+) -> FastAPI:
+    if profile not in {"minicpm-native", "qwen3-turn"}:
+        raise ValueError(f"Unknown profile: {profile}")
+    if adapter not in {"stt", "vad"}:
+        raise ValueError(f"Unknown adapter: {adapter}")
+    if profile == "qwen3-turn" and ref_audio:
+        raise ValueError("Qwen3 profile does not accept --ref-audio")
+    native = profile == "minicpm-native"
+    ws_backend = ws_backend or ("ws://127.0.0.1:8099" if native else "ws://127.0.0.1:8091")
+    model = model or ("openbmb/MiniCPM-o-4_5" if native else "Qwen/Qwen3-Omni-30B-A3B-Instruct")
+    app = FastAPI(title="Omni Realtime Web UI")
+    index_path = APP_DIR / "index.html"
+    app_version_hash = hashlib.sha256()
+    for asset_path in (
+        STATIC_DIR / "app.js",
+        STATIC_DIR / "pcm_worklet.js",
+        STATIC_DIR / "playback_worklet.js",
+        *sorted((STATIC_DIR / "profiles").glob("*.js")),
+    ):
+        app_version_hash.update(asset_path.read_bytes())
+    app_version = app_version_hash.hexdigest()[:12]
+
+    ref_audio_uri: str | None = None
+    if ref_audio:
+        ref_path = Path(ref_audio)
+        if not ref_path.is_file():
+            raise SystemExit(f"--ref-audio not found: {ref_audio}")
+        encoded = base64.b64encode(ref_path.read_bytes()).decode("ascii")
+        ref_audio_uri = f"data:audio/wav;base64,{encoded}"
+
+    @app.get("/", response_class=HTMLResponse)
+    def index() -> HTMLResponse:
+        config = json.dumps(
+            {
+                "model": model,
+                "profile": profile,
+                "adapter": adapter,
+                "realtimePath": public_realtime_url or "v1/realtime",
+                "refAudio": ref_audio_uri,
+                "appVersion": app_version,
+            },
+            ensure_ascii=True,
+        ).replace("<", "\\u003c")
+        html = (
+            index_path.read_text(encoding="utf-8")
+            .replace("__FULL_DUPLEX_CONFIG__", config)
+            .replace("__FULL_DUPLEX_APP_VERSION__", app_version)
+        )
+        return HTMLResponse(html, headers={"Cache-Control": "no-store"})
+
+    @app.get("/healthz")
+    def healthz() -> Response:
+        return Response(content="ok", media_type="text/plain")
+
+    @app.websocket("/v1/realtime")
+    async def realtime_proxy(websocket: WebSocket) -> None:
+        await websocket.accept()
+        query = urlencode(websocket.query_params.multi_items())
+        backend_url = _join_ws_url(ws_backend, "/v1/realtime", query)
+        logger.info("Proxying Realtime WebSocket to %s", backend_url)
+        try:
+            async with websockets.connect(
+                backend_url,
+                max_size=64 * 1024 * 1024,
+            ) as backend:
+                tasks = {
+                    asyncio.create_task(_pump_client_to_backend(websocket, backend)),
+                    asyncio.create_task(_pump_backend_to_client(websocket, backend)),
+                }
+                done, pending = await asyncio.wait(
+                    tasks,
+                    return_when=asyncio.FIRST_COMPLETED,
+                )
+                for task in pending:
+                    task.cancel()
+                await asyncio.gather(*pending, return_exceptions=True)
+                for result in await asyncio.gather(*done, return_exceptions=True):
+                    if isinstance(result, BaseException) and not _expected_proxy_close(result):
+                        raise result
+        except (WebSocketDisconnect, websockets.ConnectionClosed):
+            return
+        except (OSError, TimeoutError) as exc:
+            message = (
+                f"Cannot connect to realtime backend {ws_backend}. "
+                "Check that vllm serve is running and --backend matches its --port. "
+                "For containers or remote hosts, use the backend address reachable from the UI server."
+            )
+            logger.warning("%s (%s)", message, exc)
+            with contextlib.suppress(Exception):
+                await websocket.send_json({"type": "error", "code": "backend_unavailable", "error": message})
+                await websocket.close(code=1011)
+        except Exception:
+            logger.exception("Realtime WebSocket proxy failed")
+            with contextlib.suppress(Exception):
+                await websocket.close(code=1011)
+        finally:
+            with contextlib.suppress(Exception):
+                await websocket.close()
+
+    app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
+    return app
+
+
+def main(default_profile: str = "minicpm-native") -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--host", default="0.0.0.0")
+    parser.add_argument("--port", type=int, default=7862)
+    parser.add_argument("--profile", choices=("minicpm-native", "qwen3-turn"), default=default_profile)
+    parser.add_argument("--ws-backend", "--backend", dest="ws_backend")
+    mode = parser.add_mutually_exclusive_group()
+    mode.add_argument("--stt", dest="adapter", action="store_const", const="stt")
+    mode.add_argument("--vad", dest="adapter", action="store_const", const="vad")
+    parser.set_defaults(adapter="stt")
+    parser.add_argument(
+        "--public-realtime-url",
+        help="Browser-visible ws:// or wss:// Realtime URL; defaults to the same-origin proxy.",
+    )
+    parser.add_argument("--model")
+    parser.add_argument(
+        "--ref-audio",
+        help=(
+            "Reference voice wav for TTS voice cloning, e.g. the "
+            "official MiniCPM-o-Demo assets/ref_audio/ref_minicpm_signature.wav"
+        ),
+    )
+    args = parser.parse_args()
+    native = args.profile == "minicpm-native"
+    if native and not args.ref_audio:
+        parser.error("--ref-audio is required for the minicpm-native profile")
+    if not native and args.ref_audio:
+        parser.error("--ref-audio is only supported by minicpm-native")
+    args.model = args.model or ("openbmb/MiniCPM-o-4_5" if native else "Qwen/Qwen3-Omni-30B-A3B-Instruct")
+    args.ws_backend = args.ws_backend or ("ws://127.0.0.1:8099" if native else "ws://127.0.0.1:8091")
+
+    logging.basicConfig(level=logging.INFO)
+    uvicorn.run(
+        build_app(
+            ws_backend=args.ws_backend,
+            model=args.model,
+            public_realtime_url=args.public_realtime_url,
+            ref_audio=args.ref_audio,
+            profile=args.profile,
+            adapter=args.adapter,
+        ),
+        host=args.host,
+        port=args.port,
+        log_level="info",
+    )
+
+
+if __name__ == "__main__":
+    main()

examples/online_serving/streaming_video_generation/README.md

@@ -21,7 +21,7 @@ The `--diffusion-streaming-output` CLI flag is forwarded as `streaming_output=Tr
 | Direction | Message | Format | Description |
 | --- | --- | --- | --- |
 | Client to server | `session.start` | JSON text: `{"type":"session.start","model":"...","prompt":"...","format":"m4s"}` | Starts generation. `format` is optional and accepts `m4s` (default). Sampling fields such as `width`, `height`, `fps`, `num_frames`, and `extra_params` may be included. Image-conditioned models take a first frame as `image_reference`: `{"image_url":"https://..."}` or a `data:` URL. |
-| Client to server | `session.interaction` | JSON text: `{"type":"session.interaction","interaction":{"event_id":"xxx","event":{"prompt":"..."},"transition_chunks":3}}` | Updates the active prompt midway through generation. `event_id` is optional and `transition_chunks` defaults to the model setting. |
+| Client to server | `session.interaction` | JSON text: `{"type":"session.interaction","interaction":{"event_id":"xxx","event":{"prompt":"..."},"transition_chunks":3}}` | Updates the active prompt and/or camera midway through generation. Camera at `event.multi_modal_data.camera` is structural SE3 (`mode` + `translation`/`rotation`). `event_id` is optional and `transition_chunks` defaults to the model setting. |
 | Server to client | `video.start` | JSON text: `{"type":"video.start","request_id":"...","format":"m4s","config":{...}}` | Confirms the session and mirrors the accepted `format`. |
 | Server to client | `video.chunk_metadata` | JSON text: `{"type":"video.chunk_metadata","request_id":"...","kind":"media","transport_chunk_index":0,"generation_chunk_index":0,"num_frames":9,"byte_length":1234,"started_event_ids":[],"active_event_ids":[],"completed_event_ids":[]}` | Precedes each binary frame and describes the immediately following payload. |
 | Server to client | Video chunk | Binary WebSocket frame | Fragmented MP4 (`m4s`) video bytes. |
@@ -59,10 +59,10 @@ python streaming_video_client.py \
   --output helios_stream.mp4
 ```
 
-The client sends one `session.start` message, prints each received binary video chunk with its byte size and elapsed time, and saves the received bytes to `--output` after `session.done`.
+The example client sends one `session.start` message, prints each received binary video chunk with its byte size and elapsed time, and saves the received bytes to `--output` after `session.done`.
 The client remuxes the gathered stream to a regular progressive MP4 file so that local playback knows the video duration.
 
-Schedule midway prompt updates with `--prompt-updates`. None are sent unless you ask for them, and only pipelines that implement midway prompt updates accept them (LingBot-World, for one, does not, and rejects the whole session). Each entry uses `"at"` as seconds on the client clock after the server sends `video.start`:
+Schedule midway prompt updates with `--prompt-updates`. None are sent unless you ask for them, and only pipelines that implement midway prompt updates accept them. Each entry uses `"at"` as seconds on the client clock after the server sends `video.start`:
 
 ```bash
 python streaming_video_client.py \
@@ -73,6 +73,21 @@ python streaming_video_client.py \
   ]'
 ```
 
+Schedule midway camera updates with `--camera-updates` when the served model
+registers the camera modality (e.g., LingBot World 2).
+It accepts `["camera"]["data"]["translation"/"rotation"]` in the endpoint's required SE3 format.
+It also accepts `["actions"]` at the update level and converts keystrokes to the SE3 format.
+
+```bash
+python streaming_video_client.py \
+  --model robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+  --image-reference /path/to/first_frame.png \
+  --camera-updates '[
+    {"at": 1.0, "actions": ["w"]},
+    {"at": 3.0, "camera": {"mode": "target", "data": {"translation": [0.0, 0.0, 1.0], "rotation": [0.0, 0.0, 0.0, 1.0]}}, "transition_chunks": 2}
+  ]'
+```
+
 The Helios demo continues the default prompt into an underwater storm:
 
 ```bash
@@ -144,6 +159,7 @@ python streaming_video_client.py \
   --output lingbot_world_v2_stream.mp4
 ```
 
-`camera_action_script` carries one three-latent-frame WASD action list per generated
-chunk. Mid-session `session.interaction` only updates the prompt, so a LingBot rollout
-follows the camera script it started with.
+`camera_action_script` is an optional request-scoped WASD script (one
+three-latent-frame action list per generated chunk). Prefer mid-session
+`session.interaction` camera control via `--camera-updates` when you need
+live motion; the engine wire format is structural SE3 only.

examples/online_serving/streaming_video_generation/camera_utils.py

@@ -0,0 +1,70 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Client-side helpers that map WASD/IJKL keys to structural SE3 camera payloads.
+Convenience for demos.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+# Unity-frame (+X right, +Y up, +Z forward) step sizes used by the optional
+# WASD helper. Magnitudes mirror LingBot's script integrator; the engine never
+# sees key tokens — only structural SE3 payloads.
+_WASD_STEP = 0.05
+_WASD_PITCH_DEG = 4.0
+_WASD_YAW_DEG = 6.0
+# Match LingBot's per-frame pitch clamp; without absolute pose this only limits
+# a single helper call, not cumulative held velocity.
+_WASD_PITCH_LIMIT_DEG = 85.0
+
+
+def _quat_from_axis_angle(axis: tuple[float, float, float], degrees: float) -> list[float]:
+    """Return a unit quaternion ``[x, y, z, w]`` for a right-handed axis-angle."""
+    radians = math.radians(degrees)
+    half = 0.5 * radians
+    sine = math.sin(half)
+    return [axis[0] * sine, axis[1] * sine, axis[2] * sine, math.cos(half)]
+
+
+def wasd_to_camera_payload(actions: list[str], *, mode: str = "velocity") -> dict[str, Any]:
+    """Map WASD/IJKL key tokens to a structural camera interaction payload.
+
+    This is a client convenience only. Servers reject ``data.actions``.
+
+    The delta is chosen so that at the identity pose, ``T_base @ T_delta``
+    matches LingBot's script integrator rotation
+    ``R_y(yaw) @ R_x(pitch)``. Without the current absolute pose this remains
+    an approximation once the camera has already yawed/pitched.
+    """
+    keys = {str(action).lower() for action in actions}
+    dx = _WASD_STEP * (("d" in keys) - ("a" in keys))
+    dy = 0.0
+    dz = _WASD_STEP * (("w" in keys) - ("s" in keys))
+    pitch = _WASD_PITCH_DEG * (("i" in keys) - ("k" in keys))
+    yaw = _WASD_YAW_DEG * (("l" in keys) - ("j" in keys))
+    pitch = max(-_WASD_PITCH_LIMIT_DEG, min(_WASD_PITCH_LIMIT_DEG, pitch))
+
+    # R_delta = R_y(yaw) @ R_x(pitch): apply pitch first, then yaw.
+    qx, qy, qz, qw = 0.0, 0.0, 0.0, 1.0
+    if abs(pitch) > 0.0:
+        px, py, pz, pw = _quat_from_axis_angle((1.0, 0.0, 0.0), pitch)
+        qx, qy, qz, qw = px, py, pz, pw
+    if abs(yaw) > 0.0:
+        yx, yy, yz, yw = _quat_from_axis_angle((0.0, 1.0, 0.0), yaw)
+        # q = yaw ⊗ pitch
+        qx, qy, qz, qw = (
+            yw * qx + yx * qw + yy * qz - yz * qy,
+            yw * qy - yx * qz + yy * qw + yz * qx,
+            yw * qz + yx * qy - yy * qx + yz * qw,
+            yw * qw - yx * qx - yy * qy - yz * qz,
+        )
+
+    return {
+        "mode": mode,
+        "data": {
+            "translation": [dx, dy, dz],
+            "rotation": [qx, qy, qz, qw],
+        },
+    }

examples/online_serving/streaming_video_generation/streaming_video_client.py

@@ -26,6 +26,7 @@ from pathlib import Path
 from typing import Any
 
 import av
+from camera_utils import wasd_to_camera_payload
 
 try:
     from websockets.asyncio.client import connect  # pyright: ignore[reportMissingImports]
@@ -56,6 +57,15 @@ class ScheduledPromptUpdate:
     transition_chunks: int
 
 
+@dataclass(frozen=True)
+class ScheduledCameraUpdate:
+    """Client-side schedule entry for a midway camera ``session.interaction``."""
+
+    at: float
+    camera: dict[str, Any]
+    transition_chunks: int | None
+
+
 def _image_reference(value: str) -> dict[str, Any]:
     """Accept an http(s) URL as-is, or inline a local image as a data URL."""
     if value.startswith(("http://", "https://", "data:")):
@@ -119,6 +129,61 @@ def _parse_prompt_updates(value: str) -> list[ScheduledPromptUpdate]:
     return sorted(updates, key=lambda update: update.at)
 
 
+def _parse_camera_updates(value: str) -> list[ScheduledCameraUpdate]:
+    try:
+        parsed = json.loads(value)
+    except json.JSONDecodeError as exc:
+        raise argparse.ArgumentTypeError(f"--camera-updates must be valid JSON: {exc}") from exc
+    if not isinstance(parsed, list):
+        raise argparse.ArgumentTypeError("--camera-updates must be a JSON array")
+
+    updates: list[ScheduledCameraUpdate] = []
+    for index, item in enumerate(parsed):
+        if not isinstance(item, dict):
+            raise argparse.ArgumentTypeError(f"--camera-updates[{index}] must be a JSON object")
+        try:
+            at = float(item["at"])
+        except (KeyError, TypeError, ValueError) as exc:
+            raise argparse.ArgumentTypeError(f"--camera-updates[{index}] must have a valid 'at' field") from exc
+        if at < 0:
+            raise argparse.ArgumentTypeError(f"--camera-updates[{index}].at must be >= 0")
+
+        if "camera" in item:
+            camera = item["camera"]
+            if not isinstance(camera, dict):
+                raise argparse.ArgumentTypeError(f"--camera-updates[{index}].camera must be an object")
+        elif "actions" in item:
+            actions = item["actions"]
+            if not isinstance(actions, list):
+                raise argparse.ArgumentTypeError(f"--camera-updates[{index}].actions must be a list of WASD/IJKL keys")
+            mode = item.get("mode", "velocity")
+            if mode not in ("target", "velocity"):
+                raise argparse.ArgumentTypeError(f"--camera-updates[{index}].mode must be target or velocity")
+            camera = wasd_to_camera_payload(actions, mode=mode)
+        else:
+            raise argparse.ArgumentTypeError(
+                f"--camera-updates[{index}] must provide 'camera' (structural SE3) "
+                "or 'actions' (client-side WASD helper)"
+            )
+
+        transition_chunks: int | None
+        if "transition_chunks" not in item:
+            transition_chunks = None if camera.get("mode") == "velocity" else DEFAULT_TRANSITION_CHUNKS
+        else:
+            try:
+                transition_chunks = int(item["transition_chunks"])
+            except (TypeError, ValueError) as exc:
+                raise argparse.ArgumentTypeError(
+                    f"--camera-updates[{index}].transition_chunks must be an integer"
+                ) from exc
+            if transition_chunks < 0:
+                raise argparse.ArgumentTypeError(f"--camera-updates[{index}].transition_chunks must be >= 0")
+
+        updates.append(ScheduledCameraUpdate(at=at, camera=camera, transition_chunks=transition_chunks))
+
+    return sorted(updates, key=lambda update: update.at)
+
+
 def _maybe_set(payload: dict[str, Any], key: str, value: Any) -> None:
     if value is not None:
         payload[key] = value
@@ -173,7 +238,7 @@ async def _run_prompt_update_scheduler(
         payload = {
             "type": "session.interaction",
             "interaction": {
-                "event_id": f"scheduled-{update.at:.3f}",
+                "event_id": f"scheduled-prompt-{update.at:.3f}",
                 "event": {"prompt": update.prompt},
                 "transition_chunks": update.transition_chunks,
             },
@@ -183,6 +248,30 @@ async def _run_prompt_update_scheduler(
         print(f"Sent session.interaction at t={update.at:.2f}s: {json.dumps(payload, ensure_ascii=False)}")
 
 
+async def _run_camera_update_scheduler(
+    websocket: Any,
+    updates: list[ScheduledCameraUpdate],
+    *,
+    video_started_at: float,
+    send_lock: asyncio.Lock,
+) -> None:
+    for update in updates:
+        delay = (video_started_at + update.at) - time.perf_counter()
+        if delay > 0:
+            await asyncio.sleep(delay)
+
+        interaction: dict[str, Any] = {
+            "event_id": f"scheduled-camera-{update.at:.3f}",
+            "event": {"multi_modal_data": {"camera": update.camera}},
+        }
+        if update.transition_chunks is not None:
+            interaction["transition_chunks"] = update.transition_chunks
+        payload = {"type": "session.interaction", "interaction": interaction}
+        async with send_lock:
+            await websocket.send(json.dumps(payload, ensure_ascii=False))
+        print(f"Sent session.interaction at t={update.at:.2f}s: {json.dumps(payload, ensure_ascii=False)}")
+
+
 def _count_decoded_frames(video_bytes: bytes, *, stream_format: str) -> int | None:
     """Return the number of decodable video frames in a streamed byte sequence, if known."""
     if not video_bytes:
@@ -241,6 +330,7 @@ async def stream_video(args: argparse.Namespace) -> None:
     ws = None
     send_lock = asyncio.Lock()
     prompt_update_task: asyncio.Task[None] | None = None
+    camera_update_task: asyncio.Task[None] | None = None
     pending_chunk_metadata: dict[str, Any] | None = None
 
     try:
@@ -295,8 +385,8 @@ async def stream_video(args: argparse.Namespace) -> None:
                 if msg_type == "video.start":
                     stream_format = msg.get("format") or stream_format
                     print(f"Video session started: request_id={msg.get('request_id')} format={msg.get('format')}")
+                    video_started_at = time.perf_counter()
                     if args.prompt_updates and prompt_update_task is None:
-                        video_started_at = time.perf_counter()
                         prompt_update_task = asyncio.create_task(
                             _run_prompt_update_scheduler(
                                 websocket,
@@ -305,6 +395,15 @@ async def stream_video(args: argparse.Namespace) -> None:
                                 send_lock=send_lock,
                             )
                         )
+                    if args.camera_updates and camera_update_task is None:
+                        camera_update_task = asyncio.create_task(
+                            _run_camera_update_scheduler(
+                                websocket,
+                                args.camera_updates,
+                                video_started_at=video_started_at,
+                                send_lock=send_lock,
+                            )
+                        )
                 elif msg_type == "video.chunk_metadata":
                     # Save the metadata for the incoming video chunk
                     pending_chunk_metadata = msg
@@ -315,6 +414,8 @@ async def stream_video(args: argparse.Namespace) -> None:
                     done = True
                     if prompt_update_task is not None:
                         prompt_update_task.cancel()
+                    if camera_update_task is not None:
+                        camera_update_task.cancel()
                     break
                 elif msg_type == "error":
                     print(f"ERROR: {msg.get('message', msg)}")
@@ -335,6 +436,10 @@ async def stream_video(args: argparse.Namespace) -> None:
             prompt_update_task.cancel()
             with contextlib.suppress(asyncio.CancelledError):
                 await prompt_update_task
+        if camera_update_task is not None:
+            camera_update_task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await camera_update_task
         print(
             "Saving video... (May take a while. Remuxing concatenated chunks into one progressive MP4 file with total duration metadata in file header)"
         )
@@ -405,9 +510,9 @@ def parse_args() -> argparse.Namespace:
         default=None,
         help="JSON object merged into extra_params; overrides preset keys on conflict.",
     )
-    # Midway prompt updates are pipeline-specific: a model that does not implement
-    # them (LingBot-World, for one) rejects the whole session on the first one, so
-    # nothing is scheduled unless the caller asks for it.
+    # Midway interactions are pipeline-specific: unsupported modalities reject
+    # the update (and may leave the session running). Nothing is scheduled unless
+    # the caller asks for it.
     parser.add_argument(
         "--prompt-updates",
         type=_parse_prompt_updates,
@@ -419,6 +524,19 @@ def parse_args() -> argparse.Namespace:
             f"'transition_chunks' defaults to {DEFAULT_TRANSITION_CHUNKS}."
         ),
     )
+    parser.add_argument(
+        "--camera-updates",
+        type=_parse_camera_updates,
+        default=[],
+        help=(
+            "JSON array of scheduled camera updates for models that register the "
+            "camera modality (e.g. LingBot-World). Each object needs 'at' and either "
+            "'camera' (structural SE3: mode + translation/rotation) or "
+            "'actions' (client-side WASD/IJKL helper converted before send). "
+            "Optional 'transition_chunks' defaults to None for velocity and "
+            f"{DEFAULT_TRANSITION_CHUNKS} for target."
+        ),
+    )
     return parser.parse_args()
 
 

examples/online_serving/text_to_speech/README.md

@@ -27,7 +27,6 @@ For the full list of supported architectures across all modalities, see
 | OmniVoice | `k2-fsa/OmniVoice` | ✓ | — | — | — |
 | Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | ✓ (Base) | ✓ (PCM + WebSocket) | ✓ (presets + `/v1/audio/voices` upload) | ✓ (standard + FastRTC) |
 | VoxCPM2 | `openbmb/VoxCPM2` | ✓ | ✓ (AudioWorklet via gradio) | — | ✓ |
-| dots.tts | `dots-studio/dots.tts-soar` | — (text-only) | ✓ (PCM stream) | — (default placeholder only) | — |
 | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✓ (gated upstream) | ✓ | ✓ (presets) | ✓ |
 
 CosyVoice3 is intentionally absent: no online example exists for it yet. See its [offline section](../../offline_inference/text_to_speech/README.md#cosyvoice3) instead.
@@ -143,42 +142,6 @@ python examples/online_serving/text_to_speech/gepard/speech_client.py \
 - Concurrent requests at `max_num_seqs: 4` are supported. Native-AR recompute preemption is a known limitation of this architecture (a request that is preempted mid-generation can resume incorrectly); keep concurrency at or below `max_num_seqs` and treat preemption as out of scope until the platform fix lands.
 - Optional comparison against the upstream Gepard reference server needs Blackwell/Hopper + CUDA 13 + Postgres and is not part of CI.
 
-## dots.tts
-
-Single-stage, text-only TTS at 48 kHz. The current integration supports
-no-reference synthesis; voice cloning, reference audio, named voices, and
-precomputed speaker embeddings are not supported yet.
-
-### Launch
-
-```bash
-vllm serve dots-studio/dots.tts-soar --omni --trust-remote-code --port 8091
-```
-
-### Sending requests
-
-```bash
-# Non-streaming WAV
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, this is a dots TTS online-serving test.",
-        "voice": "default",
-        "response_format": "wav"
-    }' --output output.wav
-
-# Streaming 48 kHz mono PCM
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, this is a streaming dots TTS test.",
-        "voice": "default",
-        "stream": true,
-        "stream_format": "audio",
-        "response_format": "pcm"
-    }' --no-buffer | play -t raw -r 48000 -e signed -b 16 -c 1 -
-```
-
 ---
 
 ## GLM-TTS

recipes/Bagel/BAGEL-7B-MoT.md

@@ -120,12 +120,19 @@ python examples/offline_inference/image_to_image/image_edit.py \
 The `--extra-args` JSON forwards BAGEL-specific parameters (e.g. `cfg_text_scale`,
 `cfg_img_scale`, `cfg_interval`, `cfg_renorm_type`) into
 `OmniDiffusionSamplingParams.extra_args` via the model-extras registry.
-For BAGEL img2img, the pipeline derives the generated image size from the input
-image: it preserves the input aspect ratio, aligns dimensions to the latent
-stride, and applies the checkpoint size limit. Explicit `--height` and
-`--width` values do not override that BAGEL-specific resize policy. Step-mode
-admission resolves this effective size before batching so differently shaped
-requests are scheduled separately.
+For BAGEL img2img, an explicit `--height`/`--width` (or `size` on
+`/v1/images/edits`) sets the output canvas, in both the single-stage and the
+two-stage deployment; the input image is still resized to a stride-aligned size
+for the VAE/ViT prefill. The latent grid needs multiples of 16, so a requested
+side that is not one is floored to the next multiple (1000x700 generates
+992x688) with a server-side warning, and the response reports the generated
+size. A side above the checkpoint limit is rejected with HTTP 400. When no size
+is requested
+(`size=auto`, or no `height`/`width`), the pipeline derives the canvas from the
+input image: it preserves the input aspect ratio, aligns dimensions to the
+latent stride, and applies the checkpoint size limit. Step-mode admission
+resolves this effective size before batching so differently shaped requests are
+scheduled separately.
 
 Run text-to-text with the shared understanding example. BAGEL's default
 `bagel.yaml` deploy config is discovered from the checkpoint, so no model-specific

recipes/Boogu/Boogu-Image.md

@@ -132,18 +132,19 @@ with `guidance_scale=1.0` remain valid on the same server: every rank evaluates
 only the positive branch, no negative embeddings are built, and no guidance is
 applied.
 
-### 1 x RTX 4090 48 GB (FP8)
+### 1 x RTX 4090 24 GB / 48 GB (FP8)
 
-Boogu-Image supports two ways to quantize or load the DiT transformer:
+Boogu-Image supports the following FP8 loading paths:
 
-| Path               | Base model                       | Edit model                       | Method                       | Scheme                  |
-| ------------------ | -------------------------------- | -------------------------------- | ---------------------------- | ----------------------- |
-| Serialized TorchAO | `Boogu/Boogu-Image-0.1-Base-fp8` | `Boogu/Boogu-Image-0.1-Edit-fp8` | `torchao_float8_weight_only` | FP8 weight-only (W8A16) |
-| Native online FP8  | `Boogu/Boogu-Image-0.1-Base`     | `Boogu/Boogu-Image-0.1-Edit`     | `fp8`                        | Dynamic W8A8            |
+| Path | Checkpoints | MLLM | DiT |
+| --- | --- | --- | --- |
+| Pre-quantized FP8 | `Boogu/Boogu-Image-0.1-{Base,Edit}-fp8` | HF checkpoint FP8 configuration | TorchAO FP8 weight-only (W8A16) |
+| Online FP8 | `Boogu/Boogu-Image-0.1-{Base,Edit}` | Optional 128x128 block FP8: Triton W8A8 (vLLM 0.28.0); Marlin W8A16 (vLLM 0.29.0) | Native dynamic FP8 (CUTLASS W8A8) |
 
 #### Environment
 
-The following configuration was validated:
+The following isolated environments were used for Base T2I and Edit I2I
+measurements at 512x512:
 
 - OS: Linux
 - Python: 3.12
@@ -152,7 +153,9 @@ The following configuration was validated:
 - vLLM-Omni version or commit: Use a commit that contains TorchAO FP8
   checkpoint loading for diffusion models
 - TorchAO version: 0.17.0
-- `kernels` / `kernels-data`: 0.16.1 / matching kernels-data
+- `kernels` / `kernels-data`: 0.16.1 / 0.16.1
+- Transformers: 5.17.0
+- Tokenizers: 0.23.1
 
 #### Command
 
@@ -167,19 +170,61 @@ vllm serve Boogu/Boogu-Image-0.1-Base-fp8 \
   '{"transformer":{"method":"torchao_float8_weight_only"}}'
 ```
 
-Native online FP8 (currently applied only to eligible vLLM-quantizable linear
-layers in the DiT transformer):
+Native online FP8:
 
 ```bash
+# DiT only
 vllm serve Boogu/Boogu-Image-0.1-Base \
   --omni \
   --port 8091 \
   --diffusion-quantization-config \
-  '{"transformer":{"method":"fp8"}}'
+  '{"mllm":null,"transformer":"fp8","vae":null}'
+
+# MLLM + DiT
+vllm serve Boogu/Boogu-Image-0.1-Base \
+  --omni \
+  --port 8091 \
+  --quantization fp8
+
+# MLLM + DiT (using --diffusion-quantization-config)
+vllm serve Boogu/Boogu-Image-0.1-Base \
+  --omni \
+  --port 8091 \
+  --diffusion-quantization-config \
+  '{"mllm":"fp8","transformer":"fp8","vae":null}'
+
 ```
 
-For image editing, use the matching Edit model ID from the table and keep the
-corresponding quantization configuration.
+For image editing, replace `Base` with `Edit` and keep the corresponding
+quantization configuration.
+
+#### Notes
+
+- **Memory usage:**
+    - The serialized TorchAO FP8 path uses approximately 21.1 GiB at 512x512
+      and 24.2-24.6 GiB at 1024x1024.
+    - The DiT-only online FP8 path uses approximately 26.9 GiB at 512x512
+      and 29.9-30.9 GiB at 1024x1024.
+    - The MLLM + DiT online FP8 path uses approximately 20.5-20.6 GiB at
+      512x512 and 23.6-24.5 GiB at 1024x1024.
+
+- **Quantization scope:** For original Base/Edit checkpoints, `--quantization fp8`
+  quantizes the MLLM's language-model linear layers through per-block FP8
+  and the DiT's linear layers through native online FP8. The vision encoder,
+  embeddings, normalization layers and VAE remain unquantized.
+
+- **Performance:** In the 48 GB Base/Edit benchmarks at 512x512, both DiT-only
+  and MLLM + DiT online FP8 reduced E2E latency by over 10 seconds compared with
+  official FP8 and vLLM-Omni pre-quantized FP8. The main runtime difference lies in the DiT quantization implementation, which switches from TorchAO weight-only FP8 (W8A16) to native CUTLASS
+  W8A8 kernels. This reduces additional dequantization overhead and enables
+  FP8 Tensor Core acceleration.
+
+- **Known Limitation:**
+    - **Output consistency:** With MLLM + DiT online FP8, same-seed outputs may visibly differ from BF16 for some Base prompts, but these differences do not mean lower image quality.
+
+    - **Import compatibility conflict between TorchAO and Diffusers::** Loading pre-quantized checkpoints requires `torchao>=0.17.0`, which has a compatibility conflict with `diffusers==0.40.0`. This produces a warning when Diffusers is imported, but currently does not affect Boogu-Image weight loading.
+
+    - **Conflict between the default kernels and Transformers versions:** The default Transformers version range (`>=5.10.1,<5.15`) conflicts with `kernels==0.16.1` when loading pre-quantized FP8 MLLM weights through Hugging Face `FP8Linear`. When using this loading path with `kernels==0.16.1`, please upgrade Transformers to `5.17.0`.
 
 #### Verification
 
@@ -190,19 +235,6 @@ ID passed to `vllm serve`. For example, use
 corresponding Edit model.
 
 Base text-to-image:
-## Fast text-to-image (Boogu-Image-0.1-Turbo)
-
-Turbo is the distilled text-to-image checkpoint. Its `model_index.json`
-declares `BooguImageTurboPipeline`, which runs the four-step DMD path instead
-of the scheduler-driven path used by Base.
-
-### Command
-
-```bash
-vllm serve Boogu/Boogu-Image-0.1-Turbo --omni --port 8091
-```
-
-### Verification
 
 ```bash
 curl -X POST http://localhost:8091/v1/images/generations \
@@ -231,22 +263,24 @@ curl -X POST http://localhost:8091/v1/images/edits \
   | jq -r '.data[0].b64_json' | base64 -d > edited_fp8.png
 ```
 
-#### Notes
+## Fast text-to-image (Boogu-Image-0.1-Turbo)
 
-- **Memory usage:**
-    - The serialized TorchAO FP8 path uses approximately 21.1 GiB at 512x512
-      and 24.2-24.6 GiB at 1024x1024.
-    - The native online FP8 path uses approximately 31.5 GiB for Base and
-      30.7 GiB for Edit at 1024x1024. It uses more memory because only eligible
-      DiT linear layers are quantized online, while the MLLM from the regular
-      Base/Edit checkpoint remains in BF16. Most of the observed difference
-      comes from the MLLM precision, although differences in the two DiT FP8
-      representations and kernels may also contribute.
-- **Quantization scope:** The transformer-scoped configuration applies to the
-  DiT. With the serialized `-fp8` checkpoints, the MLLM follows its own
-  checkpoint-declared FP8 configuration. With the regular checkpoints used for
-  online FP8, the MLLM remains in BF16. The VAE and scheduler are outside this
-  quantization routing.
+Turbo is the distilled text-to-image checkpoint. Its `model_index.json`
+declares `BooguImageTurboPipeline`, which runs the four-step DMD path instead
+of the scheduler-driven path used by Base.
+
+### Command
+
+```bash
+vllm serve Boogu/Boogu-Image-0.1-Turbo --omni --port 8091
+```
+
+### Verification
+
+```bash
+curl -X POST http://localhost:8091/v1/images/generations \
+  -H "Content-Type: application/json" \
+  -d '{
     "model": "Boogu/Boogu-Image-0.1-Turbo",
     "prompt": "A mountain lake at sunset, photorealistic, cinematic lighting",
     "size": "1024x1024",

recipes/MammothModa2/MammothModa2.md

@@ -18,11 +18,18 @@ The generic example formats the AR prompt, drives the AR → DiT stage pipeline,
 and forwards MammothModa2-specific generation parameters through the
 pipeline-declared `extra_body` contract.
 
-MammothModa2's DiT stage consumes its inputs through the multi-stage kwargs
-interface (not `OmniDiffusionRequest`), so its generation knobs
-(`text_guidance_scale`, `cfg_range`, `num_inference_steps`) are passed via
-`--extra-body` rather than the standard `--num-inference-steps` / `--cfg-scale`
-flags. Image size uses the standard `--height` / `--width` flags.
+MammothModa2's DiT stage runs in the shared diffusion runtime in request mode.
+The first integration intentionally supports one request and one image per
+forward only (`max_num_seqs: 1`, `num_outputs_per_prompt: 1`). Request-level
+batching, step execution, continuous batching, cache acceleration,
+compilation, quantization, parallelism, and offload are not enabled by this
+recipe.
+
+Image size, seed, guidance, and denoising steps use the standard diffusion
+request fields. `cfg_range` remains a MammothModa2-specific `extra_body`
+parameter. For compatibility, the runtime also accepts the former
+`text_guidance_scale` and `num_inference_steps` keys in `extra_body`; when
+present and non-null, those keys take precedence over the standard fields.
 
 ## References
 
@@ -41,11 +48,11 @@ flags. Image size uses the standard `--height` / `--width` flags.
 
 ## Hardware Support
 
-The default deploy config runs both the AR and DiT stages on a single GPU
-(`devices: "0"`). The committed `gpu_memory_utilization` split (stage-0 AR `0.5`,
-stage-1 DiT `0.3`) is sized for an ~80 GB GPU. The model also fits on a 48 GB GPU
-after rebalancing the split so the AR weights (~23 GB) leave room for the KV
-cache — see the note under *1x L40S 48GB*.
+The default deploy config places both the AR and DiT stages on one GPU
+(`devices: "0"`). Its committed `gpu_memory_utilization` split is 0.5 for
+stage 0 and 0.3 for stage 1. The A800 validation section below also shows a
+two-GPU placement with one stage per GPU for attributable timing and memory;
+the measured results are summarized below.
 
 ## GPU
 
@@ -102,12 +109,13 @@ not implied.
 > set the stage-0 (AR) value to `0.8` and the stage-1 (DiT) value to `0.16`
 > before running. (On an ~80 GB GPU, leave the defaults unchanged.)
 
+### 1x NVIDIA A800 80GB
+
 #### Environment
 
 - OS: Linux
 - Python: Match the repository requirements for your checkout
-- Driver / runtime: NVIDIA CUDA environment with one L40S 48 GB (verified) or an
-  ~80 GB GPU for the default config
+- Driver / runtime: NVIDIA CUDA environment with one A800 80 GB
 - vLLM version: Match the repository requirements for your checkout
 - vLLM-Omni version or commit: Use the commit you are deploying from
 
@@ -120,8 +128,7 @@ hf download bytedance-research/MammothModa2-Preview --local-dir ./MammothModa2-P
 ```
 
 Run text-to-image with the shared offline example from the repository root. The
-deploy config sets `trust_remote_code`, so no extra flag is needed. Forward the
-MammothModa2 generation parameters as a JSON object through `--extra-body`:
+deploy config sets `trust_remote_code`, so no extra flag is needed:
 
 ```bash
 python examples/offline_inference/text_to_image/text_to_image.py \
@@ -130,23 +137,25 @@ python examples/offline_inference/text_to_image/text_to_image.py \
   --prompt "A stylish woman riding a motorcycle in NYC, movie poster style" \
   --height 1024 \
   --width 1024 \
-  --extra-body '{"text_guidance_scale": 4.0, "cfg_range": [0.0, 1.0], "num_inference_steps": 50}' \
+  --seed 42 \
+  --guidance-scale 4.0 \
+  --num-inference-steps 50 \
+  --extra-body '{"cfg_range": [0.0, 1.0]}' \
   --output mammoth_t2i.png
 ```
 
-The `--extra-body` JSON forwards MammothModa2-specific parameters into
-`OmniDiffusionSamplingParams.extra_args`. Keys are filtered against the model's
-declared `extra_body_params` (see
+The standard diffusion request fields are `height`, `width`, `seed`,
+`guidance_scale`, and `num_inference_steps`; use their corresponding CLI flags
+shown above. `--height` and `--width` must be multiples of 16.
+
+`cfg_range` is the only recommended MammothModa2 field in `--extra-body`; it
+sets the relative step range `[start, end]` over which CFG is applied (default
+`[0.0, 1.0]`). For compatibility, `text_guidance_scale` and
+`num_inference_steps` remain accepted `extra_body` aliases and, when non-null,
+take precedence over the standard request fields. Model extras are filtered
+against the declared `extra_body_params` (see
 [`vllm_omni/model_extras/mammothmodal2_preview.py`](../../vllm_omni/model_extras/mammothmodal2_preview.py)),
-so unknown keys for MammothModa2 are silently dropped:
-
-- `text_guidance_scale` — classifier-free guidance scale for the DiT stage
-  (default `9.0`; CFG is active only when `> 1.0`).
-- `cfg_range` — relative step range `[start, end]` over which CFG is applied
-  (default `[0.0, 1.0]`).
-- `num_inference_steps` — number of DiT denoising steps (default `50`).
-
-`--height` and `--width` must be multiples of 16.
+so unknown MammothModa2 extras may be dropped.
 
 Run text-to-text through the shared understanding example. It recognizes the
 MammothModa2 checkpoint and automatically selects `mammoth_moda2_ar.yaml`:
@@ -177,7 +186,49 @@ ls -lh mammoth_t2i.png
 python -c "from PIL import Image; print(Image.open('mammoth_t2i.png').size)"
 ```
 
-### 1x AMD MI300X, MammothModa2 Preview
+### 2x NVIDIA A800 80GB validation
+
+Use one A800 per stage so AR and DiT memory and timing are attributable. The
+per-stage override changes placement only; both stages remain single-rank.
+
+```bash
+VLLM_LOGGING_LEVEL=DEBUG vllm serve ./MammothModa2-Preview --omni \
+  --deploy-config vllm_omni/deploy/mammoth_moda2.yaml \
+  --stage-overrides '{"0":{"devices":"0"},"1":{"devices":"1"}}' \
+  --port 8099 \
+  --log-stats
+```
+
+Startup logs should identify stage 1 as `StageDiffusionClient` and resolve it
+to `MammothModa2DiTPipeline`. `DiffusionEngine` step timing is a DEBUG-level,
+per-request message, so it appears only after sending a text-to-image request
+with `VLLM_LOGGING_LEVEL=DEBUG`; it is not a startup marker. Seeing the legacy
+generation model runner for stage 1 is a failed migration.
+
+#### Migration benchmark
+
+The request-mode migration was checked on 2x NVIDIA A800 80GB PCIe with AR on
+GPU 0 and DiT on GPU 1. Each revision ran one warmup followed by 10 serial
+measured requests in the same initialized process. Both used BF16 eager mode,
+1024x1024 output, 50 denoising steps, guidance scale 4.0, seed 42, and no
+diffusion cache. The baseline was the pre-migration revision `caed3061`; the
+candidate was `19de562a`. Lower latency is better.
+
+| Metric | Baseline p50 | Baseline p95 | Candidate p50 | Candidate p95 |
+| --- | ---: | ---: | ---: | ---: |
+| End-to-end latency | 105.36 s | 106.01 s | 104.94 s | 105.79 s |
+| AR stage latency | 86.97 s | 87.61 s | 86.26 s | 87.11 s |
+| DiT stage latency | 18.27 s | 18.41 s | 18.60 s | 18.62 s |
+
+Peak sampled device memory was 39,209 MiB on the AR GPU for both revisions.
+The DiT GPU used 11,089 MiB for the baseline and 10,967 MiB for the candidate.
+The candidate's shared runtime reported 372.02 ms p50 per denoising step and a
+5.89 ms p50 AR-to-diffusion adapter time. All measured requests completed and
+both revisions produced valid, prompt-aligned 1024x1024 RGB images. The small
+latency differences are regression evidence, not a statistically significant
+speedup claim.
+
+### 1x AMD MI300X, MammothModa2 Preview (pre-migration baseline)
 
 #### Environment
 

recipes/MiniMaxAI/MiniMax-H3-NPU-950PR.md

@@ -1,195 +1,224 @@
-# MiniMax-H3 on a single NPU 950PR
+# MiniMax H3 on Ascend 950PR / 950DT
 
-This recipe runs MiniMax-H3 with online INT8 quantization on one NPU 950PR
-NPU (128 GB HiBL 1.0 HBM). It covers the single-card T2VA configuration at
-1024x576. For the eight-card Atlas 800I A3 BF16 route at 768P, see
-[MiniMax-H3-NPU.md](MiniMax-H3-NPU.md).
+> Joint video and audio generation — Ascend NPU deployment guide for the
+> four-card Ascend 950PR / 950DT route
 
-## Capacity requirements
+## Summary
 
-| Resource | Requirement |
-| --- | ---: |
-| NPU | 1x NPU 950PR |
-| NPU HBM | 128 GiB (131,072 MiB reported by `npu-smi`) |
-| Observed HBM high-water mark | 118,442 MiB (90.4% of capacity) |
-| Checkpoint storage | 135 GiB per partition, local disk strongly preferred |
-| Container shared memory | 8 GiB, or a writable local filesystem for spill |
-
-The observed high-water mark leaves about 12.3 GiB of headroom, so this card is
-sized for exactly one resident task partition at 1024x576. Do not expect a
-larger output shape or concurrency greater than one to fit.
-
-Because a single 950PR holds the whole partition in HBM, this route does not
-need layerwise offload, and the 200 GiB system-RAM floor carried by the 72 GiB
-GPU recipes does not apply. The validated run completed inside a container
-limited to 32 GiB of RAM. Passing `--enable-layerwise-offload` on this card is
-actively harmful: it stages weights in non-reclaimable host memory and the
-container's OOM killer terminates the server with exit code -9.
+- Vendor: MiniMaxAI
+- Model: [`MiniMaxAI/MiniMax-H3`](https://huggingface.co/MiniMaxAI/MiniMax-H3)
+- Tasks: T2VA, FL2VA, and Ref2VA
+- Mode: OpenAI-compatible `/v1/videos` HTTP serving
+- Hardware: Ascend 950PR / Ascend 950DT, 4x NPU (128 GB HBM per device)
+- Maintainer: Community
 
+This recipe covers the four-card 950PR / 950DT configuration at 1344x768.
+For the eight-card Atlas 800I A2 / A3 route, see
+[MiniMax-H3-NPU.md](MiniMax-H3-NPU.md); checkpoint layout, container
+preparation, MindIE-SD installation, and the ffmpeg/decord dependencies are
+described there and apply unchanged.
 
 ## Environment
 
-- Host architecture: x86_64
--  driver: 25.7.rc1.6 (hal 7.35.23)
--  firmware: 9.0.0.105.229
-- CANN toolkit: 9.1.0 (`/usr/local//cann-9.1.0`)
-- npu-smi: 25.7.rc1.6
-- Python: 3.12.13
-- PyTorch: 2.10.0+cpu
-- torch_npu: 2.10.0.post2
-- vLLM: 0.26.0
-- vLLM-Omni: 0.26.1.dev103+g584d78c67.npu (commit `584d78c6`)
-
-Install vLLM-Omni from a checkout with MiniMax-H3 support:
+Use the A5 variant of the official vLLM-Omni NPU images (see
+[NPU installation](../../docs/getting_started/installation/npu.md)):
 
 ```bash
-uv venv
-source .venv/bin/activate
-uv pip install -e .
+export IMAGE=quay.io/ascend/vllm-omni:v0.29.0-a5
 ```
 
+Check the [tag list](https://quay.io/repository/ascend/vllm-omni?tab=tags)
+for newer releases. Inside the container, install MindIE-SD and the optional
+Ref2VA media dependencies as described in
+[MiniMax-H3-NPU.md § Environment](MiniMax-H3-NPU.md#environment).
+
 ## Start a server
 
+Task selection follows the eight-card recipe: pass `--task-type fl2va` or
+`--task-type ref2va` to pick the task partition. The configurations below use
+4-card USP, 4-card text-encoder TP, and VAE `tile` parallelism.
+
+The two recommended configurations differ in how they fit memory: the
+lossless configuration enables distributed layerwise offload (DLO) to avoid
+activation OOM at longer durations, while the lossy configuration fits the
+maximum supported MiniMax-H3 workload shape without OOM, so DLO is omitted
+for performance.
+
+### Recommended lossless configuration (Ascend 950PR / 950DT)
+
 ```bash
-export MODEL=/path/to/MiniMax-H3/FL2VA
-export PORT=8000
+export PORT=9098
+export MODEL=/path/to/MiniMax-H3
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-export VLLM_OMNI_VIDEO_SYNC_TIMEOUT=14400
+export VLLM_OMNI_VIDEO_SYNC_TIMEOUT=4000
+export PYTHONDONTWRITEBYTECODE=1
+export HF_HUB_OFFLINE=1
+export TRANSFORMERS_OFFLINE=1
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+export HCCL_NPU_SOCKET_PORT_RANGE="auto"
 
 vllm serve "${MODEL}" \
   --omni \
   --host 0.0.0.0 \
   --port "${PORT}" \
   --trust-remote-code \
-  --num-gpus 1 \
-  --tensor-parallel-size 1 \
-  --usp 1 \
+  --task-type fl2va \
+  --num-gpus 4 \
+  --usp 4 \
   --ring 1 \
-  --text-encoder-tp-size 1 \
-  --vae-patch-parallel-size 1 \
+  --text-encoder-tp-size 4 \
   --vae-parallel-mode tile \
   --vae-use-tiling \
-  --init-timeout 14400 \
-  --stage-init-timeout 14400 \
-  --quantization int8
+  --vae-patch-parallel-size 4 \
+  --enable-diffusion-pipeline-profiler \
+  --enable-distributed-layerwise-offload \
+  --diffusion-attention-backend FLASH_ATTN
 ```
 
-Do not pass `--diffusion-attention-backend CUDNN_ATTN`. cuDNN attention has no
-NPU implementation and fails at the first denoise step. Leave the backend
-unset: it resolves to `TORCH_SDPA`, or to `FLASH_ATTN` when MindIE-SD is
-installed (see [MiniMax-H3-NPU.md § Environment](MiniMax-H3-NPU.md#environment)).
+H3 is CFG-distilled, so `--cfg-parallel-size` must remain 1. The first
+request includes regional compilation; warm the server once before measuring
+steady-state latency.
 
-H3 is CFG-distilled, so `--cfg-parallel-size` must remain 1.
+### Recommended lossy configuration (Ascend 950PR / 950DT)
 
-## Validated evidence
-
-Measured on one NPU 950PR with the server command above plus
-`--enable-diffusion-pipeline-profiler`, generating a 5 s 1024x576 T2VA clip from
-the request in [§ T2VA request example](#t2va-request-example). MiniMax-H3
-requested 60 denoise steps and executed 59 denoise updates, so per-step latency
-is `denoise / 59`. `ffprobe` confirms every returned MP4 is 1024x576, 24 fps,
-124 video frames, 5.175 s, with a 32 kHz audio track of 165,600 samples.
-
-Five requests were issued back to back with no idle time between them. The card
-is measurably faster on the first request than in steady state, so both regimes
-are reported.
-
-| Stage | First request, cold card | Thermal steady state |
-| --- | ---: | ---: |
-| End-to-end request | 462.13 s | 506.38 s |
-| Text encode | 0.55 s | 0.05 s |
-| Denoise (59 updates) | 429.29 s | 471.74 s |
-| Per denoise update | 7,276 ms | 7,996 ms |
-| VAE decode | 8.43 s | 8.14 s |
-| Worker-to-server handoff | 24 s | 27 s |
-| MP4 muxing | 0.57 s | 0.50 s |
-| Server-reported `denoise_step_latency_ms` | 7,702 ms | 8,440 ms |
-| Peak HBM (`npu-smi`) | 118,442 MiB | 118,442 MiB |
-| Average NPU power | 566.9 W | 556.9 W |
-| Peak NPU temperature | 104 C | 104 C |
-
-Sample counts differ per row. The cold column is a single request. In the steady
-column, text encode, denoise and VAE decode are means over the four subsequent
-requests — denoise spanned 470.37 to 473.21 s, a 0.60% spread — while
-end-to-end, handoff and muxing are means over the two of those four that
-survived the handoff timeout described above.
-
-**Report the steady-state column.** The cold card completes denoise 9.9% faster
-while drawing only 1.8% more average power, and both regimes reach the same
-104 C ceiling, so the first request is buying a short window of higher clocks
-before the die saturates. Any benchmark that issues a single request against an
-idle card will overstate throughput by about 10%.
-
-**Do not read `denoise_step_latency_ms` as a per-step time.** The server divides
-the entire stage wall time by the 60 *requested* steps, so it absorbs text
-encoding, VAE decode and the 837 MiB handoff. All three successful requests
-satisfy `denoise_step_latency_ms == e2e_stage_wall_time_ms / 60` exactly. The
-per-update figure in the table is `diffuse / 59` from the pipeline profiler.
-
-Denoise accounts for 93% of the steady-state request and the handoff accounts
-for 5.3%, so keeping the transfer in shared memory should bring end-to-end down
-to roughly 480 s.
-
-Peak HBM is sampled externally with `npu-smi` at a 3 s interval. It is a
-reserved high-water mark rather than live allocation: it reads 111,272 MiB after
-weight loading and before the first request, and does not fall back after
-requests complete. Re-measure for longer outputs, a different output shape, or
-concurrency greater than one.
-
-### Startup
-
-Weight loading took **31 min 50 s** on the validated host (first log line
-08:04:59, `Application startup complete` 08:36:49), with the checkpoint on a
-shared GlusterFS network mount. This is why the server command sets
-`--init-timeout 14400` and `--stage-init-timeout 14400`; the stock 600 s and
-1800 s timeouts both abort mid-load with `TimeoutError` and exit code 143.
-Reduce both timeouts when the partition is staged on local disk.
-
-## T2VA request example
+Adds EQBSA sparse attention (block-sparse with Q/K INT8 + V FP8 mixed
+precision) and MXFP8 online quantization, and drops DLO:
 
 ```bash
-export API_URL="http://127.0.0.1:${PORT}/v1/videos/sync"
+export PORT=9098
+export MODEL=/path/to/MiniMax-H3
+export VLLM_WORKER_MULTIPROC_METHOD=spawn
+export VLLM_OMNI_VIDEO_SYNC_TIMEOUT=4000
+export PYTHONDONTWRITEBYTECODE=1
+export HF_HUB_OFFLINE=1
+export TRANSFORMERS_OFFLINE=1
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+export HCCL_NPU_SOCKET_PORT_RANGE="auto"
 
-curl -sS --max-time 1800 -X POST "${API_URL}" \
-  -F 'prompt=At night, three cats march into a bedroom playing tiny brass instruments, then abruptly file out, with synchronized room ambience.' \
-  -F 'width=1024' \
-  -F 'height=576' \
-  -F 'aspect_ratio=16:9' \
-  -F 'fps=24' \
-  -F 'num_inference_steps=60' \
-  -F 'flow_shift=12' \
-  -F 'seed=1101' \
-  -F 'extra_params={"task":"t2va","duration":5,"audio_flow_shift":3.0}' \
-  -o t2va.mp4
+vllm serve "${MODEL}" \
+  --omni \
+  --host 0.0.0.0 \
+  --port "${PORT}" \
+  --trust-remote-code \
+  --task-type fl2va \
+  --num-gpus 4 \
+  --usp 4 \
+  --ring 1 \
+  --text-encoder-tp-size 4 \
+  --vae-parallel-mode tile \
+  --vae-use-tiling \
+  --vae-patch-parallel-size 4 \
+  --enable-diffusion-pipeline-profiler \
+  --diffusion-attention-config '{"default":{"backend":"RAINFUSION_ATTN","block_sparse":{"sparsity":0.8,"precision":"mix","start_step":8,"end_step":12}}}' \
+  --diffusion-quantization-config '{"transformer":{"method":"mxfp8"}}'
 ```
 
+The recommended lossy configuration uses a **start=8 / end=12** dense-fallback
+window; see below for more aggressive settings.
+
+### Optional optimizations
+
+The following sections list only the **increment or replacement** relative to
+the recommended lossless configuration; environment variables and the
+remaining flags stay unchanged.
+
+#### Tuning the EQBSA dense-fallback window
+
+`start_step` and `end_step` are the numbers of leading/trailing steps that
+fall back to dense attention — not step indices. `precision` defaults to
+`bf16`; EQBSA requires setting it to `mix` explicitly. For 1344x768, 50-step
+T2VA:
+
+| Configuration | `start_step` | `end_step` | Notes |
+| --- | ---: | ---: | --- |
+| Quality first (recommended) | 8 | 13 | General and complex-motion scenes |
+| Balanced | 0 | 13 | Balanced |
+| Speed first | 0 | 0 | Simple, low-motion scenes |
+
+When frames look discontinuous or details unstable, increase `end_step`
+first, then `start_step`; do not compensate by lowering `sparsity` alone.
+These pairings were validated at 1344x768 / 50 steps only; re-evaluate for
+other resolutions or step counts.
+
+#### Cache-DiT
+
+Append to enable DiT block caching with TaylorSeer extrapolation — see the
+[Cache-DiT guide](../../docs/user_guide/diffusion/cache_acceleration/cache_dit.md):
+
+```bash
+  --cache-backend cache_dit \
+  --enable-cache-dit-summary \
+  --cache-config '{"Fn_compute_blocks":2,"Bn_compute_blocks":1,"max_warmup_steps":4,"residual_diff_threshold":0.4,"max_continuous_cached_steps":4,"enable_taylorseer":true,"taylorseer_order":2}'
+```
+
+#### MXFP4 online quantization
+
+Append to switch the DiT to MXFP4 online quantization:
+
+```bash
+  --diffusion-quantization-config '{"transformer":{"method":"mxfp4"}}'
+```
+
+and remove `--enable-distributed-layerwise-offload`.
+
+## Request examples
+
+Identical to the eight-card recipe; see
+[MiniMax-H3-NPU.md § HTTP API examples](MiniMax-H3-NPU.md#http-api-examples)
+and the GPU recipe's full parameter table.
+
+## Benchmarks (Ascend 950PR)
+
+Measured with `--enable-diffusion-pipeline-profiler` on vLLM-Omni 0.28.0:
+lossless rows use the recommended lossless configuration (with DLO), lossy
+rows the recommended lossy configuration (EQBSA + MXFP8, without DLO). Memory
+figures are per device.
+
+| Task | Resolution | Frames | Duration (s) | Config | E2E (s) | DiT total (s) | DiT per-step (s) | DiT steps | Text encode (s) | Ref video encode (s) | Ref audio encode (s) | VAE decode (s) | Ref preprocess (s) | Post-process (s) | CPU MP4 (s) | Resident weights (GB) | Peak memory (GB) |
+| ---- | ---- | ---- | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
+| t2va | 1344x768 | 124 | 5 | lossless | 203.95 | 198.10 | 4.04 | 49 | 0.57 | NA | NA | 5.05 | NA | 0.001 | 0.23 | 3.97 | 22.2 |
+| t2va | 1344x768 | 362 | 15 | lossless | 1489.17 | 1474.35 | 30.08 | 49 | 0.69 | NA | NA | 13.45 | NA | 0.001 | 0.82 | 3.97 | 32.14 |
+| t2va | 1344x768 | 124 | 5 | lossy | 113.86 | 109.81 | 2.24 | 49 | 0.03 | NA | NA | 3.82 | NA | 0.001 | 0.21 | 56.99 | 64.38 |
+| t2va | 1344x768 | 362 | 15 | lossy | 762.17 | 749.40 | 15.29 | 49 | 0.03 | NA | NA | 12.19 | NA | 0.001 | 0.61 | 56.99 | 74.26 |
+| ref2va | 1344x768 | 124 | 5 | lossless | 745.43 | 730.22 | 14.9 | 49 | 1.16 | 8.15 | 0.13 | 5.12 | 0.41 | 0.001 | 0.28 | 3.97 | 23.82 |
+| ref2va | 1344x768 | 362 | 15 | lossless | 5887.24 | 5847.57 | 119.33 | 49 | 2.54 | 22.07 | 0.23 | 13.34 | 0.81 | 0.001 | 0.70 | 3.97 | 32.53 |
+| ref2va | 1344x768 | 124 | 5 | lossy | 400.40 | 387.82 | 7.91 | 49 | 0.95 | 7.05 | 0.09 | 3.91 | 0.41 | 0.001 | 0.21 | 56.6 | 66.03 |
+| ref2va | 1344x768 | 362 | 15 | lossy | 3037.84 | 3001.31 | 61.24 | 49 | 2.28 | 20.64 | 0.15 | 12.1 | 0.81 | 0.001 | 0.70 | 56.6 | 74.76 |
+
+The resident-weight figures reflect the DLO strategy difference: with DLO
+(lossless rows) DiT weights live in host memory and only the active layer
+stays on the device, while without DLO (lossy rows) the quantized weights are
+resident in HBM. Peak memory stays well inside the 128 GB per-device capacity
+in both cases. These numbers describe the validated shapes rather than a
+general throughput guarantee.
+
 ## Known limitations
 
-- INT8 online quantization is validated for T2VA on this card. Use BF16 for
-  FL2VA and Ref2VA, or re-measure before relying on it.
-- Single-card serving loads one task partition at a time. For Ref2VA, stop the
-  server and restart it against the `Ref2VA` directory.
-- Sustained load drives the die to 104 C and costs about 10% of denoise
-  throughput relative to a cold card. Warm the server with at least one full
-  request before measuring, and treat steady state as the reportable number.
-- Requests can fail during result handoff on hosts where the IPC spill path is
-  slow. See [§ Result handoff and shared
-  memory](#result-handoff-and-shared-memory).
-- Loading the checkpoint from a network mount takes over 30 minutes and forces
-  very large init timeouts. Stage the partition on local disk when possible.
-- The configuration measured here is 1024x576 at 60 steps, which differs from
-  the 1344x768 at 50 steps used by the GPU recipes in this directory. Denoise
-  cost scales superlinearly with token count, so the numbers are not directly
-  comparable across recipes.
-- The image ships a `triton` package whose  backend is not built
-  (`No module named 'triton._C.libtriton.'`). vLLM logs this as an error
-  at startup and disables Triton; the diffusion path does not need it and the
-  run completes normally.
+- Task serving is partitioned by `--task-type`: T2VA/FL2VA and Ref2VA load
+  different DiTs, so switching between them requires a restart with the other
+  partition.
+- H3 currently executes one generation request per diffusion batch.
+- The first regional-compile request is a warmup and should not be included
+  in steady-state performance measurements.
+- VAE patch parallelism requires size 1 or the full DiT group size and
+  supports the H3 native `tile` mode only.
+- The lossy configuration (EQBSA + MXFP8) is validated for T2VA and Ref2VA;
+  use the lossless configuration for FL2VA or re-validate first.
+- The lossless configuration trades throughput for activation headroom via
+  DLO; if your workload never approaches the maximum duration, dropping
+  `--enable-distributed-layerwise-offload` may improve latency at the risk of
+  OOM on longer durations.
 
 ## Additional resources
 
 - [MiniMax-H3.md](MiniMax-H3.md) — full GPU guide
-- [MiniMax-H3-NPU.md](MiniMax-H3-NPU.md) — eight-card Atlas 800I A3 BF16 guide
-- [Int8 quantization](../../docs/user_guide/quantization/int8.md)
+- [MiniMax-H3-NPU.md](MiniMax-H3-NPU.md) — eight-card Atlas 800I A2 / A3
+  guide (checkpoint, container, and dependency details)
+- [NPU installation](../../docs/getting_started/installation/npu.md)
+- [RainFusion attention](../../docs/user_guide/diffusion/attention_backends/rainfusion.md)
+  — block-sparse knobs and tuning
+- [Online quantization](../../docs/user_guide/quantization/online.md)
+  — INT8 / MXFP8 / MXFP4
+- [Cache-DiT guide](../../docs/user_guide/diffusion/cache_acceleration/cache_dit.md)
 - [Supported models](../../docs/models/supported_models.md)
 - [Video API](../../docs/serving/videos_api.md)

recipes/MiniMaxAI/MiniMax-H3-NPU.md

@@ -1,7 +1,8 @@
 # MiniMax H3 on Ascend NPU
 
 > Joint video and audio generation with text, first/last keyframes, and mixed
-> image/video/audio references — Ascend NPU deployment guide
+> image/video/audio references — Ascend NPU deployment guide for Atlas 800I A2
+> and Atlas 800I A3
 
 ## Summary
 
@@ -9,7 +10,7 @@
 - Model: [`MiniMaxAI/MiniMax-H3`](https://huggingface.co/MiniMaxAI/MiniMax-H3)
 - Tasks: T2VA, FL2VA, and Ref2VA
 - Mode: OpenAI-compatible `/v1/videos` HTTP serving
-- Hardware: Atlas 800I A3
+- Hardware: Atlas 800I A2 / Atlas 800I A3, 8x NPU
 - Maintainer: Community
 
 This recipe adapts [MiniMax-H3.md](MiniMax-H3.md) for Ascend NPU
@@ -22,103 +23,288 @@ environments. Differences from the GPU path:
   automatically falls back to soundfile / ffmpeg for wav/mp3/m4a/mp4 audio
   inputs.
 
+For the four-card Ascend 950PR / 950DT route, see
+[MiniMax-H3-NPU-950PR.md](MiniMax-H3-NPU-950PR.md).
+
 ## Prerequisites
 
 ### Checkpoint
 
-Same as the GPU recipe — Hugging Face access approval is required. Authenticate
-once; `vllm serve` downloads the required nested components automatically:
+Download the weights once; `vllm serve` accepts the repository ID or a local
+path and resolves the nested components automatically:
 
 ```bash
-hf auth login
-export MODEL=MiniMaxAI/MiniMax-H3
+export MODEL=/path/to/MiniMax-H3
+hf download MiniMaxAI/MiniMax-H3 --local-dir "${MODEL}"
+```
+
+The checkpoint directory layout is:
+
+```text
+MiniMax-H3/
+├── FL2VA/          # shared by T2VA/FL2VA tasks
+├── Ref2VA/         # Ref2VA task weights
+└── model_index.json
 ```
 
 ### Environment
 
-- Ascend driver & firmware: 25.5.1
-- CANN toolkit: 9.0.1
-- Python: 3.12
-- PyTorch: 2.10.0+cpu
-- torch_npu: 2.10.0.post2
-- Install vLLM-Omni from a checkout with MiniMax H3 support:
+The recommended deployment uses the official vLLM-Omni NPU images, which ship
+the matching vLLM / vLLM-Ascend / vLLM-Omni stack (see
+[NPU installation](../../docs/getting_started/installation/npu.md)). Pick the
+image matching the hardware — `quay.io/ascend/vllm-omni:v0.29.0` for
+Atlas 800I A2, `quay.io/ascend/vllm-omni:v0.29.0-a3` for Atlas 800I A3 — and
+check the
+[tag list](https://quay.io/repository/ascend/vllm-omni?tab=tags) for newer
+releases:
 
 ```bash
-uv venv
-source .venv/bin/activate
-uv pip install -e .
+export IMAGE=quay.io/ascend/vllm-omni:v0.29.0-a3
+export CONTAINER_NAME=h3
+
+docker run -it -u root --name ${CONTAINER_NAME} \
+  --privileged=true \
+  --shm-size=2000g \
+  --net=host \
+  --device /dev/davinci0 \
+  --device /dev/davinci1 \
+  --device /dev/davinci2 \
+  --device /dev/davinci3 \
+  --device /dev/davinci4 \
+  --device /dev/davinci5 \
+  --device /dev/davinci6 \
+  --device /dev/davinci7 \
+  --device /dev/davinci_manager \
+  --device /dev/devmm_svm \
+  --device /dev/hisi_hdc \
+  -v /home:/home \
+  -v /usr/local/dcmi:/usr/local/dcmi \
+  -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
+  -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
+  -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
+  -v /etc/ascend_install.info:/etc/ascend_install.info \
+  -v /root/.cache:/root/.cache \
+  ${IMAGE} \
+  /bin/bash
 ```
 
-Install the **mindie-sd** operator library to enable Ascend-optimized fused
-operators (`adalayernorm`, etc.) and the RainFusion `rf_v2` block-sparse
-attention kernel:
+Inside the container:
+
+- **MindIE-SD** provides the Ascend-optimized fused operators
+  (`adalayernorm`, etc.), the LaserAttention kernel, and the RainFusion
+  block-sparse attention kernel. If the image does not ship it, build it from
+  source:
 
 ```bash
-git clone https://gitcode.com/Ascend/MindIE-SD.git && cd MindIE-SD
-
-# Comment out the tik_ops build step (not needed for this use case)
-sed -i 's|^\(\s*\)source ${current_script_dir}/build_tik_ops.sh|\1# source ${current_script_dir}/build_tik_ops.sh|' build/build_ops.sh
-
+git clone https://gitcode.com/Ascend/MindIE-SD.git
+cd MindIE-SD
 python setup.py bdist_wheel
-cd dist
-pip install mindiesd-*.whl
+pip install dist/mindiesd-*.whl
 ```
 
-- `ffmpeg` and `ffprobe` must be available on `PATH`. They are used for
-  reference-video preparation and MP4 output.
-- Reference-video decoding uses `decord` when available and falls back to
-  PyAV otherwise.
-- Audio inputs do not require TorchCodec: wav/mp3/m4a/mp4 files are loaded
-  at native sample rate through the soundfile / ffmpeg fallback.
+- To track the latest vLLM-Omni, install it from a checkout:
+
+```bash
+git clone https://github.com/vllm-project/vllm-omni.git
+cd vllm-omni
+VLLM_OMNI_TARGET_DEVICE=npu pip install -e . --no-build-isolation
+```
+
+- `ffmpeg` and `ffprobe` must be available on `PATH` (reference-video
+  preparation and MP4 output). Reference-video decoding uses `decord` when
+  available and falls back to PyAV otherwise; `decord` is required only for
+  the Ref2VA workflow and can be built with:
+
+```bash
+apt update
+apt install -y ffmpeg pkg-config libavcodec-dev libavformat-dev \
+  libavutil-dev libswscale-dev libavfilter-dev libavdevice-dev
+git clone --depth 1 --recursive https://github.com/dmlc/decord.git
+cd decord
+mkdir -p build && cd build
+cmake .. -DUSE_CUDA=0 -DDECODE_FFMPEG=1 -DCMAKE_BUILD_TYPE=Release
+make
+cd ../python
+pip3 install . --no-build-isolation
+```
 
 ## Start a server
 
-Pass the repository ID directly. The pipeline loads the two nested DiTs while
-sharing the tokenizer, processor, text encoder, and VAEs from `FL2VA`.
+T2VA and FL2VA share one set of weights; the Ref2VA weights differ. Select
+the task partition at startup with **`--task-type`** (`fl2va` or `ref2va`).
+When unset, only the T2VA/FL2VA weights are loaded. All remaining flags are
+task-agnostic.
 
-### Multi-NPU: 768P combined-service configuration
+### Recommended lossless configuration (Atlas 800I A2 / A3)
 
-Use Ulysses sequence parallelism degree 8, text-encoder tensor parallelism
-degree 8, native tiled VAE patch parallelism degree 8, and distributed
-layerwise offload:
+Lossless means fused operators plus the high-performance FLASH_ATTN backend
+only. The configuration uses 8-card USP, 8-card text-encoder TP, distributed
+layerwise offload (DLO), and VAE `tile` parallelism:
 
 ```bash
-export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
 export PORT=9098
-export MODEL=MiniMaxAI/MiniMax-H3
+export MODEL=/path/to/MiniMax-H3
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-export VLLM_OMNI_VIDEO_SYNC_TIMEOUT=1800
+export VLLM_OMNI_VIDEO_SYNC_TIMEOUT=4000
 export PYTHONDONTWRITEBYTECODE=1
+export HF_HUB_OFFLINE=1
+export TRANSFORMERS_OFFLINE=1
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+export MINDIE_SD_FA_TYPE="ascend_laser_attention"
+export HCCL_NPU_SOCKET_PORT_RANGE="auto"
 
 vllm serve "${MODEL}" \
   --omni \
   --host 0.0.0.0 \
   --port "${PORT}" \
   --trust-remote-code \
+  --task-type fl2va \
   --num-gpus 8 \
   --usp 8 \
   --ring 1 \
   --text-encoder-tp-size 8 \
-  --enable-distributed-layerwise-offload \
   --vae-parallel-mode tile \
   --vae-use-tiling \
   --vae-patch-parallel-size 8 \
+  --enable-distributed-layerwise-offload \
+  --enable-diffusion-pipeline-profiler \
   --diffusion-attention-backend FLASH_ATTN
 ```
 
-Do not add `--enforce-eager`. The first request includes regional
-compilation; warm the server once before measuring steady-state latency.
-H3 is CFG-distilled, so `--cfg-parallel-size` must remain 1.
+Notes:
 
-The same endpoint accepts `task=t2va`, `task=fl2va`, and `task=ref2va`; no
-partition restart is required. Layerwise offload applies to both DiTs.
+- Do not add `--enforce-eager`. The first request includes regional
+  compilation; warm the server once before measuring steady-state latency.
+- H3 is CFG-distilled, so `--cfg-parallel-size` must remain 1.
+- Keep `--ring 1` when switching to RainFusion attention (below): the
+  block-sparse kernel ranks key blocks over the whole sequence, so ring
+  parallelism would split away the keys it needs. Scale with `--usp` instead.
+- DLO offloads DiT layers with parameters gathered across the parallel group
+  instead of replicated per rank, which is what fits the combined 768P
+  service into 64 GB HBM per device (Atlas 800I A3). Host (CPU) memory usage
+  is high with this option.
 
-On Atlas 800I A3 (64 GB HBM per device) the combined service does not fit at
-768P without offloading or sharding: use distributed layerwise offload (as
-above) or HSDP — see
-[§ Memory and attention optimizations](#memory-and-attention-optimizations-a3).
+### Recommended lossy configuration (Atlas 800I A2 / A3)
 
-### CPU MP4 response encoding (Atlas A2)
+Adds RainFusion block-sparse attention and INT8 online quantization on top of
+the lossless configuration. Validated for T2VA and Ref2VA; see
+[Benchmarks](#benchmarks-atlas-800i-a2--a3):
+
+```bash
+vllm serve "${MODEL}" \
+  --omni \
+  --host 0.0.0.0 \
+  --port "${PORT}" \
+  --trust-remote-code \
+  --task-type fl2va \
+  --num-gpus 8 \
+  --usp 8 \
+  --ring 1 \
+  --text-encoder-tp-size 8 \
+  --vae-parallel-mode tile \
+  --vae-use-tiling \
+  --vae-patch-parallel-size 8 \
+  --enable-distributed-layerwise-offload \
+  --enable-diffusion-pipeline-profiler \
+  --diffusion-quantization-config '{"transformer":{"method":"int8"}}' \
+  --diffusion-attention-config '{"default": {"backend": "RAINFUSION_ATTN",
+      "block_sparse": {"sparsity": 0.8, "start_step": 12}}}'
+```
+
+Online INT8 quantization can be combined with distributed layerwise offload
+while AllGather is enabled. Keep AllGather enabled for SP>1; use
+`--dlo-no-use-allgather` only when collective communication is unavailable.
+
+### Optional optimizations
+
+The following sections list only the **increment or replacement** relative to
+the recommended lossless configuration; environment variables and the
+remaining flags stay unchanged.
+
+#### Cache-DiT
+
+Append to enable DiT block caching with TaylorSeer extrapolation — see the
+[Cache-DiT guide](../../docs/user_guide/diffusion/cache_acceleration/cache_dit.md):
+
+```bash
+  --cache-backend cache_dit \
+  --enable-cache-dit-summary \
+  --cache-config '{"Fn_compute_blocks":2,"Bn_compute_blocks":1,"max_warmup_steps":4,"residual_diff_threshold":0.4,"max_continuous_cached_steps":4,"enable_taylorseer":true,"taylorseer_order":2}'
+```
+
+#### EQBSA sparse attention
+
+EQBSA applies Q/K INT8 with V FP8 mixed quantization on top of block-sparse
+attention. Replace `--diffusion-attention-backend FLASH_ATTN` with:
+
+```bash
+  --diffusion-attention-config '{"default":{"backend":"RAINFUSION_ATTN","block_sparse":{"sparsity":0.8,"start_step":8,"end_step":13,"precision":"mix"}}}'
+```
+
+`precision` defaults to `bf16`; enabling EQBSA requires setting it to `mix`
+explicitly. `start_step` and `end_step` are the numbers of leading/trailing
+steps that fall back to dense attention — not step indices. For 1344x768,
+50-step T2VA:
+
+| Configuration | `start_step` | `end_step` | Notes |
+| --- | ---: | ---: | --- |
+| Quality first (recommended) | 8 | 13 | General and complex-motion scenes |
+| Balanced | 0 | 13 | Balanced |
+| Speed first | 0 | 0 | Simple, low-motion scenes |
+
+When frames look discontinuous or details unstable, increase `end_step`
+first, then `start_step`; do not compensate by lowering `sparsity` alone.
+These pairings were validated at 1344x768 / 50 steps only; re-evaluate for
+other resolutions or step counts.
+
+## HTTP API examples
+
+The request contract is identical to the GPU recipe; see
+[MiniMax-H3.md § HTTP API examples](MiniMax-H3.md#http-api-examples). Use the
+validated 768P shapes (e.g. `width=1344 height=768`) on NPU. Quick start
+against the synchronous endpoint:
+
+```bash
+export API_URL="http://127.0.0.1:${PORT}/v1/videos/sync"
+
+curl -sS -X POST "${API_URL}" \
+  -F 'prompt=In a snowy blue-purple forest, Ori carefully walks past a sleeping giant; footsteps crunch in the snow while the creature breathes and softly snorts.' \
+  -F 'width=1344' \
+  -F 'height=768' \
+  -F 'aspect_ratio=16:9' \
+  -F 'fps=24' \
+  -F 'num_inference_steps=50' \
+  -F 'flow_shift=12.0' \
+  -F 'audio_flow_shift=3.0' \
+  -F 'seed=1101' \
+  -F 'extra_params={"task":"t2va","duration":5.0}' \
+  -o t2va.mp4
+```
+
+Switch tasks via `extra_params.task` (`t2va` / `fl2va` / `ref2va`); FL2VA and
+Ref2VA additionally pass `input_references` (first-frame image, reference
+video/audio). The full parameter table lives in the GPU recipe.
+
+## FlashGen 4-step online LoRA (T2VA)
+
+The FlashGen 4-step weights are a native-layout LoRA single file
+(`key_format=minimax-h3-native`) loaded at runtime — no merge into the base
+checkpoint. Download them from
+[FlashGen/Minimax-H3-4step-lora-flashgen](https://modelscope.cn/models/FlashGen/Minimax-H3-4step-lora-flashgen),
+then on top of the recommended configuration above:
+
+- append `--lora-backend peft --lora-path "${FLASHGEN_LORA}"`,
+- use `--diffusion-attention-backend FLASH_ATTN` (dense; drop the RainFusion
+  attention config),
+- issue T2VA requests with `num_inference_steps=4` and the `lora` request
+  field (`name` / `path` / `scale`) — see
+  [MiniMax-H3.md § FlashGen native LoRA](MiniMax-H3.md#flashgen-native-lora).
+
+T2VA only; FL2VA and Ref2VA are not supported, and neither
+`--enable-cpu-offload` nor `--enable-layerwise-offload` may be combined with
+it.
+
+## CPU MP4 response encoding (Atlas A2)
 
 Non-streaming MP4 responses use one public automatic encoder. It checks the
 runtime frame shape, common dtype, and RGB channel-plane contiguity for every
@@ -127,121 +313,65 @@ inputs fall back to the legacy muxer before opening the PyAV container. No CLI
 flag, model declaration, or user configuration is required. Streaming fMP4
 output remains on its existing incremental path.
 
-Current performance and correctness validation is limited to one Atlas A2 host
-with 8x Ascend 910B4-1 NPUs, the `FL2VA`/`t2va` partition, one request at a
-time, 1344x768 at 24 fps, and 5, 8.7, and 15 second requests. This optimization
-only changes CPU MP4 response encoding; it does not change DiT execution or
-stage 0.
+## Ascend-optimized components
 
-### Optional optimizations
+The optimizations stacked by the recommended configurations above:
 
-Two independent optimizations may be enabled on top of the configuration
-above. Both are validated for T2VA only.
+1. Ascend-friendly attention: the MindIE-SD `ascend_laser_attention` fused
+   kernel selected through `MINDIE_SD_FA_TYPE`, carried by the FLASH_ATTN
+   backend with a mask-free packed varlen path and K/V prefix slicing driven
+   by the packed `cu_seqlens` metadata.
+2. Ascend-friendly fused operators: RMSNorm, AddRMSNorm, GQA, SwiGLU, and
+   rotary position embedding from MindIE-SD.
+3. RainFusion block-sparse attention, optionally with EQBSA mixed INT8/FP8
+   precision — see
+   [RainFusion](../../docs/user_guide/diffusion/attention_backends/rainfusion.md).
+4. Online quantization: INT8, MXFP8, and MXFP4 — see
+   [online quantization](../../docs/user_guide/quantization/online.md).
+5. Parallelism: USP, text-encoder TP, and native VAE `tile` patch
+   parallelism, plus distributed layerwise offload.
+6. Post-training: few-step distillation via the FlashGen online LoRA.
 
-**RainFusion block-sparse attention** — switch the attention backend, keeping
-every other flag unchanged:
+## Benchmarks (Atlas 800I A2 / A3)
 
-```bash
-  --diffusion-attention-backend RAINFUSION_ATTN
-```
+Measured with `--enable-diffusion-pipeline-profiler` on vLLM-Omni 0.28.0:
+lossless rows use the recommended lossless configuration, lossy rows the
+recommended lossy configuration. Memory figures are per device.
 
-**INT8 online quantization** — add one flag to the server command above:
+### Atlas 800I A2
 
-```bash
-  --quantization int8
-```
+| Task | Resolution | Frames | Duration (s) | Config | E2E (s) | DiT total (s) | DiT per-step (s) | DiT steps | Text encode (s) | Ref video encode (s) | Ref audio encode (s) | VAE decode (s) | Ref preprocess (s) | Post-process (s) | CPU MP4 (s) | Resident weights (GB) | Peak memory (GB) |
+| ---- | ---- | ---- | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
+| t2va | 1344x768 | 124 | 5 | lossless | 150.06 | 139.9 | 2.85 | 49 | 0.83 | NA | NA | 6.76 | NA | 0.462 | 1.18 | 2.97 | 24.7 |
+| t2va | 1344x768 | 362 | 15 | lossless | 786.24 | 761.36 | 15.53 | 49 | 1.20 | NA | NA | 15.87 | NA | 1.98 | 2.95 | 2.97 | 36.21 |
+| t2va | 1344x768 | 124 | 5 | lossy | 108.1 | 95.85 | 1.95 | 49 | 0.9 | NA | NA | 6.73 | NA | 0.745 | 1.04 | 2.55 | 22.47 |
+| t2va | 1344x768 | 362 | 15 | lossy | 552.51 | 526.04 | 10.73 | 49 | 1.28 | NA | NA | 16.07 | NA | 1.90 | 3.76 | 2.55 | 35.11 |
+| ref2va | 1344x768 | 124 | 5 | lossless | 474.89 | 421.68 | 8.6 | 49 | 2.42 | 13.31 | 0.38 | 6.47 | 3.94 | 0.489 | 1.16 | 3.4 | 23.45 |
+| ref2va | 1344x768 | 362 | 15 | lossless | 3414.18 | 3336.98 | 68.08 | 49 | 6.48 | 30.85 | 0.59 | 15.8 | 12.59 | 1.99 | 3.08 | 3.4 | 36.28 |
+| ref2va | 1344x768 | 124 | 5 | lossy | 380.92 | 326.50 | 6.66 | 49 | 2.47 | 13.48 | 0.38 | 6.36 | 4.68 | 0.591 | 1.22 | 2.55 | 22.57 |
+| ref2va | 1344x768 | 362 | 15 | lossy | 2332.04 | 2249.63 | 45.89 | 49 | 6.33 | 35.1 | 0.59 | 15.72 | 11.07 | 1.35 | 2.3 | 2.55 | 35.58 |
 
-Keep `--ring 1` when using RainFusion: the `rf_v2` kernel ranks key blocks
-over the whole sequence, so ring parallelism would split away the keys it
-needs. Scale with `--usp` instead.
+### Atlas 800I A3
 
-## Memory and attention optimizations (A3)
+| Task | Resolution | Frames | Duration (s) | Config | E2E (s) | DiT total (s) | DiT per-step (s) | DiT steps | Text encode (s) | Ref video encode (s) | Ref audio encode (s) | VAE decode (s) | Ref preprocess (s) | Post-process (s) | CPU MP4 (s) | Resident weights (GB) | Peak memory (GB) |
+| ---- | ---- | ---- | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
+| t2va | 1344x768 | 124 | 5 | lossless | 114.19 | 105.87 | 2.16 | 49 | 0.48 | NA | NA | 6.56 | NA | 0.001 | 0.46 | 3.49 | 24.7 |
+| t2va | 1344x768 | 362 | 15 | lossless | 599.29 | 580.38 | 11.84 | 49 | 1.83 | NA | NA | 14.95 | NA | 0.001 | 1.02 | 3.49 | 36.21 |
+| t2va | 1344x768 | 124 | 5 | lossy | 84.82 | 77.47 | 1.58 | 49 | 0.57 | NA | NA | 6.23 | NA | 0.001 | 0.61 | 2.63 | 22.47 |
+| t2va | 1344x768 | 362 | 15 | lossy | 401.43 | 383.56 | 7.82 | 49 | 1.72 | NA | NA | 14.83 | NA | 0.001 | 1.14 | 2.63 | 35.11 |
+| ref2va | 1344x768 | 124 | 5 | lossless | 355.53 | 311.86 | 6.36 | 49 | 1.89 | 12.37 | 0.24 | 5.55 | 2.49 | 0.001 | 0.43 | 3.49 | 23.45 |
+| ref2va | 1344x768 | 362 | 15 | lossless | 2339.56 | 2277.61 | 46.47 | 49 | 5.61 | 34.46 | 0.36 | 13.61 | 6.83 | 0.001 | 1.11 | 3.49 | 36.28 |
+| ref2va | 1344x768 | 124 | 5 | lossy | 306.82 | 247.72 | 5.05 | 49 | 1.88 | 12.71 | 0.24 | 5.24 | 2.45 | 0.001 | 0.54 | 2.63 | 22.57 |
+| ref2va | 1344x768 | 362 | 15 | lossy | 1695.26 | 1634.58 | 33.35 | 49 | 5.57 | 33.27 | 0.36 | 13.51 | 6.96 | 0.001 | 1.22 | 2.63 | 35.58 |
 
-### Fitting 768P into 64 GB HBM: distributed layerwise offload or HSDP
-
-Each Atlas 800I A3 NPU has 64 GB of HBM, which is not enough for the
-combined MiniMax-H3 service at 768P. Enable **one** of the following at
-server startup:
-
-**Distributed layerwise offload** — DiT layers are offloaded with parameters
-gathered across the parallel group instead of replicated per rank:
-
-```bash
-  --enable-distributed-layerwise-offload
-```
-
-Measured peak NPU memory is about 45 GB per device for Ref2VA with a 13.88 s
-input video generating a 15 s 768P (1344x768) output video. Host (CPU) memory
-usage is high with this option.
-
-**HSDP** — hybrid sharded data parallelism for the DiT parameters. The
-multi-stream memory-reuse knob is mandatory with this flag:
-
-```bash
-export MULTI_STREAM_MEMORY_REUSE=2
-```
-
-```bash
-  --use-hsdp
-```
-
-Host memory usage is small, but large shapes may still OOM; HBM usage
-optimizations for this path are ongoing.
-
-### FLASH_ATTN backend with MindIE-SD
-
-Keep `--diffusion-attention-backend FLASH_ATTN` and install MindIE-SD (see
-[§ Environment](#environment)). On NPU this backend carries most of the
-memory-reduction and performance work: a mask-free packed varlen path that
-never materializes the quadratic `full_qk` padding mask, and K/V prefix
-slicing, both driven by the packed `cu_seqlens` metadata emitted by the H3
-transformer.
-
-### Optional: LaserAttention fused kernel
-
-For an additional attention speedup, select the Ascend LaserAttention fused
-kernel before starting the server:
-
-```bash
-export MINDIE_SD_FA_TYPE="ascend_laser_attention"
-```
-
-This requires the FLASH_ATTN backend and MindIE-SD. H3 automatically applies
-exact power-of-two input pre-scaling (`laser_input_scale=256`) so the
-kernel's fp16 workspace cannot overflow on outlier activations. Measured
-speedup numbers will be added here.
-
-
-## HTTP API examples
-
-The request format is identical to the GPU recipe; see
-[MiniMax-H3.md § HTTP API examples](MiniMax-H3.md#http-api-examples).
-Use the validated 768P shapes (e.g. `width=1344 height=768`) on NPU.
-
-## Key parameters
-
-Same as the GPU recipe; see
-[MiniMax-H3.md § Key parameters](MiniMax-H3.md#key-parameters).
-The validated resolution on NPU is 768P (e.g. 1344x768).
-
-## Validated NPU evidence
-
-Measured on an Atlas 800I A3 server (8x NPU) with CANN 9.0.1,
-PyTorch 2.10.0+cpu, and torch_npu 2.10.0.post2, using the multi-NPU
-configuration above:
-
-| Workload | Configuration |
-|----------|---------------|
-| T2VA, 209 frames, 1344x768 | TE TP8, distributed layerwise offload, Ulysses 8, VPP8 tile, regional compile |
-| Ref2VA (prompt + video), 124 frames, 1344x768 | TE TP8, distributed layerwise offload, Ulysses 8, VPP8 tile, regional compile |
-
-These measurements describe the validated shapes rather than a general
-throughput guarantee.
+These numbers describe the validated shapes rather than a general throughput
+guarantee.
 
 ## Known limitations
 
-- Combined serving requires sibling `FL2VA` and `Ref2VA` directories, loads
-  both task-specific DiTs, and loads one copy of every shared component.
+- Task serving is partitioned by `--task-type`: T2VA/FL2VA and Ref2VA load
+  different DiTs, so switching between them requires a restart with the other
+  partition.
 - H3 currently executes one generation request per diffusion batch.
 - The first regional-compile request is a warmup and should not be included
   in steady-state performance measurements.
@@ -250,17 +380,19 @@ throughput guarantee.
   the same HTTP request contract.
 - VAE patch parallelism requires size 1 or the full DiT group size and
   supports the H3 native `tile` mode only.
-- RainFusion block-sparse attention and INT8 quantization are validated for
-  T2VA only; use the BF16 dense configuration for FL2VA and Ref2VA.
-- Online INT8 quantization can be combined with distributed layerwise offload
-  while AllGather is enabled. Keep AllGather enabled for SP>1; use
-  `--dlo-no-use-allgather` only when collective communication is unavailable.
+- The lossy configuration (RainFusion + INT8) is validated for T2VA and
+  Ref2VA; use the lossless configuration for FL2VA or re-validate first.
 
 ## Additional resources
 
 - [MiniMax-H3.md](MiniMax-H3.md) — full GPU guide
-- [Attention backends § RAINFUSION_ATTN](../../docs/user_guide/diffusion/attention_backends.md#rainfusion_attn-backend-and-block-sparse-video-attention)
-  — RainFusion knobs and tuning
-- [Int8 quantization](../../docs/user_guide/quantization/int8.md)
+- [MiniMax-H3-NPU-950PR.md](MiniMax-H3-NPU-950PR.md) — four-card
+  Ascend 950PR / 950DT guide
+- [NPU installation](../../docs/getting_started/installation/npu.md)
+- [RainFusion attention](../../docs/user_guide/diffusion/attention_backends/rainfusion.md)
+  — block-sparse knobs and tuning
+- [Online quantization](../../docs/user_guide/quantization/online.md)
+  — INT8 / MXFP8 / MXFP4
+- [Cache-DiT guide](../../docs/user_guide/diffusion/cache_acceleration/cache_dit.md)
 - [Supported models](../../docs/models/supported_models.md)
 - [Video API](../../docs/serving/videos_api.md)