[Test] Align CosyVoice3 hardware mocks across platform layers

Signed-off-by: Sy03 <1370724210@qq.com>

.buildkite/amd/test-amd-merge.yml

@@ -41,7 +41,33 @@ steps:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+        # Keep multi-GPU tests out of this one-GPU pool even when their module
+        # also carries the cpu marker for unit tests.
+        - "pytest -sv tests/diffusion -m 'core_model and cpu and not (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)' --ignore=tests/diffusion/cache/test_teacache_extractors.py --ignore=tests/diffusion/models/pi05/test_pi05_units.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+    # This module takes about 22 minutes on MI300 and was consistently assigned to shard 4,
+    # pushing that shard to the 45-minute timeout. Keep its module-scoped fixture in one lane.
+    - label: "Simple · Pi0.5 CPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 45
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/diffusion/models/pi05/test_pi05_units.py -m 'core_model and cpu'"
+
+    - label: "Diffusion · AuK VAE GPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 40
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - >-
+          timeout --signal=TERM --kill-after=2m 35m
+          pytest -sv tests/diffusion/models/auk/test_auk_vae_fastpath.py
+          -m 'core_model and rocm and cards_1' --run-level core_model
 
     - label: "Simple · Engine&Entrypoints Test"
       agent_pool: mi300_1
@@ -131,6 +157,11 @@ steps:
       grade: Blocking
       commands:
         - pytest -s -v tests/diffusion/distributed/test_tensor_parallel.py
+        - >-
+          timeout 20m pytest -s -v
+          tests/diffusion/attention/test_ulysses_uaa.py::test_ulysses_uaa_2d_mask_layout_matches_baseline
+          -m 'core_model and cards_2'
+          --run-level "core_model"
 
     - label: "Diffusion Model Test"
       agent_pool: mi300_1
@@ -138,7 +169,9 @@ steps:
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
-        - timeout 30m pytest -s -v tests/e2e/offline_inference/test_z_image.py
+        # A cold Z-Image run spends about six minutes loading weights before
+        # the first-request AITER compilation, which can itself exceed 23m.
+        - timeout 55m pytest -s -v tests/e2e/offline_inference/test_z_image.py
 
     - label: "Diffusion GPU Worker Test"
       agent_pool: mi300_2

.buildkite/amd/test-amd-ready.yml

@@ -41,7 +41,33 @@ steps:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+        # Keep multi-GPU tests out of this one-GPU pool even when their module
+        # also carries the cpu marker for unit tests.
+        - "pytest -sv tests/diffusion -m 'core_model and cpu and not (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)' --ignore=tests/diffusion/cache/test_teacache_extractors.py --ignore=tests/diffusion/models/pi05/test_pi05_units.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+    # This module takes about 22 minutes on MI300 and was consistently assigned to shard 4,
+    # pushing that shard to the 45-minute timeout. Keep its module-scoped fixture in one lane.
+    - label: "Simple · Pi0.5 CPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 45
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/diffusion/models/pi05/test_pi05_units.py -m 'core_model and cpu'"
+
+    - label: "Diffusion · AuK VAE GPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 40
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - >-
+          timeout --signal=TERM --kill-after=2m 35m
+          pytest -sv tests/diffusion/models/auk/test_auk_vae_fastpath.py
+          -m 'core_model and rocm and cards_1' --run-level core_model
 
     - label: "Simple · Engine&Entrypoints Test"
       agent_pool: mi300_1
@@ -141,6 +167,11 @@ steps:
         - export VLLM_LOGGING_LEVEL=DEBUG
         - export VLLM_WORKER_MULTIPROC_METHOD=spawn
         - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
+        - >-
+          timeout 20m pytest -s -v
+          tests/diffusion/attention/test_ulysses_uaa.py::test_ulysses_uaa_2d_mask_layout_matches_baseline
+          -m 'core_model and cards_2'
+          --run-level "core_model"
         - >-
           timeout 20m pytest -s -v
           tests/diffusion/models/ltx2/test_ltx2_transformer_ulysses.py
@@ -165,6 +196,7 @@ steps:
         # registered multi-GPU marker from this one-GPU job.
         - >-
           timeout 15m pytest -sv tests/diffusion/models/
+          --ignore=tests/diffusion/models/auk/test_auk_vae_fastpath.py
           -m 'core_model and cuda and not
           (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
           --run-level "core_model"

.buildkite/common/ci_source_file_dependencies.yml

@@ -16,10 +16,14 @@ x-qwen3-tts: &qwen3_tts
   - vllm_omni/worker/gpu_model_runner.py
   - vllm_omni/model_executor/models/qwen3_tts/
   - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+  - vllm_omni/model_executor/models/common/fused_sampling.py
+  - vllm_omni/model_executor/models/common/short_kv_attention.py
+  - vllm_omni/model_executor/models/common/talker_first_audio.py
   - vllm_omni/model_executor/models/common/snake_activation.py
   - vllm_omni/model_executor/models/common/alias_free_activation.py
   - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
   - vllm_omni/deploy/qwen3_tts.yaml
+  - vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2_single_gpu.yaml
   - vllm_omni/entrypoints/openai/serving_speech.py
   - vllm_omni/entrypoints/openai/tts_adapters/
   - vllm_omni/engine/stage_pool.py
@@ -30,6 +34,9 @@ x-qwen3-tts: &qwen3_tts
   - vllm_omni/config/omni_config.py
   - vllm_omni/engine/orchestrator.py
   - vllm_omni/engine/stage_engine_core_proc.py
+  - vllm_omni/engine/cuda_mps.py
+  - vllm_omni/engine/stage_runtime.py
+  - vllm_omni/engine/stage_init_utils.py
   - vllm_omni/deploy/qwen3_tts_mrv2.yaml
   - vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2.yaml
   - vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2_b4.yaml
@@ -81,11 +88,16 @@ x-qwen3-omni: &qwen3_omni
   - vllm_omni/entrypoints/openai/serving_chat.py
   - vllm_omni/model_executor/models/qwen3_omni/
   - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+  - vllm_omni/model_executor/models/common/fused_sampling.py
+  - vllm_omni/model_executor/models/common/short_kv_attention.py
+  - vllm_omni/model_executor/models/common/talker_first_audio.py
   - vllm_omni/model_executor/models/common/snake_activation.py
   - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
   - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
   - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
   - vllm_omni/deploy/qwen3_omni_moe.yaml
+  - vllm_omni/deploy/qwen3_omni_moe_mrv2.yaml
+  - vllm_omni/deploy/qwen3_omni_moe_mrv2_h200.yaml
   - vllm_omni/deploy/qwen3_omni_moe_thinking.yaml
 
 x-wan22: &wan22
@@ -188,6 +200,10 @@ source_file_dependencies:
 
   # --- models ---
   # --- omni / tts jobs ---
+  entrypoints_function:
+    - tests/entrypoints/
+    - vllm_omni/entrypoints/
+
   omni_qwen3_omni_function:
     - *qwen3_omni
     - tests/e2e/offline_inference/test_qwen3_omni.py
@@ -236,6 +252,14 @@ source_file_dependencies:
     - *minicpmo_4_5
     - tests/e2e/accuracy/minicpmo_4_5/
 
+  omni_duplex_eval_ci:
+    - *minicpmo_4_5
+    - *minicpmo_4_5_duplex
+    - tests/e2e/accuracy/omni_duplex_eval/**
+    - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
+    - vllm_omni/benchmarks/duplex/**
+    - vllm_omni/entrypoints/cli/benchmark/omni_duplex_eval.py
+
   omni_minicpmo_4_5_perf:
     - *minicpmo_4_5
     - tests/dfx/perf/scripts/run_benchmark.py
@@ -390,6 +414,8 @@ source_file_dependencies:
   diffusion_hunyuan_image3_accuracy:
     - *hunyuan_image3
     - tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py
+    - tests/e2e/accuracy/test_hunyuan_image3_lora.py
+    - vllm_omni/diffusion/lora/
 
   diffusion_hunyuan_image3_perf:
     - *hunyuan_image3
@@ -404,6 +430,7 @@ source_file_dependencies:
 
   diffusion_minimax_h3_function:
     - *minimax_h3
+    - tests/helpers/assertions.py
     - tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py
     - tests/e2e/online_serving/minimax_h3/
     - tests/diffusion/models/minimax_h3/

.buildkite/cuda/test-merge.yml

@@ -60,11 +60,7 @@ steps:
     depends_on: upload-merge-pipeline
     steps:
       - label: "Entrypoint Test with H100"
-        # TEMPORARY 60 -> 90: engine startup/sleep-wake is ~13-15 min slower per
-        # run on the vLLM 0.26 image (model loading ~2x, init ~2x; see builds
-        # 2915 vs 2916), which pushed this suite past 60 min. Revert once the
-        # 0.26 startup regression is understood and fixed.
-        timeout_in_minutes: 90
+        timeout_in_minutes: 60
         commands:
           # Turn-based server VAD over /v1/realtime is deferred to RFC #7181 PR 5
           # (see the P0 thread on #7413); the test stays in the tree and runs again there.
@@ -73,10 +69,25 @@ steps:
 
       - label: "Entrypoint Test with L4"
         timeout_in_minutes: 60
+        # Workaround https://github.com/vllm-project/vllm-omni/issues/8016: shutdown()
+        # leaves inline UniProc DiffusionWorker VRAM mapped in the pytest parent.
+        # This job does not collect the ready leftover
+        # (test_diffusion_generate_request_id, core_model); isolate the
+        # advanced_model DiT handshake instead. Revert to a single sweep after
+        # the product shutdown fix.
         commands:
-          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model"
+          - |
+            set -eu
+            pytest -s -v tests/entrypoints/test_omni_sleep_mode.py::TestOmniDiffusionSleepMode::test_diffusion_sleep_handshake -m "advanced_model and cuda and L4 and cards_1" --run-level "advanced_model"
+            pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4 and cards_1" --run-level "advanced_model" --deselect tests/entrypoints/test_omni_sleep_mode.py::TestOmniDiffusionSleepMode::test_diffusion_sleep_handshake
         mirror_hardwares: l4_1
 
+      - label: "Entrypoint Test with L4 · 2-GPU"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4 and cards_2" --run-level "advanced_model"
+        mirror_hardwares: l4_2
+
   - group: ":card_index_dividers: E2E Test"
     depends_on: upload-merge-pipeline
     steps:
@@ -113,6 +124,30 @@ steps:
           - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py -m 'advanced_model and cuda' --run-level 'advanced_model'
         mirror_hardwares: h100_2
 
+      - label: "Omni · Qwen3-Omni Async-Chunk SHM Leak"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_omni_shm_leak.py
+          - tests/helpers/runtime.py
+          - tests/helpers/stage_config.py
+          - vllm_omni/core/sched/omni_ar_scheduler.py
+          - vllm_omni/distributed/omni_connectors/connectors/shm_connector.py
+          - vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py
+          - vllm_omni/engine/orchestrator.py
+          - vllm_omni/engine/stage_engine_core_proc.py
+          - vllm_omni/engine/stage_pool.py
+          - vllm_omni/entrypoints/openai/serving_chat.py
+          - vllm_omni/model_executor/models/qwen3_omni/
+          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+          - vllm_omni/model_executor/models/common/snake_activation.py
+          - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
+          - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
+          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
+          - vllm_omni/deploy/qwen3_omni_moe.yaml
+        timeout_in_minutes: 40
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_shm_leak.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+        mirror_hardwares: h100_2
+
       - label: "Omni · MiniCPM-o 4.5 Test"
         source_file_dependencies: omni_minicpmo_4_5_function
         timeout_in_minutes: 50
@@ -197,3 +232,14 @@ steps:
             timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_turbo_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
             timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_fasth3_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
         mirror_hardwares: h100_2
+
+      - label: "Diffusion · MiniMax-H3 Latent Upscale and Refine"
+        source_file_dependencies: diffusion_minimax_h3_function
+        timeout_in_minutes: 40
+        commands:
+          - |
+            set -eu
+            VLLM_TEST_MINIMAX_H3_LATENT_UPSCALER=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id="LBH-123-AI/Minimax_h3_latent_Upscaler", filename="minimax_h3_latent_upscaler_3d_conv_v1/minimax_h3_latent_upscaler_3d_conv_v1_bf16.safetensors"))')
+            export VLLM_TEST_MINIMAX_H3_LATENT_UPSCALER
+            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+        mirror_hardwares: h100_2

.buildkite/cuda/test-nightly.yml

@@ -21,7 +21,7 @@ steps:
           - omni_minicpmo_4_5_function
         timeout_in_minutes: 120
         commands:
-          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py --ignore=tests/e2e/online_serving/test_minicpmo_4_5_window.py
 
       - label: ":full_moon: Omni · Function Test with H100 · 2-GPU"
         source_file_dependencies:
@@ -33,9 +33,10 @@ steps:
 
       - label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
         source_file_dependencies: omni_minicpmo_4_5_duplex_function
-        timeout_in_minutes: 50
+        timeout_in_minutes: 120
         commands:
           - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and B200 and omni and cards_1" --run-level "full_model"
+          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_window.py -m "full_model and cuda and H100 and B200 and omni and cards_1" --run-level "full_model"
 
       - label: ":full_moon: Omni · Doc Test with L4 · 4-GPU"
         source_file_dependencies: omni_qwen2_5_omni_doc
@@ -70,6 +71,22 @@ steps:
           - export SEED_TTS_EVAL_DEVICE=cuda:1
           - pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level full_model
 
+      - label: ":full_moon: Omni · Omni-DuplexEval CI Guard"
+        source_file_dependencies: omni_duplex_eval_ci
+        timeout_in_minutes: 240
+        env:
+          VLLM_DUPLEX_EVAL_JUDGE_URL: "http://127.0.0.1:8001"
+        commands:
+          - |
+            set +e
+            pytest -s -v tests/e2e/accuracy/omni_duplex_eval/test_omni_duplex_eval_ci.py \
+              -m "full_model and H100 and B200 and cuda and cards_2" \
+              --run-level full_model \
+              --basetemp /tmp/omni_duplex_ci
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/e2e/accuracy/omni_duplex_eval/results/*.json"
+            exit $$EXIT
+
       - label: ":full_moon: Omni · Perf Test · No Async Chunk"
         source_file_dependencies: omni_qwen3_omni_perf
         key: nightly-omni-performance-no-async-chunk
@@ -116,6 +133,20 @@ steps:
         commands:
           - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model and H100 and B200 and cards_4" --run-level "core_model"
 
+  - group: ":card_index_dividers: Entrypoints Test"
+    key: nightly-entrypoints-test-group
+    depends_on: upload-nightly-pipeline
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
+    steps:
+
+      - label: ":full_moon: Entrypoint Test with H100"
+        source_file_dependencies: entrypoints_function
+        # Merge collects advanced_model (5 duplex interruptions). Nightly
+        # collects full_model (20 interruptions + BAGEL sleep).
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/entrypoints/ -m "full_model and cuda and H100 and B200 and cards_2" --run-level "full_model"
+
   - group: ":card_index_dividers: TTS Model Test"
     key: nightly-tts-test-group
     depends_on: upload-nightly-pipeline
@@ -199,17 +230,6 @@ steps:
             -m "full_model and diffusion and H100 and B200 and cards_1"
             --run-level "full_model"
 
-      - label: ":robot_face: Robot Policy OpenPI · π0.5 · H100 · Single-GPU"
-        timeout_in_minutes: 120
-        commands:
-          - pip install --no-deps "openpi-client @ git+https://github.com/Physical-Intelligence/openpi.git@215abfb217dbac7d5f1273282331b9b1866c0479#subdirectory=packages/openpi-client"
-          - >-
-            pytest -sv
-            tests/e2e/online_serving/test_pi05_expansion.py
-            -m "full_model and diffusion and H100 and cards_1"
-            --run-level "full_model"
-        mirror_hardwares: h100_1
-
       - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU"
         source_file_dependencies: diffusion_qwen_image_function
         timeout_in_minutes: 120
@@ -247,7 +267,7 @@ steps:
         commands:
           - export HUNYUAN_IMAGE3_MODEL="tencent/HunyuanImage-3.0-Instruct"
           - export HUNYUAN_IMAGE3_DEVICES="0,1,2,3"
-          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m "full_model and H100 and B200 and cards_4" --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py tests/e2e/accuracy/test_hunyuan_image3_lora.py -m "full_model and H100 and B200 and cards_4" --run-level full_model
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · Single-GPU"
         source_file_dependencies: diffusion_qwen_image_perf

.buildkite/cuda/test-ready.yml

@@ -16,6 +16,25 @@ steps:
           - "pytest -sv tests/model_executor -m 'core_model and cpu'"
         mirror_hardwares: l4_1
 
+      - label: "Simple · CosyVoice3 Packed Flow Test"
+        timeout_in_minutes: 15
+        source_file_dependencies:
+          - tests/model_executor/models/cosyvoice3/test_cosyvoice3_full_batch.py
+          - vllm_omni/model_executor/models/cosyvoice3/
+        commands:
+          - "pytest -sv tests/model_executor/models/cosyvoice3/test_cosyvoice3_full_batch.py -k packed -m 'core_model and cuda' --run-level=core_model"
+        mirror_hardwares: h100_1
+
+      - label: "Simple · MRV2 FA3 Graph Bounds Test"
+        timeout_in_minutes: 15
+        source_file_dependencies:
+          - tests/worker_v2/test_capture_query_bound_gpu.py
+          - tests/worker_v2/test_decode_query_bound_gpu.py
+          - vllm_omni/worker_v2/model_states/omni_model_state.py
+        commands:
+          - "pytest -sv tests/worker_v2/test_capture_query_bound_gpu.py tests/worker_v2/test_decode_query_bound_gpu.py -m 'core_model and cuda and H100' --run-level=core_model"
+        mirror_hardwares: h100_1
+
       - label: "Simple · Diffusion Test"
         timeout_in_minutes: 30
         commands:
@@ -69,6 +88,7 @@ steps:
         source_file_dependencies: diffusion_distributed_attention
         commands:
           - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_2' --run-level "core_model"
+          - timeout 10m pytest -sv tests/diffusion/attention/test_ulysses_uaa.py -m 'core_model and cuda and L4 and cards_2' --run-level "core_model"
         mirror_hardwares: l4_2
 
       - label: "Diffusion · Distributed & Attention Test · 4-GPU"
@@ -117,11 +137,18 @@ steps:
 
   - label: "Entrypoints Test"
     depends_on: upload-ready-pipeline
+    # Workaround https://github.com/vllm-project/vllm-omni/issues/8016: shutdown()
+    # leaves inline UniProc DiffusionWorker VRAM mapped in the pytest parent.
+    # Isolate that leftover case in its own process so the driver drops the
+    # mapping; the rest stays a directory sweep with --deselect. Revert to a
+    # single sweep after the product shutdown fix.
     commands:
       - |
         timeout 30m bash -c "
+          set -eu
           export VLLM_WORKER_MULTIPROC_METHOD=spawn
-          pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'
+          pytest -sv tests/entrypoints/test_async_omni.py::test_diffusion_generate_request_id -m 'core_model and cuda' --run-level 'core_model'
+          pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model' --deselect tests/entrypoints/test_async_omni.py::test_diffusion_generate_request_id
         "
     mirror_hardwares: l4_1
 
@@ -188,6 +215,16 @@ steps:
           - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda'
         mirror_hardwares: h100_1
 
+      - label: "TTS · Higgs v3 Full Response Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_higgs_audio_v3.py
+          - vllm_omni/model_executor/models/higgs_audio_v3/
+          - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py
+          - vllm_omni/deploy/higgs_multimodal_qwen3.yaml
+        commands:
+          - timeout 20m pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py::test_full_response_v1_preserves_audio -m 'core_model and cuda' --run-level 'core_model'
+        mirror_hardwares: h100_1
+
       - label: "TTS · Qwen3-TTS CustomVoice Test"
         source_file_dependencies: tts_qwen3_tts_function
         commands:
@@ -204,7 +241,7 @@ steps:
           - |
             timeout 40m bash -c "
               export VLLM_IMAGE_FETCH_TIMEOUT=60
-              pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
+              pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model and cuda' --run-level 'core_model'
             "
         mirror_hardwares: h100_1
 

.buildkite/cuda/test-weekly.yml

@@ -152,15 +152,122 @@ steps:
             --run-level "full_model"
         mirror_hardwares: h100_4
 
-      - label: "TTS · L4"
-        timeout_in_minutes: 120
+      - label: "TTS · Moss · L4"
+        timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
           - export VLLM_USE_DEEP_GEMM="0"
           - export VLLM_MOE_USE_DEEP_GEMM="0"
           - |
             failed=0
-            for f in $(pytest --collect-only -q tests/e2e/ -m "slow and L4 and tts" --run-level "full_model" | grep '\.py::' | sed -E 's|::.*||' | sort -u); do
+            for f in \
+              tests/e2e/offline_inference/test_moss_tts_expansion.py \
+              tests/e2e/offline_inference/test_moss_tts_nano_expansion.py \
+              tests/e2e/offline_inference/test_moss_tts_realtime.py \
+              tests/e2e/offline_inference/test_moss_tts_realtime_expansion.py \
+              tests/e2e/online_serving/test_moss_tts_expansion.py \
+              tests/e2e/online_serving/test_moss_tts_nano_expansion.py
+            do
+              pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
+            done
+            exit $$failed
+        mirror_hardwares: l4_1
+
+      - label: "TTS · VoxCPM2 · Voxtral · L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - |
+            failed=0
+            for f in \
+              tests/e2e/offline_inference/test_voxcpm2_tts.py \
+              tests/e2e/online_serving/test_voxcpm2_tts.py \
+              tests/e2e/online_serving/test_voxcpm2_tts_expansion.py \
+              tests/e2e/online_serving/test_voxcpm2_tts_lora.py \
+              tests/e2e/offline_inference/test_voxtral_tts_expansion.py \
+              tests/e2e/online_serving/test_voxtral_tts_expansion.py
+            do
+              pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
+            done
+            exit $$failed
+        mirror_hardwares: l4_1
+
+      - label: "TTS · Higgs · L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - |
+            failed=0
+            for f in \
+              tests/e2e/offline_inference/test_higgs_audio_v2_expansion.py \
+              tests/e2e/online_serving/test_higgs_audio_v2_expansion.py \
+              tests/e2e/online_serving/test_higgs_audio_v3.py
+            do
+              pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
+            done
+            exit $$failed
+        mirror_hardwares: l4_1
+
+      - label: "TTS · OmniVoice · Step-Audio · L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - |
+            failed=0
+            for f in \
+              tests/e2e/offline_inference/test_omnivoice_expansion.py \
+              tests/e2e/online_serving/test_omnivoice_expansion.py \
+              tests/e2e/online_serving/test_omnivoice_parity.py \
+              tests/e2e/offline_inference/test_step_audio2_expansion.py
+            do
+              pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
+            done
+            exit $$failed
+        mirror_hardwares: l4_1
+
+      - label: "TTS · Ming · GLM · IndexTTS · Breeze · L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - |
+            failed=0
+            for f in \
+              tests/e2e/offline_inference/test_ming_tts_expansion.py \
+              tests/e2e/online_serving/test_ming_tts_expansion.py \
+              tests/e2e/offline_inference/test_glm_tts_expansion.py \
+              tests/e2e/online_serving/test_glm_tts_expansion.py \
+              tests/e2e/offline_inference/test_indextts2_expansion.py \
+              tests/e2e/online_serving/test_indextts2_expansion.py \
+              tests/e2e/online_serving/test_breeze_tts_2.py
+            do
+              pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
+            done
+            exit $$failed
+        mirror_hardwares: l4_1
+
+      - label: "TTS · Audex · CosyVoice · Qwen3-TTS · L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - |
+            failed=0
+            for f in \
+              tests/e2e/offline_inference/test_audex_tts.py \
+              tests/e2e/offline_inference/test_audex_s2t.py \
+              tests/e2e/offline_inference/test_audex_tta.py \
+              tests/e2e/offline_inference/test_cosyvoice3_expansion.py \
+              tests/e2e/online_serving/test_qwen3_tts_word_timestamps.py
+            do
               pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
             done
             exit $$failed
@@ -224,6 +331,17 @@ steps:
 
       # Split the old shared H100 Single-GPU sweep: one pytest process collected
       # every slow+cards_1 diffusion file and exceeded 150 minutes.
+      - label: ":robot_face: Robot Policy OpenPI · π0.5 · H100 · Single-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - pip install --no-deps "openpi-client @ git+https://github.com/Physical-Intelligence/openpi.git@215abfb217dbac7d5f1273282331b9b1866c0479#subdirectory=packages/openpi-client"
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_pi05_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
       - label: "Diffusion · Qwen-Image · H100 · Single-GPU"
         timeout_in_minutes: 120
         commands:
@@ -416,6 +534,7 @@ steps:
           - >-
             pytest -sv
             tests/e2e/online_serving/test_bagel_expansion.py
+            tests/e2e/offline_inference/test_bagel_expansion.py
             -m "slow and diffusion and H100 and cards_2"
             --run-level "full_model"
         mirror_hardwares: h100_2

.buildkite/npu/test-npu-nightly.yml

@@ -64,6 +64,11 @@ steps:
             tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
             -m "npu and A3 and omni"
             --run-level full_model
+          - >-
+            pytest -s -v
+            tests/e2e/online_serving/test_minicpmo_4_5_window.py
+            -m "full_model and npu and A3 and omni"
+            --run-level full_model
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
         source_file_dependencies: omni_minicpmo_4_5_accuracy
         timeout_in_minutes: 180
@@ -85,6 +90,25 @@ steps:
             EXIT=$$?
             buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
             exit $$EXIT
+      - label: ":full_moon: Omni · Omni-DuplexEval CI Guard"
+        source_file_dependencies: omni_duplex_eval_ci
+        timeout_in_minutes: 240
+        mirror_hardwares: a3_npu_2
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+          VLLM_DUPLEX_EVAL_JUDGE_URL: "http://127.0.0.1:8001"
+        commands:
+          - |
+            set +e
+            pytest -s -v tests/e2e/accuracy/omni_duplex_eval/test_omni_duplex_eval_ci.py \
+              -m "full_model and npu and A3 and cards_2" \
+              --run-level full_model \
+              --basetemp /tmp/omni_duplex_ci
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/e2e/accuracy/omni_duplex_eval/results/*.json"
+            exit $$EXIT
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"
         source_file_dependencies: omni_minicpmo_4_5_perf
         key: nightly-omni-performance-minicpmo-4-5-npu
@@ -228,6 +252,42 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · Single-NPU"
+        source_file_dependencies: diffusion_qwen_image_perf
+        key: nightly-diffusion-x2iat-performance-qwen-image-single-npu
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_2
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HF_TOKEN: "${HF_TOKEN}"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "npu and A3 and cards_1" --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · 4-NPU"
+        source_file_dependencies: diffusion_qwen_image_perf
+        key: nightly-diffusion-x2iat-performance-qwen-image-4npu
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HF_TOKEN: "${HF_TOKEN}"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "npu and A3 and cards_4" --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
   - group: ":card_index_dividers: Diffusion X2V Model Test"
     key: nightly-diffusion-x2v-group
     depends_on: upload-nightly-pipeline
@@ -258,13 +318,12 @@ steps:
           HF_HUB_OFFLINE: "1"
           DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
         commands:
-          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
             EXIT=$$?
-            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
-            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
       - label: ":full_moon: Diffusion X2V · Wan 2.2 Function Test"
         source_file_dependencies: diffusion_wan22_function

.buildkite/npu/test-npu-ready.yml

@@ -55,8 +55,8 @@ steps:
             -m 'core_model and npu and 310P'
             --run-level 'core_model'
 
-  - group: ":card_index_dividers: Omni Model Test"
-    key: ready-omni-group
+  - group: ":card_index_dividers: E2E Test"
+    key: ready-e2e-group
     depends_on: upload-ready-pipeline
     steps:
       - label: "Omni · MiniCPM-o 4.5 Test"
@@ -66,6 +66,7 @@ steps:
           VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
+
       - label: "Omni · MiniCPM-o 4.5 Duplex Test"
         mirror_hardwares: a3_npu_2
         env:
@@ -73,6 +74,7 @@ steps:
           VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
         commands:
           - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
+
       - label: "Omni · MiniCPM-o 4.5 Perf Test"
         timeout_in_minutes: 60
         mirror_hardwares: a3_npu_2
@@ -83,10 +85,15 @@ steps:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and npu and A3 and omni'
 
-  - group: ":card_index_dividers: TTS Model Test"
-    key: ready-tts-group
-    depends_on: upload-ready-pipeline
-    steps:
+      - label: "Omni · Qwen3-Omni Test"
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
+        timeout_in_minutes: 60
+
       - label: ":sound: TTS · Qwen3-TTS CustomVoice Test"
         mirror_hardwares: a3_npu_2
         env:
@@ -95,3 +102,55 @@ steps:
           HF_TOKEN: "${HF_TOKEN}"
         commands:
           - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model and npu' --run-level 'core_model'
+
+      - label: "Diffusion · HunyuanVideo-1.5 Test"
+        mirror_hardwares: a3_npu_2
+        env:
+            VLLM_WORKER_MULTIPROC_METHOD: spawn
+            VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+        commands:
+            - |
+              timeout 40m bash -c "
+                export VLLM_IMAGE_FETCH_TIMEOUT=60
+                pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'core_model and diffusion and npu' --run-level 'core_model'
+                "
+        timeout_in_minutes: 60
+
+      - label: "Diffusion · Qwen Image Test"
+        mirror_hardwares: a3_npu_2
+        env:
+            VLLM_WORKER_MULTIPROC_METHOD: spawn
+            VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+        commands:
+            - |
+                timeout 40m bash -c "
+                export VLLM_IMAGE_FETCH_TIMEOUT=60
+                pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model and npu' --run-level 'core_model'
+                "
+        timeout_in_minutes: 60
+
+      - label: "Diffusion · Wan22 Test"
+        mirror_hardwares: a3_npu_2
+        env:
+            VLLM_WORKER_MULTIPROC_METHOD: spawn
+            VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+        commands:
+            - |
+                timeout 40m bash -c "
+                export VLLM_IMAGE_FETCH_TIMEOUT=60
+                pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model and npu' --run-level 'core_model'
+                "
+        timeout_in_minutes: 60
+
+      - label: "Diffusion · Hunyuan Image Test"
+        mirror_hardwares: a3_npu_4
+        env:
+            VLLM_WORKER_MULTIPROC_METHOD: spawn
+            VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+        commands:
+            - |
+                timeout 40m bash -c "
+                export VLLM_IMAGE_FETCH_TIMEOUT=60
+                pytest -s -v tests/e2e/online_serving/test_hunyuan_image.py -m 'core_model and diffusion and npu' --run-level 'core_model'
+                "
+        timeout_in_minutes: 60

.github/PULL_REQUEST_TEMPLATE.md

@@ -12,4 +12,7 @@ PLEASE FILL IN THE PR DESCRIPTION HERE.
 ## Test Result
 
 **BEFORE SUBMITTING:** read [CONTRIBUTING.md](https://github.com/vllm-project/vllm-omni/blob/main/CONTRIBUTING.md) and run the [precheck-pr skill](https://github.com/vllm-project/vllm-omni/blob/main/.claude/skills/precheck-pr/SKILL.md) with the code agent for a self-check against project conventions.
+
+> [!NOTE]
+> To avoid agent spam, vLLM-Omni limits contributors without write access to **5 open PRs**. If this blocks well-intentioned critical work, please contact a committer to request bypass list access.
 (anything written below this line will be removed by GitHub Actions)

.github/workflows/build_wheel.yml

@@ -56,6 +56,35 @@ jobs:
         run: |
           bash scripts/build_wheel.sh --python python
 
+      - name: Validate runtime package data from source archive
+        run: |
+          set -euo pipefail
+          archive_dir="$(mktemp -d)"
+          install_dir="$(mktemp -d)"
+          vcs_wheels=(dist/*.whl)
+          if [[ ${#vcs_wheels[@]} -ne 1 || ! -f "${vcs_wheels[0]}" ]]; then
+            echo "Expected exactly one VCS-built wheel" >&2
+            exit 1
+          fi
+          git archive HEAD | tar -x -C "${archive_dir}"
+          (
+            cd "${archive_dir}"
+            VLLM_OMNI_TARGET_DEVICE=cuda \
+              VLLM_OMNI_VERSION_OVERRIDE=0.0.0 \
+              python -m build --wheel --outdir archive-dist
+          )
+          wheels=("${archive_dir}"/archive-dist/*.whl)
+          if [[ ${#wheels[@]} -ne 1 || ! -f "${wheels[0]}" ]]; then
+            echo "Expected exactly one archive-built wheel" >&2
+            exit 1
+          fi
+          python -m pip install --no-deps --target "${install_dir}" "${wheels[0]}"
+          python tools/check_wheel_package_data.py \
+            --source-root "${archive_dir}" \
+            --wheel "${wheels[0]}" \
+            --vcs-wheel "${vcs_wheels[0]}" \
+            --installed-root "${install_dir}"
+
       - name: Upload Wheel Artifact
         uses: actions/upload-artifact@v7
         with:

.gitignore

@@ -243,9 +243,15 @@ datasets/
 *.csv
 *.json
 !recipes/cosmos3/negative_prompt.json
+!benchmarks/mammoth_moda2/reference_results.json
 !tests/dfx/perf/tests/*.json
 !apps/ComfyUI-vLLM-Omni/example_workflows/*.json
+# Omni-DuplexEval CI config JSON (e.g. accuracy/omni_duplex_eval)
+!tests/e2e/accuracy/omni_duplex_eval/*.json
 *.jsonl
+# Shared Hunyuan IT2I input for accuracy tests and vllm bench.
+!tests/assets/hunyuan_image3/it2i.jsonl
+!tests/assets/livekit/client_vad_replay.jsonl
 *.parquet
 
 # Output files

CONTRIBUTING.md

@@ -11,3 +11,5 @@ how to extend allowlists) are documented in the
 section of that guide.
 
 Before submitting a PR, run the [precheck-pr skill](https://github.com/vllm-project/vllm-omni/blob/main/.claude/skills/precheck-pr/SKILL.md) with the code agent for a self-check against project conventions.
+
+vLLM-Omni limits contributors without write access to **5 open PRs**; see the [Pull Request Limit](docs/contributing/README.md#pull-request-limit) section of the contributing guide.

README.md

@@ -16,6 +16,7 @@ Easy, fast, and cheap omni-modality model serving for everyone
 
 *Latest News* 🔥
 
+- [2026/09] We released [0.30.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.30.0), rebased onto vLLM 0.30.0, featuring a unified full-duplex serving framework around engine-owned sessions for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md) and AURA, native cross-stage KV and multimodal payload transfer (Mooncake AR-to-DiT handoff and NIXL connectors), interactive world-model serving with [LingBot World](recipes/Robbyant/LingBot-World-2.0.md), and realtime [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) Turbo inference on NVIDIA Blackwell and Ascend 950.
 - [2026/08] We released [0.28.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.28.0), featuring production-ready [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) serving on GPU and NPU, a unified AR/DiT paged KV cache runtime, and enhanced realtime full-duplex serving for the [MiniCPM-o series](recipes/OpenBMB/MiniCPM-o-4_5.md).
 - [2026/08] [VeRL-Omni](https://github.com/verl-project/verl-omni) `v0.2.0` is released: faster diffusion RL powered by vLLM-Omni (request-level/step-wise batching with FA3), rebuilt Qwen3-Omni multimodal training (DPO & GSPO), plus LTX-2.3, Qwen-Image-Edit support and more. See the [release notes](https://github.com/verl-project/verl-omni/releases/tag/v0.2.0).
 - [2026/08] We released [0.26.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.26.0) - aligned with the vLLM 0.26 release line, featuring [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) joint video/audio generation, an experimental full-duplex realtime runtime for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md), distributed layerwise diffusion offload, and broader model, hardware, streaming, TTS, and quantization support.
@@ -58,9 +59,9 @@ vLLM-Omni is flexible and easy to use with:
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
 - **Omni-modality models** (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
-- **TTS models** (e.g. Qwen3-TTS, IndexTTS 2.5, CosyVoice3)
-- **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
-- **Robot-policy and action models** (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
+- **TTS models** (e.g. Qwen3-TTS, Tencent AuK, Breeze-TTS-2, CosyVoice3)
+- **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, LingBot World, MAGI-2, LTX-2.5, Wan2.2)
+- **Robot-policy and action models** (e.g. π0.5, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
 
 ## Getting Started
 

benchmarks/README.md

@@ -66,6 +66,13 @@ Accuracy benchmarks for image generation/editing models, adapting external suite
 - **Layout**: `accuracy/text_to_image/` (GEBench), `accuracy/image_to_image/` (GEdit-Bench)
 - **Method**: generation and judge scoring both run through local `vllm-omni serve` endpoints
 
+### [MammothModa2](mammoth_moda2/README.md) — Startup and Weight Loading
+
+Startup / model-loading benchmark for the two-stage MammothModa2 (AR → DiT) deployment, plus a raw safetensors loading micro-benchmark.
+
+- **Layout**: `mammoth_moda2/bench_startup.py` (single-process startup and first/steady request timing), `mammoth_moda2/parse_startup_log.py` (per-stage breakdown from engine logs), `mammoth_moda2/bench_storage_scenarios.sh` (cold/warm page cache on local disk vs. network storage), `mammoth_moda2/raw_load_bench.py` (safetensors → GPU without vLLM)
+- **Key metrics**: time to engine ready, per-stage spawn/init/weight-load/profile time, first vs. steady-state request latency
+
 ### Common serving metrics framework
 
 `vllm_omni/benchmarks/` extends `vllm bench serve --omni` with Omni-specific datasets, backends, and multimodal metrics. Key metrics include:

benchmarks/accuracy/common.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 from __future__ import annotations
 
 import base64
@@ -200,7 +203,16 @@ class VllmOmniImageClient:
             )
             last_response = response
             if response.status_code != 404:
-                response.raise_for_status()
+                try:
+                    response.raise_for_status()
+                except requests.HTTPError as error:
+                    # Keep the server's validation detail in the exception;
+                    # otherwise multipart image tests only report a bare
+                    # ``400 Bad Request`` and hide the actionable cause.
+                    raise requests.HTTPError(
+                        f"{error}: {response.text}",
+                        response=response,
+                    ) from error
                 return decode_base64_image(response.json()["data"][0]["b64_json"])
 
         assert last_response is not None

benchmarks/diffusion/README.md

@@ -164,3 +164,22 @@ batch may still pay compile or CUDA-graph capture cost.
 
 For a Qwen-Image continuous-batching replay example, see
 [`performance_dashboard/qwen_image_serving_performance.md`](./performance_dashboard/qwen_image_serving_performance.md).
+
+## HunyuanImage3 reference-prefix reuse
+
+The prefix-cache benchmark uses the unified `vllm bench serve --omni` runner,
+not this legacy diffusion benchmark. It reuses the two-image IT2I input on a
+single DiT stage, comparing dense, paged without caching, and paged with caching.
+See [Shared-reference benchmark](../../docs/design/feature/prefix_caching.md#shared-reference-benchmark)
+for the configuration, commands, and separate partial-hit accuracy coverage.
+
+The accuracy regression compares paged-no-cache, partial-hit and exact-repeat
+outputs against the checked-in official-repository reference-image goldens at
+**50 steps, CFG=2.5, seeds 43/45** (both CFG branches remain on CFGP1).
+It uses the AR-to-DiT image criteria: CLIP ≥ 90, SSIM ≥ 0.26, PSNR ≥ 12.5 dB.
+`HUNYUAN_IMAGE3_INCLUDE_DENSE=1` adds dense as another comparison to the goldens.
+It also requires actual reference-image hits and model-side query slicing,
+and checks bitwise repeatability for repeated requests at the same hit boundary.
+Images, logs, deployment YAMLs, hit traces and quality metrics are saved under
+pytest's temporary output directory. `HUNYUAN_IMAGE3_MODEL` selects a local model;
+DFX uses the repository's normal model/cache resolution.

benchmarks/diffusion/backends.py

@@ -107,6 +107,8 @@ async def async_request_image_edits(
 
     if extra_body.get("guidance_scale") is not None:
         form.add_field("guidance_scale", str(extra_body["guidance_scale"]))
+    if extra_body.get("guidance_scale_2") is not None:
+        form.add_field("guidance_scale_2", str(extra_body["guidance_scale_2"]))
     if extra_body.get("negative_prompt") is not None:
         form.add_field("negative_prompt", str(extra_body["negative_prompt"]))
     if extra_body.get("true_cfg_scale") is not None:

benchmarks/diffusion/bench_wan_vae_decode.py

@@ -0,0 +1,324 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Benchmark and exactness check for the Wan VAE decoder fast path.
+
+Loads a diffusers Wan VAE (by default the Cosmos3 one), decodes seeded latents
+for the requested video size at each ``--fast-path`` level and reports wall
+time, peak memory and, against the ``off`` level, bitwise equality, max abs
+difference and PSNR. ``--profile`` adds a torch.profiler CUDA kernel table with
+the convolution / layout-transpose share.
+
+Single GPU::
+
+    python benchmarks/diffusion/bench_wan_vae_decode.py --model nvidia/Cosmos3-Nano
+    python benchmarks/diffusion/bench_wan_vae_decode.py --model nvidia/Cosmos3-Nano \
+        --fast-path off,lossless --frames 33 --profile
+    python benchmarks/diffusion/bench_wan_vae_decode.py --tiny --frames 9 --size 256x256
+
+Multi-GPU VAE parallel decode (launch one process per GPU with ``torchrun``; the
+decode is timed across all ranks and only rank 0 prints)::
+
+    torchrun --nproc-per-node 2 benchmarks/diffusion/bench_wan_vae_decode.py --model nvidia/Cosmos3-Nano \
+        --vae-patch-parallel-size 2 --vae-parallel-mode tile
+
+``tile`` distributes the VAE's own spatial tiles over the ranks (it needs the
+video to exceed the tile size, 256x256 pixels by default) and ``spatial_shard_*``
+shards every decoder feature map along height or width with halo exchange. Both
+return the full video on rank 0 only, so equality and PSNR are computed there.
+"""
+
+from __future__ import annotations
+
+import argparse
+import math
+import os
+import time
+from collections.abc import Iterable
+
+import torch
+import torch.distributed as dist
+
+from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import (
+    DistributedAutoencoderKLWan,
+    OmniAutoencoderKLWan,
+)
+from vllm_omni.diffusion.distributed.autoencoders.wan_vae_fastpath import (
+    VAE_FAST_PATH_LEVELS,
+    install_wan_vae_fastpath,
+)
+
+TINY_CONFIG = dict(
+    base_dim=32,
+    decoder_base_dim=64,
+    z_dim=48,
+    dim_mult=[1, 2, 4, 4],
+    num_res_blocks=2,
+    temperal_downsample=[False, True, True],
+    is_residual=True,
+    patch_size=2,
+    in_channels=12,
+    out_channels=12,
+    scale_factor_temporal=4,
+    scale_factor_spatial=16,
+)
+
+DTYPES = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}
+VAE_PARALLEL_MODES = ("tile", "spatial_shard_height", "spatial_shard_width")
+_CONV_TAGS = ("conv", "cudnn", "gemm", "cutlass", "xmma")
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--model", default="nvidia/Cosmos3-Nano", help="HF id or local path of the pipeline")
+    parser.add_argument("--subfolder", default="vae")
+    parser.add_argument("--tiny", action="store_true", help="Use a small random VAE instead of loading weights")
+    parser.add_argument("--size", default="1280x720", help="Output video WxH")
+    parser.add_argument("--frames", type=int, default=189, help="Output frame count (1 + 4k)")
+    parser.add_argument("--dtype", choices=sorted(DTYPES), default="bf16")
+    parser.add_argument("--fast-path", default=",".join(VAE_FAST_PATH_LEVELS), help="Comma-separated levels")
+    parser.add_argument("--warmup", type=int, default=1)
+    parser.add_argument("--iters", type=int, default=2)
+    parser.add_argument("--seed", type=int, default=0)
+    parser.add_argument("--profile", action="store_true", help="Print a torch.profiler CUDA kernel table")
+    parser.add_argument("--profile-rows", type=int, default=25)
+    parser.add_argument("--save-output", default=None, help="Save the `off` decode output to this .pt path")
+    parser.add_argument(
+        "--vae-patch-parallel-size",
+        type=int,
+        default=1,
+        help="VAE parallel decode over this many ranks (run under torchrun with as many processes)",
+    )
+    parser.add_argument(
+        "--vae-parallel-mode",
+        choices=VAE_PARALLEL_MODES,
+        default="tile",
+        help="tile: distribute the VAE's spatial tiles; spatial_shard_*: shard feature maps with halo exchange",
+    )
+    return parser.parse_args()
+
+
+def init_distributed() -> tuple[int, int]:
+    """Join the torchrun process group (if any) and return ``(rank, world_size)``."""
+    world_size = int(os.environ.get("WORLD_SIZE", "1"))
+    if world_size <= 1:
+        return 0, 1
+    from vllm_omni.diffusion.distributed.parallel_state import (
+        init_distributed_environment,
+        initialize_model_parallel,
+    )
+
+    rank = int(os.environ["RANK"])
+    local_rank = int(os.environ.get("LOCAL_RANK", rank))
+    init_distributed_environment(world_size=world_size, rank=rank, local_rank=local_rank)
+    initialize_model_parallel(sequence_parallel_size=world_size, ulysses_degree=world_size)
+    return rank, world_size
+
+
+def load_vae(
+    args: argparse.Namespace, dtype: torch.dtype, device: torch.device, parallel: bool
+) -> OmniAutoencoderKLWan:
+    cls = DistributedAutoencoderKLWan if parallel else OmniAutoencoderKLWan
+    if args.tiny:
+        torch.manual_seed(args.seed)
+        vae = cls(**TINY_CONFIG).to(device=device, dtype=dtype).eval()
+        if parallel:
+            vae.init_distributed()
+    else:
+        vae = cls.from_pretrained(args.model, subfolder=args.subfolder, torch_dtype=dtype).to(device).eval()
+    if parallel:
+        # Same setup as the registry: parallel decode needs tiling on.
+        vae.use_tiling = True
+        vae.set_parallel_size(args.vae_patch_parallel_size, mode=args.vae_parallel_mode)
+    return vae
+
+
+def make_latents(vae: OmniAutoencoderKLWan, args: argparse.Namespace, dtype: torch.dtype, device: torch.device):
+    width, height = (int(v) for v in args.size.lower().split("x"))
+    spatial = int(getattr(vae.config, "scale_factor_spatial", 16))
+    temporal = int(getattr(vae.config, "scale_factor_temporal", 4))
+    if width % spatial or height % spatial or (args.frames - 1) % temporal:
+        raise SystemExit(f"size must be a multiple of {spatial} and frames must be 1 + {temporal}k")
+    shape = (1, vae.config.z_dim, (args.frames - 1) // temporal + 1, height // spatial, width // spatial)
+    generator = torch.Generator(device="cpu").manual_seed(args.seed)
+    latents = torch.randn(shape, generator=generator).to(device=device, dtype=dtype)
+    mean = getattr(vae.config, "latents_mean", None)
+    std = getattr(vae.config, "latents_std", None)
+    if mean is not None and std is not None:
+        mean = torch.as_tensor(mean, device=device, dtype=dtype).view(1, -1, 1, 1, 1)
+        std = torch.as_tensor(std, device=device, dtype=dtype).view(1, -1, 1, 1, 1)
+        latents = latents * std + mean
+    return latents
+
+
+@torch.inference_mode()
+def decode(vae: OmniAutoencoderKLWan, latents: torch.Tensor) -> torch.Tensor:
+    return vae.decode(latents, return_dict=False)[0]
+
+
+def sync() -> None:
+    """Wait for this rank's GPU and, under torchrun, for every other rank."""
+    torch.accelerator.synchronize()
+    if dist.is_initialized():
+        dist.barrier()
+
+
+def psnr(a: torch.Tensor, b: torch.Tensor) -> float:
+    mse = torch.mean((a.float() - b.float()) ** 2).item()
+    return math.inf if mse == 0 else 10 * math.log10(4.0 / mse)  # outputs live in [-1, 1]
+
+
+def bits_equal(a: torch.Tensor, b: torch.Tensor) -> bool:
+    if a.shape != b.shape or a.dtype != b.dtype:
+        return False
+    int_dtype = torch.int16 if a.element_size() == 2 else torch.int32
+    return bool((a.contiguous().view(int_dtype) == b.contiguous().view(int_dtype)).all())
+
+
+def profile_decode(vae: OmniAutoencoderKLWan, latents: torch.Tensor, rows: int, verbose: bool) -> None:
+    from torch.profiler import ProfilerActivity, profile
+
+    with profile(activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], record_shapes=False) as prof:
+        decode(vae, latents)
+        sync()
+    if not verbose:
+        return
+    events = [e for e in prof.key_averages() if e.device_time_total > 0]
+    events.sort(key=lambda e: e.device_time_total, reverse=True)
+    total = sum(e.device_time_total for e in events if e.device_type.name == "CUDA")
+    if total == 0:
+        total = sum(e.device_time_total for e in events)
+    conv = sum(
+        e.device_time_total
+        for e in events
+        if e.device_type.name == "CUDA" and any(tag in e.key.lower() for tag in _CONV_TAGS)
+    )
+    transpose = sum(e.device_time_total for e in events if e.device_type.name == "CUDA" and "nhwc" in e.key.lower())
+    print(
+        f"  GPU kernel time: {total / 1e3:.1f} ms  conv-like: {100 * conv / total:.1f}%  "
+        f"nchw<->nhwc transposes: {100 * transpose / total:.1f}%"
+    )
+    print(f"  {'kernel':90s} {'ms':>9s} {'%':>6s} {'calls':>7s}")
+    shown = 0
+    for e in events:
+        if e.device_type.name != "CUDA":
+            continue
+        share = 100 * e.device_time_total / total
+        print(f"  {e.key[:90]:90s} {e.device_time_total / 1e3:9.2f} {share:6.1f} {e.count:7d}")
+        shown += 1
+        if shown >= rows:
+            break
+
+
+def run_level(
+    args: argparse.Namespace,
+    level: str,
+    dtype: torch.dtype,
+    device: torch.device,
+    *,
+    rank: int,
+    parallel: bool,
+) -> tuple[torch.Tensor, dict]:
+    vae = load_vae(args, dtype, device, parallel)
+    report = install_wan_vae_fastpath(vae, level=level)
+    latents = make_latents(vae, args, dtype, device)
+    for _ in range(args.warmup):
+        decode(vae, latents)
+    sync()
+    torch.accelerator.reset_peak_memory_stats()
+    timings = []
+    output = None
+    for _ in range(args.iters):
+        sync()
+        start = time.perf_counter()
+        output = decode(vae, latents)
+        sync()
+        timings.append(time.perf_counter() - start)
+    peak = torch.tensor(torch.accelerator.max_memory_allocated() / 2**30, device=device)
+    if dist.is_initialized():
+        dist.all_reduce(peak, op=dist.ReduceOp.MAX)  # the largest rank decides whether the config fits
+    stats = {
+        "installed": report.installed,
+        "reason": report.reason,
+        "fused_silu": report.fused_silu_dtypes,
+        "patched": dict(report.patched),
+        "time_s": min(timings),
+        "peak_gib": peak.item(),
+    }
+    if args.profile:
+        if rank == 0:
+            print(f"[{level}] profile")
+        profile_decode(vae, latents, args.profile_rows, verbose=rank == 0)
+    assert output is not None
+    del vae
+    torch.accelerator.empty_cache()
+    return output.detach(), stats
+
+
+def main() -> None:
+    args = parse_args()
+    if not torch.cuda.is_available():
+        raise SystemExit("CUDA device required")
+    rank, world_size = init_distributed()
+    parallel = args.vae_patch_parallel_size > 1
+    if parallel and world_size < args.vae_patch_parallel_size:
+        raise SystemExit(
+            f"--vae-patch-parallel-size {args.vae_patch_parallel_size} needs torchrun with at least that many "
+            f"processes (WORLD_SIZE={world_size})"
+        )
+    device = torch.device("cuda", torch.accelerator.current_device_index()) if world_size > 1 else torch.device("cuda")
+    dtype = DTYPES[args.dtype]
+    levels: Iterable[str] = [level.strip() for level in args.fast_path.split(",") if level.strip()]
+    for level in levels:
+        if level not in VAE_FAST_PATH_LEVELS:
+            raise SystemExit(f"unknown fast-path level {level!r}; choose from {VAE_FAST_PATH_LEVELS}")
+    if "off" not in levels:
+        levels = ["off", *levels]
+
+    if rank == 0:
+        parallel_desc = f" vae_parallel={args.vae_parallel_mode}x{args.vae_patch_parallel_size}" if parallel else ""
+        print(
+            f"model={'tiny' if args.tiny else args.model} size={args.size} frames={args.frames} "
+            f"dtype={args.dtype} world_size={world_size}{parallel_desc}"
+        )
+    golden = None
+    baseline_time = None
+    try:
+        for level in levels:
+            output, stats = run_level(args, level, dtype, device, rank=rank, parallel=parallel)
+            if level == "off":
+                golden = output
+                baseline_time = stats["time_s"]
+                if args.save_output and rank == 0:
+                    torch.save(output.cpu(), args.save_output)
+            assert golden is not None
+            if rank != 0:
+                # Parallel decode assembles the video on rank 0 only; the other
+                # ranks just keep the collectives in lockstep.
+                del output
+                continue
+            speedup = baseline_time / stats["time_s"] if baseline_time else float("nan")
+            line = (
+                f"[{level:13s}] decode {stats['time_s'] * 1e3:9.1f} ms  speedup {speedup:5.2f}x  "
+                f"peak {stats['peak_gib']:5.2f} GiB  installed={stats['installed']}"
+            )
+            if level != "off":
+                equal = torch.equal(output, golden)
+                line += (
+                    f"  torch_equal={equal} bits_equal={bits_equal(output, golden)} "
+                    f"max_abs_diff={(output.float() - golden.float()).abs().max().item():.3e} "
+                    f"psnr={psnr(output, golden):.2f} dB"
+                )
+                if stats["installed"]:
+                    line += f"  fused_silu={stats['fused_silu'] or 'off'}"
+                else:
+                    line += f"  reason={stats['reason']}"
+            print(line)
+            del output
+            torch.accelerator.empty_cache()
+    finally:
+        if dist.is_initialized():
+            dist.destroy_process_group()
+
+
+if __name__ == "__main__":
+    main()

benchmarks/diffusion/diffusion_benchmark_serving.py

@@ -1107,6 +1107,11 @@ def calculate_metrics(
     error_outputs = [o for o in outputs if not o.success]
 
     num_success = len(success_outputs)
+    error_log_path = os.environ.get("BENCH_ERROR_LOG")
+    if error_outputs and error_log_path:
+        with open(error_log_path, "a") as f:
+            for output in error_outputs:
+                f.write(f"ERROR latency={output.latency:.3f} detail={output.error!r}\n")
     latencies = [o.latency for o in success_outputs]
     peak_memories = [o.peak_memory_mb for o in success_outputs if o.peak_memory_mb > 0]
 

benchmarks/kernels/_magi2_gpu_bench.py

@@ -0,0 +1,194 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Cold-cache torch.cuda timing and correctness gates for MAGI-2 microbenchmarks."""
+
+import argparse
+import importlib
+import importlib.metadata
+import json
+import logging
+import math
+import os
+import subprocess
+import sys
+from collections.abc import Callable
+from pathlib import Path
+from types import ModuleType
+from typing import Any
+
+LOGGER = logging.getLogger(__name__)
+
+
+def emit(**fields: Any) -> None:
+    LOGGER.info(json.dumps(fields, sort_keys=True))
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--tokens", type=int, nargs="+", default=[1, 4, 16, 64, 129, 4096])
+    parser.add_argument("--dtype", choices=["bfloat16", "float16", "float32"], default="bfloat16")
+    parser.add_argument("--num-tests", type=int, default=30, help="Measured samples per implementation.")
+    parser.add_argument("--warmup", type=int, default=10, help="Interleaved warmup calls before sampling.")
+    parser.add_argument("--seed", type=int, default=42)
+
+
+def load_musa(args: argparse.Namespace, parser: argparse.ArgumentParser) -> Any:
+    if min(*args.tokens, args.num_tests) <= 0 or args.warmup < 0:
+        parser.error("tokens and num-tests must be positive; warmup must be nonnegative")
+    logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout)
+    torch = importlib.import_module("torch")
+    cuda = getattr(torch, "cuda")
+    # torchada is needed only on MUSA; after it is loaded, use torch.cuda for
+    # both CUDA and MUSA as documented by torchada.
+    if hasattr(torch.version, "musa") and torch.version.musa is not None:
+        try:
+            importlib.import_module("torchada")
+        except ModuleNotFoundError as error:
+            if error.name != "torchada":
+                raise
+    has_gpu = cuda.is_available() or (
+        hasattr(torch.version, "musa") and torch.version.musa is not None and torch.musa.is_available()
+    )
+    if not has_gpu:
+        emit(status="skip", reason="CUDA/MUSA device is unavailable", torch=torch.__version__)
+        return None
+    torch.manual_seed(args.seed)
+    cuda.manual_seed_all(args.seed)
+    return torch
+
+
+def report_environment(torch: Any, args: argparse.Namespace, module: ModuleType) -> None:
+    root = Path(__file__).resolve().parents[2]
+    packages = {}
+    for name in ("torch", "torch_musa", "torchada", "triton", "vllm", "vllm-omni"):
+        try:
+            packages[name] = importlib.metadata.version(name)
+        except importlib.metadata.PackageNotFoundError:
+            packages[name] = None
+    cuda = getattr(torch, "cuda")
+    properties = cuda.get_device_properties(cuda.current_device())
+    try:
+        source_sha = subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True).strip()
+        source_dirty = bool(subprocess.check_output(["git", "-C", str(root), "status", "--porcelain"], text=True))
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        source_sha = os.environ.get("MAGI2_BENCH_SOURCE_SHA", "unknown")
+        source_dirty = None
+    emit(
+        status="environment",
+        command=sys.argv,
+        source_root=str(root),
+        source_sha=source_sha,
+        source_dirty=source_dirty,
+        kernel_source=module.__file__,
+        device=cuda.get_device_name(cuda.current_device()),
+        sm_count=getattr(properties, "multi_processor_count", None),
+        packages=packages,
+        env={name: os.environ.get(name) for name in ("MUSA_VISIBLE_DEVICES", "TORCHDYNAMO_DISABLE")},
+        config=vars(args),
+        mode="eager operator diagnostic; no compile or graph capture",
+        comparison="forward_native versus the default CustomOp dispatch",
+        timing="torch.cuda.Event; alternating native/fused calls; vanilla 512 MiB flush outside timed regions",
+        timing_scope="whole callable, including device launch gaps; not isolated kernel duration",
+        l2_flush=True,
+        l2_flush_bytes=512 * 1024 * 1024,
+    )
+
+
+class _LaunchProbe:
+    def __init__(self, kernel: Any) -> None:
+        self.kernel = kernel
+        self.count = 0
+
+    def __getitem__(self, grid: Any) -> Any:
+        self.count += 1
+        return self.kernel[grid]
+
+
+def check_pair(
+    torch: Any,
+    reference: Callable,
+    fused: Callable,
+    module: ModuleType,
+    kernels: tuple[str, ...],
+    *,
+    rtol: float,
+    atol: float,
+) -> dict:
+    cuda = getattr(torch, "cuda")
+    expected = reference()
+    probes = {name: _LaunchProbe(getattr(module, name)) for name in kernels}
+    try:
+        for name, probe in probes.items():
+            setattr(module, name, probe)
+        actual = fused()
+        cuda.synchronize()
+    finally:
+        for name, probe in probes.items():
+            setattr(module, name, probe.kernel)
+    if any(probe.count != 1 for probe in probes.values()):
+        raise RuntimeError(f"Expected one launch per fused kernel; got {[(k, p.count) for k, p in probes.items()]}")
+    expected = expected if isinstance(expected, tuple) else (expected,)
+    actual = actual if isinstance(actual, tuple) else (actual,)
+    errors = []
+    for result, baseline in zip(actual, expected, strict=True):
+        for value in (baseline, result):
+            if not torch.isfinite(value).all().item():
+                raise AssertionError("Nonfinite benchmark output")
+            if value.numel() > 1 and (value == value.flatten()[0]).all().item():
+                raise AssertionError("Constant benchmark output")
+        torch.testing.assert_close(result, baseline, rtol=rtol, atol=atol)
+        difference = (result.float() - baseline.float()).abs()
+        errors.append(
+            {
+                "max_abs": difference.max().item(),
+                "max_rel": (difference / baseline.float().abs().clamp_min(1e-6)).max().item(),
+            }
+        )
+    return {"parity": "pass", "errors": errors, "rtol": rtol, "atol": atol, "kernels": list(kernels)}
+
+
+def _percentile(samples: list[float], quantile: float) -> float:
+    ordered = sorted(samples)
+    position = (len(ordered) - 1) * quantile
+    lower, upper = math.floor(position), math.ceil(position)
+    return ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower)
+
+
+def measure_pair(torch: Any, reference: Callable, fused: Callable, args: argparse.Namespace) -> dict:
+    cuda = getattr(torch, "cuda")
+    functions = (reference, fused)
+    order: list[int] = []
+
+    def alternate() -> Any:
+        mode = len(order) % 2
+        order.append(mode)
+        return functions[mode]()
+
+    times_ms = []
+    for _ in range(args.warmup):
+        alternate()
+    cuda.synchronize()
+    order.clear()
+    flush = torch.empty(512 * 1024 * 1024, dtype=torch.uint8, device="cuda")
+    for _ in range(2 * args.num_tests):
+        flush.zero_()
+        cuda.synchronize()
+        start = cuda.Event(enable_timing=True)
+        end = cuda.Event(enable_timing=True)
+        start.record()
+        alternate()
+        end.record()
+        end.synchronize()
+        times_ms.append(start.elapsed_time(end))
+    samples: tuple[list[float], list[float]] = ([], [])
+    for mode, milliseconds in zip(order[-len(times_ms) :], times_ms, strict=True):
+        if not math.isfinite(milliseconds) or milliseconds <= 0:
+            raise RuntimeError(f"Invalid GPU event sample: {milliseconds}")
+        samples[mode].append(milliseconds * 1000)
+    stats = {}
+    for label, values in zip(("native", "fused"), samples, strict=True):
+        if len(values) != args.num_tests:
+            raise RuntimeError("GPU timing harness returned an unexpected sample count")
+        p50 = _percentile(values, 0.5)
+        stats[label] = {"median_us": p50, "p50_us": p50, "p90_us": _percentile(values, 0.9), "samples": len(values)}
+    stats["speedup_p50"] = stats["native"]["p50_us"] / stats["fused"]["p50_us"]
+    return stats

benchmarks/kernels/_magi2_musa_bench.py

@@ -0,0 +1,194 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Cold-cache torch.cuda timing and correctness gates for MAGI-2 microbenchmarks."""
+
+import argparse
+import importlib
+import importlib.metadata
+import json
+import logging
+import math
+import os
+import subprocess
+import sys
+from collections.abc import Callable
+from pathlib import Path
+from types import ModuleType
+from typing import Any
+
+LOGGER = logging.getLogger(__name__)
+
+
+def emit(**fields: Any) -> None:
+    LOGGER.info(json.dumps(fields, sort_keys=True))
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--tokens", type=int, nargs="+", default=[1, 4, 16, 64, 129, 4096])
+    parser.add_argument("--dtype", choices=["bfloat16", "float16", "float32"], default="bfloat16")
+    parser.add_argument("--num-tests", type=int, default=30, help="Measured samples per implementation.")
+    parser.add_argument("--warmup", type=int, default=10, help="Interleaved warmup calls before sampling.")
+    parser.add_argument("--seed", type=int, default=42)
+
+
+def load_musa(args: argparse.Namespace, parser: argparse.ArgumentParser) -> Any:
+    if min(*args.tokens, args.num_tests) <= 0 or args.warmup < 0:
+        parser.error("tokens and num-tests must be positive; warmup must be nonnegative")
+    logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout)
+    torch = importlib.import_module("torch")
+    cuda = getattr(torch, "cuda")
+    # torchada is needed only on MUSA; after it is loaded, use torch.cuda for
+    # both CUDA and MUSA as documented by torchada.
+    if hasattr(torch.version, "musa") and torch.version.musa is not None:
+        try:
+            importlib.import_module("torchada")
+        except ModuleNotFoundError as error:
+            if error.name != "torchada":
+                raise
+    has_gpu = cuda.is_available() or (
+        hasattr(torch.version, "musa") and torch.version.musa is not None and torch.musa.is_available()
+    )
+    if not has_gpu:
+        emit(status="skip", reason="CUDA/MUSA device is unavailable", torch=torch.__version__)
+        return None
+    torch.manual_seed(args.seed)
+    cuda.manual_seed_all(args.seed)
+    return torch
+
+
+def report_environment(torch: Any, args: argparse.Namespace, module: ModuleType) -> None:
+    root = Path(__file__).resolve().parents[2]
+    packages = {}
+    for name in ("torch", "torch_musa", "torchada", "triton", "vllm", "vllm-omni"):
+        try:
+            packages[name] = importlib.metadata.version(name)
+        except importlib.metadata.PackageNotFoundError:
+            packages[name] = None
+    cuda = getattr(torch, "cuda")
+    properties = cuda.get_device_properties(cuda.current_device())
+    try:
+        source_sha = subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True).strip()
+        source_dirty = bool(subprocess.check_output(["git", "-C", str(root), "status", "--porcelain"], text=True))
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        source_sha = os.environ.get("MAGI2_BENCH_SOURCE_SHA", "unknown")
+        source_dirty = None
+    emit(
+        status="environment",
+        command=sys.argv,
+        source_root=str(root),
+        source_sha=source_sha,
+        source_dirty=source_dirty,
+        kernel_source=module.__file__,
+        device=cuda.get_device_name(cuda.current_device()),
+        sm_count=getattr(properties, "multi_processor_count", None),
+        packages=packages,
+        env={name: os.environ.get(name) for name in ("MUSA_VISIBLE_DEVICES", "TORCHDYNAMO_DISABLE")},
+        config=vars(args),
+        mode="eager operator diagnostic; no compile or graph capture",
+        comparison="forward_native versus the default CustomOp dispatch",
+        timing="torch.cuda.Event; alternating native/fused calls; vanilla 512 MiB flush outside timed regions",
+        timing_scope="whole callable, including device launch gaps; not isolated kernel duration",
+        l2_flush=True,
+        l2_flush_bytes=512 * 1024 * 1024,
+    )
+
+
+class _LaunchProbe:
+    def __init__(self, kernel: Any) -> None:
+        self.kernel = kernel
+        self.count = 0
+
+    def __getitem__(self, grid: Any) -> Any:
+        self.count += 1
+        return self.kernel[grid]
+
+
+def check_pair(
+    torch: Any,
+    reference: Callable,
+    fused: Callable,
+    module: ModuleType,
+    kernels: tuple[str, ...],
+    *,
+    rtol: float,
+    atol: float,
+) -> dict:
+    cuda = getattr(torch, "cuda")
+    expected = reference()
+    probes = {name: _LaunchProbe(getattr(module, name)) for name in kernels}
+    try:
+        for name, probe in probes.items():
+            setattr(module, name, probe)
+        actual = fused()
+        cuda.synchronize()
+    finally:
+        for name, probe in probes.items():
+            setattr(module, name, probe.kernel)
+    if any(probe.count != 1 for probe in probes.values()):
+        raise RuntimeError(f"Expected one launch per fused kernel; got {[(k, p.count) for k, p in probes.items()]}")
+    expected = expected if isinstance(expected, tuple) else (expected,)
+    actual = actual if isinstance(actual, tuple) else (actual,)
+    errors = []
+    for result, baseline in zip(actual, expected, strict=True):
+        for value in (baseline, result):
+            if not torch.isfinite(value).all().item():
+                raise AssertionError("Nonfinite benchmark output")
+            if value.numel() > 1 and (value == value.flatten()[0]).all().item():
+                raise AssertionError("Constant benchmark output")
+        torch.testing.assert_close(result, baseline, rtol=rtol, atol=atol)
+        difference = (result.float() - baseline.float()).abs()
+        errors.append(
+            {
+                "max_abs": difference.max().item(),
+                "max_rel": (difference / baseline.float().abs().clamp_min(1e-6)).max().item(),
+            }
+        )
+    return {"parity": "pass", "errors": errors, "rtol": rtol, "atol": atol, "kernels": list(kernels)}
+
+
+def _percentile(samples: list[float], quantile: float) -> float:
+    ordered = sorted(samples)
+    position = (len(ordered) - 1) * quantile
+    lower, upper = math.floor(position), math.ceil(position)
+    return ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower)
+
+
+def measure_pair(torch: Any, reference: Callable, fused: Callable, args: argparse.Namespace) -> dict:
+    cuda = getattr(torch, "cuda")
+    functions = (reference, fused)
+    order: list[int] = []
+
+    def alternate() -> Any:
+        mode = len(order) % 2
+        order.append(mode)
+        return functions[mode]()
+
+    times_ms = []
+    for _ in range(args.warmup):
+        alternate()
+    cuda.synchronize()
+    order.clear()
+    flush = torch.empty(512 * 1024 * 1024, dtype=torch.uint8, device="cuda")
+    for _ in range(2 * args.num_tests):
+        flush.zero_()
+        cuda.synchronize()
+        start = cuda.Event(enable_timing=True)
+        end = cuda.Event(enable_timing=True)
+        start.record()
+        alternate()
+        end.record()
+        end.synchronize()
+        times_ms.append(start.elapsed_time(end))
+    samples: tuple[list[float], list[float]] = ([], [])
+    for mode, milliseconds in zip(order[-len(times_ms) :], times_ms, strict=True):
+        if not math.isfinite(milliseconds) or milliseconds <= 0:
+            raise RuntimeError(f"Invalid GPU event sample: {milliseconds}")
+        samples[mode].append(milliseconds * 1000)
+    stats = {}
+    for label, values in zip(("native", "fused"), samples, strict=True):
+        if len(values) != args.num_tests:
+            raise RuntimeError("GPU timing harness returned an unexpected sample count")
+        p50 = _percentile(values, 0.5)
+        stats[label] = {"median_us": p50, "p50_us": p50, "p90_us": _percentile(values, 0.9), "samples": len(values)}
+    stats["speedup_p50"] = stats["native"]["p50_us"] / stats["fused"]["p50_us"]
+    return stats

benchmarks/kernels/benchmark_magi2_bf16_moe.py

@@ -0,0 +1,310 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Compare MAGI-2 routed MoE with equal preprocessing boundaries.
+
+python -m benchmarks.kernels.benchmark_magi2_bf16_moe --tokens 2 4096 10000
+
+Default (`--mode dispatch`): the same MAGI-2 layer is timed with its native
+reference and default BF16 fast path, including routing and all preprocessing.
+With `--mode routed`, routing probabilities/IDs are inputs to BOTH timed paths. Sorting,
+padding, packing, allocations and reduction remain inside their respective
+calls. The reference uses deterministic routed storage in diagnostic mode.
+Use --mode dispatch to measure the
+actual layer with the native and default fast paths, including the router. Unsupported
+dispatch fails explicitly; it is never silently replaced by another baseline.
+
+Stage timings are standalone diagnostics, not additive or env speedup claims.
+These synthetic local shapes are not full-model performance measurements.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib
+import json
+import math
+import os
+import statistics
+import subprocess
+import time
+from contextlib import contextmanager
+from pathlib import Path
+from unittest.mock import patch
+
+
+def parse_args(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--tokens", type=int, nargs="+", default=[2, 4096])
+    parser.add_argument("--heads", type=int, default=3, help="EP4 local head count for MAGI-2 Preview.")
+    parser.add_argument("--experts", type=int, default=256, help="Released experts per head.")
+    parser.add_argument("--top-k", type=int, default=6)
+    parser.add_argument("--warmup", type=int, default=10)
+    parser.add_argument("--iters", type=int, default=50)
+    parser.add_argument("--seed", type=int, default=419)
+    parser.add_argument("--mode", choices=["routed", "dispatch"], default="dispatch")
+    parser.add_argument(
+        "--native-deterministic",
+        action="store_true",
+        help="Use the portable deterministic native reference (needed by Triton-MUSA 3.2).",
+    )
+    parser.add_argument("--stages", action="store_true")
+    parser.add_argument("--output", type=Path)
+    args = parser.parse_args(argv)
+    if min(*args.tokens, args.heads, args.experts, args.top_k, args.iters) < 1 or args.warmup < 0:
+        parser.error("Shapes/iters must be positive; warmup must be nonnegative")
+    if args.top_k > args.experts:
+        parser.error("top-k must not exceed experts per head")
+    return args
+
+
+@contextmanager
+def env_switch(enabled):
+    # Keep the legacy benchmark context API; dispatch is no longer opt-in.
+    with patch.dict(os.environ, MAGI2_DETERMINISTIC="0"):
+        yield
+
+
+def stats(values):
+    if not values or any(not math.isfinite(x) or x <= 0 for x in values):
+        raise ValueError("Missing/non-positive/non-finite timing sample")
+    ordered = sorted(values)
+    return {
+        "mean_us": statistics.mean(values),
+        "p50_us": statistics.median(values),
+        "p90_us": ordered[math.ceil(0.9 * len(ordered)) - 1],
+        "samples_us": values,
+    }
+
+
+def measure(torch, calls, args):
+    """Alternating AB/BA; cold-cache flush is outside each timed region."""
+    cuda = getattr(torch, "cuda")
+    names = list(calls)
+    sequence = names + names[::-1]
+    wall_samples = []
+    labels = []
+    counter = 0
+
+    def invoke():
+        nonlocal counter
+        name = sequence[counter % len(sequence)]
+        counter += 1
+        cuda.synchronize()
+        start = time.perf_counter_ns()
+        output = calls[name]()
+        cuda.synchronize()
+        elapsed = (time.perf_counter_ns() - start) / 1000
+        wall_samples.append(elapsed)
+        labels.append(name)
+        return output
+
+    count = len(names) * args.iters
+    cache = torch.empty(512 * 1024 * 1024, device="cuda", dtype=torch.uint8)
+    gpu_ms = []
+    for index in range(len(names) * args.warmup + count):
+        cache.zero_()
+        cuda.synchronize()
+        begin, end = cuda.Event(enable_timing=True), cuda.Event(enable_timing=True)
+        begin.record()
+        invoke()
+        end.record()
+        end.synchronize()
+        if index >= len(names) * args.warmup:
+            gpu_ms.append(begin.elapsed_time(end))
+    labels, wall_samples = labels[-count:], wall_samples[-count:]
+    result = {}
+    for name in names:
+        wall = [x for label, x in zip(labels, wall_samples) if label == name]
+        event = [float(x) * 1000 for label, x in zip(labels, gpu_ms) if label == name]
+        if len(wall) != args.iters or len(event) != args.iters:
+            raise RuntimeError("Timing harness did not return the requested sample counts")
+        result[name] = {"synchronized_wall": stats(wall), "gpu_event": stats(event)}
+    return result
+
+
+def make_inputs(torch, tokens, args):
+    # Fan-in-scaled synthetic weights. Unlike the old script, W_up and W_gate
+    # have the same distribution and top-k IDs are unique within each token.
+    torch.manual_seed(args.seed)
+    x = torch.randn(tokens, args.heads, 256, device="cuda").to(torch.bfloat16)
+    shape = (args.heads * args.experts, 256, 1280)
+    gate, up = [(torch.randn(shape, device="cuda") / 16).to(torch.bfloat16) for _ in range(2)]
+    down = (torch.randn(shape[0], 1280, 256, device="cuda") / math.sqrt(1280)).to(torch.bfloat16)
+    logits = torch.randn(args.heads, tokens, args.experts, device="cuda")
+    selected = logits.topk(args.top_k, dim=-1)
+    probabilities = selected.values.softmax(-1)
+    return x, probabilities, selected.indices, gate, up, down
+
+
+def check_and_capture(torch, moe, kernels, data, args):
+    x, probabilities, indices, gate, up, down = data
+    gather, weights, offsets = moe.global_sort_routes(probabilities, indices, args.experts)
+    sorted_routes, launches = [], []
+    scatter, invoke = kernels._deterministic_scatter, moe.invoke_fused_moe_bf16
+
+    def capture_scatter(routes, *params):
+        sorted_routes.append(routes.detach().clone())
+        return scatter(routes, *params)
+
+    def capture_launch(*params, **kwargs):
+        launches.append((params, kwargs))
+        return invoke(*params, **kwargs)
+
+    with patch.object(kernels, "_deterministic_scatter", capture_scatter):
+        moe.triton_mh_moe_forward(x, gather, weights, offsets, gate, up, down, deterministic=True)
+    packed_w13 = moe._pack_bf16_w13(gate, up)
+    route_buffers = moe._allocate_bf16_route_buffers(
+        x.shape[0] * args.heads * args.top_k,
+        args.heads * args.experts,
+        128,
+        x.device,
+    )
+    with patch.object(moe, "invoke_fused_moe_bf16", capture_launch):
+        actual = moe._bf16_fused_moe_forward(x, probabilities, indices, packed_w13, down, route_buffers)
+    if len(sorted_routes) != 1 or len(launches) != 2:
+        raise RuntimeError("Unexpected kernel dispatch: benchmark contract needs updating")
+    flat_experts = (indices + torch.arange(args.heads, device=x.device)[:, None, None] * args.experts).flatten()
+    order = flat_experts.argsort(stable=True)
+    new_routes = launches[1][0][2].reshape(-1, 256).index_select(0, order)
+    scatter_ids = gather.long() * args.heads + flat_experts[order] // args.experts
+    expected = torch.zeros(x.shape[0] * args.heads, 256, device=x.device, dtype=torch.float32)
+    expected.index_add_(0, scatter_ids, sorted_routes[0].float())
+    expected = expected.to(x.dtype).view_as(x)
+    errors = []
+    # Same expert/FP32 reduction contract as test_bf16_moe_wiring.py, rather
+    # than relaxing tolerances on the inherited BF16 scatter reduction.
+    for result, reference in ((new_routes, sorted_routes[0]), (actual, expected)):
+        if not (torch.isfinite(result).all() and torch.isfinite(reference).all()):
+            raise RuntimeError("Non-finite MoE output")
+        torch.testing.assert_close(result, reference, rtol=2e-2, atol=0.5)
+        relative_l2 = (result.float() - reference.float()).norm() / reference.float().norm().clamp_min(1e-6)
+        if relative_l2.item() >= 1e-3:
+            raise AssertionError(f"MoE relative L2 {relative_l2.item()} >= 1e-3")
+        errors.append(
+            {"max_abs": (result.float() - reference.float()).abs().max().item(), "relative_l2": relative_l2.item()}
+        )
+    return launches, errors
+
+
+def benchmark(torch, moe, kernels, tokens, args):
+    data = make_inputs(torch, tokens, args)
+    x, probabilities, indices, gate, up, down = data
+    launches, errors = check_and_capture(torch, moe, kernels, data, args)
+    packed_w13 = moe._pack_bf16_w13(gate, up)
+    route_count = tokens * args.heads * args.top_k
+    route_buffers = moe._allocate_bf16_route_buffers(route_count, args.heads * args.experts, 128, x.device)
+
+    def reference():
+        gather, weights, offsets = moe.global_sort_routes(probabilities, indices, args.experts)
+        return moe.triton_mh_moe_forward(x, gather, weights, offsets, gate, up, down, deterministic=True)
+
+    calls = {
+        "reference_deterministic_routed": reference,
+        "candidate_routed": lambda: moe._bf16_fused_moe_forward(
+            x, probabilities, indices, packed_w13, down, route_buffers
+        ),
+    }
+    if args.mode == "dispatch":
+        config = moe.Magi2MultiHeadMoEConfig(
+            args.heads * 256, args.heads, args.experts, args.top_k, 1280, torch.bfloat16
+        )
+        layer = moe.Magi2MultiHeadMoE(config).to(x.device)
+        for parameter, value in ((layer.W_gate, gate), (layer.W_up, up), (layer.W_down, down)):
+            parameter.copy_(value)
+        layer.gate.normal_(std=1 / 16)
+
+        flat_x = x.reshape(tokens, -1)
+
+        def native_dispatch():
+            x_heads = flat_x.view(tokens, args.heads, 256)
+            probs, idx = layer._route(x_heads)
+            gather, weights, offsets = moe.global_sort_routes(probs, idx, args.experts)
+            return moe.triton_mh_moe_forward(
+                x_heads,
+                gather,
+                weights,
+                offsets,
+                layer.W_gate,
+                layer.W_up,
+                layer.W_down,
+                deterministic=args.native_deterministic,
+            ).reshape(tokens, -1)
+
+        calls = {"native_dispatch": native_dispatch, "fused_dispatch": lambda: layer(flat_x)}
+        torch.testing.assert_close(calls["fused_dispatch"](), calls["native_dispatch"](), rtol=2e-2, atol=0.5)
+    timings = measure(torch, calls, args)
+    names = list(calls)
+    result = {"status": "pass", "tokens": tokens, "mode": args.mode, "correctness": errors, "timings": timings}
+    result["speedup_p50_wall"] = (
+        timings[names[0]]["synchronized_wall"]["p50_us"] / timings[names[1]]["synchronized_wall"]["p50_us"]
+    )
+    if args.stages:
+        route_ids = (indices.int() + torch.arange(args.heads, device=x.device)[:, None, None] * args.experts).reshape(
+            -1, args.top_k
+        )
+
+        def gems():
+            for params, kwargs in launches:
+                moe.invoke_fused_moe_bf16(*params, **kwargs)
+
+        result["standalone_stages_not_additive"] = measure(
+            torch,
+            {
+                "reference_sort": lambda: moe.global_sort_routes(probabilities, indices, args.experts),
+                "candidate_align": lambda: moe._align_bf16_routes(route_ids, args.heads * args.experts, 128),
+                "candidate_pack_w13": lambda: torch.stack((gate.transpose(1, 2), up.transpose(1, 2)), dim=2).reshape(
+                    args.heads * args.experts, 2560, 256
+                ),
+                "candidate_two_gemms_prepared": gems,
+            },
+            args,
+        )
+    return result
+
+
+def main():
+    args = parse_args()
+    import torch
+
+    if hasattr(torch.version, "musa") and torch.version.musa is not None:
+        importlib.import_module("torchada")  # before importing vLLM / capturing CUDA APIs
+    cuda = getattr(torch, "cuda")
+    has_gpu = cuda.is_available() or (
+        hasattr(torch.version, "musa") and torch.version.musa is not None and torch.musa.is_available()
+    )
+    if not has_gpu:
+        print(json.dumps({"status": "skip", "reason": "No visible CUDA/MUSA device"}))
+        return
+    import vllm_omni.diffusion.models.magi2.fused_moe_kernels as kernels
+    import vllm_omni.diffusion.models.magi2.mh_moe as moe
+
+    root = Path(__file__).resolve().parents[2]
+    try:
+        sha = subprocess.check_output(
+            ["git", "-C", str(root), "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL
+        ).strip()
+    except (OSError, subprocess.CalledProcessError):
+        sha = os.environ.get("MAGI2_BENCH_SOURCE_SHA", "unknown")
+    report = {
+        "device": cuda.get_device_name(),
+        "sm_count": cuda.get_device_properties(0).multi_processor_count,
+        "source_sha": sha,
+        "module_path": moe.__file__,
+        "torch": torch.__version__,
+        "args": {**vars(args), "output": str(args.output)},
+        "cache": "512 MiB flush outside each region",
+        "capture": "eager; JIT warmed; no graph/compile",
+        "results": [],
+    }
+    with torch.inference_mode():
+        for tokens in args.tokens:
+            result = benchmark(torch, moe, kernels, tokens, args)
+            report["results"].append(result)
+            print(json.dumps(result), flush=True)
+    if args.output:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(json.dumps(report, indent=2) + "\n")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/kernels/benchmark_magi2_mhc.py

@@ -0,0 +1,141 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Compare native and fused MAGI-2 mHC post-processing on CUDA or MUSA.
+
+Run from the checkout root with PyTorch and the selected backend dependencies.
+torchada is loaded automatically only on MUSA::
+
+    python -m benchmarks.kernels.benchmark_magi2_mhc \
+        --tokens 1 4 16 64 129 4096 --hidden-size 3072 --num-tests 30
+
+The released config supplies hidden=3072, four streams and 20 Sinkhorn steps.
+Token counts are synthetic; supply per-rank sizes from the target trace when
+comparing production shapes. FP32 post/residual logits are strided views of the
+same [tokens, 24] projection, as in MHCHandler.compute_logits. Mix inputs use
+the selected dtype. Both mix implementations consume identical coefficients.
+
+The three comparisons cover coefficient preparation, stream mixing, and their
+combined pipeline (including intermediate allocations, excluding projection,
+normalization and apply_pre). This is an eager operator diagnostic, without
+compile or graph capture. It does not measure full model performance.
+No CUDA/MUSA device (or no PyTorch) produces a JSON skip record and exit status zero.
+"""
+
+import argparse
+import math
+from functools import partial
+from typing import Any
+
+from benchmarks.kernels._magi2_gpu_bench import (
+    add_arguments,
+    check_pair,
+    emit,
+    load_musa,
+    measure_pair,
+    report_environment,
+)
+
+
+def benchmark_shape(torch: Any, mhc: Any, args: argparse.Namespace, tokens: int) -> None:
+    dtype = getattr(torch, args.dtype)
+    device = torch.device("cuda")
+    packed = torch.randn(tokens, 24, device=device, dtype=torch.float32)
+    # Preserve the token stride of the model's shared projection allocation.
+    post_inputs = (
+        packed[:, 4:8],
+        packed[:, 8:].view(tokens, 4, 4),
+        torch.tensor([0.8], device=device),
+        torch.randn(4, device=device),
+        torch.tensor([1.1], device=device),
+        torch.randn(4, 4, device=device),
+    )
+    streams = torch.randn(tokens, 4, args.hidden_size, device=device, dtype=dtype)
+    branch = torch.randn(tokens, args.hidden_size, device=device, dtype=dtype)
+    post_op, mix_op = mhc.MHCPostResidual(), mhc.MHCMix()
+    scale = 1 / math.sqrt(4 * args.hidden_size)
+    kwargs = dict(scale=scale, iterations=args.sinkhorn_iterations, epsilon=1e-12, out_dtype=dtype)
+    prepare_reference = partial(post_op.forward_native, *post_inputs, **kwargs)
+    prepare_fused = partial(post_op, *post_inputs, **kwargs)
+    coefficients = prepare_reference()
+    mix_reference = partial(mix_op.forward_native, streams, branch, *coefficients)
+    mix_fused = partial(mix_op, streams, branch, *coefficients)
+    post_kernel, mix_kernel = "_mhc_post_residual_kernel", "_mhc_mix_kernel"
+    cases = {
+        "post_residual": (prepare_reference, prepare_fused, (post_kernel,)),
+        "mix": (mix_reference, mix_fused, (mix_kernel,)),
+        "pipeline": (
+            lambda: mix_op.forward_native(streams, branch, *prepare_reference()),
+            lambda: mix_op(streams, branch, *prepare_fused()),
+            (post_kernel, mix_kernel),
+        ),
+    }
+    rtol = {torch.bfloat16: 1.6e-2, torch.float16: 2e-3, torch.float32: 1e-5}[dtype]
+    for name, (reference, fused, kernels) in cases.items():
+        if args.op != "all" and args.op != name:
+            continue
+        # BF16 MUSA custom ops can quantize values near zero to the adjacent
+        # representable value; retain the strict relative gate while allowing
+        # one BF16 ulp in the absolute comparison.
+        atol = (
+            (1e-2 if dtype == torch.bfloat16 else 1e-6)
+            if name == "post_residual"
+            else (1e-2 if dtype == torch.bfloat16 else 1e-5)
+        )
+        parity = check_pair(torch, reference, fused, mhc, kernels, rtol=rtol, atol=atol)
+        timing = measure_pair(torch, reference, fused, args)
+        # Logical minimum traffic: input reads plus output writes. Native
+        # intermediates and repeat parameter loads can add actual DRAM traffic.
+        element_bytes = streams.element_size()
+        prepare_bytes = tokens * 20 * (4 + element_bytes) + 22 * 4
+        mix_bytes = tokens * (9 * args.hidden_size + 20) * element_bytes
+        minimum_bytes = {"post_residual": prepare_bytes, "mix": mix_bytes, "pipeline": prepare_bytes + mix_bytes}[name]
+        # Sigmoid/exp costs and max reductions are not represented as FLOPs.
+        prepare_flops = tokens * (72 + 64 * args.sinkhorn_iterations)
+        mix_flops = tokens * args.hidden_size * 36
+        scalar_flops = {"post_residual": prepare_flops, "mix": mix_flops, "pipeline": prepare_flops + mix_flops}[name]
+        emit(
+            status="pass",
+            op=name,
+            tokens=tokens,
+            hidden_size=args.hidden_size,
+            num_streams=4,
+            dtype=args.dtype,
+            logits_dtype="float32",
+            packed_shape=list(packed.shape),
+            post_stride=list(post_inputs[0].stride()),
+            residual_stride=list(post_inputs[1].stride()),
+            scale=scale,
+            sinkhorn_iterations=args.sinkhorn_iterations,
+            epsilon=1e-12,
+            minimum_io_bytes=minimum_bytes,
+            scalar_flops_excluding_exp_and_max=scalar_flops,
+            effective_fused_gbps=minimum_bytes / timing["fused"]["p50_us"] / 1000,
+            **parity,
+            **timing,
+        )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    add_arguments(parser)
+    parser.add_argument("--hidden-size", type=int, default=3072)
+    parser.add_argument("--sinkhorn-iterations", type=int, default=20)
+    parser.add_argument("--op", choices=["all", "post_residual", "mix", "pipeline"], default="all")
+    args = parser.parse_args()
+    if args.hidden_size <= 0 or not 0 <= args.sinkhorn_iterations <= 64:
+        parser.error("hidden-size must be positive; sinkhorn-iterations must be in [0, 64]")
+    torch = load_musa(args, parser)
+    if torch is None:
+        return
+    cuda = getattr(torch, "cuda")
+    from vllm_omni.diffusion.layers import mhc
+
+    report_environment(torch, args, mhc)
+    with torch.inference_mode():
+        for tokens in args.tokens:
+            benchmark_shape(torch, mhc, args, tokens)
+            cuda.empty_cache()
+
+
+if __name__ == "__main__":
+    main()

benchmarks/kernels/benchmark_magi2_swiglu7.py

@@ -0,0 +1,87 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Compare native and fused MAGI-2 SwiGLU7 operators on CUDA or MUSA.
+
+Run from the checkout root in an environment with PyTorch and the selected
+backend dependencies.  torchada is loaded automatically only on MUSA::
+
+    python -m benchmarks.kernels.benchmark_magi2_swiglu7 \
+        --tokens 1 4 16 64 129 4096 --intermediate-size 1280 --num-tests 30
+
+The released MAGI-2 config supplies the 1280 intermediate width. Token counts
+are a synthetic sweep; pass per-rank sizes from the target trace for production
+comparisons. Input is contiguous interleaved gate/up [tokens, 2 * intermediate].
+Both paths evaluate FP32 arithmetic with one cast to the requested output dtype.
+This measures eager operator latency, not compiled serving performance.
+No CUDA/MUSA device (or no PyTorch) produces a JSON skip record and exit status zero.
+"""
+
+import argparse
+from functools import partial
+
+from benchmarks.kernels._magi2_musa_bench import (
+    add_arguments,
+    check_pair,
+    emit,
+    load_musa,
+    measure_pair,
+    report_environment,
+)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    add_arguments(parser)
+    parser.add_argument("--intermediate-size", type=int, default=1280)
+    parser.add_argument("--out-dtype", choices=["bfloat16", "float16", "float32"], default=None)
+    args = parser.parse_args()
+    if args.intermediate_size <= 0:
+        parser.error("intermediate-size must be positive")
+    torch = load_musa(args, parser)
+    if torch is None:
+        return
+    cuda = getattr(torch, "cuda")
+    from vllm_omni.diffusion.layers import swiglu7
+
+    report_environment(torch, args, swiglu7)
+    dtype = getattr(torch, args.dtype)
+    out_dtype = getattr(torch, args.out_dtype or args.dtype)
+    device = torch.device("cuda")
+    rtol = {torch.bfloat16: 1.6e-2, torch.float16: 2e-3, torch.float32: 1e-5}[out_dtype]
+    atol = 1e-5 if out_dtype == torch.float32 else 2e-3
+    op = swiglu7.SwiGLU7()
+    with torch.inference_mode():
+        for tokens in args.tokens:
+            # Values outside both clamp boundaries exercise the released formula.
+            x = (torch.randn(tokens, 2 * args.intermediate_size, device=device) * 10).to(dtype)
+            reference = partial(op.forward_native, x, out_dtype=out_dtype)
+            fused = partial(op, x, out_dtype=out_dtype)
+            parity = check_pair(torch, reference, fused, swiglu7, ("_swiglu7_kernel",), rtol=rtol, atol=atol)
+            timing = measure_pair(torch, reference, fused, args)
+            output_elements = tokens * args.intermediate_size
+            minimum_io_bytes = (
+                x.numel() * x.element_size() + output_elements * torch.empty((), dtype=out_dtype).element_size()
+            )
+            emit(
+                status="pass",
+                op="swiglu7",
+                input_shape=list(x.shape),
+                input_stride=list(x.stride()),
+                dtype=args.dtype,
+                out_dtype=args.out_dtype or args.dtype,
+                alpha=1.702,
+                limit=7.0,
+                minimum_io_bytes=minimum_io_bytes,
+                # Six scalar arithmetic operations and one exp per output;
+                # clamps and transcendental costs are not modeled as FLOPs.
+                scalar_flops_excluding_exp_and_clamps=6 * output_elements,
+                effective_fused_gbps=minimum_io_bytes / timing["fused"]["p50_us"] / 1000,
+                **parity,
+                **timing,
+            )
+            del x, reference, fused
+            cuda.empty_cache()
+
+
+if __name__ == "__main__":
+    main()

benchmarks/mammoth_moda2/README.md

@@ -0,0 +1,124 @@
+# MammothModa2 startup benchmark
+
+Measures `Omni()` initialization and first/subsequent image requests for
+MammothModa2-Preview. This is a measurement baseline for
+[#7075](https://github.com/vllm-project/vllm-omni/issues/7075); it changes no runtime code.
+
+## Run
+
+From the repository root, with the model downloaded and vLLM-Omni installed:
+
+```bash
+set -o pipefail
+python benchmarks/mammoth_moda2/bench_startup.py \
+    --model /path/MammothModa2-Preview \
+    --deploy-config vllm_omni/deploy/mammoth_moda2.yaml \
+    --height 512 --width 512 --seed 42 \
+    --extra-body '{"text_guidance_scale": 4.0, "cfg_range": [0.0, 1.0], "num_inference_steps": 8}' \
+    --repeat 1 --label baseline --output-json baseline.json --save-image baseline.png \
+    2>&1 | tee baseline.log
+python benchmarks/mammoth_moda2/parse_startup_log.py baseline.log --require-complete --markdown
+```
+
+Use `OMP_NUM_THREADS=16` to set CPU threads and `--parallel-stage-init` to
+initialize stages concurrently. For first/subsequent requests, use
+`--height 1024 --width 1024 --repeat 3` and 50 inference steps. Memory polling
+is opt-in (`--sample-memory`); collect it in separate runs to avoid perturbing
+the timing comparison. `--save-image` saves only the first request's image.
+Compare that image across runs; a fixed seed alone does not guarantee equality
+across execution modes or software versions.
+
+## Measurement boundaries
+
+| Metric | Boundary / limitation |
+| --- | --- |
+| Imports | Before torch import → benchmark imports complete; excludes interpreter startup |
+| Time to ready | `Omni()` call → return; excludes client imports and requests |
+| Time to first image | Benchmark clock start → first `generate()` return, including request setup |
+| First / subsequent request | Each `generate()` call → return; same prompt and settings |
+| Stage spawn / device init | Log timestamps: launch → `world_size=` → `Starting to load model`; 1 s resolution |
+| Weight load | Engine-reported `Loading weights took`; read/cast/copy are merged |
+| Model setup estimate | Reported model-load time minus weight-load time; not independently instrumented |
+| Profile + KV + warmup | One merged interval after loading; compilation/capture may be nested within it |
+| Compile / graph capture | Explicit engine log durations when present; N/A for eager execution |
+| Memory | Optional 1 s samples: sum of process-tree RSS and maximum device memory used across GPUs reported by `nvidia-smi`; includes other GPU users and can miss transient peaks |
+
+Stage timelines overlap with parallel initialization; do not sum them to
+calculate time to ready. Missing log fields stay unmeasured. The parser marks
+logs without `BENCH_JSON` as incomplete; `--require-complete` makes this an error.
+
+## Reference results
+
+A800-SXM4-80GB; Ubuntu 22.04; 16-CPU cgroup quota on a 128-core host;
+vLLM 0.28.0+cu129, torch 2.13.0+cu129, vLLM-Omni `34aa1d2a`.
+Checkpoint: 34.5 GiB, 8 shards, mostly fp32, on local NVMe.
+Committed eager deploy config, GPU memory fractions 0.5 / 0.3.
+
+The 2026-09-10 rerun used the corrected benchmark: one warm-up process,
+then three rounds of four configurations with order rotated between rounds.
+The checkpoint page cache was warmed before the matrix; no cache eviction
+was requested during it. Each fresh process sent one 512×512, 8-step request
+(guidance 4.0, seed 42). Memory polling was disabled. All 12 measured runs
+are retained in [the results](reference_results.json).
+Values below are `Omni()` seconds, mean ± sample standard deviation, n=3;
+client imports took approximately another 10 s.
+
+| CPU threads | Serial init | Parallel init |
+| --- | --- | --- |
+| Default (64) | 60.1 ± 0.2 | 41.1 ± 1.0 |
+| 16 | 56.5 ± 0.9 | 36.7 ± 0.2 |
+
+On this configuration, parallel initialization reduced time to ready by about
+32%; combining it with 16 threads reduced it by about 39%. This supports
+trying these settings on similar deployments, not changing defaults for all
+models or hardware. Fixed 512×512 samples matched across all 12 measured runs;
+this is a smoke check, not a broad quality evaluation.
+
+For context, in the earlier serial baseline AR / DiT spent approximately 11 / 11 s in
+spawn and imports, 6 / 6 s in device setup, 6 / 4 s loading weights, and
+5 / 2 s in profile + KV + warmup. About 6 s followed the last stage becoming
+ready. These coarse log intervals locate work; they are not independent
+operator timings.
+
+Separate observations from the earlier experiment (not rerun in this matrix):
+
+- Requested page-cache eviction: 80.8 s; FlashInfer JIT cache removed:
+  129.1 s (means, n=3 each). These are separate cache conditions, not guaranteed
+  cold-machine startup or isolated disk/JIT costs.
+- Three 1024×1024, 50-step eager requests in one process took
+  95.2 / 94.9 / 94.6 s. No large first-request penalty was observed in that run.
+- With AR `enforce_eager: false` and serial init, time to ready was
+  62.5 ± 0.1 s (warm caches, n=3); logged graph capture was 4 s / 0.20 GiB.
+  The model reported torch.compile unsupported. KV capacity fell from about
+  15.7 to 2.0 GiB, and one parallel-init trial failed. The reported non-KV
+  profiling difference is not a measurement of graph allocation. Graph-mode
+  samples differed from eager. This needs separate memory and quality analysis
+  before recommending the configuration.
+
+## Loading diagnostic and storage scenarios
+
+```bash
+OMP_NUM_THREADS=16 python benchmarks/mammoth_moda2/raw_load_bench.py \
+    --model /path/MammothModa2-Preview --shards 6,7,8 --mode cpu_cast
+# Compare --mode gpu_cast in a fresh process with the same cache conditions.
+
+MODEL_LOCAL=/local/MammothModa2-Preview OUT_DIR=./startup-bench \
+    bash benchmarks/mammoth_moda2/bench_storage_scenarios.sh
+# Optional MODEL_NFS=/nfs/MammothModa2-Preview; SCENARIOS="local-cold local-warm".
+```
+
+The diagnostic clones each mmap-backed tensor into owned CPU memory before
+casting. `materialize_s` includes page faults, CPU allocation and copying;
+`cast_s` measures dtype conversion; `h2d_s` includes GPU allocation and copy.
+CUDA context/operation warmup is outside the timer. Total also includes file
+opening and bookkeeping. This extra CPU copy differs from the engine loader,
+so these numbers do not decompose its loading time or measure disk bandwidth.
+The revised diagnostic was run three times each for CPU cast with 64 / 16
+threads and GPU cast with 16 threads; per-run values are in the results file.
+Both CPU materialization and conversion were faster with 16 threads here; this
+does not isolate how much of the engine-level improvement comes from either.
+
+Storage scenarios explicitly read shards before warm runs and request
+`posix_fadvise(DONTNEED)` before cold runs. Eviction is not guaranteed, and
+server-side NFS caches are uncontrolled. The script stops on failure and
+summarizes only logs from the current invocation.

benchmarks/mammoth_moda2/__init__.py

@@ -1,4 +1,2 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
-
-"""Qwen3-Omni turn-based browser call UI."""

benchmarks/mammoth_moda2/bench_startup.py

@@ -0,0 +1,338 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Startup / loading benchmark for multi-stage omni models (MammothModa2 first).
+
+Measures, from a single fresh process:
+  * python import time (torch + vllm_omni)
+  * Omni() construction wall time (all stages: weight load, profiling, KV cache, warmup)
+  * first request latency (includes JIT / lazy init that only happens on first run)
+  * steady-state request latency (repeat N-1 more requests)
+  * optional host RSS / device memory samples (--sample-memory, once a second)
+
+Per-stage breakdown (weight loading, model-load memory, engine init) is emitted by the
+stage engine-core subprocesses into the log; use parse_startup_log.py on the captured
+stdout/stderr to merge them with the JSON written here.
+
+Usage (keep the log: the per-stage breakdown is parsed from it afterwards):
+  python benchmarks/mammoth_moda2/bench_startup.py --model /path/MammothModa2-Preview \
+      --deploy-config vllm_omni/deploy/mammoth_moda2.yaml \
+      --height 1024 --width 1024 --seed 42 \
+      --extra-body '{"text_guidance_scale": 4.0, "cfg_range": [0.0, 1.0], "num_inference_steps": 50}' \
+      --repeat 2 --label nfs-warm --output-json out.json 2>&1 | tee run.log
+  python benchmarks/mammoth_moda2/parse_startup_log.py run.log --markdown
+"""
+
+import argparse
+import json
+import os
+import platform
+import subprocess
+import threading
+import time
+
+T_PROCESS_START = time.perf_counter()
+
+import torch  # noqa: E402
+
+from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args  # noqa: E402
+from vllm_omni.entrypoints.omni import Omni  # noqa: E402
+from vllm_omni.entrypoints.openai.stage_params import clone_sampling_params  # noqa: E402
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams  # noqa: E402
+from vllm_omni.model_extras import (  # noqa: E402
+    build_text_to_image_prompt as build_model_text_to_image_prompt,
+)
+from vllm_omni.model_extras import (  # noqa: E402
+    get_extra_body_params,
+    get_model_class_name,
+    should_init_extra_args_for_non_diffusion_stages,
+)
+from vllm_omni.platforms import current_omni_platform  # noqa: E402
+
+T_IMPORTS_DONE = time.perf_counter()
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--model", required=True)
+    p.add_argument("--deploy-config", default=None)
+    p.add_argument("--prompt", default="A stylish woman riding a motorcycle in NYC, movie poster style")
+    p.add_argument("--negative-prompt", default=None)
+    p.add_argument("--height", type=int, default=1024)
+    p.add_argument("--width", type=int, default=1024)
+    p.add_argument("--seed", type=int, default=42)
+    p.add_argument("--extra-body", type=json.loads, default=None, help="JSON object forwarded as model extra_body")
+    p.add_argument(
+        "--repeat", type=int, default=2, help="number of generate() calls; 1st = first-request, rest = steady"
+    )
+    p.add_argument("--label", default="run", help="free-form label, e.g. nfs-cold / nfs-warm / local-nvme")
+    p.add_argument("--output-json", default=None)
+    p.add_argument("--save-image", default=None, help="optional path to save the first request's image")
+    p.add_argument("--sample-memory", action="store_true", help="sample memory once a second (adds overhead)")
+    p.add_argument(
+        "--parallel-stage-init",
+        action="store_true",
+        help="initialize stages that share a GPU concurrently (VllmOmniOrchestratorConfig.parallel_stage_init)",
+    )
+    args = p.parse_args()
+    if args.repeat < 1:
+        p.error("--repeat must be at least 1")
+    return args
+
+
+def gpu_info() -> dict:
+    info = {"torch": torch.__version__, "cuda": torch.version.cuda, "python": platform.python_version()}
+    try:
+        import vllm
+
+        info["vllm"] = vllm.__version__
+    except Exception:  # pragma: no cover
+        pass
+    try:
+        import vllm_omni
+
+        info["vllm_omni"] = getattr(vllm_omni, "__version__", "src")
+    except Exception:  # pragma: no cover
+        pass
+    try:
+        out = subprocess.run(
+            ["nvidia-smi", "--query-gpu=name,driver_version,memory.total", "--format=csv,noheader"],
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+        info["gpu"] = out.stdout.strip()
+    except Exception:
+        info["gpu"] = "n/a"
+    return info
+
+
+class PeakSampler:
+    """Poll host RSS of this process tree and GPU memory in use once a second (benchmark-side only)."""
+
+    def __init__(self, interval_s: float = 1.0):
+        self.interval_s = interval_s
+        self.host_rss_peak_gib = 0.0
+        self.gpu_used_peak_gib = 0.0
+        self._stop = threading.Event()
+        self._thread = threading.Thread(target=self._run, daemon=True)
+
+    def _sample(self) -> None:
+        try:
+            import psutil
+
+            root = psutil.Process()
+            rss = root.memory_info().rss + sum(
+                c.memory_info().rss for c in root.children(recursive=True) if c.is_running()
+            )
+            self.host_rss_peak_gib = max(self.host_rss_peak_gib, rss / 2**30)
+        except Exception:
+            pass
+        try:
+            out = subprocess.run(
+                ["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"],
+                capture_output=True,
+                text=True,
+                timeout=5,
+            )
+            used = max(float(x) for x in out.stdout.split() if x.strip())
+            self.gpu_used_peak_gib = max(self.gpu_used_peak_gib, used / 1024)
+        except Exception:
+            pass
+
+    def _run(self) -> None:
+        while not self._stop.is_set():
+            self._sample()
+            self._stop.wait(self.interval_s)
+
+    def start(self) -> "PeakSampler":
+        self._thread.start()
+        return self
+
+    def stop(self) -> dict:
+        self._stop.set()
+        self._thread.join(timeout=10)
+        self._sample()
+        return {
+            "host_rss_peak_gib": round(self.host_rss_peak_gib, 2),
+            "gpu_used_peak_gib": round(self.gpu_used_peak_gib, 2),
+        }
+
+
+def fs_type(path: str) -> str:
+    try:
+        out = subprocess.run(["stat", "-f", "-c", "%T", path], capture_output=True, text=True, timeout=10)
+        return out.stdout.strip()
+    except Exception:
+        return "unknown"
+
+
+def build_request(omni: Omni, args: argparse.Namespace, generator: torch.Generator):
+    model_class_name = get_model_class_name(omni)
+    declared = get_extra_body_params(model_class_name)
+
+    prompt_dict = {"prompt": args.prompt}
+    if args.negative_prompt:
+        prompt_dict["negative_prompt"] = args.negative_prompt
+    prompt_dict = build_model_text_to_image_prompt(
+        model_class_name=model_class_name, prompt=prompt_dict, height=args.height, width=args.width
+    )
+
+    diffusion_params = OmniDiffusionSamplingParams(
+        height=args.height,
+        width=args.width,
+        seed=args.seed,
+        generator=generator,
+    )
+    user_extra = dict(args.extra_body or {})
+    if declared:
+        apply_declared_extra_args(diffusion_params, declared, user_extra)
+    else:
+        diffusion_params.extra_args.update({k: v for k, v in user_extra.items() if v is not None})
+
+    init_non_diffusion = should_init_extra_args_for_non_diffusion_stages(model_class_name)
+    sampling_params_list = [clone_sampling_params(p) for p in (omni.default_sampling_params_list or [])]
+    if not sampling_params_list:
+        sampling_params_list = [diffusion_params]
+
+    replaced = False
+    for idx, params in enumerate(sampling_params_list):
+        if isinstance(params, OmniDiffusionSamplingParams):
+            sampling_params_list[idx] = diffusion_params
+            replaced = True
+        elif init_non_diffusion and hasattr(params, "extra_args"):
+            params.extra_args = {**(params.extra_args or {}), **(diffusion_params.extra_args or {})}
+            if hasattr(params, "seed"):
+                params.seed = args.seed
+            # MammothModa2 AR stage: one visual token per grid cell + one EOL per row + 1 look-ahead.
+            info = prompt_dict.get("additional_information", {})
+            if idx == 0 and info.get("omni_task") == ["t2i"]:
+                ar_w = int(info.get("ar_width", [0])[0])
+                ar_h = int(info.get("ar_height", [0])[0])
+                if ar_w > 0 and ar_h > 0:
+                    params.max_tokens = ar_h * (ar_w + 1) + 1
+    if not replaced and len(sampling_params_list) == 1:
+        sampling_params_list = [diffusion_params]
+    return prompt_dict, sampling_params_list
+
+
+def stage_metrics(outputs) -> dict:
+    """Pull vllm-omni's per-stage timing out of the request output when present."""
+    res = {}
+    for out in outputs or []:
+        metrics = getattr(out, "metrics", None)
+        if isinstance(metrics, dict) and "stage_metrics" in metrics:
+            for sid, m in metrics["stage_metrics"].items():
+                if not isinstance(m, dict):
+                    continue
+                res[str(sid)] = {
+                    k: m.get(k)
+                    for k in ("stage_gen_time_ms", "num_tokens_in", "num_tokens_out", "vllm_ttft_ms", "vllm_tpot_ms")
+                    if k in m
+                }
+    return res
+
+
+def main() -> None:
+    args = parse_args()
+    result = {
+        "label": args.label,
+        "model": args.model,
+        "prompt": args.prompt,
+        "negative_prompt": args.negative_prompt,
+        "sample_memory": args.sample_memory,
+        "model_fs": fs_type(args.model),
+        "deploy_config": args.deploy_config,
+        "height": args.height,
+        "width": args.width,
+        "seed": args.seed,
+        "extra_body": args.extra_body,
+        "parallel_stage_init": args.parallel_stage_init,
+        "omp_num_threads": os.environ.get("OMP_NUM_THREADS"),
+        "env": gpu_info(),
+        "phases_s": {"imports": round(T_IMPORTS_DONE - T_PROCESS_START, 3)},
+        "requests": [],
+    }
+
+    omni_kwargs = {"model": args.model, "mode": "text-to-image", "log_stats": True}
+    if args.deploy_config:
+        omni_kwargs["deploy_config"] = args.deploy_config
+    if args.parallel_stage_init:
+        omni_kwargs["parallel_stage_init"] = True
+
+    sampler = PeakSampler().start() if args.sample_memory else None
+    t0 = time.perf_counter()
+    omni = None
+    try:
+        omni = Omni(**omni_kwargs)
+        t1 = time.perf_counter()
+        result["peak_mem_startup"] = sampler.stop() if sampler else {}
+        sampler = None
+        result["phases_s"]["engine_init"] = round(t1 - t0, 3)
+        result["phases_s"]["process_to_engine_ready"] = round(t1 - T_PROCESS_START, 3)
+        print(
+            f"BENCH engine_init={t1 - t0:.2f}s process_to_engine_ready={t1 - T_PROCESS_START:.2f}s "
+            f"peak_mem_startup={result['peak_mem_startup']}",
+            flush=True,
+        )
+        sampler = PeakSampler().start() if args.sample_memory else None
+
+        generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
+        outputs = None
+        first_outputs = None
+        first_image_elapsed = 0.0
+        for i in range(args.repeat):
+            prompt_dict, sampling_params_list = build_request(omni, args, generator)
+            tr0 = time.perf_counter()
+            outputs = omni.generate(prompt_dict, sampling_params_list=sampling_params_list)
+            tr1 = time.perf_counter()
+            if first_outputs is None:
+                first_outputs = outputs
+                first_image_elapsed = tr1 - T_PROCESS_START
+            rec = {
+                "index": i,
+                "kind": "first" if i == 0 else "steady",
+                "latency_s": round(tr1 - tr0, 3),
+                "stages": stage_metrics(outputs),
+            }
+            result["requests"].append(rec)
+            print(f"BENCH request[{i}] {rec['kind']} latency={tr1 - tr0:.2f}s stages={rec['stages']}", flush=True)
+
+        result["peak_mem_requests"] = sampler.stop() if sampler else {}
+        sampler = None
+        steady = [r["latency_s"] for r in result["requests"] if r["kind"] == "steady"]
+        result["summary_s"] = {
+            "imports": result["phases_s"]["imports"],
+            "engine_init": result["phases_s"]["engine_init"],
+            "first_request": result["requests"][0]["latency_s"],
+            "steady_request_avg": round(sum(steady) / len(steady), 3) if steady else None,
+            "time_to_first_image_from_process_start": round(first_image_elapsed, 3),
+        }
+
+        for key in ("host_rss_peak_gib", "gpu_used_peak_gib"):
+            result["summary_s"][key] = (
+                max(result["peak_mem_startup"][key], result["peak_mem_requests"][key]) if args.sample_memory else None
+            )
+
+        from vllm_omni.diffusion.utils.image_output import extract_images_from_outputs
+
+        images = extract_images_from_outputs(first_outputs)
+        if not images:
+            raise RuntimeError("First request returned no image")
+        if args.save_image:
+            os.makedirs(os.path.dirname(os.path.abspath(args.save_image)), exist_ok=True)
+            images[0].save(args.save_image)
+
+        print("BENCH_JSON " + json.dumps(result), flush=True)
+        if args.output_json:
+            with open(args.output_json, "w") as f:
+                json.dump(result, f, indent=2)
+            print(f"BENCH wrote {args.output_json}", flush=True)
+    finally:
+        if sampler:
+            sampler.stop()
+        if omni is not None:
+            omni.close()
+
+
+if __name__ == "__main__":
+    main()

benchmarks/mammoth_moda2/bench_storage_scenarios.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+# MODEL_LOCAL=/local/model MODEL_NFS=/nfs/model OUT_DIR=./startup-bench bash "$0"
+# Optional SCENARIOS: local-cold local-warm nfs-cold nfs-warm.
+set -euo pipefail
+HERE=$(cd "$(dirname "$0")" && pwd)
+OUT_DIR=${OUT_DIR:-./startup-bench}
+DEPLOY_CONFIG=${DEPLOY_CONFIG:-vllm_omni/deploy/mammoth_moda2.yaml}
+STEPS=${STEPS:-50}
+SIZE=${SIZE:-1024}
+REPEAT=${REPEAT:-2}
+SEED=${SEED:-42}
+EXTRA=${EXTRA:-'{"text_guidance_scale": 4.0, "cfg_range": [0.0, 1.0], "num_inference_steps": '$STEPS'}'}
+mkdir -p "$OUT_DIR/logs" "$OUT_DIR/json" "$OUT_DIR/images"
+rm -f "$OUT_DIR/summary.json" "$OUT_DIR/summary.md"
+
+prepare_cache() {
+  python - "$1" "$2" <<'PY'
+import os
+import sys
+from pathlib import Path
+
+files = sorted(Path(sys.argv[1]).glob("*.safetensors"))
+if not files:
+    raise SystemExit("no checkpoint shards found")
+for path in files:
+    with path.open("rb") as f:
+        if sys.argv[2] == "cold":
+            os.posix_fadvise(f.fileno(), 0, 0, os.POSIX_FADV_DONTNEED)
+        else:
+            while f.read(8 * 1024 * 1024):
+                pass
+print("[bench]", "requested eviction of" if sys.argv[2] == "cold" else "read", len(files), "shards")
+PY
+}
+
+default_scenarios=""
+if [ -n "${MODEL_LOCAL:-}" ]; then default_scenarios="local-cold local-warm"; fi
+if [ -n "${MODEL_NFS:-}" ]; then default_scenarios="$default_scenarios nfs-cold nfs-warm"; fi
+read -r -a scenarios <<< "${SCENARIOS:-$default_scenarios}"
+if [ "${#scenarios[@]}" -eq 0 ]; then echo "Set MODEL_LOCAL or MODEL_NFS" >&2; exit 1; fi
+logs=()
+for label in "${scenarios[@]}"; do
+  case "$label" in
+    local-cold|local-warm) model=${MODEL_LOCAL:?Set MODEL_LOCAL} ;;
+    nfs-cold|nfs-warm) model=${MODEL_NFS:?Set MODEL_NFS} ;;
+    *) echo "Unknown scenario: $label" >&2; exit 1 ;;
+  esac
+  prepare_cache "$model" "${label##*-}"
+  log="$OUT_DIR/logs/$label.log"
+  # Remove previous outputs for this label so a failed rerun cannot leave stale results.
+  rm -f "$OUT_DIR/json/$label.json" "$OUT_DIR/images/$label.png"
+  if python "$HERE/bench_startup.py" --model "$model" --deploy-config "$DEPLOY_CONFIG" \
+    --height "$SIZE" --width "$SIZE" --seed "$SEED" --extra-body "$EXTRA" --repeat "$REPEAT" \
+    --label "$label" --output-json "$OUT_DIR/json/$label.json" --save-image "$OUT_DIR/images/$label.png" \
+    > "$log" 2>&1; then
+    logs+=("$log")
+  else
+    rc=$?
+    echo "[bench] $label failed (exit $rc); see $log" >&2
+    exit "$rc"
+  fi
+done
+python "$HERE/parse_startup_log.py" "${logs[@]}" --require-complete --markdown \
+  --json-out "$OUT_DIR/summary.json" | tee "$OUT_DIR/summary.md"
+echo BENCH_ALL_DONE

benchmarks/mammoth_moda2/parse_startup_log.py

@@ -0,0 +1,245 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Merge per-stage startup numbers from a bench_startup.py log with its BENCH_JSON line.
+
+The stage engine cores run in subprocesses and report weight-loading / init timings
+only through log lines such as:
+
+  (StageEngineCoreProc_stage0_replica0 pid=1) INFO ... Loading weights took 39.66 seconds
+  (StageEngineCoreProc_stage0_replica0 pid=1) INFO ... Model loading took 21.45 GiB memory and 40.59 seconds
+  (StageEngineCoreProc_stage0_replica0 pid=1) INFO ... init engine (profile, create kv cache, warmup model) took 7.00 s
+  (StageEngineCoreProc_stage0_replica0 pid=1) INFO ... Filesystem type for checkpoints: NFS4.
+      Checkpoint size: 34.52 GiB. Available RAM: 22.76 GiB.
+  INFO ... [Omni] AsyncOmniEngine initialized in 159.74 seconds
+
+Usage:
+  python benchmarks/mammoth_moda2/parse_startup_log.py run.log [more.log ...] [--markdown] [--json-out summary.json]
+"""
+
+import argparse
+import json
+import re
+import sys
+from datetime import datetime
+from typing import Any
+
+STAGE = r"\(StageEngineCoreProc_stage(?P<stage>\d+)_replica\d+ pid=\d+\)"
+PATTERNS = {
+    "weights_load_s": re.compile(STAGE + r".*Loading weights took (?P<v>[\d.]+) seconds"),
+    "model_load_s": re.compile(STAGE + r".*Model loading took (?P<mem>[\d.]+) GiB memory and (?P<v>[\d.]+) seconds"),
+    "engine_init_s": re.compile(
+        STAGE + r".*init engine \(profile, create kv cache, warmup model\) took (?P<v>[\d.]+) s"
+    ),
+    # Only present when the stage does not run with enforce_eager.
+    "graph_capture_s": re.compile(
+        STAGE + r".*Graph capturing finished in (?P<v>[\d.]+) secs, took (?P<mem>[\d.]+) GiB"
+    ),
+    "compile_s": re.compile(STAGE + r".*torch\.compile takes (?P<v>[\d.]+) s in total"),
+    "ckpt_fs": re.compile(
+        STAGE
+        + r".*Filesystem type for checkpoints: (?P<fs>\w+)\. Checkpoint size: (?P<size>[\d.]+) GiB\."
+        + r" Available RAM: (?P<ram>[\d.]+) GiB"
+    ),
+}
+ENGINE_READY = re.compile(r"AsyncOmniEngine initialized in (?P<v>[\d.]+) seconds")
+PREFETCH_SKIP = re.compile(STAGE + r".*exceeds 90% of available RAM")
+COMPILE_UNSUPPORTED = re.compile(STAGE + r".*`torch\.compile` is turned on, but the model .* does not support it")
+
+# Startup timeline: (milestone key, regex). Timestamps in vllm logs are "MM-DD HH:MM:SS" (1 s resolution).
+TS = re.compile(r"(?:INFO|WARNING|ERROR) (?P<ts>\d\d-\d\d \d\d:\d\d:\d\d)")
+MILESTONES = [
+    ("omni_init_start", re.compile(r"\[Omni\] Initializing with model")),
+    ("stage{stage}_launch", re.compile(r"Stage-(?P<stage>\d+) set runtime devices")),
+    ("stage{stage}_proc_up", re.compile(STAGE + r".*world_size=\d+ rank=\d+")),
+    ("stage{stage}_load_start", re.compile(STAGE + r".*Starting to load model")),
+    ("stage{stage}_load_done", re.compile(STAGE + r".*Loading weights took")),
+    ("stage{stage}_init_done", re.compile(STAGE + r".*init engine \(profile, create kv cache, warmup model\) took")),
+    ("stage{stage}_ready", re.compile(r"\[StageRuntime\] Stage (?P<stage>\d+) initialized")),
+    ("engine_ready", re.compile(r"AsyncOmniEngine initialized in")),
+]
+
+
+def _ts_seconds(ts: str) -> int:
+    # Use a leap year because engine timestamps do not include the year.
+    value = datetime.strptime("2000-" + ts, "%Y-%m-%d %H:%M:%S")
+    return int((value - datetime(2000, 1, 1)).total_seconds())
+
+
+def timeline(lines: list[str]) -> dict:
+    """Return milestone timestamps (s, relative to omni_init_start) and derived per-stage phase durations."""
+    marks: dict[str, int] = {}
+    for line in lines:
+        tsm = TS.search(line)
+        if not tsm:
+            continue
+        for key, pat in MILESTONES:
+            m = pat.search(line)
+            if m:
+                k = key.format(stage=m.group("stage")) if "{stage}" in key else key
+                marks.setdefault(k, _ts_seconds(tsm.group("ts")))
+                break
+    if "omni_init_start" not in marks:
+        return {}
+    t0 = marks["omni_init_start"]
+    rel = {k: (v - t0) % (366 * 86400) for k, v in marks.items()}
+    phases: dict[str, Any] = {}
+    names = ("launch", "proc_up", "load_start", "load_done", "init_done", "ready")
+    for s in sorted({k.split("_")[0] for k in rel if k.startswith("stage")}):
+        found = {name: rel.get(f"{s}_{name}") for name in names}
+        if any(v is None for v in found.values()):
+            continue
+        ts = {name: int(v) for name, v in found.items() if v is not None}
+        phases[s] = {
+            "spawn_import_config_s": ts["proc_up"] - ts["launch"],
+            "device_dist_init_s": ts["load_start"] - ts["proc_up"],
+            "model_and_weights_load_s": ts["load_done"] - ts["load_start"],
+            "profile_kv_warmup_s": ts["init_done"] - ts["load_done"],
+            "ready_handoff_s": ts["ready"] - ts["init_done"],
+            "total_s": ts["ready"] - ts["launch"],
+        }
+    if "engine_ready" in rel:
+        last_ready = max((v for k, v in rel.items() if k.endswith("_ready") and k != "engine_ready"), default=0)
+        phases["post_stages_wiring_s"] = rel["engine_ready"] - last_ready
+        phases["engine_ready_s"] = rel["engine_ready"]
+    return {"marks_rel_s": rel, "phases": phases}
+
+
+def parse(path: str) -> dict:
+    stages: dict[str, dict] = {}
+    rec: dict[str, Any] = {"log": path, "stages": stages}
+    with open(path, errors="replace") as f:
+        lines = f.readlines()
+    rec["timeline"] = timeline(lines)
+    for line in lines:
+        if line.startswith("BENCH_JSON "):
+            rec["bench"] = json.loads(line[len("BENCH_JSON ") :])
+            continue
+        m = ENGINE_READY.search(line)
+        if m:
+            rec["engine_ready_s_reported"] = float(m.group("v"))
+            continue
+        m = PREFETCH_SKIP.search(line)
+        if m:
+            stages.setdefault(m.group("stage"), {})["page_cache_prefetch_skipped"] = True
+            continue
+        m = COMPILE_UNSUPPORTED.search(line)
+        if m:
+            stages.setdefault(m.group("stage"), {})["torch_compile_unsupported"] = True
+            continue
+        for key, pat in PATTERNS.items():
+            m = pat.search(line)
+            if not m:
+                continue
+            st = stages.setdefault(m.group("stage"), {})
+            if key == "ckpt_fs":
+                st["checkpoint_fs"] = m.group("fs")
+                st["checkpoint_size_gib"] = float(m.group("size"))
+                st["available_ram_gib"] = float(m.group("ram"))
+            elif key == "graph_capture_s":
+                st["graph_capture_s"] = float(m.group("v"))
+                st["graph_capture_mem_gib"] = float(m.group("mem"))
+            elif key == "model_load_s":
+                st["model_load_s"] = float(m.group("v"))
+                st["model_load_mem_gib"] = float(m.group("mem"))
+                # "Model loading took" covers model construction + weight loading; "Loading weights took" only
+                # the weights. The difference also includes loader setup/teardown overhead.
+                if "weights_load_s" in st:
+                    st["model_setup_estimate_s"] = round(st["model_load_s"] - st["weights_load_s"], 2)
+            else:
+                st[key] = float(m.group("v"))
+            break
+    rec["status"] = "complete" if "bench" in rec else "incomplete"
+    return rec
+
+
+def _table(columns: dict[str, str], records: list[dict]) -> str:
+    def row(values):
+        return "| " + " | ".join("" if v is None else str(v) for v in values) + " |"
+
+    lines = [row(columns.values()), row(["---"] * len(columns))]
+    lines.extend(row(record.get(key) for key in columns) for record in records)
+    return "\n".join(lines)
+
+
+def to_markdown(recs: list[dict]) -> str:
+    summary, stages, phases, warnings = [], [], [], []
+    for r in recs:
+        b = r.get("bench", {})
+        label = b.get("label", r["log"])
+        if r.get("status") == "incomplete":
+            warnings.append(f"Incomplete log: {r['log']} (no BENCH_JSON).")
+        summary.append({"label": label, **b.get("summary_s", {})})
+        for sid, st in sorted(r["stages"].items()):
+            stages.append({"label": label, "stage": sid, **st})
+            if st.get("torch_compile_unsupported"):
+                stages[-1]["compile_s"] = "unsupported"
+        ph = r.get("timeline", {}).get("phases", {})
+        for sid, values in sorted(ph.items()):
+            if sid.startswith("stage"):
+                phases.append({"label": label, "stage": sid, **values})
+        if "post_stages_wiring_s" in ph:
+            phases.append({"label": label, "stage": "after last stage", "total_s": ph["post_stages_wiring_s"]})
+    tables = [
+        (
+            {
+                "label": "label",
+                "imports": "imports (s)",
+                "engine_init": "engine ready (s)",
+                "first_request": "first request (s)",
+                "steady_request_avg": "subsequent avg (s)",
+                "time_to_first_image_from_process_start": "process→first image (s)",
+                "host_rss_peak_gib": "sampled host RSS (GiB)",
+                "gpu_used_peak_gib": "sampled GPU used (GiB)",
+            },
+            summary,
+        ),
+        (
+            {
+                "label": "label",
+                "stage": "stage",
+                "model_setup_estimate_s": "model setup estimate (s)",
+                "weights_load_s": "weights load (s)",
+                "compile_s": "compile (s)",
+                "graph_capture_s": "graph capture (s)",
+                "engine_init_s": "profile/KV/warmup reported (s)",
+            },
+            stages,
+        ),
+        (
+            {
+                "label": "label",
+                "stage": "stage",
+                "spawn_import_config_s": "spawn/import (s)",
+                "device_dist_init_s": "device init (s)",
+                "model_and_weights_load_s": "model+weights load (s)",
+                "profile_kv_warmup_s": "profile/KV/warmup interval (s)",
+                "total_s": "total (s)",
+            },
+            phases,
+        ),
+    ]
+    return "\n\n".join(warnings + [_table(columns, rows) for columns, rows in tables])
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("logs", nargs="+")
+    ap.add_argument("--markdown", action="store_true")
+    ap.add_argument("--json-out", default=None)
+    ap.add_argument("--require-complete", action="store_true", help="fail if any log has no BENCH_JSON")
+    args = ap.parse_args()
+    recs = [parse(p) for p in args.logs]
+    if args.require_complete and any(r["status"] != "complete" for r in recs):
+        ap.error("incomplete benchmark log: missing BENCH_JSON")
+    if args.json_out:
+        with open(args.json_out, "w") as f:
+            json.dump(recs, f, indent=2)
+    if args.markdown:
+        print(to_markdown(recs))
+    else:
+        json.dump(recs, sys.stdout, indent=2)
+        print()
+
+
+if __name__ == "__main__":
+    main()

benchmarks/mammoth_moda2/raw_load_bench.py

@@ -0,0 +1,127 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Raw safetensors -> GPU micro-benchmark, without vLLM's loader.
+
+Compares CPU and GPU dtype conversion after materializing each mmap-backed tensor.
+Materialization includes page faults, CPU allocation and copying; it is not a disk
+bandwidth measurement. CUDA setup is excluded; transfer includes GPU allocation.
+
+  --mode cpu_cast         materialize -> cast to bf16 on the CPU -> copy to GPU
+  --mode gpu_cast         materialize -> copy source dtype to GPU -> cast on the GPU
+
+  python benchmarks/mammoth_moda2/raw_load_bench.py --model /path/MammothModa2-Preview --shards 6,7,8
+  OMP_NUM_THREADS=4 python benchmarks/mammoth_moda2/raw_load_bench.py --model /path --shards 6,7,8 --mode cpu_cast
+"""
+
+import argparse
+import json
+import os
+import re
+import time
+
+import torch
+from safetensors import safe_open
+
+SHARD_NO = re.compile(r"-(\d+)-of-\d+\.safetensors$")
+
+
+def shard_files(model: str, shards: str) -> list[str]:
+    """Resolve ``--shards`` to files, refusing empty or partial matches so a typo cannot pass as a fast load."""
+    files = sorted(f for f in os.listdir(model) if f.endswith(".safetensors"))
+    if not files:
+        raise SystemExit(f"no .safetensors files under {model}")
+    if shards != "all":
+        want = {int(x) for x in shards.split(",")}
+        found = {int(m.group(1)): f for f in files if (m := SHARD_NO.search(f))}
+        missing = sorted(want - found.keys())
+        if missing:
+            raise SystemExit(f"shard(s) {missing} not found under {model}; available: {sorted(found)}")
+        files = [found[n] for n in sorted(want)]
+    return [os.path.join(model, f) for f in files]
+
+
+def sync() -> None:
+    if torch.cuda.is_available():
+        torch.accelerator.synchronize()
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--model", required=True)
+    ap.add_argument("--shards", default="all", help="comma list of 1-based shard numbers, or 'all'")
+    ap.add_argument("--mode", choices=("cpu_cast", "gpu_cast"), default="cpu_cast")
+    ap.add_argument("--label", default="raw")
+    ap.add_argument("--no-cuda", action="store_true", help="read + CPU cast only (mode must be cpu_cast)")
+    a = ap.parse_args()
+    if a.no_cuda and a.mode != "cpu_cast":
+        ap.error("--no-cuda only makes sense with --mode cpu_cast")
+
+    paths = shard_files(a.model, a.shards)
+    total_bytes = sum(os.path.getsize(p) for p in paths)
+
+    if not a.no_cuda:
+        # Initialize the context and exercise both operations outside the timer.
+        warmup = torch.zeros(1).to("cuda")
+        warmup = warmup.to(torch.bfloat16)
+        sync()
+        del warmup
+
+    t0 = time.perf_counter()
+    n_tensors, cpu_bytes = 0, 0
+    dtypes: dict[str, int] = {}
+    t_materialize = t_cast = t_h2d = 0.0
+    for p in paths:
+        with safe_open(p, framework="pt", device="cpu") as f:
+            for k in f.keys():
+                r0 = time.perf_counter()
+                t = f.get_tensor(k).clone()  # touch every page and own the CPU storage
+                t_materialize += time.perf_counter() - r0
+                dtypes[str(t.dtype)] = dtypes.get(str(t.dtype), 0) + 1
+                cpu_bytes += t.numel() * t.element_size()
+                needs_cast = t.is_floating_point() and t.dtype != torch.bfloat16
+                if a.mode == "cpu_cast":
+                    c0 = time.perf_counter()
+                    if needs_cast:
+                        t = t.to(torch.bfloat16)
+                    t_cast += time.perf_counter() - c0
+                    if not a.no_cuda:
+                        h0 = time.perf_counter()
+                        g = t.to("cuda")
+                        sync()
+                        t_h2d += time.perf_counter() - h0
+                        del g
+                else:
+                    h0 = time.perf_counter()
+                    g = t.to("cuda")
+                    sync()
+                    t_h2d += time.perf_counter() - h0
+                    c0 = time.perf_counter()
+                    if needs_cast:
+                        g = g.to(torch.bfloat16)
+                    sync()
+                    t_cast += time.perf_counter() - c0
+                    del g
+                n_tensors += 1
+                del t
+    total = time.perf_counter() - t0
+    res = {
+        "label": a.label,
+        "mode": a.mode,
+        "files": [os.path.basename(p) for p in paths],
+        "file_bytes_gib": round(total_bytes / 2**30, 2),
+        "tensor_bytes_gib": round(cpu_bytes / 2**30, 2),
+        "n_tensors": n_tensors,
+        "dtypes": dtypes,
+        "torch_threads": torch.get_num_threads(),
+        "cpu_affinity": len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else None,
+        "materialize_s": round(t_materialize, 3),
+        "cast_s": round(t_cast, 3),
+        "h2d_s": round(t_h2d, 3),
+        "total_s": round(total, 3),
+        "cuda_setup_included": False,
+    }
+    print("RAWLOAD_JSON " + json.dumps(res))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/mammoth_moda2/reference_results.json

@@ -0,0 +1,39 @@
+{
+  "date": "2026-09-10",
+  "runtime_commit": "34aa1d2afa59547945fc59f88ab6f4f25f52a777",
+  "environment": {"torch": "2.13.0+cu129", "cuda": "12.9", "python": "3.12.14", "vllm": "0.28.0", "vllm_omni": "0.28.1.dev81+g34aa1d2af.d20260908", "gpu": "NVIDIA A800-SXM4-80GB, 580.126.09, 81920 MiB"},
+  "method": "One excluded warm-up/canary; warm page cache; three rounds of the 2x2 eager configurations with rotated order; fresh process for every run; memory polling disabled.",
+  "request": {"height": 512, "width": 512, "seed": 42, "steps": 8, "guidance": 4.0, "prompt": "A stylish woman riding a motorcycle in NYC, movie poster style"},
+  "gpu_memory_utilization": [0.5, 0.3],
+  "cpu_quota": 16,
+  "startup_columns": ["configuration", "round", "engine_init_s", "imports_s", "first_request_s", "time_to_first_image_s", "image_md5"],
+  "startup_runs": [
+    ["default-serial", 1, 60.022, 10.158, 20.827, 91.089, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["default-serial", 2, 60.015, 10.082, 20.766, 90.909, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["default-serial", 3, 60.34, 9.911, 21.089, 91.381, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["omp-serial", 1, 55.45, 9.965, 20.711, 86.169, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["omp-serial", 2, 57.01, 9.667, 20.697, 87.418, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["omp-serial", 3, 57.044, 9.707, 20.415, 87.211, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["default-parallel", 1, 40.258, 10.112, 20.392, 70.829, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["default-parallel", 2, 42.177, 10.135, 20.692, 73.093, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["default-parallel", 3, 40.848, 9.997, 20.995, 71.887, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["omp-parallel", 1, 36.88, 9.666, 20.319, 66.932, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["omp-parallel", 2, 36.534, 9.6, 20.314, 66.517, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["omp-parallel", 3, 36.689, 9.582, 20.467, 66.802, "60c1fe4d464fba8c6ffd1b752fbbd40c"]
+  ],
+  "raw_loading_columns": ["configuration", "round", "torch_threads", "n_tensors", "materialize_s", "cast_s", "h2d_s", "total_s"],
+  "raw_loading_runs": [
+    ["cpu64", 1, 64, 620, 3.174, 2.309, 1.217, 7.796],
+    ["cpu64", 2, 64, 620, 2.957, 2.443, 1.433, 7.846],
+    ["cpu64", 3, 64, 620, 2.916, 2.613, 1.067, 7.942],
+    ["cpu16", 1, 16, 620, 0.857, 1.137, 0.713, 3.643],
+    ["cpu16", 2, 16, 620, 0.871, 1.174, 0.746, 3.608],
+    ["cpu16", 3, 16, 620, 0.814, 1.142, 0.724, 3.487],
+    ["gpu16", 1, 16, 620, 0.938, 0.028, 1.41, 3.74],
+    ["gpu16", 2, 16, 620, 0.926, 0.029, 1.467, 3.83],
+    ["gpu16", 3, 16, 620, 0.87, 0.029, 1.395, 3.661]
+  ],
+  "raw_loading_workload": {"shards": [6, 7, 8], "source_dtype": "float32", "tensor_gib": 13.3, "warmup": "CUDA context, copy, cast and synchronize before timer", "order_per_round": ["cpu64", "cpu16", "gpu16"]},
+  "executed_script_sha256": {"parse_startup_log.py": "aa260de2cfcaec2fc4a678b245bb7c16fa464543aaec72b947fd4f769d547011", "raw_load_bench.py": "1f70f6b3c205402b2c2c7c52b224e1be3c04a12d12d11044a86f1b9b6c7a1d0f", "bench_startup.py": "04b1e8b9398b4d9885fbb6fe341e8a9ee98792d0be200c15c57e1fc5183c6ff0", "__init__.py": "b1d617f45f34dbe6f5f58d792d90c84af8d8d0a7e7def77c48170ca93495ca0d"},
+  "notes": ["Standard deviations use ddof=1.", "Only first-request ONGs are compared.", "Raw materialization includes CPU allocation/copy; H2D includes GPU allocation; not engine-loader timings."]
+}

benchmarks/tts/bench_tts.py

@@ -1,4 +1,7 @@
 #!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 """Universal TTS benchmark CLI for vllm-omni.
 
 Runs ``vllm bench serve --omni`` with model-aware defaults loaded from
@@ -87,6 +90,7 @@ def build_bench_args(
     served_model_name: str | None = None,
     num_warmups: int = 2,
     request_seed: int | None = None,
+    duration_seconds: float | None = None,
 ) -> list[str]:
     """Build the ``vllm bench serve --omni`` command for one (task, concurrency) run."""
     dataset_name = _TASK_TO_DATASET[task]
@@ -95,7 +99,10 @@ def build_bench_args(
     task_extra_body: dict[str, Any] = dict((model_cfg.get("task_extra_body") or {}).get(task) or {})
     if request_seed is not None:
         task_extra_body["seed"] = request_seed
-
+    if duration_seconds is not None:
+        if not math.isfinite(duration_seconds) or duration_seconds <= 0:
+            raise ValueError("duration_seconds must be finite and positive")
+        task_extra_body["duration_seconds"] = duration_seconds
     # Resolve dataset path
     if dataset_path:
         resolved_dataset_path = dataset_path
@@ -194,7 +201,7 @@ def print_summary_table(results: list[dict[str, Any]]) -> None:
     if not results:
         return
     header = (
-        f"{'Task':<16} {'Concurrency':>11} {'RTF mean':>10} "
+        f"{'Task':<16} {'Task Type':<12} {'Concurrency':>11} {'RTF mean':>10} "
         f"{'TTFP (ms)':>10} {'Throughput':>12} {'WER':>7} {'SIM':>7} {'UTMOS':>7}"
     )
     print(f"\n{'=' * len(header)}")
@@ -204,6 +211,7 @@ def print_summary_table(results: list[dict[str, Any]]) -> None:
     print("-" * len(header))
     for r in results:
         task = r.get("_task", "?")
+        task_type = r.get("_task_type", "-")
         conc = r.get("_concurrency", "?")
         rtf = r.get("mean_audio_rtf", float("nan"))
         ttfp = r.get("mean_audio_ttfp_ms", float("nan"))
@@ -216,7 +224,7 @@ def print_summary_table(results: list[dict[str, Any]]) -> None:
             return f"{v:.{digits}f}" if not math.isnan(v) else "  n/a"
 
         print(
-            f"{task:<16} {str(conc):>11} {fmt(rtf):>10} {fmt(ttfp, 0):>10} "
+            f"{task:<16} {task_type:<12} {str(conc):>11} {fmt(rtf):>10} {fmt(ttfp, 0):>10} "
             f"{fmt(throughput):>12} {fmt(wer):>7} {fmt(sim):>7} {fmt(utmos):>7}"
         )
     print("=" * len(header))
@@ -249,6 +257,12 @@ def main() -> None:
         help="Number of prompts per run. If one value, applied to all concurrency levels.",
     )
     parser.add_argument("--num-warmups", type=int, default=2, help="Warmup requests before each measured run")
+    parser.add_argument(
+        "--duration-seconds",
+        type=float,
+        default=None,
+        help="Target audio duration per request (AuK defaults to 5 seconds). Independent of --output-len.",
+    )
     parser.add_argument(
         "--request-seed",
         type=int,
@@ -277,6 +291,8 @@ def main() -> None:
 
     if args.num_warmups < 0:
         parser.error("--num-warmups cannot be negative")
+    if args.duration_seconds is not None and (not math.isfinite(args.duration_seconds) or args.duration_seconds <= 0):
+        parser.error("--duration-seconds must be finite and positive")
 
     model_configs = load_model_configs(Path(args.model_configs))
     if args.model not in model_configs:
@@ -333,11 +349,16 @@ def main() -> None:
                 served_model_name=args.served_model_name,
                 num_warmups=args.num_warmups,
                 request_seed=args.request_seed,
+                duration_seconds=args.duration_seconds,
             )
             result = run_one_benchmark(cmd)
             if result is not None:
                 result["_task"] = task
                 result["_concurrency"] = concurrency
+                task_body = model_cfg.get("task_extra_body", {}).get(task, {})
+                task_type = task_body.get("task_type")
+                if task_type is not None:
+                    result["_task_type"] = task_type
                 all_results.append(result)
                 # Persist the metadata so plot_results.py can pick it up.
                 if args.output_dir and result_filename:

benchmarks/tts/evidence/breeze_tts_2_20260918.json

@@ -0,0 +1,757 @@
+{
+  "measurement_scope": "Historical measurements for PR7752, not a main-branch comparison or a measurement of subsequent review fixes.",
+  "environment": {
+    "gpu": "NVIDIA H100 80 GB",
+    "gpu_count": 1,
+    "python": "3.12",
+    "pytorch": "2.13.0+cu132",
+    "vllm": "0.29.0",
+    "transformers": "5.14.1",
+    "nsight_systems": "2026.5.1"
+  },
+  "model": {
+    "id": "BreezeBlue/Breeze-TTS-2",
+    "revision": "3e28c5151381a722f1d8661b4118c298caa77aa4"
+  },
+  "workload": {
+    "texts": [
+      "Hello, this is Breeze TTS 2 running on vLLM Omni.",
+      "你好，今天天气很好，我们一起出去走走吧。"
+    ],
+    "alternating_texts": true,
+    "seed": 42,
+    "guidance_scale": 1.0,
+    "repetition_penalty": 1.1,
+    "top_p": 1.0,
+    "sampled": {
+      "temperature": 0.9,
+      "top_k": 50
+    },
+    "greedy": {
+      "temperature": 0.0,
+      "top_k": 0
+    },
+    "stream": false,
+    "response_format": "pcm",
+    "sample_rate": 24000,
+    "channels": 1,
+    "pcm_bits": 16
+  },
+  "cumulative_diagnostic": {
+    "baseline_sha": "5bda85c371e8c0c7728c2f7f5865d3ddd79b9d99",
+    "combined_sha": "37a04897cf8566c63e82eb13c27c064b9bcd4222",
+    "intermediate_variants": "Ordered patch subsets of the combined commit, not independent commits.",
+    "patch_sha256": {
+      "batched_eos": "2f107ea8e890e03cc138ef2f69e693e3474d3aaf0eb4051c00886cbaa9785b3d",
+      "greedy_graph": "4e783313260ddc2e36c8a74ba9d95dc5d78338443612ebf89854e31b8be6933b",
+      "batched_embedding": "90557e4ec73fdedcff51fd717d5e8c6477e71087fd97ac52534542234f5d1d3a"
+    },
+    "sequence_slots": 8,
+    "kv_gib": 2,
+    "codec_chunk_frames": 16,
+    "max_new_tokens": 48,
+    "temperature": 0.0,
+    "cohorts_per_variant": 1,
+    "nsight_injection_present": true,
+    "collection_enabled": false,
+    "warmup": "Two eight-request full-response warmups, then four full-response 64-request cohorts, each preceded by one concurrency-sized warmup.",
+    "variants": [
+      {
+        "variant": "baseline",
+        "makespan_s": 0.9672094583511353,
+        "requests": 8,
+        "concurrency": 8,
+        "audio_s": 30.72,
+        "failures": 0
+      },
+      {
+        "variant": "plus_batched_eos",
+        "makespan_s": 0.9280132218264043,
+        "requests": 8,
+        "concurrency": 8,
+        "audio_s": 30.72,
+        "failures": 0
+      },
+      {
+        "variant": "plus_greedy_graph",
+        "makespan_s": 0.8857334200292826,
+        "requests": 8,
+        "concurrency": 8,
+        "audio_s": 30.72,
+        "failures": 0
+      },
+      {
+        "variant": "plus_batched_embedding",
+        "makespan_s": 0.8435833756811917,
+        "requests": 8,
+        "concurrency": 8,
+        "audio_s": 30.72,
+        "failures": 0
+      }
+    ],
+    "limits": [
+      "Single cumulative diagnostic samples; no repeat spread.",
+      "Nsight injection remains when collection is disabled.",
+      "The final variant used graph tracing configuration; prior variants used node tracing configuration.",
+      "Do not interpret these timings as isolated uninstrumented per-optimization gains."
+    ]
+  },
+  "mechanical_attribution": {
+    "batched_eos": {
+      "stage0_stream_synchronization_calls": [
+        773,
+        438
+      ],
+      "device_to_host_copies": [
+        841,
+        506
+      ],
+      "capture_mode": "node"
+    },
+    "greedy_graph": {
+      "radix_sort_kernels": [
+        735,
+        0
+      ],
+      "capture_mode": "node"
+    },
+    "combined_graph_capture": {
+      "captured_wall_ms": [
+        955.4114495404065,
+        855.6996420957148
+      ],
+      "gpu_envelope_ms": [
+        926.633897,
+        826.396835
+      ],
+      "activity_upper_ms": [
+        651.130806,
+        628.864685
+      ],
+      "idle_lower_ms": [
+        275.50309100000004,
+        197.53215
+      ],
+      "stage0_explicit_kernel_launch_count": [
+        7087,
+        6198
+      ],
+      "stage0_graph_launch_count": [
+        156,
+        156
+      ]
+    },
+    "limits": [
+      "Host API durations overlap GPU activity and are not additional idle time.",
+      "Graph envelopes hide internal gaps: activity is an upper bound and idle is a lower bound.",
+      "Stage 1 has eight versus nine graph launches across the graph-mode traces.",
+      "No separate final-capacity stage breakdown was measured."
+    ]
+  },
+  "combined_uninstrumented_ab": {
+    "baseline_sha": "5bda85c371e8c0c7728c2f7f5865d3ddd79b9d99",
+    "head_sha": "37a04897cf8566c63e82eb13c27c064b9bcd4222",
+    "sequence_slots": 8,
+    "kv_gib": 2,
+    "codec_chunk_frames": 16,
+    "requests_per_group": 64,
+    "max_new_tokens": 384,
+    "baseline_rounds": 2,
+    "head_rounds": 1,
+    "profiler_configured": false,
+    "nsight_injection_present": false,
+    "warmup": "One excluded 64-request full-response wave for each temperature/concurrency family, plus an excluded 2*C-request full-response cohort before every group.",
+    "rows": [
+      {
+        "build": "plain-baseline",
+        "phase": "round-0",
+        "temperature": 0.0,
+        "concurrency": 8,
+        "requests": 64,
+        "makespan_s": 10.77213971177116,
+        "req_per_s": 5.941252314994074,
+        "aggregate_rtf": 0.030157166046391827,
+        "audio_s": 357.2,
+        "caps": 0,
+        "failures": 0,
+        "endpoint_trimmed_audio_s": 305.88,
+        "endpoint_trimmed_aggregate_rtf": 0.03521688149526337,
+        "quiet_tail_at_least_3s": 7,
+        "max_quiet_tail_s": 3.24
+      },
+      {
+        "build": "plain-baseline",
+        "phase": "round-0",
+        "temperature": 0.0,
+        "concurrency": 16,
+        "requests": 64,
+        "makespan_s": 11.076066234149039,
+        "req_per_s": 5.778224745774739,
+        "aggregate_rtf": 0.029092420240988227,
+        "audio_s": 380.72,
+        "caps": 2,
+        "failures": 0,
+        "endpoint_trimmed_audio_s": 315.48,
+        "endpoint_trimmed_aggregate_rtf": 0.035108616185333584,
+        "quiet_tail_at_least_3s": 6,
+        "max_quiet_tail_s": 26.04
+      },
+      {
+        "build": "plain-baseline",
+        "phase": "round-0",
+        "temperature": 0.9,
+        "concurrency": 8,
+        "requests": 64,
+        "makespan_s": 8.603743681218475,
+        "req_per_s": 7.438622345259851,
+        "aggregate_rtf": 0.03389435739528236,
+        "audio_s": 253.84,
+        "caps": 0,
+        "failures": 0,
+        "endpoint_trimmed_audio_s": 239.02,
+        "endpoint_trimmed_aggregate_rtf": 0.035995915325991445,
+        "quiet_tail_at_least_3s": 0,
+        "max_quiet_tail_s": 1.3
+      },
+      {
+        "build": "plain-baseline",
+        "phase": "round-0",
+        "temperature": 0.9,
+        "concurrency": 16,
+        "requests": 64,
+        "makespan_s": 8.66502623911947,
+        "req_per_s": 7.386013409984042,
+        "aggregate_rtf": 0.033143460216950235,
+        "audio_s": 261.44,
+        "caps": 0,
+        "failures": 0,
+        "endpoint_trimmed_audio_s": 242.14,
+        "endpoint_trimmed_aggregate_rtf": 0.03578519137325296,
+        "quiet_tail_at_least_3s": 0,
+        "max_quiet_tail_s": 1.3
+      },
+      {
+        "build": "plain-baseline",
+        "phase": "round-1",
+        "temperature": 0.0,
+        "concurrency": 8,
+        "requests": 64,
+        "makespan_s": 13.904579282738268,
+        "req_per_s": 4.602800178172402,
+        "aggregate_rtf": 0.0392784725501081,
+        "audio_s": 354.0,
+        "caps": 1,
+        "failures": 0,
+        "endpoint_trimmed_audio_s": 285.36,
+        "endpoint_trimmed_aggregate_rtf": 0.048726448285457906,
+        "quiet_tail_at_least_3s": 8,
+        "max_quiet_tail_s": 25.38
+      },
+      {
+        "build": "plain-baseline",
+        "phase": "round-1",
+        "temperature": 0.0,
+        "concurrency": 16,
+        "requests": 64,
+        "makespan_s": 10.060411099344492,
+        "req_per_s": 6.361569061941222,
+        "aggregate_rtf": 0.02935460754944121,
+        "audio_s": 342.72,
+        "caps": 0,
+        "failures": 0,
+        "endpoint_trimmed_audio_s": 289.04,
+        "endpoint_trimmed_aggregate_rtf": 0.03480629359031446,
+        "quiet_tail_at_least_3s": 10,
+        "max_quiet_tail_s": 3.16
+      },
+      {
+        "build": "plain-baseline",
+        "phase": "round-1",
+        "temperature": 0.9,
+        "concurrency": 8,
+        "requests": 64,
+        "makespan_s": 8.535738762002438,
+        "req_per_s": 7.4978864494894575,
+        "aggregate_rtf": 0.033363581777683074,
+        "audio_s": 255.84,
+        "caps": 0,
+        "failures": 0,
+        "endpoint_trimmed_audio_s": 237.9,
+        "endpoint_trimmed_aggregate_rtf": 0.03587952401009852,
+        "quiet_tail_at_least_3s": 0,
+        "max_quiet_tail_s": 2.12
+      },
+      {
+        "build": "plain-baseline",
+        "phase": "round-1",
+        "temperature": 0.9,
+        "concurrency": 16,
+        "requests": 64,
+        "makespan_s": 8.584469985216856,
+        "req_per_s": 7.4553234049642105,
+        "aggregate_rtf": 0.033355882752629996,
+        "audio_s": 257.36,
+        "caps": 0,
+        "failures": 0,
+        "endpoint_trimmed_audio_s": 242.57999999999998,
+        "endpoint_trimmed_aggregate_rtf": 0.035388201769382704,
+        "quiet_tail_at_least_3s": 0,
+        "max_quiet_tail_s": 1.94
+      },
+      {
+        "build": "plain-batched",
+        "phase": "round-0",
+        "temperature": 0.0,
+        "concurrency": 8,
+        "requests": 64,
+        "makespan_s": 12.983129982370883,
+        "req_per_s": 4.929473870083891,
+        "aggregate_rtf": 0.03320153943936907,
+        "audio_s": 391.04,
+        "caps": 2,
+        "failures": 0,
+        "endpoint_trimmed_audio_s": 312.72,
+        "endpoint_trimmed_aggregate_rtf": 0.041516788124747,
+        "quiet_tail_at_least_3s": 11,
+        "max_quiet_tail_s": 28.4
+      },
+      {
+        "build": "plain-batched",
+        "phase": "round-0",
+        "temperature": 0.0,
+        "concurrency": 16,
+        "requests": 64,
+        "makespan_s": 9.772029620129615,
+        "req_per_s": 6.549304749155183,
+        "aggregate_rtf": 0.027698496655696188,
+        "audio_s": 352.8,
+        "caps": 1,
+        "failures": 0,
+        "endpoint_trimmed_audio_s": 286.26,
+        "endpoint_trimmed_aggregate_rtf": 0.034136902187275955,
+        "quiet_tail_at_least_3s": 6,
+        "max_quiet_tail_s": 26.04
+      },
+      {
+        "build": "plain-batched",
+        "phase": "round-0",
+        "temperature": 0.9,
+        "concurrency": 8,
+        "requests": 64,
+        "makespan_s": 8.171920725144446,
+        "req_per_s": 7.831696140061215,
+        "aggregate_rtf": 0.03169376638669115,
+        "audio_s": 257.84,
+        "caps": 0,
+        "failures": 0,
+        "endpoint_trimmed_audio_s": 241.72,
+        "endpoint_trimmed_aggregate_rtf": 0.03380738344011437,
+        "quiet_tail_at_least_3s": 0,
+        "max_quiet_tail_s": 1.22
+      },
+      {
+        "build": "plain-batched",
+        "phase": "round-0",
+        "temperature": 0.9,
+        "concurrency": 16,
+        "requests": 64,
+        "makespan_s": 8.02551719499752,
+        "req_per_s": 7.974563937124526,
+        "aggregate_rtf": 0.030886380830501534,
+        "audio_s": 259.84000000000003,
+        "caps": 0,
+        "failures": 0,
+        "endpoint_trimmed_audio_s": 243.0,
+        "endpoint_trimmed_aggregate_rtf": 0.03302681973250008,
+        "quiet_tail_at_least_3s": 0,
+        "max_quiet_tail_s": 1.78
+      }
+    ],
+    "limits": [
+      "Ranges compare the head sample with two baseline samples; they are not confidence intervals.",
+      "Full-response greedy output lengths and frame-limit incidence vary, so no general greedy throughput gain is claimed.",
+      "No transcription was run on these historical A/B outputs."
+    ]
+  },
+  "capacity": {
+    "runtime_sha": "7bc0b3e3a2e41ef44e8efb30fd927e93bf6a48a7",
+    "main_base_sha": "6bda549ac23a667efbe44a54cf3f6d57e6e24da2",
+    "sequence_slots": [
+      8,
+      16,
+      32
+    ],
+    "kv_gib": [
+      2,
+      4,
+      8
+    ],
+    "codec_graph_buckets": [
+      [
+        1,
+        2,
+        4,
+        8
+      ],
+      [
+        1,
+        2,
+        4,
+        8,
+        16
+      ],
+      [
+        1,
+        2,
+        4,
+        8,
+        16,
+        32
+      ]
+    ],
+    "backbone_capture_sizes": [
+      1,
+      2,
+      4,
+      8,
+      32,
+      64,
+      128,
+      256
+    ],
+    "codec_chunk_frames": 16,
+    "requests_per_group": 128,
+    "cohorts_per_point": 1,
+    "excluded_warmups_per_configuration": {
+      "cohorts": 4,
+      "requests_per_cohort": 64,
+      "covers": "Both client concurrency levels and both sampling settings"
+    },
+    "profiler_configured": false,
+    "nsight_injection_present": false,
+    "rows": [
+      {
+        "slots": 8,
+        "concurrency": 8,
+        "observed_peak_client_requests": 8,
+        "temperature": 0.0,
+        "max_new_tokens": 384,
+        "requests": 128,
+        "failures": 0,
+        "req_per_s": 6.7812081269297915,
+        "makespan_s": 18.87569259107113,
+        "aggregate_rtf": 0.02841355459879445,
+        "audio_s": 664.32,
+        "near_cap_count": 0,
+        "exact_cap_count": 0,
+        "fixed48_verified": null
+      },
+      {
+        "slots": 8,
+        "concurrency": 16,
+        "observed_peak_client_requests": 16,
+        "temperature": 0.0,
+        "max_new_tokens": 384,
+        "requests": 128,
+        "failures": 0,
+        "req_per_s": 5.8798058352164615,
+        "makespan_s": 21.76942633604631,
+        "aggregate_rtf": 0.029975526459636356,
+        "audio_s": 726.24,
+        "near_cap_count": 2,
+        "exact_cap_count": 2,
+        "fixed48_verified": null
+      },
+      {
+        "slots": 8,
+        "concurrency": 8,
+        "observed_peak_client_requests": 8,
+        "temperature": 0.9,
+        "max_new_tokens": 384,
+        "requests": 128,
+        "failures": 0,
+        "req_per_s": 7.72991106982811,
+        "makespan_s": 16.559052082709968,
+        "aggregate_rtf": 0.0320365502296664,
+        "audio_s": 516.88,
+        "near_cap_count": 0,
+        "exact_cap_count": 0,
+        "fixed48_verified": null
+      },
+      {
+        "slots": 8,
+        "concurrency": 16,
+        "observed_peak_client_requests": 16,
+        "temperature": 0.9,
+        "max_new_tokens": 384,
+        "requests": 128,
+        "failures": 0,
+        "req_per_s": 7.945057322591292,
+        "makespan_s": 16.110645248088986,
+        "aggregate_rtf": 0.030958196095482296,
+        "audio_s": 520.4,
+        "near_cap_count": 0,
+        "exact_cap_count": 0,
+        "fixed48_verified": null
+      },
+      {
+        "slots": 8,
+        "concurrency": 8,
+        "observed_peak_client_requests": 8,
+        "temperature": 0.0,
+        "max_new_tokens": 48,
+        "requests": 128,
+        "failures": 0,
+        "req_per_s": 9.850579526557848,
+        "makespan_s": 12.994159344118088,
+        "aggregate_rtf": 0.026436684863521503,
+        "audio_s": 491.52,
+        "near_cap_count": 128,
+        "exact_cap_count": 128,
+        "fixed48_verified": true
+      },
+      {
+        "slots": 8,
+        "concurrency": 16,
+        "observed_peak_client_requests": 16,
+        "temperature": 0.0,
+        "max_new_tokens": 48,
+        "requests": 128,
+        "failures": 0,
+        "req_per_s": 10.665146583658053,
+        "makespan_s": 12.00171033712104,
+        "aggregate_rtf": 0.02441754218978076,
+        "audio_s": 491.52,
+        "near_cap_count": 128,
+        "exact_cap_count": 128,
+        "fixed48_verified": true
+      },
+      {
+        "slots": 16,
+        "concurrency": 16,
+        "observed_peak_client_requests": 16,
+        "temperature": 0.0,
+        "max_new_tokens": 384,
+        "requests": 128,
+        "failures": 0,
+        "req_per_s": 7.754874623816668,
+        "makespan_s": 16.505747186020017,
+        "aggregate_rtf": 0.023979758231665527,
+        "audio_s": 688.32,
+        "near_cap_count": 1,
+        "exact_cap_count": 1,
+        "fixed48_verified": null
+      },
+      {
+        "slots": 16,
+        "concurrency": 32,
+        "observed_peak_client_requests": 32,
+        "temperature": 0.0,
+        "max_new_tokens": 384,
+        "requests": 128,
+        "failures": 0,
+        "req_per_s": 10.104109318018903,
+        "makespan_s": 12.668113138061017,
+        "aggregate_rtf": 0.018577125085143444,
+        "audio_s": 681.92,
+        "near_cap_count": 1,
+        "exact_cap_count": 1,
+        "fixed48_verified": null
+      },
+      {
+        "slots": 16,
+        "concurrency": 16,
+        "observed_peak_client_requests": 16,
+        "temperature": 0.9,
+        "max_new_tokens": 384,
+        "requests": 128,
+        "failures": 0,
+        "req_per_s": 10.920674745703012,
+        "makespan_s": 11.72088748915121,
+        "aggregate_rtf": 0.021831484669108944,
+        "audio_s": 536.88,
+        "near_cap_count": 0,
+        "exact_cap_count": 0,
+        "fixed48_verified": null
+      },
+      {
+        "slots": 16,
+        "concurrency": 32,
+        "observed_peak_client_requests": 32,
+        "temperature": 0.9,
+        "max_new_tokens": 384,
+        "requests": 128,
+        "failures": 0,
+        "req_per_s": 11.210975235811437,
+        "makespan_s": 11.417383172083646,
+        "aggregate_rtf": 0.02120613516360261,
+        "audio_s": 538.4,
+        "near_cap_count": 0,
+        "exact_cap_count": 0,
+        "fixed48_verified": null
+      },
+      {
+        "slots": 16,
+        "concurrency": 16,
+        "observed_peak_client_requests": 16,
+        "temperature": 0.0,
+        "max_new_tokens": 48,
+        "requests": 128,
+        "failures": 0,
+        "req_per_s": 15.14487085057202,
+        "makespan_s": 8.451706274878234,
+        "aggregate_rtf": 0.017195040435543284,
+        "audio_s": 491.52,
+        "near_cap_count": 128,
+        "exact_cap_count": 128,
+        "fixed48_verified": true
+      },
+      {
+        "slots": 16,
+        "concurrency": 32,
+        "observed_peak_client_requests": 32,
+        "temperature": 0.0,
+        "max_new_tokens": 48,
+        "requests": 128,
+        "failures": 0,
+        "req_per_s": 16.563937910502617,
+        "makespan_s": 7.727630995213985,
+        "aggregate_rtf": 0.015721905507840955,
+        "audio_s": 491.52,
+        "near_cap_count": 128,
+        "exact_cap_count": 128,
+        "fixed48_verified": true
+      },
+      {
+        "slots": 32,
+        "concurrency": 32,
+        "observed_peak_client_requests": 32,
+        "temperature": 0.0,
+        "max_new_tokens": 384,
+        "requests": 128,
+        "failures": 0,
+        "req_per_s": 15.246181432777554,
+        "makespan_s": 8.395544849336147,
+        "aggregate_rtf": 0.012390119317202106,
+        "audio_s": 677.6,
+        "near_cap_count": 0,
+        "exact_cap_count": 0,
+        "fixed48_verified": null
+      },
+      {
+        "slots": 32,
+        "concurrency": 64,
+        "observed_peak_client_requests": 64,
+        "temperature": 0.0,
+        "max_new_tokens": 384,
+        "requests": 128,
+        "failures": 0,
+        "req_per_s": 11.166074835908939,
+        "makespan_s": 11.463294119108468,
+        "aggregate_rtf": 0.015826284127331107,
+        "audio_s": 724.32,
+        "near_cap_count": 1,
+        "exact_cap_count": 1,
+        "fixed48_verified": null
+      },
+      {
+        "slots": 32,
+        "concurrency": 32,
+        "observed_peak_client_requests": 32,
+        "temperature": 0.9,
+        "max_new_tokens": 384,
+        "requests": 128,
+        "failures": 0,
+        "req_per_s": 17.17219888865503,
+        "makespan_s": 7.453908543102443,
+        "aggregate_rtf": 0.014336645143680651,
+        "audio_s": 519.92,
+        "near_cap_count": 0,
+        "exact_cap_count": 0,
+        "fixed48_verified": null
+      },
+      {
+        "slots": 32,
+        "concurrency": 64,
+        "observed_peak_client_requests": 64,
+        "temperature": 0.9,
+        "max_new_tokens": 384,
+        "requests": 128,
+        "failures": 0,
+        "req_per_s": 17.514343665613783,
+        "makespan_s": 7.3082955572754145,
+        "aggregate_rtf": 0.013677750331777613,
+        "audio_s": 534.32,
+        "near_cap_count": 0,
+        "exact_cap_count": 0,
+        "fixed48_verified": null
+      },
+      {
+        "slots": 32,
+        "concurrency": 32,
+        "observed_peak_client_requests": 32,
+        "temperature": 0.0,
+        "max_new_tokens": 48,
+        "requests": 128,
+        "failures": 0,
+        "req_per_s": 25.82908453652114,
+        "makespan_s": 4.955653763841838,
+        "aggregate_rtf": 0.010082303393232905,
+        "audio_s": 491.52,
+        "near_cap_count": 128,
+        "exact_cap_count": 128,
+        "fixed48_verified": true
+      },
+      {
+        "slots": 32,
+        "concurrency": 64,
+        "observed_peak_client_requests": 64,
+        "temperature": 0.0,
+        "max_new_tokens": 48,
+        "requests": 128,
+        "failures": 0,
+        "req_per_s": 28.81314277607874,
+        "makespan_s": 4.442417163401842,
+        "aggregate_rtf": 0.00903812085653044,
+        "audio_s": 491.52,
+        "near_cap_count": 128,
+        "exact_cap_count": 128,
+        "fixed48_verified": true
+      }
+    ],
+    "limits": [
+      "This scales slots, KV reservation and codec graph buckets together.",
+      "One cohort per point; no repeated capacity samples or confidence intervals.",
+      "The 48-frame cohorts deliberately truncate speech and are excluded from full-utterance quality claims.",
+      "Peak device memory was not measured.",
+      "No capacity claim extends to cloning, CFG guidance above one, long prompts or other GPUs."
+    ]
+  },
+  "quality": {
+    "full_response_pcm_checked": 1536,
+    "sampled_full_responses": 768,
+    "sampled_frame_limit_hits": 0,
+    "sampled_quiet_tails_at_least_three_seconds": 0,
+    "greedy_frame_limit_hits": 5,
+    "asr_selected_unique_clips": 47,
+    "asr_selected_sampled_clips": 24,
+    "sampled_asr": "All selected sampled transcripts matched after accounting for TTS 2 versus TTS2 spacing.",
+    "endpoint_method": "Trim leading/trailing quiet using 20 ms RMS windows at or below -40 dBFS; keep internal pauses and original makespans.",
+    "limits": [
+      "ASR selects median-duration and longest English/Chinese clips per group, deduplicated by WAV hash.",
+      "Selected ASR and endpoint measurements do not establish perceptual quality parity.",
+      "Capped greedy clips have extra ASR words with unreliable tail alignment.",
+      "No listening or MOS assessment was performed."
+    ]
+  },
+  "metrics": {
+    "requests_per_second": "Completed requests divided by cohort wall time",
+    "aggregate_rtf": "Cohort wall time divided by total returned audio duration",
+    "timing_exclusions": [
+      "PCM validation",
+      "hashing",
+      "WAV file writes",
+      "post-timing audio analysis"
+    ]
+  }
+}

benchmarks/tts/model_configs.yaml

@@ -7,6 +7,33 @@
 # is tracked here.
 
 models:
+  tencent/AuK: &auk
+    supported_tasks: [default_voice, voice_clone]
+    backend: openai-audio-speech
+    endpoint: /v1/audio/speech
+    trust_remote_code: true
+    task_extra_body:
+      default_voice:
+        duration_seconds: 5.0
+        voice: default
+        task_type: CustomVoice
+      voice_clone:
+        duration_seconds: 5.0
+        voice: default
+        task_type: Base
+  tencent/AuK-Flash: *auk
+
+  BreezeBlue/Breeze-TTS-2:
+    supported_tasks: [default_voice, voice_design, voice_clone]
+    backend: openai-audio-speech
+    endpoint: /v1/audio/speech
+    task_extra_body:
+      default_voice: {}
+      voice_design:
+        task_type: VoiceDesign
+      voice_clone:
+        task_type: Base
+
   IndexTeam/IndexTTS-2.5:
     supported_tasks: [voice_clone]
     backend: openai-audio-speech

docker/Dockerfile.npu

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_VERSION=v0.29.0
+ARG VLLM_ASCEND_VERSION=v0.30.0
 ARG VLLM_ASCEND_DEVICE_SUFFIX=
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_VERSION}${VLLM_ASCEND_DEVICE_SUFFIX}
 

docker/Dockerfile.npu.ci

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_VERSION=v0.29.0
+ARG VLLM_ASCEND_VERSION=v0.30.0
 ARG VLLM_ASCEND_DEVICE_SUFFIX=
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_VERSION}${VLLM_ASCEND_DEVICE_SUFFIX}
 

docs/.nav.yml

@@ -15,7 +15,7 @@ nav:
         - Image Edit: serving/image_edit_api.md
         - Videos: serving/videos_api.md
       - Streaming and Realtime:
-        - Realtime Audio: serving/realtime_api.md
+        - Full Duplex OpenAI: serving/realtime_openai_duplex_api.md
         - Full Duplex: serving/full_duplex_api.md
         - Realtime Duplex Reference: serving/realtime_duplex_api.md
         - Streaming Video Input: serving/video_stream_api.md
@@ -47,6 +47,7 @@ nav:
     - configuration/*
   - Models:
     - models/supported_models.md
+    - SeedVR2: models/seedvr2.md
   - Features:
     - Overview: features/README.md
     - Runtime and Stage Execution:
@@ -98,6 +99,7 @@ nav:
         - Hugging Face Hub: user_guide/diffusion/attention_backends/huggingface_hub.md
         - RainFusion: user_guide/diffusion/attention_backends/rainfusion.md
       - Regional Compilation: user_guide/diffusion/regional_compilation.md
+      - Wan VAE Decoder Fast Path: user_guide/diffusion/vae_fast_path.md
       - Frame Interpolation: user_guide/diffusion/frame_interpolation.md
       - Startup and Loading: user_guide/diffusion/startup_and_loading.md
       - LoRA: user_guide/diffusion/lora.md
@@ -150,6 +152,7 @@ nav:
           - Hybrid Sharded Data Parallel (HSDP): design/feature/hsdp.md
           - Pipeline Parallel: design/feature/pipeline_parallel.md
           - Sequence Parallel: design/feature/sequence_parallel.md
+          - Window Sequence Parallel: design/feature/window_sequence_parallel.md
           - Tensor Parallel: design/feature/tensor_parallel.md
           - VAE Patch Parallelism: design/feature/vae_parallel.md
         - Attention Optimization:

docs/README.md

@@ -56,9 +56,9 @@ vLLM-Omni is flexible and easy to use with:
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
 - Omni-modality models (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
-- TTS models (e.g. Qwen3-TTS, IndexTTS 2.5, CosyVoice3)
-- Diffusion models — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
-- Robot-policy and action models (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
+- TTS models (e.g. Qwen3-TTS, Tencent AuK, Breeze-TTS-2, CosyVoice3)
+- Diffusion models — image, video, and audio generation (e.g. MiniMax H3, LingBot World, MAGI-2, LTX-2.5, Wan2.2)
+- Robot-policy and action models (e.g. π0.5, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
 
 For more information, checkout the following:
 

docs/cli/bench/serve.md

@@ -302,6 +302,21 @@ vllm bench serve --omni \
 selects all and oversize values use all available cases. Reference audio is required, and OmniInteract uses the
 `/v1/realtime` endpoint.
 
+To replay an existing sample set (for example `sampled_cases.jsonl` from a prior run), pass
+`--omniinteract-video-list` instead of discovering cases. List order is preserved; `--num-prompts` takes the prefix of
+the list (`0` or a value larger than the list length runs every row). Do not combine video-list mode with `--seed`,
+`--dataset-path`, `--omniinteract-scenario-tags`, or `--omniinteract-scenario-focus`.
+
+Use `--omniinteract-scenario-tags` to steer small runs toward paper-table scenarios (`realtime`, `proactive`,
+`nested`, `interrupted`, `1qna`). By default the sampler covers each requested tag with at least one matching case
+(when available under `--omniinteract-subsets`), then fills the remaining `--num-prompts` budget from other cases.
+Add `--omniinteract-scenario-focus` to run only cases that match those tags. Example smoke coverage:
+
+```bash
+  --num-prompts 5 \
+  --omniinteract-scenario-tags realtime proactive nested interrupted 1qna
+```
+
 Audio is replayed as 16 kHz PCM16 in 200 ms chunks and video at 1 FPS with real-time pacing. All selected media is decoded
 before timing and remains in client memory for the run, so `--max-concurrency` does not limit media preparation memory; use
 explicit `--num-prompts 0` only when the client has enough RAM for the full dataset. Media commands are bounded by
@@ -309,7 +324,9 @@ explicit `--num-prompts 0` only when the client has enough RAM for the full data
 options apply. Use `--omniinteract-require-response` only for functional E2E cases; LISTEN is a valid benchmark result.
 
 Each completed case writes `output.wav`, `wav_transcript.json`, `events.json`, `result.json`, and a final `.done` marker under
-`--omniinteract-output-dir`. The root also contains `batch_summary.json` and `official_eval_manifest.jsonl`; failed cases write
+`--omniinteract-output-dir`. The root also contains `batch_summary.json`, `official_eval_manifest.jsonl` (eligible cases only),
+and `sampled_cases.jsonl` (every sampled case as absolute `video_path` / official `output_name` / `subset` rows for Lucky-Lance
+OmniInteract `batch_inference_minicpmo.py --video_list`); failed cases write
 `.failed.json`. Runs sharing one output root are serialized. Completion validates transport, response lifecycle, and artifacts,
 not answer accuracy. Transcript timestamps are serialized playback-queue times. Playback ACKs report cumulative progress
 incrementally along that serialized clock, like a live listener; the first ack for a response goes out as soon as its audio
@@ -318,6 +335,48 @@ arrives, checkpointing the response's history position so a later committed user
 outputs are ineligible and omitted from the official manifest; `audio_clipped_bytes` records output beyond the rounded video
 horizon.
 
+Accuracy evaluation is opt-in and requires an already-running text judge with an OpenAI-compatible Chat Completions API. The
+benchmark does not launch or stop the judge server. Add the following options to the command above:
+
+```bash
+vllm serve Qwen/Qwen2.5-7B-Instruct \
+  --served-model-name Qwen2.5-7B-Instruct \
+  --port 8001
+```
+
+Then add to the benchmark command:
+
+```bash
+  --omniinteract-evaluate \
+  --omniinteract-judge-base-url http://127.0.0.1:8001 \
+  --omniinteract-judge-model Qwen2.5-7B-Instruct
+```
+
+`--omniinteract-judge-model` must match that judge process's `--served-model-name` (or its `--model`
+string if `--served-model-name` is omitted). It is not a checkpoint path to load, and it is not the
+Omni DUT `--served-model-name`. Point `--omniinteract-judge-base-url` at the judge, not at the Omni
+realtime endpoint.
+
+The early / core / interrupted-partial judge prompts follow the official English
+templates in [Lucky-Lance/OmniInteract](https://github.com/Lucky-Lance/OmniInteract)
+`eval/evaluation/llm_judge.py` (commit `de304cef35fd9a50a5caadb5090c34cfbf0dd868`).
+They correspond to the OmniInteract paper appendix
+([arXiv:2605.26485](https://arxiv.org/abs/2605.26485)) Listing A.1 (early-stage),
+Listing A.2 (interrupted partial quality), and Listing A.3 (core-stage).
+A local OpenAI-compatible judge is not the paper's GPT-4o judge, so reported
+IA-QTF1 numbers are protocol-compatible rather than official paper-table scores.
+
+After artifact publication, the evaluator builds `[start, t_a, end)` slots, judges early and core response text, writes
+per-case details plus `evaluation/unified_eval_summary.json`, and prints IA-QTF1 columns
+(1Q1A realtime / proactive / nested / Global, 1QnA, All Global), interruption diagnostics (NOR / PAQ / CSM), and nested
+metrics (NCCS, inner / outer IA-QTF1, missed outer). Slices and sections with no slots in the evaluated sample are
+omitted from the terminal report. The header lists how many evaluated cases carry each scenario tag
+(`realtime`, `proactive`, `nested`, `interrupted`, `1qna`). Realtime and proactive
+exclude nested inner/outer slots; 1Q1A Global recomputes F1 from those three TP/FP/FN aggregates; All Global includes every
+scored slot plus unmatched-chunk false positives. Plain `wav_transcript.json` timestamps provide chunk-level approximate
+timing. If transcript chunks include `aligned_words`, the evaluator splits boundary-crossing chunks and derives trigger
+timing from word alignment.
+
 Per-response TTFT and TTFP start when the server begins executing the native model-turn request that owns the response. RTF
 continues to use client receipt of `response.created` through the last audio packet, divided by emitted audio duration. Global
 TTFT, TTFP, and RTF cover the complete input-stream window. TPOT/ITL use engine stage-0 timing; ITL is emitted only when every
@@ -325,9 +384,12 @@ token interval is present within a continuous generation segment. Model-unit pac
 excluded from TPOT/ITL. Raw request metrics retain `response_created_to_first_text_ms` and
 `response_created_to_first_audio_ms` as client-envelope diagnostics.
 
-The checked-in local performance configuration measures four deterministic cases from each OmniInteract subset (12 videos
-total), with no benchmark warmups and a maximum concurrency of two. Each subset also sends one readiness request before its
-measured cases. Run it from the repository root with:
+The checked-in local performance configuration measures four deterministic cases from `1q1a`, `1q1a_math`, and
+`1qna` (12 videos total), with no benchmark warmups and a maximum concurrency of two. Each subset also sends one readiness request before its
+measured cases. The runner starts MiniCPM-o 4.5 on the first visible GPU and a text judge
+(`Qwen/Qwen2.5-7B-Instruct`) on the second (`CUDA_VISIBLE_DEVICES=1` relative to the process). After artifacts land it
+scores All Global IA-QTF1 through that local judge. This is protocol-compatible with the paper metric, not a GPT-4o
+paper-table score. Run it from the repository root with two visible GPUs:
 
 ```bash
 export HF_HOME=/path/to/persistent/huggingface-cache
@@ -338,12 +400,14 @@ bash tools/nightly/run_nightly_jobs.sh \
   --label-substr minicpmo_4_5_omniinteract
 ```
 
-The first run downloads the pinned OmniInteract archive into `HF_HOME`; later runs reuse that cache.
+The first run downloads the pinned OmniInteract archive and the judge weights into `HF_HOME`; later runs reuse that cache.
 
 It requires every case to commit its input, complete any emitted response lifecycles, and publish the expected WAV,
 transcript, event, and result artifacts without errors. A valid LISTEN-only case may have no response audio or transcript
 chunks. Official-manifest eligibility is reported separately because clipped or cancelled output is a benchmark-quality
-signal, not a transport failure. This local performance test does not score answer accuracy.
+signal, not a transport failure. Accuracy must finish with `status=ok` on every subset. After all three subsets finish,
+All Global IA-QTF1 is recomputed from pooled `Global_TP` / `Global_FP` / `Global_FN` and must be at or above
+`omniinteract_aggregate_min_ia_qtf1` (checked in as `0.2`).
 
 ### Video-MME Benchmark
 

docs/configuration/README.md

@@ -2,7 +2,7 @@
 
 This section lists the most common options for running vLLM-Omni.
 
-For options within a vLLM Engine, please refer to the [vLLM 0.29 configuration guide](https://docs.vllm.ai/en/v0.29.0/configuration/index.html).
+For options within a vLLM Engine, please refer to the [vLLM 0.30 configuration guide](https://docs.vllm.ai/en/v0.30.0/configuration/index.html).
 
 Each model defines fixed topology in a registered `PipelineConfig` and runtime
 overrides in a deploy YAML.

docs/configuration/environment_variables.md

@@ -64,6 +64,7 @@ The lifecycle labels used below are:
 | Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
 | --- | --- | --- | --- | --- |
 | `DIFFUSION_ATTENTION_BACKEND` | Backend name or `auto`; default `auto` (platform selection) | Diffusion stages; read when `OmniDiffusionConfig` is constructed | `diffusion_attention_config.default` wins. An unknown backend fails during backend resolution. | Stable fallback |
+| `DIFFUSION_ATTENTION_QUANT` | `<dtype_qk>:<dtype_vo>[:<q_block_size>:<k_block_size>]`; default unset | Diffusion stages; read when `OmniDiffusionConfig` is constructed | Used only when `DIFFUSION_ATTENTION_BACKEND` sets a non-`auto` backend and no explicit attention default exists. Quantization support varies by backend. Malformed fields raise `ValueError`. | Stable fallback |
 | `DIFFUSION_CACHE_BACKEND` | `none`, `cache_dit`, `tea_cache`, `mag_cache`, `step_cache`, `stepcache`, or `step_cache_dit`; default `none` | Diffusion runner startup | Explicit `cache_backend` in config wins. Otherwise this name wins over the deprecated alias. Unsupported values raise `ValueError` during runner setup. | Stable fallback |
 | `DIFFUSION_CACHE_ADAPTER` | Same values as `DIFFUSION_CACHE_BACKEND`; default `none` | Diffusion runner startup | Used only when neither explicit `cache_backend` nor `DIFFUSION_CACHE_BACKEND` is set. Unsupported values raise `ValueError`. | Deprecated; use `DIFFUSION_CACHE_BACKEND` |
 | `OMNI_DIFFUSION_PROMPT_EMBED_CACHE` | Boolean: `1`, `true`, `yes`, `on`, `0`, `false`, `no`, or `off`; default disabled | Each diffusion runner; resolved during model setup | A recognized environment value overrides the explicit enable setting. An unrecognized value is ignored. | Experimental |
@@ -77,6 +78,13 @@ Backend names for `DIFFUSION_ATTENTION_BACKEND` are the members of
 `SAGE_ATTN`, `FLASHINFER_ATTN`, and `TRTLLM_ATTN`. Platform support still
 depends on the installed kernels and model path.
 
+For `DIFFUSION_ATTENTION_QUANT`, the dtype fields accept `float16`,
+`bfloat16`, `int8`, or `fp8_e4m3`; the selected backend may support only a
+subset. For example, `DIFFUSION_ATTENTION_BACKEND=TRTLLM_ATTN` with
+`DIFFUSION_ATTENTION_QUANT=int8:bfloat16:1:16` selects INT8 Q/K SageAttention.
+Explicit `diffusion_attention_config.default` takes precedence over both
+environment variables.
+
 ### Serving and runtime
 
 | Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
@@ -130,6 +138,22 @@ settings.
 Omni-owned variable, even though vLLM-Omni supplies a persistent default when
 it is unset.
 
+### Torch compilation
+
+| Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
+| --- | --- | --- | --- | --- |
+| `VLLM_OMNI_TORCH_DYNAMO_RECOMPILE_LIMIT` | Positive integer; unset preserves the current Torch setting | Process default for Torch Dynamo; read when `vllm_omni` is imported | Sets `torch._dynamo.config.recompile_limit` to the requested value. A non-positive or non-integer value raises `ValueError`. Later backend-specific overrides still take precedence. | Diagnostic |
+
+Set this variable before launching Omni, for example:
+
+```bash
+export VLLM_OMNI_TORCH_DYNAMO_RECOMPILE_LIMIT=64
+```
+
+The setting applies across model families and platforms. vLLM compilation
+contexts and backends can subsequently apply their own limits; this variable
+does not replace those backend-specific policies.
+
 ## Per-stage environment
 
 Deploy configurations can set arbitrary environment keys for one stage:
@@ -162,7 +186,7 @@ their keys only.
 ## Inherited vLLM variables
 
 vLLM-Omni also reads variables through its aligned vLLM dependency. Refer to
-the [vLLM 0.29 environment-variable reference](https://docs.vllm.ai/en/v0.29.0/configuration/env_vars.html)
+the [vLLM 0.30 environment-variable reference](https://docs.vllm.ai/en/v0.30.0/configuration/env_vars.html)
 for their definitions. This includes vLLM launch, cache, logging, plugin, ROCm,
 XPU, ModelScope, and FlashInfer workspace settings.
 
@@ -190,7 +214,7 @@ collection, examples, bug reports, or logs.
 
 ## Model-specific variables
 
-The audit found 58 variables read by a single model or pipeline family. They are
+The inventory includes 66 variables read by a single model or pipeline family. They are
 not listed as public usage options here because doing so would turn implementation
 escape hatches into an accidental compatibility contract.
 
@@ -199,11 +223,11 @@ Every audited model-specific name has a migration disposition in the
 
 | Disposition | Count | Required outcome |
 | --- | ---: | --- |
-| Promote | 35 | Move a stable setting into typed stage or model configuration. |
-| Request scope | 6 | Move request-varying behavior into a declared request-option schema. |
+| Promote | 38 | Move a stable setting into typed stage or model configuration. |
+| Request scope | 5 | Move request-varying behavior into a declared request-option schema. |
 | External | 0 | Retain only when a supported third-party library owns the contract. |
-| Internalize | 11 | Keep a debug or diagnostic switch out of public documentation and configuration. |
-| Deprecate/remove | 6 | Remove a compatibility escape hatch that has no continuing contract. |
+| Internalize | 16 | Keep a debug or diagnostic switch out of public documentation and configuration. |
+| Deprecate/remove | 7 | Remove a compatibility escape hatch that has no continuing contract. |
 
 The disposition is a migration target, not a statement that the existing
 environment switch is stable. Promote or request-scope work should land in

docs/configuration/hsdp_pre_sharded_loading.md

@@ -0,0 +1,22 @@
+# Pre-sharded HSDP loading
+
+Use `--use-hsdp --hsdp-shard-size 8 --hsdp-weight-load-strategy pre_sharded`
+to shard transformer parameters before loading checkpoint values. Set the shard
+size to the available GPU count (or configure replication explicitly).
+The default strategy, `full`, retains the existing load-then-shard behavior.
+The same `hsdp_weight_load_strategy` option is available through Python and stage
+engine configuration.
+
+The pre-sharded strategy reads rank-local slices from ordinary Hugging Face
+safetensors checkpoints into FSDP-owned storage. No converted checkpoint is
+required. Cosmos 3 and Cosmos 3 Edge transformer blocks release their initialized
+parameter storage to meta as they are constructed; nonpersistent buffers retain
+their values.
+Encoders and VAEs follow their existing loading paths.
+
+This strategy requires the default diffusion pipeline loader, complete dedicated
+transformer checkpoint sources, and supported checkpoint key mappings. Tensor
+layouts must match the runtime layout; tensor transforms are unsupported.
+Checkpoint tensor dtypes must match runtime tensor dtypes at preflight; dtype conversion is unsupported.
+Quantization and LoRA are unsupported. Every worker needs checkpoint access.
+`num_weight_load_threads` controls the safetensors reader's threads per rank.

docs/configuration/stage_configs.md

@@ -80,6 +80,19 @@ Note: for the diffusion path, an omitted `distributed_executor_backend` selects
 segments) and `mp` when `num_gpus > 1`. Set `mp` explicitly to keep a worker
 subprocess on one GPU. `ray` / `external_launcher` are not fully supported yet.
 
+### Stage-level runner selection
+
+`model_runner: v1` or `v2` at the deploy level sets the default runner.
+A `model_runner` on an individual stage overrides that default, allowing
+one stage to migrate or roll back independently. This does not change
+`PipelineConfig` topology, input adapters, or the connector payload contract.
+An omitted stage override preserves the deploy-level selection.
+
+MRv2 native downstream receivers currently support turn-based requests only;
+streaming sessions and resumable input require the V1 prompt-replacement path.
+Selecting a runner does not add the session capabilities it lacks. Platform
+fallbacks and stage overrides are validated after configuration resolution.
+
 ### Stage fields
 
 Each entry under `stages:` accepts any `StageDeployConfig` field directly (no nested `engine_args:`). Only fields whose value legitimately varies across stages live here; pipeline-wide settings (trust_remote_code, distributed_executor_backend, dtype, quantization, prefix/chunked prefill, DP/PP sizes) are declared at the top level and applied to every stage. Unknown keys fall through to `engine_extras:` and are forwarded to the engine. Frequently used fields are listed below; the source-of-truth schema is `StageDeployConfig` in `vllm_omni/config/stage_config.py`.
@@ -306,7 +319,9 @@ ROCm and MUSA. Select one of these deployment profiles:
 | `qwen3_tts_mrv2.yaml` | V2 | B1 | Explicit MRV2 profile (same runner selection as the default) |
 | `qwen3_tts_high_concurrency_mrv2.yaml` | V2 | B1, B2 | Opt-in throughput tuning |
 | `qwen3_tts_high_concurrency_mrv2_b4.yaml` | V2 | B1, B2, B3, B4 | Experimental throughput / buffered playback |
+| `qwen3_tts_high_concurrency_mrv2_single_gpu.yaml` | V2 | B1–B8 | Experimental two-stage deployment on one GPU, with MPS |
 | `qwen3_tts_high_concurrency.yaml` | V1 | Existing defaults | V1 high-concurrency control |
+| `qwen3_tts_fused_single_gpu.yaml` | V2 | N/A (in-Talker decoder) | Opt-in single-stage CUDA pipeline on one GPU |
 
 ```bash
 # MRV2 is the default; pass a copy with `model_runner: v1` to force V1.
@@ -337,6 +352,102 @@ supply `mtp_sampling_params` and `get_mtp_seed(sampling_params)` for model-local
 (with `mtp_sample_steps` and `mtp_sample_vocab_size`) as needed. Qwen3-TTS retains
 its existing `talker_mtp` entry point for V1.
 
+### Selecting one or two Qwen3-TTS stages
+
+Choose the deployment profile explicitly with `--deploy-config`. The default
+`qwen3_tts` pipeline retains two stages: the Talker publishes codec chunks to
+Code2Wav, which produces audio. Its stage devices can be placed on the same
+GPU or on separate GPUs. `qwen3_tts_fused` is a separate, opt-in pipeline for
+one CUDA GPU: the Talker owns the stateful codec decoder and emits PCM directly,
+without a Code2Wav engine or stage connector.
+
+```bash
+# Two stages on one GPU; the two-stage pipeline remains independently selectable.
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni \
+  --deploy-config vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2_single_gpu.yaml
+
+# One stage on one GPU; no inter-stage transport or Code2Wav scheduling.
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni \
+  --deploy-config vllm_omni/deploy/qwen3_tts_fused_single_gpu.yaml
+```
+
+The single-stage profile requires CUDA, V2, asynchronous chunks, TP=PP=1,
+an in-process worker and disabled prefix caching. Select the full profile;
+adding `talker_stream_decode: true` to a two-stage Talker is rejected because
+its latent output belongs to the Code2Wav input contract. Single-stage options
+live in `stages[].additional_config`; two-stage transport and first-frame
+options live in `connectors.*.extra`.
+
+The single-stage profile defaults `talker_stream_first_audio` to `false` for
+throughput. To opt into a separate first-frame delivery queue, save this
+overlay beside the bundled deployment files and pass its path to
+`--deploy-config`:
+
+```yaml
+base_config: qwen3_tts_fused_single_gpu.yaml
+stages:
+  - stage_id: 0
+    additional_config:
+      talker_stream_decode: true
+      talker_stream_first_audio: true
+      ref_code_context_frames: 72
+      code_predictor_kv_cache: true
+      code_predictor_fused_sampling: true
+      code_predictor_fused: true
+```
+
+This queue uses PCM from the single-stage decoder. The two-stage
+`talker_first_audio` option below uses its own first-frame decoder and does
+not require single-stage execution. The two options cannot be enabled
+together. Single-stage prefix-cache reset with
+`reset_running_requests=True` returns `False` while requests are active;
+wait for completion or abort them before retrying.
+
+Reference-cloning context is selected independently by each profile. The
+single-stage profile explicitly uses 72 reference frames, matching the
+`qwen3_tts.yaml` context boundary. A shorter
+`additional_config.ref_code_context_frames` (for example 25) can change
+timbre continuity even when WER is similar;
+the throughput measurements for CustomVoice do not qualify Base voice
+cloning quality. Reference layout, grouping and priming are model hooks;
+the shared worker does not select Qwen's context length or codebook layout.
+
+### Qwen3-TTS first-frame delivery and rollback
+
+The standard `qwen3_tts.yaml`, `qwen3_tts_high_concurrency.yaml`,
+`qwen3_tts_mrv2.yaml` and `qwen3_tts_high_concurrency_mrv2.yaml` profiles
+enable the `talker_first_audio` connector option, as does the experimental
+two-stage `qwen3_tts_high_concurrency_mrv2_single_gpu.yaml` profile.
+This changes the default CUDA MRV2 streaming path:
+residual prediction runs eagerly after the Talker sample, the Talker loads
+an additional first-frame decoder with its weights and CUDA graphs, and the
+orchestrator orders its audio before subsequent Code2Wav chunks.
+
+Code2Wav retains full-audio prefix graphs alongside the optional state-only
+graphs. Each request's delivery marker selects the graph and audio trimming;
+enabling the option alone never suppresses audio or disables prefix batching
+for requests that retain regular codec delivery.
+
+The path requires asynchronous chunks, TP/PP 1, an in-process executor and
+disabled prefix caching. Requests with reference codes and unsupported
+runners or platforms retain regular Code2Wav delivery.
+
+To restore regular codec delivery and avoid loading the Talker's additional
+decoder, use a deploy overlay with the relevant base profile:
+
+```yaml
+base_config: qwen3_tts.yaml
+connectors:
+  connector_of_shared_memory:
+    extra:
+      talker_first_audio: false
+```
+
+First-packet latency measures when PCM starts arriving. Time to first audible
+audio (TTFA) also includes any leading silence in the generated audio. An
+earlier first packet therefore does not necessarily improve TTFA; measure
+both for the intended voice and workload.
+
 ### Included performance work
 
 - Request snapshots have a fast path for immutable scalar leaves, including
@@ -372,14 +483,34 @@ speaker similarity before adopting either batching preset for a production
 workload. Floating-point decoder outputs can differ across batch sizes; this PR
 does not claim bitwise or quality equivalence.
 
-### Optional MPS deployment
+### Experimental MPS deployment
 
-NVIDIA MPS is an optional operator setting for colocated CUDA processes, not a
-YAML option or a library default. This PR does not establish a throughput or
-first-packet latency benefit from MPS. Measure the exact deployment with and
-without MPS before enabling it.
+NVIDIA MPS lets colocated CUDA stage processes share GPU execution resources.
+It is disabled by default. The experimental
+`qwen3_tts_high_concurrency_mrv2_single_gpu.yaml` profile sets `cuda_mps: true`,
+alongside cached residual prediction, fused sampling, time-major codec
+convolutions, first-frame delivery, and larger graph batches.
 
-Use only assigned GPUs and an independent MPS pipe directory. A private MPS
-server does not provide exclusive GPU ownership or MIG isolation. For a
-single-GPU deployment, explicitly place both stages on that GPU; the supplied
-high-concurrency profile places its two stages on different GPUs by default.
+The runtime requires `nvidia-cuda-mps-control` on `PATH` and one explicit CUDA
+GPU per local EngineCore stage, with `parallel_stage_init: false`.
+Use numeric GPU ordinals for stage placement and visibility so initialization
+locks identify the physical GPU before MPS remaps it. It starts a private MPS daemon for each selected
+GPU and stops its own daemon after the stages exit. If
+`CUDA_MPS_PIPE_DIRECTORY` already names an operator-managed daemon, the runtime
+reuses it without stopping it. Diffusion and remote stages are unsupported.
+Set `cuda_mps: false` in a deploy overlay to disable automatic MPS management.
+
+Stage `env.CUDA_MPS_PIPE_DIRECTORY` overrides the parent setting when selecting
+the daemon; an explicit empty string selects a private daemon. Stages sharing
+a GPU reuse the first stage's daemon. Later stages may omit the setting or
+explicitly match that policy; a conflicting explicit setting fails before
+that stage starts. The parent environment is unchanged.
+
+MPS can improve throughput under concurrent load while increasing first-packet
+or first-audible-audio latency, particularly at low request rates. For a
+latency-sensitive workload, compare the same profile with `cuda_mps: false`.
+An inherited operator-managed MPS daemon must also be disabled by its owner
+for that comparison; this switch only controls automatic MPS management.
+
+MPS does not reserve a GPU. Use only assigned GPUs, explicitly place stages on
+the intended GPU, and warm the complete pipeline before measuring performance.

docs/contributing/README.md

@@ -295,6 +295,10 @@ If you encounter a bug or have a feature request, please search existing issues
 
 Thank you for your contribution to vLLM-Omni! Before submitting the pull request, please ensure the PR meets the following criteria. This helps vLLM-Omni maintain the code quality and improve the efficiency of the review process.
 
+### Pull Request Limit
+
+To avoid agent spam, vLLM-Omni uses GitHub's [pull request limit](https://github.blog/open-source/maintainers/how-pull-request-limits-are-cutting-down-the-noise/) for contributors without write access. The current cap is 5 open PRs. If this blocks well-intentioned critical work, contact a committer to request bypass list access.
+
 ### DCO and Signed-off-by
 
 When contributing changes to this project, you must agree to the [DCO](https://developercertificate.org/). Commits must include a `Signed-off-by:` header which certifies agreement with the terms of the DCO.

docs/design/feature/attention_backend_selection.md

@@ -4,6 +4,9 @@ This document defines the selection and extension contract for diffusion
 attention backends. User-facing backend choices, installation, and tuning are
 in the [attention backend guides](../../user_guide/diffusion/attention_backends.md).
 
+The [execution contract PoC](attention_execution_contract_poc.md) describes the
+experimental post-selection capability and lifecycle contract from RFC #7226.
+
 ## Scope
 
 The contract applies to DiT and other diffusion attention layers. It is

docs/design/feature/attention_execution_contract_poc.md

@@ -0,0 +1,133 @@
+# Diffusion attention execution contract PoC
+
+This prototype accompanies [RFC #7226](https://github.com/vllm-project/vllm-omni/issues/7226)
+and extends the [attention selection design](attention_backend_selection.md).
+It demonstrates path-specific capabilities with dense BF16 FA4, NPU/ROCm
+routing examples, and tensor-state lifetime with a test-only attention module.
+
+## Execution contract
+
+`ExecutionContext` describes the requested execution path. `ExecutionPathResult`
+reports its identity, support status and reason, and compilation mode.
+`AttentionBackend.resolve_capabilities()` provides conservative pre-construction
+results. After initialization, `Attention.resolve_execution_path()` supplies the
+active parallel, paged-KV, and HSDP context; the backend combines it with the
+selected kernel and normalized metadata.
+
+Resolution runs outside compiled execution. FA4 reports `SUPPORTED` and
+`CUSTOM_OP` for dense, noncausal BF16 without parallel or HSDP boundaries when
+its kernel accepts the head dimensions. Dimension validation delegates to FA4's
+architecture-specific rules through `backends/utils/fa.py`. Kernel rejections
+report `UNSUPPORTED` with an actionable reason. Missing private validators and
+other unmigrated paths report `UNMIGRATED` with advisory `EAGER_ONLY` defaults.
+`requested_support()` checks a fullgraph request; it does not enforce selection
+or change existing execution.
+
+FA4 execution uses an opaque custom op. Its fake output preserves Q's batch,
+sequence, and head count and uses V's head dimension. Output is contiguous even
+when Q is noncontiguous. Metadata normalization is
+shared with dispatch. Producers must update published mask semantics when masks
+change; unpublished masks remain runtime-dependent to avoid synchronization.
+Callers resolve again when execution metadata changes.
+
+## NPU and ROCm worked examples
+
+The same `FLASH_ATTN` backend resolves MindIE paths on NPU and AITER paths on
+ROCm. Both examples report `UNMIGRATED`: their route is inspected, while hardware
+correctness and compilation remain unvalidated. `EAGER_ONLY` is an advisory
+default, not a measured compiler limitation. Existing execution is unchanged.
+
+| Input | NPU / MindIE | ROCm / AITER |
+| --- | --- | --- |
+| Dense | `npu_dense` | `rocm_dense` |
+| Padding mask | `npu_masked`: expand the key mask to `[B, 1, Q, K]` | `rocm_masked_varlen`: unpad, call varlen attention, then restore rows |
+| Packed inputs | Opt-in `[real, pad]` contract; choose varlen or prefix-KV slicing from the environment | Forward complete packed-document boundaries to varlen attention |
+| Incomplete packing | Rebuild a mask when possible; otherwise reject before the kernel | Shared CUDA-like normalization rejects incomplete metadata |
+| Unpublished mask semantics | Route to the masked wrapper | Keep resolution runtime-dependent without reading mask values |
+
+NPU resolution reuses `_resolve_packed_seq_npu`; ROCm reuses the metadata
+normalization used by dispatch. AITER identity comes from initialization rather
+than caller-provided context. Quantized, paged, piecewise, and parallel paths
+remain outside these worked examples.
+
+The new L1 tests use CPU tensors and replace only vendor kernel calls. They
+check path identity, forwarded masks/boundaries, fallback rejection, and
+conservative fullgraph-request handling. They do not test vendor numerics or
+compiler support. Existing L1 CI already collects this test file.
+
+```bash
+python -m pytest tests/diffusion/attention/test_flash_attn.py \
+  -k 'npu_contract or rocm_contract' -m 'core_model and cpu' -q
+```
+
+### Hardware validation required before migration
+
+On an Ascend/MindIE or ROCm/AITER runner, record device, driver, PyTorch, vendor
+library, and compiler versions, then validate the corresponding rows above:
+
+1. Compare eager outputs with FP32 SDPA for dense and padding-mask inputs,
+   including unequal Q/K lengths. Compare valid query rows; do not assume both
+   platforms define padded-query outputs identically.
+2. Compare packed outputs with separate per-document references. On NPU, cover
+   both varlen and laser prefix-slicing modes, plus mask reconstruction and the
+   missing-mask error. A padding fallback does not establish multi-document
+   isolation. On ROCm, cover multiple real documents.
+3. Check output shape, dtype, device, finite values, and unchanged inputs.
+   Start BF16 comparisons at `atol=rtol=1e-2` and have the platform maintainer
+   confirm the tolerance against its reference tests.
+4. Run `torch.compile(fullgraph=True, dynamic=True)` with the platform's supported
+   compiler over multiple sequence lengths. Record graph breaks and recompiles;
+   choose `TRACEABLE`, `CUSTOM_OP`, or verified `EAGER_ONLY` from that evidence.
+   A custom-op path additionally needs schema/fake checks and repeated replay.
+
+Promote only the paths validated on that platform. These hardware checks have
+not been run locally; ROCm/NPU fullgraph support is not claimed.
+
+## State lifetime example
+
+`test_attention_state_lifetime.py` prepares an owned query-scale tensor eagerly
+and passes it explicitly to an opaque attention op. The compiled module retains
+its state after the caller drops its reference and releases ownership when the
+compiled callable is deleted. Equivalent instances reuse a graph with their own
+state values.
+
+This example tests ownership and release, dynamic replay across instances,
+custom-op schema/fake behavior, and Inductor execution. It adds no production
+backend or state registry. Planning keys, caching, fallback-policy declarations,
+opaque handles, CUDA graph capture, and TRTLLM integration are deferred.
+
+## Validation
+
+```bash
+python -m pytest tests/diffusion/attention/test_flash_attn_compile.py \
+  tests/diffusion/attention/test_flash_attn.py \
+  tests/diffusion/attention/test_attention_capabilities.py \
+  tests/diffusion/attention/test_attention_state_lifetime.py -q -rs
+```
+
+The focused suite covers capability decisions and metadata changes, existing
+padding/mask regressions, tensor-state lifetime, and real FA4 fullgraph execution
+against FP32 SDPA and eager FA4. Representative Q/K and V dimensions are
+(32, 32), (64, 64), (80, 48), (192, 128), and (256, 256), with batches 1 and 2
+and sequence lengths up to 1,024. Invalid dimensions are checked against actual
+kernel errors. These samples exercise the contract rather than define support.
+BF16 comparisons use `atol=rtol=1e-2`; `torch.library.opcheck` checks schema and
+fake-output correctness.
+
+Single-graph reuse is asserted in the tensor-state test. The FA4 numerical tests
+retain compiler caches across shapes within each case but do not require one
+graph across different batch sizes, Q/K length equality, or head dimensions.
+
+Tested environment: GB300, PyTorch `2.13.0+cu130`, FA4 `4.0.0b18`, CUTLASS DSL
+`4.6.2` with CUDA 13 libraries, Quack `0.6.4`, and TVM FFI `0.1.11`.
+Real-kernel tests require CUDA and CuTe FA4; their availability is checked at runtime.
+
+Before the rebase, the focused suite passed all 97 tests without skips, including
+11 CPU NPU/ROCm routing tests. After rebasing onto upstream `e3be42e05`, the
+expanded suite reports 99 passed and 11 failed in the local vLLM 0.28 environment.
+All 11 failures are import errors: upstream now requires `compute_layout_strides`
+from vLLM, which this environment lacks. Both noncontiguous FA4 regression cases
+and real-kernel schema/fake checks pass. The full suite must be rerun with the
+vLLM version required by upstream. NPU/ROCm device numerics and compilation remain
+unvalidated. Pre-commit passes with the CI hook skips. Upstream PyTorch/CUTLASS
+warnings remain.

docs/design/feature/omni_connectors/nixl_connector.md

@@ -1,5 +1,8 @@
 # NixlConnector
 
+For the reusable model-runner API, illustrated control flow, deadline semantics,
+and failure/ownership contracts, see [Cross-Stage Payload Transport](stage_payload.md).
+
 ## When to Use
 
 Multi-node or intra-node stage transfer over NIXL, which brokers RDMA/shared-memory
@@ -46,6 +49,15 @@ provides a matching torch-xpu, UCX with Level Zero support, and NIXL runtime.
 
 ## Configuration
 
+MiniMax-H3's default disaggregated and Turbo deploy configs use shared memory
+for same-host transfer. To opt into NIXL, select
+`vllm_omni/deploy/minimax_h3_disaggregated_nixl.yaml` or
+`vllm_omni/deploy/minimax_h3_disaggregated_turbo_nixl.yaml` with
+`--deploy-config`. These overlays inherit the corresponding model and
+parallel settings and switch both ends of the stage edge. They require NIXL
+and a working UCX backend; initialization failures do not automatically fall
+back to shared memory.
+
 ```yaml
 connectors:
   nixl_connector:
@@ -92,9 +104,44 @@ Parameters:
 - `transfer_timeout_s`: how long a `get()` waits for its `READ` to complete
   (default 300). NIXL 1.3 has no transfer cancellation API, so a timed-out transfer's
   buffers and registrations remain owned by the connector until NIXL reports a
-  terminal state; `close()` waits for that state before releasing them.
+  terminal state. `close()` polls remaining transfers once and returns without
+  releasing active DMA resources. A serialized background closer automatically
+  finishes cleanup after completion; no second call is required.
   `VLLM_OMNI_NIXL_XFER_TIMEOUT_S` overrides it.
 
+### Diffusion worker integration
+
+Diffusion initializes `OmniConnectorModelRunnerMixin` in synchronous mode,
+borrowing its KV transfer manager's lazy connector. This mode creates no second
+connector, starts no receive/save threads, and leaves KV key builders and
+sharding callbacks unchanged. Connector lifetime remains with the manager.
+
+The shared `recv_stage_payload` entry point and asynchronous `_poll_single_request`
+use the same key and endpoint metadata builder. Diffusion passes its actual
+incoming edge and external request ID; explicit transfer handles take precedence.
+Sender endpoints are passed per request, without mutating KV sender state.
+Diffusion-specific prompt merging, output handles and device placement live in
+`diffusion/worker/stage_payload.py`.
+
+Diffusion supplies ordered TP and SP groups to the shared payload fanout.
+Only the rank leading both groups reads the connector. Each subsequent broadcast
+uses its own group-local rank zero, including ranks that received the payload
+from the preceding group. Tensor payloads use tensor-dictionary broadcasts rather
+than serializing the full payload as a Python object. Receive misses still reach
+the same collectives. Inline fallback is allowed only when all required payload
+keys remain present; otherwise each rank raises an explicit payload error.
+
+The synchronous path passes one monotonic deadline to `get_with_deadline()` for
+discovery and DMA completion waits, rather than calling the ordinary `get()`
+with its potentially much longer transfer timeout. Native calls, serialization,
+and distributed collectives are not preemptible by this deadline.
+
+`GroupCoordinator.broadcast_tensor_dict` preserves the group-local source for
+`broadcast_object` while passing the mapped global source to PyTorch tensor
+collectives. For subgroup `[1, 3]`, local source `0` maps to global source `1`;
+passing `1` back to `broadcast_object` would incorrectly select global rank `3`.
+The nonzero-subgroup regression covers both sender and receiver behavior.
+
 ### Source ownership and failure limits
 
 Claims and expiry use the same lock. Completion identifies both a payload
@@ -105,12 +152,24 @@ transfers retain the complete destination bundle and source claim until terminal
 Zero-element tensors retain their shape/dtype/skeleton positions but never create
 DMA descriptors or registrations.
 
-A lost metadata response, lost completion ACK, or abandoned consumer can retain a
-claim indefinitely. NIXL 1.3 cannot prove remote cancellation, so neither TTL nor
+After a lost metadata response, retries on the same consumer thread reuse the
+claim ID for that endpoint, key and requested generation until a reply arrives.
+This makes claim acquisition idempotent; subsequent independent reads still get
+distinct claims. Callers retry on the same thread and call `abandon_get()` when
+giving up. Synchronous stage payload receive does this automatically. Background
+recovery reacquires the same claim solely to acknowledge it, without issuing a
+READ. Lost completion ACKs are retried against the exact payload generation.
+A permanently unreachable or dead consumer can still retain a claim indefinitely.
+NIXL 1.3 cannot prove remote cancellation, so neither TTL nor
 `cleanup()` frees those allocations. Producer `close()` rejects new work and
-retains its agent, listener and claimed allocations; a later `close()` finishes
+retains its agent, listener and claimed allocations; the background closer finishes
 teardown after claims drain. Permanently abandoned claims remain until process
-exit. This favors memory safety over bounded shutdown/memory usage. Only trusted
+exit. A consumer with unfinished local transfers likewise retains a strong reference
+to its connector, agent, tensors and registrations after `close()` returns. Its
+background closer polls and releases resources only after a terminal state;
+permanently stuck transfers retain resources until process exit. Closing connectors
+reject new work and report unhealthy. This avoids an unbounded transfer-polling loop
+in shutdown without freeing DMA-owned memory. Only trusted
 peers may access this unauthenticated control plane. Both endpoints must use the
 claim-aware protocol; rolling compatibility with older GET_META clients is not
 provided. Legacy externally-owned metadata without a generation is accepted only

docs/design/feature/omni_connectors/stage_payload.md

@@ -0,0 +1,280 @@
+# Cross-Stage Payload Transport
+
+Stage payload transport moves model outputs such as encoder conditioning between
+workers without routing their tensor contents through the orchestrator. It is
+separate from native diffusion KV transfer: a deployment can use both, either, or
+neither. MiniMax-H3 is one consumer of this interface, not its implementation.
+
+## Architecture
+
+```mermaid
+flowchart TB
+    Config[Topology and deploy configuration] --> Keys[Input and output payload keys]
+    Keys --> Producer[Producer model runner]
+    Keys --> Consumer[Consumer model runner]
+    Producer --> Adapter[DiffusionStagePayloadMixin]
+    Consumer --> Adapter
+    Adapter --> Runtime[Shared OmniConnector runtime]
+    Runtime --> Descriptor[Request key and endpoint resolution]
+    Runtime --> Fanout[TP then SP or HSDP shard then replicate fanout]
+    Runtime --> Manager[OmniKVTransferManager owns lazy connector]
+    Manager --> SHM[SharedMemoryConnector]
+    Manager --> NIXL[NixlConnector]
+    Producer -. small transfer handle .-> Orchestrator[Orchestrator]
+    Orchestrator -. prompt and handle .-> Consumer
+    Consumer --> NativeKV[Separate native diffusion KV path]
+```
+
+The diffusion adapter owns model-facing selection and validation. The shared
+runtime owns request-key construction, endpoint metadata, retry policy, and
+distributed fanout. Backends own storage, transfer progress, and allocation
+lifetime. No pipeline should instantiate a NIXL agent or implement its own
+metadata-query loop.
+
+The synchronous runtime borrows the manager's connector. It does not create
+background AR receive/save loops or replace the manager's KV key callbacks.
+The asynchronous AR path continues using `get()` through its existing poller.
+Both paths share the descriptor and fanout helpers; their scheduling policies
+are deliberately different.
+
+## Configuration and Identity
+
+`stage_output_payload_keys` selects non-`None` entries in a producer's
+`DiffusionOutput.custom_output`. `stage_input_payload_keys` declares the required
+entries in the consumer's `prompt.additional_information`. Both normalize to
+tuples in `OmniDiffusionConfig`, including lists loaded from YAML.
+
+The model topology declares these keys; deploy configuration selects the
+connector for the edge. MiniMax-H3's standard and Turbo disaggregated profiles
+default to SharedMemory. Their `_nixl.yaml` overlays opt into NIXL with
+`--deploy-config`; choosing NIXL does not imply fallback to SharedMemory.
+
+Without an explicit handle, the key is:
+
+```text
+{external_req_id or request_id}_{from_stage}_{chunk_id}
+```
+
+The synchronous complete-payload path uses chunk zero. Source and destination
+IDs come from the configured edge, not from an assumption that stages are
+adjacent or that the producer is stage zero. An explicit handle can override
+the key, source, destination, and backend metadata:
+
+```python
+{
+    "key": "request-42_2_0",
+    "from_stage": "2",
+    "to_stage": "5",
+    "size_bytes": 4096,
+    "metadata": {...},
+    "payload_keys": ["text_encoder_output"],
+}
+```
+
+This descriptor is attached under `_stage_payload_transfer`. Backend metadata
+is opaque to the pipeline. If it is absent, request-scoped sender information
+can supply `source_host` and `source_port`. `payload_sender_info` takes
+precedence over legacy `kv_sender_info`. A flat endpoint is accepted directly;
+a stage-keyed map is resolved using the actual source stage, accepting string
+and integer keys. A single unambiguous candidate remains supported. Multiple
+unmatched candidates are not arbitrarily resolved to stage zero.
+
+## API Contracts
+
+| API | Owner | Contract |
+| --- | --- | --- |
+| `init_omni_connectors(..., synchronous=True)` | Shared runtime | Borrow an existing manager; no AR polling threads or KV callback mutation. |
+| `_stage_payload_recv_spec(...)` | Shared transport | Resolve explicit handle or conventional key plus request-scoped endpoint. |
+| `recv_stage_payload(..., retry_seconds=2.0)` | Shared transport | Leader retries with one monotonic deadline, retires discovery, then fans out success or absence. |
+| `get_with_deadline(..., deadline=...)` | Connector | Bound backend-controlled waits by the supplied absolute monotonic deadline. Return `(payload, bytes)` or `None`. |
+| `abandon_get(key)` | Connector | Retire unresolved discovery on the calling thread after the retry loop. Never free an active READ. Default is a no-op for stateless discovery. |
+| `_maybe_recv_stage_payload(request)` | Diffusion adapter | Skip warmup, receive, merge declared keys on the target device, and reject missing required values. |
+| `_maybe_send_stage_payload(requests, outputs)` | Diffusion adapter | Leader publishes selected complete outputs; all ranks attach the same handle and remove only transferred fields. |
+| `OmniKVTransferManager.close()` | Manager | Drain prefetch, close only the connector already created, and disable later lazy initialization. |
+
+`get_with_deadline()` is intentionally not a wrapper around an arbitrary
+blocking `get()`. The base implementation raises `NotImplementedError`.
+An unsupported backend must implement this capability before being used for
+synchronous remote-only payloads. Existing asynchronous `get()` users are
+unchanged.
+
+The deadline covers retry sleeps, metadata socket waits, and NIXL completion
+polling. It is **not a hard real-time bound** on Python execution, allocation,
+serialization, individual native driver calls, device copies, or payload-group
+collectives. A stuck native call cannot safely be preempted with a Python
+future timeout. The implementation does not start a detached READ and discard
+its ownership when the model-thread budget expires.
+
+## Successful Transfer
+
+The synchronous diffusion producer uses an explicit handle:
+
+```mermaid
+sequenceDiagram
+    participant P as Producer leader
+    participant B as Connector
+    participant O as Orchestrator
+    participant C as Consumer leader
+    participant R as Consumer peer ranks
+    P->>B: put(key, selected custom_output)
+    B-->>P: success, bytes, backend metadata
+    P->>P: Broadcast handle; remove transferred inline keys
+    P->>O: Output carrying small handle
+    O->>C: Prompt carrying handle and endpoint
+    C->>B: get_with_deadline(key, deadline)
+    B-->>C: Complete payload or absence
+    C->>B: abandon_get(key)
+    C->>R: Delivery status, then tensors if present
+    C->>C: Merge and validate required keys
+    R->>R: Merge and validate required keys
+```
+
+MiniMax-H3 instead uses the AR encoder's asynchronous sender. It does not
+forward the diffusion producer's explicit handle:
+
+```mermaid
+sequenceDiagram
+    participant A as AR encoder leader
+    participant S as Background sender
+    participant B as Connector
+    participant O as Orchestrator
+    participant C as Diffusion consumer leader
+    participant R as Consumer peer ranks
+    A->>S: Queue encoder conditioning for conventional key
+    A->>O: Output without inline conditioning
+    O->>C: Request identity and request-scoped sender endpoint
+    par Asynchronous publication
+        S->>B: put(conventional key, conditioning), with retries
+    and Bounded receive
+        C->>B: get_with_deadline(conventional key, endpoint)
+        B-->>C: Payload or absence
+    end
+    C->>B: abandon_get(key)
+    C->>R: Delivery status, then tensors if present
+    C->>C: Merge and validate encoder_output
+    R->>R: Merge and validate encoder_output
+```
+
+Within a TP-by-SP payload grid, exactly one rank, the leader in both groups,
+accesses the connector. The shared broadcaster traverses TP and then SP in
+the same order on every rank: TP first populates the source SP row, then SP
+propagates each TP column.
+
+When initialized HSDP groups span the stage, the adapter instead uses the
+HSDP shard and replicate dimensions. The shard broadcast populates the first
+replica row; the replicate broadcast propagates every shard position to the
+other rows. Both dimensions participate in leader election. This includes
+standalone HSDP, whose TP/SP groups are singletons, as well as pure sharding
+and pure replication. The two HSDP dimensions replace, rather than overlap,
+the TP/SP fanout.
+
+The single-consumer guarantee is scoped to these configured payload groups;
+it is not a global election across arbitrary independent DP/CFG/PP groups or
+stage replicas. Followers do not initialize or query a connector. Absence is
+broadcast too, so peers do not wait for tensor broadcasts that the leader will
+never issue.
+
+Pipelines must gather sharded producer outputs before publication. Selecting
+payload keys does not gather tensor shards automatically.
+
+## Execution and Failure Policy
+
+Full forward receives before pipeline execution and sends after output
+preparation. Stepwise execution receives once when a new cached state is
+created. That state retains the external request ID; final decoded output is
+published exactly once. Intermediate denoising steps and non-final streaming
+chunks are not published under the complete-payload key. Chunk-addressed
+streaming transfer remains the asynchronous path's separate contract.
+
+Dummy request IDs (`dummy_req_id` and its slash-prefixed variants) bypass both
+directions. A pure warmup batch does not lazily initialize the connector,
+publish buffers, wait for remote data, or enter payload fanout.
+
+For the synchronous diffusion producer, failed `put()` calls preserve inline
+fields; only successful publications replace those fields with a handle.
+This guarantee does not apply to H3's AR encoder asynchronous sender. That
+path omits inline conditioning before the background `put()` completes;
+exhausting its send retries does not restore conditioning to the forwarded
+output. It uses the conventional key and request-scoped sender endpoint,
+not the diffusion producer's handle-based handoff.
+
+If a receive fails, inline continuation is valid only when **all required
+keys have non-`None` values**. Missing required values raise an explicit
+stage-payload error on each rank after fanout. H3 with omitted inline
+conditioning therefore fails explicitly if remote publication or delivery
+fails; the original prompt is not equivalent conditioning. With no declared
+input keys, an explicit handle's `payload_keys` defines the required set.
+
+## NIXL Ownership and Recovery
+
+```mermaid
+stateDiagram-v2
+    [*] --> Published: put registers source
+    Published --> Released: unclaimed lease expires
+    Published --> Claimed: atomic metadata query
+    Claimed --> Reading: consumer submits READ
+    Claimed --> Recovering: reply lost and caller abandons
+    Recovering --> Recovering: query or ACK lost
+    Recovering --> Released: same claim recovered and ACKed
+    Reading --> Deferred: deadline expires while DMA may be active
+    Reading --> Released: terminal DMA and completion ACK
+    Deferred --> Deferred: active or unknown DMA state
+    Deferred --> Released: terminal state and completion ACK
+```
+
+Claims are scoped by payload generation and UUID. Query timeouts or lost
+metadata replies retain the cached UUID for retries, since the producer may
+already have granted ownership. An explicit `META_NOT_FOUND` reply clears
+the cached claim; a subsequent query can use a new UUID. At the end of the
+synchronous retry loop, `abandon_get()`
+moves unresolved queries to background recovery. Recovery asks for the same
+claim, obtains the generation, and acknowledges it **without submitting a
+READ**. A lost completion reply is retried against that exact generation,
+not by acquiring a fresh payload with the same key. Existing claims may be
+recovered while a producer is closing; new claims may not.
+
+If a READ was submitted and its state is active or unknown, timeout does not
+release source or destination allocations. The deferred-transfer reaper keeps
+tensors, registrations, descriptor lists, and transfer handles until NIXL
+reports a terminal state. Completion ACK failure also stays in background
+retry. Producer close can therefore remain deferred until ownership drains.
+
+Recovery is eventual, not guaranteed under permanent network partition or
+consumer process death. A producer cannot safely infer that an unreachable
+consumer has stopped DMA. Conservatively retained ownership in that case is
+an explicit safety limit, distinct from a live consumer silently abandoning
+a recoverable discovery claim.
+
+## Reusing the Interface
+
+| Backend | Synchronous support | Backend responsibility |
+| --- | --- | --- |
+| SharedMemory | Implemented | Nonblocking read-lock acquisition; return absence on contention; shared runtime retries. Copy and deserialize after acquiring the lock. |
+| NIXL | Implemented | Deadline-aware socket and completion waits, generation-scoped claims, deferred active-DMA cleanup, discovery recovery. |
+| Other existing connectors | Opt-in required | Implement `get_with_deadline`; implement `abandon_get` only when discovery creates ownership. Do not inherit an unbounded blocking call. |
+
+To add a model, declare input/output keys and use ordinary prompt and output
+structures. To add a backend, implement the connector capability and ownership
+tests, register it through the existing factory, and select it in deploy
+configuration. Neither requires a backend-specific diffusion model runner.
+
+Worker teardown disables offloading, closes the manager after draining
+prefetch, then tears down native KV and distributed resources in `finally`
+blocks. The synchronous mixin does not close the borrowed connector itself.
+Repeated manager close is safe, including for an unused manager.
+
+## Validation Map
+
+| Test module | Main invariants |
+| --- | --- |
+| `tests/diffusion/test_diffusion_stage_payload.py` | Warmup bypass, strict missing-payload errors, valid inline fallback, source-stage endpoints, TP/SP fanout, standalone HSDP election and SHM fanout, handle publication. |
+| `tests/diffusion/test_diffusion_model_runner.py` | New-request receive and final-only stepwise publication with cached external identity. |
+| `tests/distributed/omni_connectors/test_nixl_connector.py` | Lost replies, same-claim recovery, deadlines, generation isolation, active DMA retention, deferred close and ACKs. |
+| `tests/distributed/omni_connectors/test_shm_connector.py` | Key and metadata reads, lock contention, expired deadline, cleanup. |
+| `tests/worker/test_omni_connector_mixin.py` | Borrowed connector ownership and shared synchronous/asynchronous helpers. |
+| `tests/distributed/omni_connectors/test_kv_async_prefetch.py` | Prefetch ordering and idempotent manager close. |
+| `tests/config/test_omni_config.py` | Tuple normalization across topology, deploy, stage CLI, and serialization. |
+
+Mocked ownership tests do not establish native transport performance or
+accelerator correctness. Native NIXL and model E2E runs are separate checks;
+their results must name the tested revision and runtime.

docs/design/feature/prefix_caching.md

@@ -8,6 +8,7 @@
 - [High-Level Approach](#high-level-approach)
 - [Example](#example)
 - [What About Multimodal Inputs?](#what-about-multimodal-inputs)
+- [Diffusion KV Prefix Caching](#diffusion-kv-prefix-caching)
 - [Implementation](#implementation)
 - [Related Files](#related-files)
 
@@ -25,7 +26,7 @@ vLLM implements automatic prefix caching for managing its kv-cache, which is bes
 - Model / stage specific multimodal data
 
 !!! note "Note 1"
-    This document describes vLLM-Omni's mechanism for caching tensor outputs that are meant to be passed between stages, when requests have common prefixes, similar to the way in which vLLM has prefix caching for the kv-cache. This works in conjunction with vLLM's multimodal encoder caching, but is distinct. See the final section for a concrete example for how they tie together in practice.
+    The following sections describe caching tensor outputs passed between AR stages. This is distinct from multimodal encoder caching and from [diffusion KV prefix caching](#diffusion-kv-prefix-caching), which reuses GPU KV pages inside a diffusion stage.
 
 ### High-Level Approach
 
@@ -45,7 +46,7 @@ Host footprint: each cached key costs `num_blocks × block_size × D × dtype_by
 ### Example
 
 !!! note "Note 3"
-    Prefix caching in vLLM-Omni currently is only supported on AutoRegressive stages with one kv-cache group. Configure it with the pipeline-wide `enable_prefix_caching` field in the deploy config.
+    The stage-output tensor cache described here supports AutoRegressive stages with one kv-cache group. Configure it with the pipeline-wide `enable_prefix_caching` field in the deploy config. Diffusion KV caching has the separate requirements below.
 
 The way in which vLLM-Omni ties into vLLM's prefix caching is best understood by example. Say that we have the following:
 
@@ -167,6 +168,65 @@ When we pass our multimodal tensors to the language model component in the same
 
 Finally, we look up the output hidden states/multimodal tensors corresponding to the prefix cache hit `Block 1` and concatenate it with the forward pass result to get the final result, which is expected to be identical to the full hidden states when prefix caching is disabled.
 
+### Diffusion KV Prefix Caching
+
+HunyuanImage3's standalone DiT pipeline can reuse stable text/reference-image KV
+across requests. Enable it on the diffusion stage in the deploy config:
+
+```yaml
+pipeline: hunyuan_image3_dit
+stages:
+  - stage_id: 0
+    diffusion_kv_mode: paged_scheduler
+    enable_prefix_caching: true
+```
+
+The Scheduler uses vLLM's native `KVCacheManager` to look up and retain complete
+prefix blocks. The Worker owns the GPU pages and block tables. On a hit, the first
+denoise forward computes only the uncached query suffix while attending to both
+cached and new KV. Dynamic target-image KV is never published as a reusable prefix.
+
+Cache identity includes token IDs, reference-image content and VAE random state,
+plus model/layout and LoRA context. The same image with different prompts can
+reuse the common leading blocks; it does not imply that every image span or CFG
+branch is interchangeable. Disabling prefix caching skips cache-identity hashing;
+`dense_legacy` remains the default. Enabling prefix caching with a mode other than
+`paged_scheduler`, without a registered model hook, or together with native
+`kv_transfer_config` raises a configuration error.
+Combining it with `enable_sleep_mode: true` is also rejected: sleep discards KV
+pages without invalidating the Scheduler's prefix-cache index. Disable either
+prefix caching or sleep mode.
+
+The current scope is local DiT reuse, not AR-imported KV or missing-page-only
+cross-stage transfer. Prefix-hit accuracy has been exercised with TP4/SP1 and
+TP2/SP2 (Ulysses), both with EP and CFGP1. Other combinations still require validation. See the
+[diffusion compatibility notes](../../user_guide/diffusion_features.md#diffusion-kv-prefix-caching).
+
+#### Shared-reference benchmark
+
+From the repository root, use the unified `vllm bench serve --omni` test runner:
+
+```bash
+pytest tests/dfx/perf/scripts/run_benchmark.py \
+  --test-config-file tests/dfx/perf/tests/test_hunyuan_image3_prefix_caching.json
+```
+
+Run GPU tests through your environment's GPU scheduler. The suite reuses the
+two-image IT2I accuracy input on a single DiT stage, comparing dense, paged without
+prefix caching, and paged with prefix caching. Each mode runs two ordinary upstream
+warmups followed by eight identical requests at CFG 2.5, 8 denoise steps and seed 42.
+Warmups are excluded from latency and request throughput. This measures full-prefix
+reuse, not the earlier distinct-prompt workload; its results must be reported separately.
+
+`tests/e2e/accuracy/test_hunyuan_image3_prefix_cache_accuracy.py` uses the same input
+at CFG 2.5, 50 steps and seed 42, comparing all modes against the existing official
+Instruct output image and IT2I thresholds (CLIP ≥90, SSIM ≥0.26, PSNR ≥12.5).
+A different prompt warms the cache; the original prompt then exercises partial
+and repeated hits, with reference-image reuse and query slicing verified.
+Only outputs for the original prompt are scored. The golden comes from AR-to-DiT;
+this single-DiT quality check does not establish identical conditioning between
+the pipelines. The existing AR-to-DiT test remains unchanged.
+
 ### Implementation
 
 The block/slot model is `vllm_omni/core/prefix_cache/`.

docs/design/feature/preprocess_phase_contract.md

@@ -42,7 +42,8 @@ Scheduler / current input batch
        prefill / other spans     single-token decode
           preprocess             preprocess or preprocess_decode_batch
               |                              |
-       prompt embedding              runner-managed talker_mtp
+       prompt embedding              decode embeddings
+              |                      talker_mtp (if present)
                   \                 /
                     model forward
 ```
@@ -96,9 +97,24 @@ computed per-row fields are not promised at that earlier boundary.
 ### Routing guarantee
 
 With `has_preprocess=True`, the batched decode hook is eligible only when it is
-callable, the runner has `talker_mtp`, the scheduled span is one token, and
+callable, the scheduled span is one token, and
 `_omni_is_prefill` is false. Otherwise the normal `preprocess` hook runs.
 
+The hook's return format depends on whether the runner manages MTP:
+
+| Runner capability | `preprocess_decode_batch` return tuple |
+| --- | --- |
+| Without `talker_mtp` | `(input_ids, inputs_embeds, updates)` |
+| With `talker_mtp` | `(input_ids, inputs_embeds, hidden_states, text_step, updates)` |
+
+Each tensor's batch dimension and each entry in the `updates` list correspond
+to one physical request row, including separate CFG branches. Non-MTP batches
+contain contiguous single-token decode spans; the runner flushes a batch before
+processing any other span. It copies embeddings into its existing input buffer
+and applies each update dictionary to the corresponding request. Returning the
+original `input_ids` object lets it skip an unchanged token-ID copy. The existing
+five-result MTP format remains supported.
+
 Runner-managed MTP is likewise restricted to single-token decode rows. A prefill
 row, including a one-token tail, is never added to that MTP batch: the runner
 does not consume its `mtp_inputs`, generate codes for it, or overwrite its
@@ -130,6 +146,8 @@ formalizes that existing behavior. Prefer upgrading the matching vLLM and
 vLLM-Omni release pair and consuming the explicit phase keys. Do not silently
 fall back to `input_ids.numel() > 1` when the model requires chunking correctness.
 For a custom or backported runtime, verify both metadata and MTP guards.
+Older runners can restrict `preprocess_decode_batch` to MTP models; a non-MTP
+model must retain its scalar `preprocess` implementation for those runtimes.
 
 ## Correctness and testing plan
 

docs/design/feature/skip_softmax.md

@@ -178,7 +178,7 @@ t = s * u / (1 + (s - 1) * u)
 ```
 
 and a large `s` pushes most positions toward `t ≈ 1`. With the default `s = 12` and
-`num_inference_steps = 50` (50 sigma points, `N = 49` denoiser forwards), the published sequence
+`num_inference_steps = 49` (50 sigma points, `N = 49` denoiser forwards), the sequence
 starts `1.000, 0.998, 0.996, 0.995, 0.993, ...` and stays above `0.9` for 28 steps, giving:
 
 | `disabled_until_timestep` | Dense steps | Skip-Softmax steps |

docs/design/feature/window_sequence_parallel.md

@@ -0,0 +1,186 @@
+# Window-Aligned Sequence Parallelism (SeedVR2)
+
+## Motivation
+
+SeedVR2's diffusion transformer (`NaDiT`) has no global attention: every layer
+partitions the video tokens into 3D windows and attends only inside a window.
+The window size is normalised against a fixed 720p reference area, so raising the
+output resolution grows the *number* of windows, never their size.
+
+Ulysses redistributes the sequence and head dimensions around attention.
+For SeedVR2, window-aligned SP offers a different communication tradeoff: one
+hidden-state exchange per layout boundary versus Q/K/V and output exchanges
+for Ulysses. The actual performance crossover requires measurement.
+Window-aligned SP instead assigns **whole windows** to ranks:
+
+* every rank keeps **all** attention heads, so no head-count divisibility is
+  required,
+* attention inside a layer needs **zero** communication,
+* regular and shifted layouts alternate at every layer, so the released
+  32-layer model has 31 inter-layer transitions, plus one text reduction per
+  layer and a final video gather. A transition can be a local permutation
+  when ownership does not change.
+
+## Window layouts
+
+`nadit.py` ports the reference windowing verbatim (Apache-2.0,
+ByteDance SeedVR2). For a post-patch token grid `(T, H, W)`:
+
+```text
+scale   = sqrt(45 * 80 / (H * W))          # 720p reference area
+wh, ww  = ceil(round(H * scale) / nh), ceil(round(W * scale) / nw)
+wt      = ceil(min(T, 30) / nt)            # BYTEDANCE_MAX_TEMPORAL_WINDOW
+```
+
+The regular method tiles with `slice(it * wt, min((it + 1) * wt, T))`; the
+shifted method moves every cut by half a window and **clips at the boundary**
+(`max(..., 0)` / `min(..., size)`) -- it is not a cyclic `torch.roll`. Both
+methods produce an exact partition of the grid; the shifted layout is genuinely
+different, so an `A -> B -> A -> B` layer schedule needs both directions of
+re-sharding.
+
+Windows are enumerated `for iw ... for ih ... for it ...` (w outermost, t
+innermost) and tokens inside a window are row-major in `(t, h, w)`. The
+canonical token id is `(t * H + h) * W + w`, which is exactly what the reference
+obtains by windowing `torch.arange`.
+
+Consequences that matter (all verified against the reference):
+
+* Window token counts are **ragged** (edge windows are clipped), so a shard must
+  be balanced by token count, not by window count.
+* The number of layout transitions follows the actual layer schedule. For the
+  released 3B checkpoint (`num_layers=32`, alternating methods) there are 31
+  inter-layer transitions, not 16; `ensure_layout(current, required)` derives
+  them instead of assuming `layer % 2`.
+
+## Planner
+
+`nadit.py` implements a deterministic LPT assignment:
+
+```text
+windows sorted by (-video_token_count, original_window_id)
+each window is placed on the rank with the smallest (assigned_tokens, rank_id)
+each rank then lists its windows in ascending original order
+```
+
+The cache key covers the post-patch token grid, the full window method and
+parameters (via a SHA-1 geometry fingerprint), the geometry version, the **SP
+world size** and the planner version, so a different degree can never reuse a
+stale assignment. Caches are LRU-bounded.
+
+## Plan A redistribution
+
+At a layout boundary the activations must move from the source assignment to the
+destination assignment. With `Z_src[r]` / `Z_dst[d]` the canonical token ids held
+by each rank and `C[r, d]` the number of rows rank `d` needs from rank `r`:
+
+* `input_split_sizes` of rank `r` is row `r` of `C`, `output_split_sizes` is
+  column `r`,
+* each peer segment is packed in **destination-local order**, so the receiver can
+  reorder with a single `argsort` of the concatenated destination positions,
+* `network_exchange_required` is computed from the global count matrix, so every
+  rank takes the same branch; a rank that happens to exchange nothing never
+  skips the collective alone.
+
+The runtime is a variable-split `torch.distributed.all_to_all_single` over an
+regular SP process group (`get_sp_group().device_group`), plus a local-permutation fast
+path when the whole group stays in place (which is the SP=1 case). The first
+version is deliberately synchronous: no extra streams, no per-layer
+`cuda.synchronize()` or barrier "fixups", and no hidden all-gather.
+
+## Text stream
+
+Text is replicated on every rank and stays identical: each layer's attention
+produces one text output per window, ranks sum their local window outputs,
+one SUM all-reduce runs on the window-SP group, and every rank divides by the
+**global** window count of the current layout.
+
+```text
+T_next = (sum_r sum_{w in Omega_r} T_w) / W_global
+```
+
+This is neither a token-weighted mean nor a mean of rank means; a rank with no
+window contributes a zero tensor (with the *same dtype as its peers* -- a dtype
+mismatch changes the collective and deadlocks NCCL, which is why the empty-rank
+path is covered by a GPU test).
+
+## Existing SP configuration
+
+Configure `ulysses_degree=N` to create the existing N-rank SP group. Pass
+`DiffusionParallelConfig` to `SeedVR2NaDiT.build_runtime(parallel_config=...)`;
+the model validates the configuration and obtains `get_sp_group().device_group`.
+This adds no framework parallel option or group-construction branch.
+
+For SeedVR2 this selects **window-aligned Plan A**, not head-sharded Ulysses.
+Every rank retains all 20 attention heads, so SP=8 does not require the head
+count to divide by eight. The model has no `_sp_plan`, and its attention uses
+`skip_sequence_parallel=True`; generic sequence hooks and Ulysses attention
+exchanges are not installed. Reports name the actual execution path explicitly.
+
+SeedVR2 rejects ring, allgather, `advanced_uaa`, and `ulysses_a2a_permute`.
+Other diffusion models keep their existing Ulysses/Ring/AllGather behavior.
+
+## SeedVR2 integration
+
+`SeedVR2WindowRuntime` owns, per request: the layer schedule, the entry
+distribution, `ensure_layout` transitions, the text reduction and the final
+reconstruction back to canonical token order. The DiT forward keeps only this
+rank's video rows from the patch embedding onwards; the patch projection,
+norms, ADA modulation, residual and MLP are all tokenwise, so they run locally.
+
+The block never calls a parallel attention strategy: the shared `Attention`
+layer is created with `skip_sequence_parallel=True`, and window attention is a
+packed-varlen call over `[video window, replicated text]` segments (with a
+grouped-SDPA fallback for backends without a packed-varlen entry point).
+
+## Reference quirks reproduced on purpose
+
+* `vid_out_ada` is declared with `layers=["out"]`, whose embedding regrouping is
+  arithmetically incompatible with its 1-D parameters. The reference never uses
+  that value: it asks the forward-wide cache for `emb_repeat_0_vid`, a key the
+  first block already wrote, and therefore applies the *block* attention
+  modulation together with the learned `out_scale` / `out_shift`. The port
+  applies that effective modulation directly; changing it would break parity.
+* `mm_layers=10` controls weight sharing only. Text is consumed and produced by
+  all 32 blocks; only the last block freezes its text MLP path.
+* RoPE positions are window-local: a video token at window-local `(dt, dh, dw)`
+  uses axial indices `(text_len + dt, dh, dw)`, and every window restarts at 0.
+* The rotary span is `3 * (rope_dim // 3)` (126 of 128 channels for the 3B), so
+  the last two channels pass through unrotated.
+
+## Checkpoint compatibility
+
+The parameter layout of the port mirrors the reference, with one explicit
+exception: the released checkpoint stores the per-block RoPE table as
+`blocks.<i>.attn.rope.rope.freqs`, while this port registers it as
+`blocks.<i>.attn.rope.freqs`. The validation loader normalizes exactly that
+suffix (and only that suffix); every other key must already match.
+
+> The validation loader explicitly normalizes the reference RoPE buffer suffix
+> from `.rope.rope.freqs` to `.rope.freqs`. Checkpoint validation fails on any
+> remaining missing or unexpected key after normalization.
+
+Normalization is restricted and collision-checked: a key that is already
+normalized is left untouched, and two source keys mapping onto the same target
+key raise instead of silently overwriting a tensor. Loading then fails on any
+remaining missing, unexpected or shape-mismatched key, so a checkpoint that does
+not match the port cannot reach inference. Full 32-layer loading is the
+acceptance path; truncating the block list is a development fixture and requires
+an explicit flag.
+
+The native serving loader applies the same suffix normalization and strict
+checkpoint check.
+
+## Integration limits
+
+The native serving pipeline passes its parallel configuration to the DiT runtime
+and returns restored video through the standard media path. The causal VAE can
+run replicated, tiled, or height-sharded across the window-SP group.
+
+## Limitations
+
+* Plan A re-shards the full activation at every layout boundary. Plan B halo
+  exchange is a follow-up with a separate crossover evaluation.
+* The packed-varlen attention kernel needs a backend that accepts
+  `cu_seqlens`; the grouped-SDPA path is the portable fallback and is what the
+  pinned L20 environment exercises.

docs/design/fullduplex-personaplex.md

@@ -1,142 +1,137 @@
-# PersonaPlex Unified Full-Duplex Design
+# PersonaPlex on the Unified Full-Duplex Framework
 
 ## Status and target
 
-This design adapts PersonaPlex PR #4771 to the unified full-duplex runtime on
-top of:
-
-- vLLM-Omni `origin/main`: `67c54777bb22e9e7e08fdf7c47a64f06b566fc47`
-- PersonaPlex PR head: `477fb7c225f0c06991bc8aa55eadbd908ba282e4`
-
-The target is the engine-native path:
+PersonaPlex (`nvidia/personaplex-7b-v1`, a Moshi finetune) is served by the
+[Unified Full-Duplex Framework](fullduplex.md) through one model plugin,
+`PersonaPlexDuplexPlugin` (`vllm_omni/model_executor/models/personaplex/duplex/plugin.py`),
+selected by `PipelineConfig.duplex_plugin`. The serving path is:
 
 ```text
 /v1/realtime?duplex=1
-  -> OpenAI Realtime session actor
-  -> DuplexRequestClient
-  -> AsyncOmni correlated RPC
-  -> DuplexControlPlane
-  -> resumable Stage 0 request
-  -> PersonaPlex Talker
-  -> streaming PersonaPlex Code2Wav
-  -> PersonaPlex data-plane projector
+  -> OmniDuplexSessionHandler (thin: websocket I/O, Realtime <-> DuplexCommand/DuplexEvent)
+  -> DuplexOmni / DuplexOmniEngine
+  -> DuplexSessionManager -> DuplexSessionRunner (one per session)
+       plugin.plan_append: one 80 ms frame -> one resumable Stage 0 append
+  -> PersonaPlex Talker (Stage 0, lockstep temporal transformer + depformer)
+  -> streaming PersonaPlex Code2Wav (Stage 1, Mimi)
+  -> plugin.data_plane: cumulative audio/text -> deltas
   -> response.output_audio.delta + response.output_audio_transcript.delta
 ```
 
-The standalone `/api/chat` and `/v1/audio/duplex` server that accompanied the
-original PR was demo-only and has been removed from the tree; the unified
-engine path above is the only serving surface. It was never evidence that the
-unified engine path works.
-
-## Why configuration-only enablement was invalid
-
-The staged pipeline as shipped by the original PR was explicitly turn based.
-Its Talker read `pplex_user_codes`, `pplex_prefill_text`, and
-`pplex_silence_codes`, but no production staged input path wrote those fields.
-The voice prompt, persona prefill, and streaming Mimi state lived only in the
-standalone `PersonaPlexEngine` (demo-only, since removed).
-
-Setting only the following fields would therefore advertise an endpoint whose
-model never receives the live microphone stream:
-
-```python
-duplex_control_enabled = True
-duplex_runtime_extension = "..."
-duplex_serving_adapter = "..."
-```
-
-The adapter must supply a real scheduler data plane, not just endpoint
-capabilities.
+The pre-framework pair (engine runtime extension plus serving adapter) and
+the standalone `/api/chat` / `/v1/audio/duplex` server are gone; the plugin is
+the only integration surface, and this document describes it.
 
 ## Supported scope
 
-The unified implementation supports:
+The integration supports:
 
-- up to two engine-owned sessions on one replica;
-- 24 kHz mono float PCM input;
-- one 1920-sample, 80 ms model frame per physical append unit;
-- continuous user input while assistant audio is generated or played;
-- bundled `.pt` voice prompts and a session persona;
-- greedy text and depformer sampling, matching the current PersonaPlex port;
-- `/v1/realtime?duplex=1` (the wire vocabulary is catalogued in the
-  [Realtime Duplex API](../serving/realtime_duplex_api.md) serving guide);
+- up to `duplex_session.max_sessions` engine-owned sessions on one replica
+  (the shipped deploy sets two);
+- 24 kHz mono float PCM input, one 1920-sample (80 ms) model frame per
+  physical append unit; clients may send any chunking, the session buffers
+  whole frames;
+- continuous user input while assistant audio is generated or played
+  (pure lockstep: the model listens while it speaks);
+- bundled `.pt` voice prompts (`voice`) and a session persona (`instructions`);
+- greedy text and depformer sampling (one temporal token per frame);
 - the public client preset
   `vllm_omni.clients.personaplex.create_duplex_session_config()` (24 kHz
-  `pcm_f32le` input format, voice prompt, persona) as the canonical
-  session-config source for `DuplexClient` consumers;
-- engine lease close, disconnect cleanup, reconnect after cleanup, and explicit
-  response cancellation without cross-session state reuse.
+  `pcm_f32le` input, voice, persona) as the canonical session-config source
+  for `DuplexClient` / `InlineDuplexClient` consumers;
+- engine lease close, disconnect cleanup and explicit response cancellation
+  without cross-session state reuse.
 
-The implementation does not claim:
+It does not claim:
 
-- more than two simultaneous PersonaPlex sessions on one replica;
-- arbitrary WAV voice cloning;
-- turn-based `response.create` semantics for an otherwise continuous model;
-- scheduler migration of live codec state between replicas;
-- exact output equality with the standalone engine after different scheduling
-  boundaries.
+- arbitrary WAV voice cloning (voices are bundled basenames, resolved by the
+  worker under the checkpoint);
+- turn-based `response.create` semantics, client commits or external turn
+  signals (`supports_client_commit=false`, `supports_external_turn_signal=false`);
+- text seeding, hence no `/v1/chat/completions` route
+  (`supports_chat_completions=false`);
+- session resume across a transport drop (`supports_session_resume=false`);
+- destructive output interruption or model-state rewind at a playback cursor
+  (`supports_barge_in=false`, `supports_audio_truncate=false`): overlapping
+  speech is native model behaviour, not a barge-in contract.
 
-The capability payload derives multi-session support from the configured
-session limit. The shipped two-session deployment reports
-`supports_multi_session=true` and `supports_multi_session_same_replica=true`.
 `duplex_session.max_sessions` is the only capacity source: config resolution
-propagates it to every stage as `duplex_max_sessions`, and both Mimi pools read
-that model-config value. Connector extras do not carry a second model-specific
-capacity knob that could drift from engine admission.
-It still reports `supports_barge_in=false`: the generic epoch fence can suppress
-stale transport output, but neither PersonaPlex nor the current MiniCPM-o 4.5
-adapter proves that model-owned streaming state can be destructively rewound or
-restarted at a playback cursor. Continuous overlapping speech is model-native
-duplex behavior, not by itself a barge-in contract.
+propagates it to every stage as `duplex_max_sessions`, and both Mimi pools
+(Stage 0 encoders, Stage 1 decoders) read that model-config value. The
+capability payload derives `supports_multi_session` /
+`supports_multi_session_same_replica` from it.
 
 ## Components and ownership
 
-### Serving adapter
+### The plugin (engine side)
 
-`PersonaPlexServingRuntimeAdapter` owns only serving-side state:
+`PersonaPlexDuplexPlugin` owns both halves of the contract.
 
-- a transactional PCM append buffer;
-- validation of `voice_prompt` and `instructions`;
-- public capabilities;
-- PersonaPlex data-plane output projection.
+Engine policy:
 
-It accepts 24 kHz `pcm_f32le`, groups client packets into whole 1920-sample
-frames, zero-pads only the final residual, and rolls a reservation back when an
-engine append fails. It never loads CUDA weights and never encodes user audio.
+- `configure_sampling_params`: Stage 0 greedy (`temperature=0`, `top_k=1`,
+  `max_tokens=1`), other stages untouched.
+- `plan_append`: validates exactly one 1920-sample 24 kHz `pcm_f32le` frame
+  (through `model_executor/common/duplex/payload.py`) and reserves one
+  scheduler slot per frame plus, on the first append of an epoch (`seq == 1`),
+  the `personaplex_prefill_slots` of the voice/persona prefill. The prompt
+  carries the fence, `seq`, the payload and the runtime config under
+  `model_intermediate_buffer["duplex"]`.
+- `decide_output`: never decides. PersonaPlex is always-clocked; what the
+  client hears comes from the final-stage data plane.
+- `silence_unit_payload`: one frame of zeros at 24 kHz
+  (`silence_continuation_samples=1920`,
+  `silence_continuation_sample_rate_hz=24000`). The runner uses it to keep a
+  model turn clocked when the client pauses; the startup warmup sends it.
 
-Client input cannot provide local filesystem paths. A voice is a bundled
-basename such as `NATF2.pt`; the worker resolves it under the local model
-checkpoint. `instructions` is the persona string.
+Session policy:
 
-### Runtime extension
+- `capabilities`: `personaplex_capabilities(max_sessions)` (80 ms units,
+  append-only, no commits, no barge-in, no resume, no chat route).
+- `prepare_runtime_config`: `voice` must be a bundled `.pt` basename,
+  `instructions` defaults to the shipped persona; the prefill slot count is
+  computed once per `(model, voice, persona)` off the orchestrator loop
+  (`personaplex_prefill_slots`, itself caching the tokenizer and the voice
+  bundle row count). The result is server-owned runtime config
+  (`personaplex_model_path`, `personaplex_voice_prompt`,
+  `personaplex_persona`, `personaplex_prefill_slots`); the same keys are
+  refused in a client's `extra_body`.
+- `runtime_config_for_update`: persona and voice are immutable for the
+  session (`persona_update_unsupported`, `voice_update_unsupported`).
+- session state, extra-body validation and data-plane context are the
+  framework defaults (`DefaultDuplexModelSessionState` with the 80 ms
+  `PersonaPlexPcmAppendBuffer`, `DuplexDataPlaneContext`).
 
-`PersonaPlexDuplexRuntimeExtension` is pure model policy. It:
+Because `supports_client_commit` is off, the session auto-responds without
+`extra_body.auto_response`: a stock Realtime client streams audio and hears
+the model without any vendor flag.
 
-- configures greedy Stage 0 sampling and bounded segment lengths;
-- maps each accepted PCM append to a scheduler prompt;
-- places immutable session identity, append sequence, PCM payload, voice, and
-  persona under `model_intermediate_buffer["duplex"]`;
-- reserves exactly one scheduler prompt slot per encoded Mimi frame, plus the
-  first-append voice/persona prefill length;
-- never performs model inference or owns session state.
+### Input framing
 
-The extension returns no turn/listen decision. PersonaPlex is an always-clocked
-model, so visible audio/text comes from the final stage data plane.
+`PersonaPlexPcmAppendBuffer` is the `FixedFramePcmAppendBuffer` of
+`model_executor/common/duplex/pcm_buffer.py` at 24 kHz / 1920 samples /
+80 ms. It accepts 24 kHz `pcm_f32le` only, groups client packets into whole
+frames, takes one frame out per append as a reservation (committed when the
+stage accepted the append, rolled back to the front of the buffer when it did
+not), zero-pads only a final residual on commit, and never encodes audio.
 
-### Stage 0 streaming runtime
+### Stage 0 streaming runtime (worker side)
 
-The Talker owns a `PersonaPlexStage0DuplexRuntime` helper, analogous to
-MiniCPM-o's Stage 0 helper but with PersonaPlex lockstep semantics.
+The Talker owns a `PersonaPlexStage0DuplexRuntime`
+(`model_executor/models/personaplex/duplex/stage0.py`), created lazily in
+`_duplex_stage0_runtime()` and released through `on_requests_finished`.
 
-For each admitted session it owns:
+For each live `(session_id, epoch)` it owns:
 
-- streaming Mimi encoder convolution and transformer state;
-- the selected voice embedding bundle;
-- persona tokenization and prefill embeddings;
-- the prior user code frame needed by the one-frame acoustic delay;
-- append identity used to make a retried scheduler update idempotent.
+- the streaming Mimi encoder convolution and transformer state;
+- the selected voice embedding bundle and the persona prefill embeddings;
+- the prior user code frames needed by the one- and two-frame acoustic
+  delays;
+- the append identity `(epoch, seq)` that makes a retried scheduler update
+  idempotent.
 
-The first append builds this ordered prefill:
+The first append of an epoch builds this ordered prefill:
 
 ```text
 voice embeddings
@@ -146,45 +141,47 @@ voice embeddings
   -> first live user frame
 ```
 
-Later appends encode only new 1920-sample frames. The helper returns the
+Later appends encode only new 1920-sample frames. The runtime returns the
 per-frame user codes and prompt embeddings through the request's
-`model_intermediate_buffer`. The normal vLLM runner remains authoritative for
-attention metadata, block tables, KV allocation, scheduling, and sampling.
+`model_intermediate_buffer`; the normal vLLM runner remains authoritative for
+attention metadata, block tables, KV allocation, scheduling and sampling.
 
-The Talker must distinguish resumable prompt prefill from decode positions.
-Prompt rows consume the exact prepared embeddings; sampled decode rows continue
-to use the existing delayed agent/user frame construction. No code path may
-fall back to an all-initial user stream for a duplex request.
+**Epochs.** The framework identifies a Stage 0 request by
+`(session_id, epoch)`; `response.cancel` and `output_audio_buffer.clear`
+advance the epoch, abort the current request and start the next append at
+`seq == 1` on a fresh request with fresh KV. The Stage 0 runtime therefore
+keys its state by `(session_id, epoch)`: when an append of a newer epoch
+arrives, any older-epoch state of the same session is closed first (its Mimi
+encoder returns to the pool, so the codec budget never counts a superseded
+epoch), and the voice/persona prefill is replayed because the plan reserved
+the slots again. The user-visible consequence is that a cancel restarts the
+model's conversation context. A late `on_requests_finished` for the aborted
+request finds nothing to close.
 
-Cleanup is keyed by the full `(session_id, incarnation)` identity. Every live
-session has an independent Mimi encoder instance; encoder convolution/KV state
-is never shared between asynchronously scheduled sessions. A finished or
+Every live session has an independent Mimi encoder instance; encoder state is
+never shared between asynchronously scheduled sessions. A finished or
 aborted scheduler request resets and returns only that session's encoder.
 
 ### Stage 1 streaming decoder
 
-The current `PersonaPlexCode2Wav` calls one-shot `MimiModel.decode` and is not
-CUDA-graph safe. Unified duplex uses eager Stage 1 and maintains an independent
-streaming Mimi decoder for every active request. Decoder ownership is keyed by
-the stable Stage 1 request id and released by `on_requests_finished`; a mixed
-batch must never advance another request's convolution or transformer state.
-
-Each Stage 0 segment emits de-delayed agent codebooks. Stage 1 decodes only the
-new code frames, emits only the new PCM suffix, and resets state when the
-request is closed. Connector chunk boundaries retain the final raw code frame
-needed to de-delay the next chunk.
-
-The deploy default sets Stage 1 `enforce_eager: true`; a default configuration
-that fails during CUDA graph capture is not an acceptable deployment profile.
+`PersonaPlexCode2Wav` runs eager (`enforce_eager: true` in the deploy) and
+maintains an independent streaming Mimi decoder for every active request,
+keyed by the Stage 1 request id and released by `on_requests_finished`. Each
+Stage 0 segment emits de-delayed agent codebooks; Stage 1 decodes only the new
+code frames and emits only the new PCM suffix. Connector chunk boundaries
+retain the final raw code frame needed to de-delay the next chunk.
 
 ### Data-plane projector
 
-`PersonaPlexDataPlaneSession` converts cumulative or delta Stage 1 output into
-model-neutral native results:
+`PersonaPlexDataPlaneSession` is the `CumulativeAudioTextDataPlane` of
+`model_executor/common/duplex/data_plane.py` with a 24 kHz default rate. It
+keeps one audio/text cursor per request so cumulative Stage 1 output cannot
+replay old audio, and yields one internal result per new suffix:
 
 ```python
 {
     "stage_role": "tts",
+    "is_listen": False,
     "data_plane_request_id": request_id,
     "text": text_delta,
     "audio_data": encoded_audio_delta,
@@ -192,82 +189,74 @@ model-neutral native results:
     "sample_rate_hz": 24000,
     "audio_duration_ms": delta_duration,
     "end_of_turn": False,
+    "runtime_impl": "scheduler_data_plane",
+    ...
 }
 ```
 
-It owns per-request audio and text cursors so a cumulative output cannot replay
-old audio. The generic projector turns each native result into the Realtime
-`response.output_audio.delta` / `response.output_audio_transcript.delta` pair (and
-`response.output_text.delta` for text) under one `response_id`; the full
-mapping is the name map in the
-[Realtime Duplex API](../serving/realtime_duplex_api.md) serving guide.
-PersonaPlex keeps one visible response open while continuous output arrives.
-Session close or cancellation terminates that response through the generic
-Realtime lifecycle; a codec segment finishing is not a conversational turn
-boundary. PersonaPlex advertises `supports_barge_in=false` and
-`supports_session_resume=false`, so `barge_in`, `turn_detection.server_vad`,
-and `session.resume` are rejected on this model.
+The runner turns each result into the Realtime
+`response.output_audio.delta` / `response.output_audio_transcript.delta` pair
+under one `response_id`. PersonaPlex keeps one visible response open while
+continuous output arrives; session close or cancellation terminates it through
+the generic Realtime lifecycle. A codec segment finishing is not a
+conversational turn boundary.
 
 ## Error and lifecycle contracts
 
-- Unsupported sample rate, malformed base64, non-finite PCM, invalid voice
-  basename, or changed format fails before scheduler submission.
+- Unsupported sample rate, malformed base64, non-finite PCM, a partial frame
+  at the plan, an invalid voice basename or a changed format fails before
+  scheduler submission, as a typed `error` event.
 - Append is prepare/submit/commit. Failure rolls back the exact reserved PCM
   bytes and does not advance the model frame cursor.
-- A repeated `operation_id` must not encode or submit the same frame twice.
-- Input iterator exceptions execute the same cleanup as explicit close.
-- Cancellation does not release the engine lease until the stage request and
-  any in-flight codec operation are actually finished.
-- A bounded cleanup timeout returns a cleanup error and keeps the session in
-  the closing admission set; it must not make the slot available while work
-  still mutates shared state.
-- New sessions cannot observe the previous voice, persona, PCM tail, Mimi
-  convolution state, or Talker delayed code frame.
-
-(The legacy `PersonaPlexDuplexRuntime.run()` and standalone server drain path,
-which followed the same exception-safe rule, have been removed from the tree.)
+- A repeated `(epoch, seq)` identity does not encode or submit the same frame
+  twice.
+- Cancellation advances the epoch; the aborted request's Stage 0 state is
+  released by the next-epoch append or by `on_requests_finished`, whichever
+  comes first.
+- Close releases the admission slot only once stage cleanup succeeded; the
+  reaper retries a failed cleanup while the slot stays held.
+- New sessions cannot observe a previous voice, persona, PCM tail, Mimi
+  convolution state or Talker delayed code frame.
 
 ## Testing and acceptance
 
-### Contract tests
+### CPU contract tests
 
-Tests first cover:
+- `tests/model_executor/models/personaplex/duplex/test_plugin.py`: pipeline
+  binding and plugin load, honest capabilities, private keys, voice/persona
+  resolution and caching, immutable updates, greedy sampling, one-slot-plus-
+  prefill planning, frame validation, the 24 kHz silence unit, and the
+  CPU-checked helpers of the E2E driver.
+- `tests/model_executor/models/personaplex/duplex/test_stage0_runtime.py`:
+  first-append prefill, depformer teacher forcing, causal user-frame delays,
+  idempotent retries, independent encoders per session, capacity, epoch
+  restart and late finish of a superseded request.
+- `tests/engine/duplex/test_session_runner_personaplex.py`: the session
+  runner with the real plugin -- one submission per frame, prefill on the
+  first append only, half frames buffered, wrong rate refused, cumulative
+  Code2Wav output projected as 24 kHz deltas, cancel restarting the epoch,
+  close aborting the request.
+- `tests/model_executor/executor_common/`: the shared toolbox (PCM helpers, payload
+  validation, fixed-frame buffer, cumulative data plane, request-output
+  readers).
 
-- pipeline registration enables the control plane and selects both PersonaPlex
-  adapters;
-- the capability payload is two-session, 80 ms, append-only, and honest;
-- PCM reservation commit/rollback, partial-frame flush, invalid input, and
-  operation idempotency;
-- runtime prompt fields and exact token budgeting;
-- first-append voice/persona prefill followed by live user codes;
-- later appends retain per-session Mimi state and do not replay prefill;
-- interleaved Stage 0 and Stage 1 work preserves independent codec histories;
-- output projection emits only audio/text deltas;
-- close, exception, timeout, and reconnect cleanup;
-- ordinary non-duplex imports do not load PersonaPlex modules.
+### GPU validation
 
-### Remote H20 validation
+`tests/e2e/online_serving/personaplex_realtime_duplex.py` (wrapped by
+`tests/e2e/online_serving/test_personaplex_duplex.py` when
+`PERSONAPLEX_MODEL_PATH` is set) drives the default `personaplex.yaml`
+deployment and requires:
 
-Validation runs in an isolated remote worktree using the ModelScope
-`nv-community/personaplex-7b-v1` checkpoint and its Mimi dependency.
-
-The required evidence is:
-
-1. default `personaplex.yaml` reaches ready without a local eager override;
-2. `/health` returns 200;
-3. `/v1/realtime?duplex=1` reports `model_native_duplex`, `chunk_period_ms=80`,
-   and two-session admission;
-4. paced 24 kHz PCM appends produce finite, non-silent 24 kHz audio deltas and
-   text deltas;
-5. microphone input continues during assistant output without cancelling the
-   scheduler request;
-6. two paced sessions simultaneously produce independent non-empty audio, and
-   a third session is rejected with `resource_exhausted`;
-7. closing either session frees only its scheduler request and GPU codec state,
-   after which a replacement session can use a different persona without state
-   leakage;
-8. malformed input returns one typed error and does not poison the next append;
-9. all focused unit tests and `git diff --check` pass.
+1. the server reaches ready without a local eager override; `/health` is 200;
+2. `/v1/realtime?duplex=1` reports `model_native_duplex`, `chunk_period_ms=80`
+   and two-session admission, with server-allocated session ids;
+3. paced 24 kHz PCM appends produce finite, non-silent, whole-frame 24 kHz
+   audio deltas and text deltas;
+4. two paced sessions simultaneously produce independent non-empty audio, and
+   a third session is refused with `resource_exhausted`;
+5. closing one session frees only its scheduler request and codec state, after
+   which a replacement session with a different persona is admitted with a
+   fresh id and no state leakage.
 
 Audio that is empty, all zero, non-finite, or only a protocol `listen` event is
 not a successful end-to-end result.

docs/design/fullduplex.md

@@ -242,10 +242,12 @@ vllm_omni/
 │           └── overlap_policy.py / commit_policy.py / playback_ledger.py
 ├── config/stage_config.py           PipelineConfig.duplex_plugin; DuplexSessionRuntimeConfig
 ├── model_executor/models/minicpmo_4_5/duplex/plugin.py   MiniCPMO45DuplexPlugin (+ data_plane, input, policy, ...)
+├── model_executor/models/nemotron_voicechat/duplex/plugin.py   NemotronVoiceChatDuplexPlugin (+ data_plane, input, capabilities)
 └── clients/
     ├── duplex.py                    DuplexClientBase (ABC), DuplexClient (websocket), client-side events
     ├── inline_duplex.py             InlineDuplexClient (in-process, over DuplexOmni)
     ├── minicpmo_4_5.py              MiniCPM-o 4.5 session preset
+    ├── nemotron_voicechat.py        Nemotron VoiceChat session preset
     └── personaplex.py               PersonaPlex session preset
 ```
 
@@ -283,8 +285,13 @@ command  handle.submit(DuplexCommand)
          -> DuplexSessionManager.dispatch: unknown_session / input_backpressure checks, then runner mailbox
 output   DuplexOrchestrator._intercept_stage_output -> runner.on_stage_output -> mailbox -> typed events
          -> output_sink (DuplexSessionEventMessage) -> DuplexOmni._route_engine_message -> handle.events()
-detach   DuplexOmni.detach_session -> touch(DETACH): engine-owned disconnect grace; expiry -> SessionExpired
-resume   DuplexOmni.resume_session(expected_lease_generation) -> lease CAS; the existing handle is re-entered
+detach   DuplexOmni.detach_session(expected_lease_generation) -> touch(DETACH): engine-owned disconnect
+         grace, refused for a lease newer than the one the caller held; expiry -> SessionExpired
+resume   DuplexOmni.resume_session(expected_lease_generation) -> lease CAS keyed by the control id (a replay
+         answers with the generation it produced); the existing handle is re-entered. A caller cancelled
+         mid-RPC observes the outcome afterwards (replaying until the engine answers) and settles a landed
+         resume: the generation goes to the resume waiting to activate, else to the connection still
+         attached, else the lease is detached again
 close    DuplexOmni.close_session -> close RPC; the manager tears the runner down, then the stage cleanup
          (abort submitted requests, release reserved ids), then SessionClosed, then the RPC result.
          SessionClosed is emitted after the cleanup attempt (in a finally), so it is also sent when the
@@ -383,7 +390,7 @@ policy that used to be two separately configured objects:
 | Half | Members |
 | --- | --- |
 | engine policy | `configure_sampling_params(runtime_config, defaults)`, `plan_append(...) -> DuplexAppendPlan` (the resumable Stage0 prompt for one unit), `decide_output(...) -> DuplexOutputDecision \| None` (e.g. the listen decision on a finished Stage0 segment) |
-| session policy | `capabilities(max_sessions)`, `validate_client_extra_body`, `prepare_runtime_config(config, model_config)` (server-owned runtime keys, reference audio resolution), `runtime_config_for_update`, `runtime_config_for_function_output`, `create_session_state() -> DuplexModelSessionState`, `data_plane: DuplexDataPlane` (projects raw stage outputs into internal events), `data_plane_context(...)` |
+| session policy | `capabilities(max_sessions)`, `validate_client_extra_body`, `prepare_runtime_config(config, model_config)` (server-owned runtime keys, reference audio resolution), `runtime_config_for_update`, `runtime_config_for_function_output`, `runtime_config_after_model_output` (consumption acknowledgement), `create_session_state() -> DuplexModelSessionState`, `data_plane: DuplexDataPlane` (projects raw stage outputs into internal events), `data_plane_context(...)` |
 
 `DuplexOmniEngine._validate_deployment` loads the plugin before any stage
 starts; `DuplexSessionManager.__init__` validates it against the stage
@@ -391,12 +398,12 @@ sampling defaults once the stage pools exist. Plugin hooks that may block
 (`prepare_runtime_config` fetching `ref_audio`) are awaited in `open()` and
 offloaded from the loop.
 
-The MiniCPM-o 4.5 plugin (`model_executor/models/minicpmo_4_5/duplex/plugin.py`)
-is the only integration in this framework version. PersonaPlex and Nemotron
-VoiceChat still carry their pre-framework duplex code (runtime extension plus
-serving adapter) and are therefore not served over this framework yet: their
-pipelines declare no `duplex_plugin`, so they run turn-based until the
-follow-up PRs port them (RFC vllm-omni#7181, PR 2/3).
+Frame-based plugins use `engine/duplex/intermediate.py::build_duplex_append_prompt`
+for the shared request identity, sequencing and config snapshots. Token budgets,
+PCM framing and model-specific worker fields remain in the plugins.
+
+See [supported models and deployments](../serving/full_duplex_api.md#enable-full-duplex)
+for the current plugin integrations and deployment configurations.
 
 ## Serving
 

docs/design/index.md

@@ -68,6 +68,7 @@ guides list selectable backends, platform defaults, installation, and tuning.
 The design contracts separate selection mechanics from backend algorithms:
 
 - [Attention Backend Selection](feature/attention_backend_selection.md)
+- [Attention Execution Contract PoC](feature/attention_execution_contract_poc.md)
 - [Skip-Softmax](feature/skip_softmax.md)
 
 #### CPU offloading
@@ -115,3 +116,6 @@ The design contracts separate selection mechanics from backend algorithms:
 The pre-#5137 pages are preserved in the
 [legacy module archive](module/archive/README.md) for historical reference and
 are not active design contracts.
+
+- [Qwen3-Omni MRv2 performance](qwen3_omni_mrv2_performance.md)
+- [MiniCPM-o 4.5 turn-mode MRv2 performance](minicpm_o45_mrv2_performance.md)

docs/design/minicpm_o45_mrv2_performance.md

@@ -0,0 +1,22 @@
+# MiniCPM-o 4.5 MRv2 profiles
+
+Turn mode uses the stage-level MRv2 contracts from #8184. Thinker emits live
+latent metadata outside graph replay; Talker keeps codec history and EOS
+control on device. Code2Wav reuses mainline Whole-Euler Flow graphs and shared
+prompt state. The previous experimental block compilation, tiled attention,
+channels-last and merged-CFM implementations are removed.
+
+`minicpmo_4_5_turn_mrv2_h200.yaml` selects Talker capacity 16 and 4 GiB KV;
+the generic MRv2 profile selects capacity 8 and 2 GiB KV. These configurations
+require new end-to-end performance measurements after the mainline integration.
+Previously reported numbers do not describe this revised codec backend.
+
+The generic MRv2 profile and the opt-in native V1 duplex H200 profile enable
+TF32 matmuls within Code2Wav forward/capture only; the previous process policy
+is restored afterwards. cuDNN's TF32 policy is independent. TF32 changes rounding.
+Duplex keeps the mainline V1 session path. Turn results do not establish duplex
+performance or interruption correctness.
+
+The shared asynchronous output snapshot and batched Talker preprocessing also
+apply to V1 when async chunking/scheduling is enabled. Default V1 behavior must
+therefore be included in end-to-end regression validation.

docs/design/module/diffusion/diffusion_model_integration.md

@@ -57,3 +57,58 @@ cross-stage routing.
 
 Test registry selection, checkpoint loading, minimal inference, input and output
 contracts, and every declared optional capability.
+
+## Exact conditioning projection reuse
+
+Models with repeated conditioning projections can use `ExactProjectionCache`
+from `vllm_omni.diffusion.cache`. It stores actual projection outputs for identical
+conditioning tensors and unchanged weights, with a byte budget and TP-wide hit
+coordination. It has no model names, sampling schedules, projection layouts, or
+artifact formats built in. MiniMax-H3 is its first model integration.
+
+Keep one cache per model instance and prepare each conditioning tensor once
+before its projections. For a leaf linear with fixed SiLU preprocessing:
+
+```python
+from torch.nn.functional import silu
+
+from vllm_omni.diffusion.cache import ExactProjectionCache
+
+cache = ExactProjectionCache(max_bytes=256 * 1024**2)
+
+# Inside an inference forward, with gradients disabled:
+cache.prepare(conditioning)
+modulation = cache.project(
+    "block.0.modulation",
+    projection,
+    conditioning,
+    lambda: projection(silu(conditioning)),
+)
+```
+
+The original computation must depend only on the supplied conditioning tensor,
+the leaf projection's weights/buffers, and fixed preprocessing. Include prompt,
+guidance, modality, or other request-dependent conditioning in the actual input;
+a timestep-only key is unsafe when those affect the result. Do not mutate the
+input between `prepare` and `project`, or modify outputs returned on cache hits.
+Use separate cache instances for concurrent forwards with different inputs.
+
+Call `clear()` when loading or switching adapters, changing projection behavior,
+or moving the model. Weight-version checks invalidate individual entries.
+Compilation, gradient-enabled execution, projection hooks, and unversioned
+inference weights use the original computation. Validate cold/warm parity,
+invalidation, and rank-local misses when integrating another model.
+Hook checks include global hooks and hooks on wrapped submodules. Numerical
+settings are captured at each projection call, so entering an autocast context
+after `prepare()` cannot reuse an output from a different precision.
+
+Validated offline results can be supplied through `_lookup_precomputed()`;
+this extension is consulted only at TP1 on runtime-cache misses. Artifact
+validation, weight binding, and model-specific layouts remain in the adapter.
+H3's adapter retains its main/Ref2VA sidecars and their eager-only admission.
+
+The cache retains projection results and leaves all original weights available.
+It does not reduce weight residency or cancel layerwise prefetch. A future
+integration that skips weight transfers must coordinate with `OffloadPlan`
+before prefetch and provide a safe miss path; moving weight storage currently
+causes conservative cache invalidation.

docs/design/qwen3_omni_mrv2_performance.md

@@ -0,0 +1,22 @@
+# Qwen3-Omni MRv2 performance
+
+This model adaptation builds on #8184's existing stage runtime and output
+contracts. It adds Talker CPU metadata parsing, sampled-token embedding
+handoff, an optional first-frame decoder, length-grouped Code2Wav graph
+buckets, and opt-in incremental short-KV code prediction. The H200 profile
+also enables conservative fused sampling and bounded cuDNN algorithm search.
+The shared predictor keeps the upstream re-prefill helper and its default
+sampling behavior; short KV is explicitly selected by the deployment profile.
+
+Use `vllm_omni/deploy/qwen3_omni_moe_mrv2_h200.yaml` for the experimental H200
+profile. Preserve a control with `code_predictor_fused_sampling=false` and
+`codec_cudnn_benchmark=false` when C64 P99 is the primary requirement.
+cuDNN search restores process settings on exit and has startup cost. Kernel
+arithmetic is not bitwise-equivalent; the first-frame decoder uses additional
+model memory. Full-duplex performance is not established by turn-mode tests.
+
+First-frame delivery requires `talker_first_audio: true` in connector extra,
+a one-frame initial codec chunk, CUDA, a local single-rank Talker and prefix caching disabled.
+Other deployments retain the regular Code2Wav path.
+`codec_fused_snake` explicitly opts CUDA profiles into fused decoder blocks;
+the base V1 profile retains its original blocks and indexed text cache.

docs/design/qwen3_omni_tts_performance_optimization.md

@@ -93,13 +93,13 @@ Each optimization stacks on the previous one. The summary plots below show the c
 
 **Benchmark environment:**
 
-| | Qwen3-Omni                  | Qwen3-TTS |
-| --- |-----------------------------| --- |
-| **GPU** | A100                        | H200 |
+| | Qwen3-Omni | Qwen3-TTS |
+| --- | ----------------------------- | --- |
+| **GPU** | A100 | H200 |
 | **Model** | Qwen3-Omni-30B-A3B-Instruct | Qwen3-TTS-12Hz-1.7B-CustomVoice |
-| **vLLM** | v0.17.0                     | v0.18.0 |
-| **vllm-omni** | commit 199f7832             | v0.18.0rc2 |
-| **CUDA** | 12.9                        | 12.8 |
+| **vLLM** | v0.17.0 | v0.18.0 |
+| **vllm-omni** | commit 199f7832 | v0.18.0rc2 |
+| **CUDA** | 12.9 | 12.8 |
 
 This post walks through each optimization in the same order they are typically enabled in practice, then ends with deployment playbooks for both models.
 
@@ -310,7 +310,68 @@ self._compiled_model_fwd = torch.compile(
 
 `mode="default"` is used instead of `mode="reduce-overhead"` to avoid conflicts with vLLM's own CUDA graph capture on the main Talker model. `dynamic=True` handles the growing sequence length without recompilation.
 
-These optimizations are always-on in the current codebase - all Qwen3-TTS benchmark results in this post include them.
+The historical benchmark results above use re-prefill. It remains the default
+for Qwen3-TTS and the shared Qwen3-Omni predictor.
+
+### Experimental Qwen3-TTS single-GPU throughput profile
+
+The opt-in `qwen3_tts_high_concurrency_mrv2_single_gpu.yaml` profile uses a
+Qwen3-TTS-specific execution path on CUDA with BF16 weights:
+
+- `code_predictor_kv_cache` reuses each layer's keys and values across the 15
+  residual-codebook predictions within one audio frame. The first step writes
+  two positions; subsequent steps write one. Scratch storage is bounded by the
+  configured batch capacity, overwritten for every frame, and never shared
+  between requests across frames. The worker's serialized CUDA stream owns it;
+  it is separate from the Talker's paged KV cache.
+- A short causal/GQA Triton kernel masks unwritten K/V loads. Dynamic-batch
+  `torch.compile` warms the predictor before the runner captures whole-MTP
+  CUDA graphs. This path creates no inner CUDA graphs. Unsupported devices and
+  dtypes retain re-prefill; FP16 retains its existing precision safeguards.
+- `code_predictor_fused_sampling` fuses temperature scaling, top-k filtering,
+  and Gumbel argmax when the runner supplies FP32 uniforms. It preserves top-k
+  ties and uses the supplied randomness. Explicit generator paths retain the
+  existing sampler. Qwen3-Omni does not read either TTS option.
+- `decode_cudnn_benchmark` selects convolution algorithms during Code2Wav
+  warmup and graph capture. The process's cuDNN flags are restored afterwards,
+  including on capture failure. Captured graphs retain the selected algorithms.
+
+- `decode_time_major_conv` keeps codec convolution activations in time-major
+  layout to reduce transposes and copies between layers.
+- `talker_first_audio` completes residual prediction immediately after the
+  first codebook sample and decodes the first frame in the Talker process.
+  The codec advances its state without repeating the delivered frame; the
+  orchestrator orders that frame before subsequent codec chunks. This path
+  requires CUDA, MRV2, asynchronous chunks, a single-process executor, TP/PP 1,
+  and disabled prefix caching. Reference-code requests retain codec delivery.
+  Supported Qwen3-TTS profiles enable it explicitly; other models do not opt in.
+
+The predictor, sampling, convolution-layout and autotuning options default to
+false outside this throughput profile. It uses fixed 1/25-frame chunks,
+72 frames of left context, Talker/codec capacities of 128/64, codec batches
+through 8, and synchronous codec scheduling. Client concurrency is independent
+of those capacities. This profile also sets `cuda_mps: true`; the runtime owns
+a private MPS daemon unless an operator supplies an existing MPS pipe directory.
+Ordinary profiles leave MPS disabled. See
+[experimental MPS deployment](../configuration/stage_configs.md#experimental-mps-deployment)
+for requirements and ownership. Startup includes compilation and graph capture.
+
+```bash
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni \
+  --deploy-config vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2_single_gpu.yaml \
+  --api-server-count 4 --stage-init-timeout 1800 --init-timeout 2400
+```
+
+KV reuse changes GEMM shapes and attention reduction order; cuDNN autotuning
+can also change waveform numerics. This path is not bitwise equivalent to
+re-prefill, and aggregate WER alone cannot establish perceptual or speaker
+similarity equivalence. Validate quality on the intended workload before
+adopting it. Qwen3-Omni keeps its existing predictor and audio-delivery behavior.
+
+For a controlled ablation, copy the profile and disable individual options while keeping capacities, chunking, hardware, client concurrency,
+and warmup identical. Report repeated runs, first-audio latency and failures
+alongside throughput. Warm requests measure a ready service; they do not measure
+cold-start latency.
 
 ---
 

docs/features/README.md

@@ -59,6 +59,7 @@ defines the shared configuration and backend extension points.
 | Distribute diffusion work across devices | [Parallelism](../user_guide/diffusion/parallelism/overview.md) | [Parallelism designs](../design/index.md#parallelism) |
 | Select dense, sparse, or quantized attention paths | [Attention Backends](../user_guide/diffusion/attention_backends.md) | [Attention Backend Selection](../design/feature/attention_backend_selection.md) |
 | Compile repeated diffusion regions | [Regional Compilation](../user_guide/diffusion/regional_compilation.md) | User-facing optimization guide |
+| Speed up Wan VAE video decode | [Wan VAE Decoder Fast Path](../user_guide/diffusion/vae_fast_path.md) | User-facing optimization guide |
 | Add generated video frames | [Frame Interpolation](../user_guide/diffusion/frame_interpolation.md) | User-facing extension guide |
 | Reduce diffusion model startup time | [Startup and Loading](../user_guide/diffusion/startup_and_loading.md) | User-facing loading guide |
 | Apply diffusion adapters | [LoRA](../user_guide/diffusion/lora.md) | User-facing extension guide |

docs/features/sleep_mode.md

@@ -251,8 +251,8 @@ curl -X POST http://localhost:8000/v1/images/generations \
 
 | Method | Arguments | Return Type | Description |
 | :--- | :--- | :--- | :--- |
-| **sleep** | `stage_ids: List[int], level: int` | `List[OmniACK]` | Triggers hibernation for specified stages. |
-| **wake_up** | `stage_ids: List[int]` | `List[OmniACK]` | Reloads weights and re-maps memory. |
+| **sleep** | `stage_ids: List[int], level: int` | `List[OmniACK]` | Triggers hibernation for specified stages. Raises `RuntimeError` if a stage fails; the stages stay marked as sleeping, so `wake_up` brings them back, unless a level-2 sleep already discarded weights on some replica. |
+| **wake_up** | `stage_ids: List[int]` | `List[OmniACK]` | Reloads weights and re-maps memory. Raises `RuntimeError` if a stage fails; that stage stays sleeping and the call can be retried. |
 | **pause_generation** | `mode: str, stage_ids: List[int]` | `None` | Stops admission; with `mode="keep"` also stops diffusion schedulers and returns after their ACK. |
 | **resume_generation** | `stage_ids: List[int]` | `None` | Reopens paused schedulers, then admission. |
 
@@ -261,7 +261,7 @@ curl -X POST http://localhost:8000/v1/images/generations \
 | Field | Type | Description |
 | :--- | :--- | :--- |
 | **task_id** | `str` | Unique identifier for the operation. |
-| **status** | `str` | `SUCCESS` or `ERROR`. |
+| **status** | `str` | `SUCCESS` or `ERROR`. `sleep` and `wake_up` raise instead of returning an `ERROR` ACK, and `/v1/omni/sleep` and `/v1/omni/wakeup` return HTTP 500 with the error. |
 | **stage_id** | `int` | The ID of the stage that responded. |
 | **rank** | `int` | The rank ID within the Tensor Parallel group. |
 | **freed_bytes** | `int` | Actual amount of physical VRAM reclaimed. |

docs/getting_started/installation/README.md

@@ -1,7 +1,7 @@
 # Installation
 
 !!! important
-    vLLM-Omni is released against the matching upstream vLLM major/minor version. The 0.30 development line requires vLLM 0.30.x; the source-install instructions pin vLLM 0.30.0. Published vLLM-Omni 0.28.0 wheels and images still require vLLM 0.28.x.
+    vLLM-Omni is released against the matching upstream vLLM major/minor version. The vLLM-Omni 0.30.x release line uses vLLM 0.30.x; the source-install instructions pin vLLM 0.30.0.
 
 vLLM-Omni supports the following hardware platforms:
 

docs/getting_started/installation/gpu.md

@@ -34,9 +34,7 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
 
 ### Pre-built wheels
 
-The source-install instructions target vLLM 0.30.0. Until vLLM-Omni 0.30.0 artifacts are published, the CUDA/ROCm pre-built examples below remain on the matching 0.28.0 release pair.
-
-Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, 0.25.0, 0.26.0, and 0.28.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
+Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, 0.25.0, 0.26.0, 0.28.0, and 0.30.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
 
 === "NVIDIA CUDA"
 

docs/getting_started/installation/gpu/cuda.inc.md

@@ -5,7 +5,7 @@
 # --8<-- [end:requirements]
 # --8<-- [start:set-up-using-python]
 
-vLLM-Omni depends on the matching major/minor release of vLLM. The 0.30 development line uses vLLM 0.30.x. Published 0.28.0 wheels and images use vLLM 0.28.x.
+vLLM-Omni depends on the matching major/minor release of vLLM. The vLLM-Omni 0.30.x release line uses vLLM 0.30.x.
 
 !!! note
     PyTorch installed via `conda` will statically link `NCCL` library, which can cause issues when vLLM tries to use `NCCL`. See <gh-issue:8420> for more details.
@@ -18,22 +18,22 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new**
 
 #### Installation of vLLM
 
-These pre-built wheel instructions install the published vLLM-Omni 0.28.0 release. For the 0.30 development line, use the source-install instructions below.
+These pre-built wheel instructions install the published vLLM-Omni 0.30.0 release.
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm==0.28.0 --torch-backend=auto
+uv pip install vllm==0.30.0 --torch-backend=auto
 ```
 
 #### Installation of vLLM-Omni
 
 ```bash
-uv pip install vllm-omni==0.28.0
+uv pip install vllm-omni==0.30.0
 ```
 
 To run Gradio demos, also install the optional extras:
 ```bash
-uv pip install 'vllm-omni[demo]==0.28.0'
+uv pip install 'vllm-omni[demo]==0.30.0'
 ```
 
 # --8<-- [end:pre-built-wheels]
@@ -111,7 +111,7 @@ docker run --runtime nvidia --gpus 2 \
     --env "HF_TOKEN=$HF_TOKEN" \
     -p 8091:8091 \
     --ipc=host \
-    vllm/vllm-omni:v0.28.0 \
+    vllm/vllm-omni:v0.30.0 \
     vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```
 

docs/getting_started/installation/gpu/rocm.inc.md

@@ -10,9 +10,8 @@
 
 For ROCm, vLLM-Omni currently recommends the setup steps through Docker Images.
 
-vLLM-Omni depends on the matching major/minor release of vLLM. The 0.30
-development line uses vLLM 0.30.x. Published 0.28.0 wheels and images use vLLM
-0.28.x.
+vLLM-Omni depends on the matching major/minor release of vLLM. The vLLM-Omni
+0.30.x release line uses vLLM 0.30.x.
 
 The Dockerfile's `BASE_IMAGE` pin applies only to Docker builds. The
 `vllm-omni` package does not install vLLM as a dependency, so non-Docker source
@@ -23,12 +22,12 @@ installing vLLM-Omni, as shown below.
 
 #### Installation of vLLM
 
-These pre-built wheel instructions install the published vLLM-Omni 0.28.0 release. For the 0.30 development line, use the source-install instructions below.
+These pre-built wheel instructions install the published vLLM-Omni 0.30.0 release.
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 
 ```bash
-uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.28.0/rocm723
+uv pip install vllm==0.30.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.30.0/rocm723
 ```
 
 #### Installation of vLLM-Omni
@@ -37,7 +36,7 @@ uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/roc
 # we need to add --no-build-isolation as the torch
 # is not obtained from pypi, we have to install using the
 # torch installed in our environment
-uv pip install vllm-omni==0.28.0
+uv pip install vllm-omni==0.30.0
 
 # Optional if want to run Qwen3 TTS
 uv pip uninstall onnxruntime # should be removed before we can install onnxruntime-rocm
@@ -150,6 +149,8 @@ vllm-omni-rocm
 
 vLLM-Omni offers an official docker image for deployment. These images are built on top of vLLM docker images and available on Docker Hub as [vllm/vllm-omni-rocm](https://hub.docker.com/r/vllm/vllm-omni-rocm/tags). The version of vLLM-Omni indicates which release of vLLM it is based on.
 
+The prebuilt ROCm image is published separately from the release pipeline: `v0.28.0` is the latest tag currently available on Docker Hub, and newer-tag availability is tracked in [#7405](https://github.com/vllm-project/vllm-omni/issues/7405).
+
 #### Launch vLLM-Omni Server
 
 Here's an example deployment command that has been verified on 2 x MI300's:

docs/getting_started/installation/npu/npu.inc.md

@@ -10,25 +10,25 @@ The recommended way to use vLLM-Omni on NPU is through the vLLM-Ascend pre-built
 === "A2"
 
     ```bash
-    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.29.0
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.30.0
     ```
 
 === "A3"
 
     ```bash
-    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.29.0-a3
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.30.0-a3
     ```
 
 === "A5"
 
     ```bash
-    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.29.0-a5
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.30.0-a5
     ```
 
 === "310P (Experimental)"
 
     ```bash
-    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.29.0-310p
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.30.0-310p
     ```
 
 ```bash
@@ -53,7 +53,7 @@ docker run --rm \
 
 # Inside the container, install vLLM-Omni from source
 cd /vllm-workspace
-git clone -b v0.29.0rc1 https://github.com/vllm-project/vllm-omni.git
+git clone -b v0.30.0 https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
 pip install -v -e . --no-build-isolation
 # or VLLM_OMNI_TARGET_DEVICE=npu pip install -v -e .
@@ -71,10 +71,10 @@ We are keeping [issue #886](https://github.com/vllm-project/vllm-omni/issues/886
 
 # --8<-- [start:installation-main]
 
-You can also install vLLM-Omni from the latest main branch if you want to use the latest features or bug fixes. Use the aligned vLLM-Ascend v0.29.0 image described above, and check [issue #886](https://github.com/vllm-project/vllm-omni/issues/886) for the status of the latest vLLM-Omni commit on NPU.
+You can also install vLLM-Omni from the latest main branch if you want to use the latest features or bug fixes. Use the aligned vLLM-Ascend v0.30.0 image described above, and check [issue #886](https://github.com/vllm-project/vllm-omni/issues/886) for the status of the latest vLLM-Omni commit on NPU.
 
 ```bash
-# Inside the vLLM-Ascend v0.29.0 container, install vLLM-Omni from main
+# Inside the vLLM-Ascend v0.30.0 container, install vLLM-Omni from main
 cd /vllm-workspace
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
@@ -103,7 +103,7 @@ Here's an example deployment command that has been verified on 4 x NPUs:
 === "A2"
 
     ```bash
-    export IMAGE=quay.io/ascend/vllm-omni:v0.29.0
+    export IMAGE=quay.io/ascend/vllm-omni:v0.30.0
     docker run --rm \
         --name vllm-omni-a2 \
         --shm-size=64g \
@@ -127,7 +127,7 @@ Here's an example deployment command that has been verified on 4 x NPUs:
 === "A3"
 
     ```bash
-    export IMAGE=quay.io/ascend/vllm-omni:v0.29.0-a3
+    export IMAGE=quay.io/ascend/vllm-omni:v0.30.0-a3
     docker run --rm \
         --name vllm-omni-a3 \
         --shm-size=64g \
@@ -151,7 +151,7 @@ Here's an example deployment command that has been verified on 4 x NPUs:
 === "A5"
 
     ```bash
-    export IMAGE=quay.io/ascend/vllm-omni:v0.29.0-a5
+    export IMAGE=quay.io/ascend/vllm-omni:v0.30.0-a5
     docker run --rm \
         --name vllm-omni-a5 \
         --shm-size=64g \
@@ -175,7 +175,7 @@ Here's an example deployment command that has been verified on 4 x NPUs:
 === "310P (Experimental)"
 
     ```bash
-    export IMAGE=quay.io/ascend/vllm-omni:v0.29.0-310p
+    export IMAGE=quay.io/ascend/vllm-omni:v0.30.0-310p
     docker run --rm \
         --name vllm-omni-310p \
         --shm-size=64g \

docs/models/seedvr2.md

@@ -0,0 +1,280 @@
+# SeedVR2 3B video restoration
+
+The native `SeedVR2Pipeline` restores an input video using the released 3B FP16
+NaDiT and s8/c16/t4 causal VAE. It uses fixed checkpoint conditioning, CFG=1,
+and one Euler step. Text prompts do not change conditioning.
+
+## Code ownership
+
+The model package groups each computation stage in one file: `nadit.py` owns
+window geometry, routing, RoPE and the DiT; `vae.py` owns the causal VAE and its
+temporal/spatial tiling; `pipeline_seedvr2.py` owns input validation, decoding,
+conditioning, the Euler step and colour transfer. `config.py` owns model limits,
+and `long_video.py` owns the optional HTTP job/window lifecycle. Framework code
+only registers the model, its pre/postprocessors, metadata and HTTP routes.
+
+`color_correction_method` is model-specific: online requests use the existing
+`extra_params` JSON object, and offline sampling uses
+`extra_args={"color_correction_method": "wavelet"}`. The default remains `lab`;
+invalid values are rejected by model-side validation before inference. It is
+not a top-level field in the common video API.
+
+## Model directory
+
+Place these files together, retaining their names:
+
+| File | SHA-256 of the validated release |
+| --- | --- |
+| `seedvr2_ema_3b_fp16.safetensors` | `2fd0e03a3dad24e07086750360727ca437de4ecd456f769856e960ae93e2b304` |
+| `ema_vae_fp16.safetensors` | `20678548f420d98d26f11442d3528f8b8c94e57ee046ef93dbb7633da8612ca1` |
+| `pos_emb.pt` | `fa07a14844314772266b66c3b95deb0027696d8fe7065721263db5176f45d799` |
+
+Add `model_index.json` containing:
+
+```json
+{"_class_name": "SeedVR2Pipeline"}
+```
+
+The loader checks every checkpoint tensor, including RoPE buffers, and rejects
+missing, duplicate, unexpected, or incompatible tensors. Install PyAV for video
+decoding. The validated environment uses PyAV 18.1, PyTorch 2.13/CUDA 13, and L20 GPUs.
+
+## Serve and restore a video
+
+```bash
+vllm serve "$MODEL_DIR" --omni \
+  --model-class-name SeedVR2Pipeline --dtype float16 --enforce-eager \
+  --num-gpus 1 --host 127.0.0.1 --port 8098
+
+curl --fail-with-body http://127.0.0.1:8098/v1/videos/sync \
+  -F 'prompt= ' \
+  -F 'input_references=@input.mp4;type=video/mp4' \
+  -F 'size=224x128' -F 'num_inference_steps=1' \
+  -F 'guidance_scale=1' -F 'seed=7723' \
+  --output restored.mp4
+```
+
+For two-rank window sequence parallelism, expose two GPUs and replace `--num-gpus 1` with:
+
+```bash
+--num-gpus 2 --distributed-executor-backend mp \
+  --stage-overrides '{"0":{"ulysses_degree":2}}'
+```
+
+Use degree 4 and four visible GPUs for SP4. Window-SP alone replicates the VAE.
+To shard VAE activations across the same ranks, use:
+
+```bash
+--vae-use-tiling --num-gpus 2 --distributed-executor-backend mp \
+  --stage-overrides '{"0":{"ulysses_degree":2,"vae_patch_parallel_size":2,"vae_parallel_mode":"spatial_shard_height"}}'
+```
+
+The VAE patch degree must match the window-SP degree.
+For a 16-frame 1280×720 input restored at its original size, use four GPUs,
+VAE tiling, and height sharding with both degrees set to four:
+
+```bash
+CUDA_VISIBLE_DEVICES=0,1,2,3 vllm serve "$MODEL_DIR" --omni \
+  --model-class-name SeedVR2Pipeline --dtype float16 --enforce-eager \
+  --vae-use-tiling --num-gpus 4 --distributed-executor-backend mp \
+  --stage-overrides '{"0":{"ulysses_degree":4,"vae_patch_parallel_size":4,"vae_parallel_mode":"spatial_shard_height"}}' \
+  --host 127.0.0.1 --port 8098
+```
+
+Request `size=1280x720` for this landscape input or `size=720x1280` for a
+portrait input of the same dimensions. Keep the requested output size equal to
+the input size when testing restoration without upscaling.
+
+The multipart API requires a prompt field; a single space supplies a blank
+prompt. Nonblank text is rejected. Omit `fps` to retain the source frame rate;
+an explicit rate must match the source. Output dimensions are explicit multiples
+of 16. Input frames are resized with antialiased bicubic interpolation. The final
+frame is repeated internally to reach 4n+1, and the decoded output is cropped back
+to the original frame count. Five frames remain five; six frames are internally
+padded to nine and return six.
+
+## Colour correction
+
+Restoration reproduces detail faithfully but shifts global colour, so the
+resized input is used to transfer colour back onto the restored frames. This is
+on by default; pass `extra_params={"color_correction_method":"wavelet"}` to choose how:
+
+| Value | Behavior |
+| --- | --- |
+| `lab` (default) | Swaps the lowest frequency band, then histogram-matches CIELAB chroma. Most faithful colour. |
+| `wavelet` | Swaps only the lowest frequency band, keeping every higher band from the restoration. |
+| `adain` | Matches per-channel mean and standard deviation. Cheapest, and corrects only a global tint. |
+| `none` | Skips the transfer and returns raw restored colour. |
+
+```bash
+curl --fail-with-body http://127.0.0.1:8098/v1/videos/sync \
+  -F 'prompt= ' -F 'input_references=@input.mp4;type=video/mp4' \
+  -F 'size=224x128' -F 'num_inference_steps=1' -F 'guidance_scale=1' \
+  -F 'seed=7723' -F 'extra_params={"color_correction_method":"wavelet"}' \
+  --output restored.mp4
+```
+
+`lab` and `wavelet` retain the restored high-frequency detail; `adain` measurably
+softens it because it rescales every frequency. The transfer runs in FP32 after
+the VAE decode and costs a fraction of a second per frame. Unrecognized values
+are rejected with 400 before the clip is admitted. Correction cannot recover
+colour lost to 4:2:0 chroma subsampling in the returned MP4, which dominates the
+remaining deviation once the transfer is applied.
+
+## Request admission
+
+The default 3B limit is 848×480 pixels per input or output frame and 2,035,200
+pixels across the requested output clip. With VAE tiling and window SP of at
+least four ranks whose `vae_patch_parallel_size` matches `ulysses_degree`, the
+per-frame limit rises to 2560×1472 and the clip budget to 18,841,600 pixels.
+Raising the degree never lowers the budget. Both budgets count temporal
+padding to 4n+1 frames: the default admits up to five 848×480 frames or 93
+192×112 frames; the sharded profile admits up to 45 848×480 frames or five
+2560×1472 frames. A 16-frame 1280×720 or 720×1280 clip pads to 17 frames and
+uses 15,667,200 of the sharded clip's 18,841,600 pixels. Both orientations passed
+original-size HTTP restoration on four RTX 5090 GPUs with VAE tiling and height
+sharding. An independent 257-frame cap bounds per-frame decoder work for tiny
+inputs. The other longer combinations above are admission bounds, not completed
+GPU validation. The decoder checks
+declared duration, frame count, and input dimensions when available, then
+enforces the limits as frames arrive. Requests outside these budgets return 400
+before building the resized whole-clip tensor.
+
+The per-frame cap is calibrated for the smallest qualified device. The clip cap
+scales with the smallest visible device's memory: it stays at the calibrated
+five 2560×1472 frames on a 32 GB device and grows on larger ones from a
+per-rank memory model measured at SP4 (see `sharded_budget` in
+`vllm_omni/diffusion/models/seedvr2/pipeline_seedvr2.py`), which keeps 15% of the device
+free and leaves room for allocator fragmentation. It stops at 2,118,057,984
+padded pixels (513 frames at 1536×2688), the largest clip validated on four
+B300 ranks; 80 GB devices get about 1.08 billion and 141 GB or larger devices
+reach the cap. Clips at the cap in both memory regimes (561 frames at 2560×1472
+and 2,049 frames at 768×1344) peaked at 54 GiB of activations per rank. Set
+these before starting the server to override the caps; each must be a positive
+integer, and validating an override on the target hardware is the operator's
+responsibility:
+
+| Variable | Default | Bounds |
+| --- | --- | --- |
+| `VLLM_OMNI_SEEDVR2_SHARDED_FRAME_PIXELS` | 3,768,320 | Pixels per frame on the sharded profile |
+| `VLLM_OMNI_SEEDVR2_SHARDED_CLIP_PIXELS` | 18,841,600, scaled up with device memory | Padded pixels per clip on the sharded profile |
+| `VLLM_OMNI_SEEDVR2_MAX_FRAMES` | 257 | Decoder-work frame cap, all profiles |
+
+The sharded profile applies to any `ulysses_degree` of four or more, so eight
+ranks inherit a budget calibrated on four. Raising these caps trades a 400 for a
+CUDA OOM: the admission check passes and the request fails later inside the DiT
+forward or the VAE decode, which can abort every in-flight request on the engine
+rather than only the oversized one. Before serving with raised caps, restore the
+largest clip they admit once and confirm it completes.
+
+A 362-frame 1536×2688 2x upscale was restored on eight ranks with
+`ulysses_degree=8`, VAE tiling and height sharding, using 4,300,000 for the
+per-frame cap, the memory-scaled clip cap and 2,000 for the frame cap. The long-video
+route sizes its windows from these caps, so the clip cap sets how many frames
+each window covers.
+
+## Long-video restoration
+
+`POST /v1/seedvr2/restore-long` accepts one uploaded 24 FPS video, a blank
+prompt, `size`, `num_frames` (up to 7,200), and optional `loop_input=true` and
+`extra_params={"color_correction_method":"wavelet"}`, which it applies to every window.
+An output frame must fit the sharded per-frame pixel cap. It returns a job ID; poll
+`GET /v1/seedvr2/restore-long/{id}` and download the completed MP4 from
+`GET /v1/seedvr2/restore-long/{id}/content`. `DELETE
+/v1/seedvr2/restore-long/{id}` asks a running job to stop; it settles as
+`cancelled` at the next window boundary. A job whose owning process is gone,
+such as after a server restart, reports `failed` on the next poll.
+
+Jobs live on the API server's local disk, so the route requires
+`--api-server-count 1` and runs one job at a time. Storage is bounded by an
+upload cap and by deleting settled jobs once they age out:
+
+| Variable | Default | Bounds |
+| --- | --- | --- |
+| `VLLM_OMNI_SEEDVR2_LONG_OUTPUT_DIR` | system temp dir | Parent of the job directories |
+| `VLLM_OMNI_SEEDVR2_LONG_MAX_UPLOAD_BYTES` | 8 GiB | Largest accepted upload |
+| `VLLM_OMNI_SEEDVR2_LONG_JOB_TTL_SECONDS` | 3,600 | Age at which a settled job is deleted |
+| `VLLM_OMNI_SEEDVR2_LONG_MAX_WINDOW` | 121 | Longest model window, in frames |
+
+Download a completed job before its TTL expires; the sweep runs on each new
+submission and removes the output with the job directory.
+
+The service sends windows through the existing SeedVR2 endpoint, blends four
+frames at each boundary, and writes one continuous MP4 encoder. Each window is
+the longest 4n+1-frame clip that the sharded clip-pixel cap, the frame cap and
+`VLLM_OMNI_SEEDVR2_LONG_MAX_WINDOW` admit at the output size. Every window pays
+a fixed round trip through the serving stack and recomputes its overlap, and a
+window of four latent frames gets no temporal attention in the DiT, so raise
+the clip cap as far as the device allows. Windows carry source pixels, which
+the model resizes on the device, and cross the endpoint losslessly; only the
+final MP4 is compressed. The next window is restored while the previous one is
+blended and encoded. It repeats source frames and audio only when
+`loop_input=true`. The sharded serving profile above (VAE tiling, VAE height
+sharding, Ulysses degree of at least four) is required. This route keeps a
+window within the whole-clip pixel budget; it does not raise the
+`/v1/videos/sync` 257-frame or pixel limits. The long route
+uses fixed one-step, CFG=1 conditioning and requires `imageio[ffmpeg]` for
+audio muxing. The 7,200-frame 768×1344 case is still undergoing full GPU E2E.
+
+## Temporal and spatial VAE tiling
+
+Add `--vae-use-tiling` to bound VAE intermediate activations along time. The
+encoder processes nine frames first, then eight per chunk; the decoder processes
+two latent frames per chunk. Each causal convolution carries its past inputs
+within that encode/decode call, with temporal stride and first-frame upsampling
+alignment preserved. GroupNorm and attention still see the entire spatial frame.
+There is no overlap blend or independent restart at chunk boundaries.
+
+Five-frame clips use one temporal chunk. On a single GPU, convolutions with
+more than 256 input rows additionally process tiles of 128 output rows with
+exact halos. GroupNorm and bottleneck attention retain full-frame statistics.
+This bounds convolution workspace; full-frame activations and the DiT's
+whole-clip activations still consume memory.
+
+With VAE patch parallelism, each rank owns a band of latent rows and the
+corresponding encoder/decoder rows. Neighbor exchanges supply convolution halos;
+all-reduced centered statistics preserve full-frame GroupNorm. Bottleneck
+attention gathers the low-resolution frame before splitting it again. Encoder
+parameters are gathered before latent sampling, and decoded pixels are gathered
+before returning the video. Unequal bands are supported; fewer latent rows than
+ranks fall back to replicated execution. Width sharding and batch slicing are
+unsupported. FP16 reduction and convolution order can change rounding, so
+parallel outputs are numerically close rather than bitwise identical.
+
+## Current integration scope
+
+| Property | Contract |
+| --- | --- |
+| Weights / dtype | Released 3B DiT and VAE, FP16 |
+| Reference semantics | C0 whole clip, colour correction on by default |
+| Video input | One uploaded file, or offline TCHW RGB floats / PIL frames |
+| Timing | Constant frame rate, increasing PTS; normalize the video origin to zero |
+| Audio | First mono/stereo track, aligned by source PTS, cropped to the video interval, re-encoded as AAC |
+| Randomness | Per-request generator; preserve reference latent strides when sampling noise |
+| Sequence parallelism | Native engine SP1/2/4 correctness verified on five-frame L20 cases |
+| VAE placement | Replicated by default; optional height sharding on the window-SP group |
+| Unsupported | VFR, multichannel audio, 7B, other sampling schedules, quantization, cache acceleration, VAE width sharding / batch slicing, CPU offload, CFG/TP/PP parallelism, compiled execution, LoRA |
+
+Unsupported engine modes are rejected before process hooks and worker creation.
+`ulysses_degree` selects the model-owned SP group. This branch assigns whole
+windows to ranks; the specialized head-sharded path is a dependent change.
+Ring and AllGather-KV are unsupported.
+
+The practical P0 reference's five-frame batching, overlap, and CPU swapping are
+separate execution semantics. Temporal tiling and VAE patch
+parallelism reduce activation peaks but do not bound the whole-clip DiT memory.
+Large output frames may still exceed device capacity.
+
+Invalid inputs return 400. A single-GPU model OOM fails that request; a
+subsequent short request succeeds. With VAE patch parallelism, restart the
+service after an OOM: another rank may still be waiting in a collective. Cancelling an in-progress video job removes it through the
+existing DELETE endpoint; in-flight GPU work may drain before memory is reusable.
+A lost SP worker fails the request and makes health return 503. Restart the
+service to restore availability; automatic rank recovery is not provided.
+
+## Reproducible deployment
+
+See the [RTX 5090 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/ByteDance/SeedVR2-RTX-5090.md) for the
+input/output contract, complete serving command, and media checks. The local
+tests in `tests/diffusion/models/seedvr2/test_seedvr2_e2e.py` exercise the full
+3B pipeline, all colour modes, and HTTP restoration with USP 1 and 8.

docs/models/supported_models.md

@@ -19,9 +19,9 @@ th {
 
 | Architecture | Models | Example HF Models | NVIDIA GPU | AMD GPU | Ascend NPU | Intel GPU | Recipe |
 | -------------- | -------- | ------------------- | ------------ | --------- | ----- | ----------- | -------- |
-| `Qwen3OmniMoeForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen3-Omni.md) |
+| `Qwen3OmniMoeForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen3-Omni.md) |
 | `Qwen2_5OmniForConditionalGeneration` | Qwen2.5-Omni | `Qwen/Qwen2.5-Omni-7B`, `Qwen/Qwen2.5-Omni-3B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `MingFlashOmniForConditionalGeneration` + `MingImagePipeline` | Ming-flash-omni-2.0 (omni-speech + imagegen<sup>1</sup>) | `Jonathan1909/Ming-flash-omni-2.0` | ✅︎ | | | | — |
+| `MingFlashOmniForConditionalGeneration` + `MingImagePipeline` | Ming-flash-omni-2.0 (omni-speech + imagegen<sup>1</sup>) | `Jonathan1909/Ming-flash-omni-2.0` | ✅︎ | | | ✅︎ | — |
 | `BagelForConditionalGeneration` | BAGEL (DiT-only) | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `InternVLAA1Pipeline` | InternVLA-A1 | `InternRobotics/InternVLA-A1-3B` | ✅︎ | ✅︎ | | | — |
 | `Gr00tN1d7Pipeline` | GR00T N1.7 | `nvidia/GR00T-N1.7-3B` | ✅︎ | | | | — |
@@ -33,9 +33,9 @@ th {
 | `QwenImageLayeredPipeline` | Qwen-Image-Layered | `Qwen/Qwen-Image-Layered` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2511 | `Qwen/Qwen-Image-Edit-2511` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `GlmImagePipeline` | GLM-Image | `zai-org/GLM-Image` | ✅︎ | ✅︎ | | | — |
-| `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `Krea2Pipeline` | Krea 2 (Raw + Turbo) | `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | ✅︎ | | | ✅︎ | — |
-| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tongyi-MAI/Z-Image-Turbo.md) |
+| `Krea2Pipeline` | Krea 2 (Raw + Turbo) | `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/krea/Krea-2.md) |
+| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Wan-AI/Wan2.2-T2V.md) |
 | `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `LingBotWorldCausalDMDPipeline` | LingBot-World 2.0 v1 (experimental) | `robbyant/lingbot-world-v2-14b-causal-fast-diffusers` | Experimental | | | | — |
 | `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V, V2V, T2V with sound, action policy | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
@@ -43,17 +43,18 @@ th {
 | `Wan22S2VPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ | | | ✅︎ | — |
-| `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | — |
-| `LTX2TwoStagePipeline` | LTX-2 / LTX-2.3 ordinary two-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` + matching Lightricks LoRA and upsampler | ✅︎ | ✅︎ | | | — |
-| `LTX2DistilledOneStagePipeline` | LTX-2 / LTX-2.3 merged-distilled one-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` | ✅︎ | ✅︎ | | | — |
-| `LTX2DistilledTwoStagePipeline` | LTX-2 / LTX-2.3 merged-distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` + matching Lightricks upsampler | ✅︎ | ✅︎ | | | — |
-| `LTX2Pipeline` | LTX-2.5 Full/SFT one-stage T2V and I2V | `Lightricks/LTX-2.5-Diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.5.md) |
+| `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.md) |
+| `LTX2TwoStagePipeline` | LTX-2 / LTX-2.3 ordinary two-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` + matching Lightricks LoRA and upsampler | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.md) |
+| `LTX2DistilledOneStagePipeline` | LTX-2 / LTX-2.3 merged-distilled one-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.md) |
+| `LTX2DistilledTwoStagePipeline` | LTX-2 / LTX-2.3 merged-distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` + matching Lightricks upsampler | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.md) |
+| `LTX2Pipeline` | LTX-2.5 Full/SFT one-stage T2V and I2V | `Lightricks/LTX-2.5-Diffusers` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.5.md) |
 | `LTX2TwoStagePipeline` | LTX-2.5 Full/SFT two-stage T2V and I2V | `Lightricks/LTX-2.5-Diffusers` + `Lightricks/LTX-2.5` LoRA | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.5.md) |
 | `LTX2DistilledOneStagePipeline` | LTX-2.5 merged-distilled one-stage T2V and I2V | `Lightricks/LTX-2.5-Diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.5.md) |
 | `LTX2DistilledTwoStagePipeline` | LTX-2.5 merged-distilled two-stage T2V and I2V | `Lightricks/LTX-2.5-Diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.5.md) |
 | `LingBotVideoPipeline` | LingBot-Video dense and MoE T2I, T2V, TI2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | | — |
-| `SanaVideoPipeline` | SANA-Video-2B T2V (native 480p/720p; Diffusers adapter validated at both resolutions) | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers`, `Efficient-Large-Model/SANA-Video_2B_720p_diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/SANA-Video-2B.md) |
+| `SanaVideoPipeline` | SANA-Video-2B T2V (native 480p/720p; Diffusers adapter validated at both resolutions) | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers`, `Efficient-Large-Model/SANA-Video_2B_720p_diffusers` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/SANA-Video-2B.md) |
 | `SanaImageToVideoPipeline` | SANA-Video-2B I2V (native and Diffusers adapter validated at 480p/720p) | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers`, `Efficient-Large-Model/SANA-Video_2B_720p_diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/SANA-Video-2B.md) |
+| `SeedVR2Pipeline` | [SeedVR2 3B](seedvr2.md) video restoration | `ByteDance-Seed/SeedVR2-3B` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/ByteDance/SeedVR2-RTX-5090.md) |
 | `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | ✅︎ | | | [Published](https://recipes.vllm.ai/MiniMaxAI/MiniMax-H3) |
 | `Magi2Pipeline` | MAGI-2 native Preview-stage T2VA and I2VA (272p/540p) | `sand-ai/MAGI-2-preview` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SandAI/MAGI-2-preview-L20X.md) |
 | `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
@@ -65,23 +66,24 @@ th {
 | `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `LongCatVideoAvatarPipeline` | LongCat-Video-Avatar-1.5 A2V/AI2V (native single-speaker and multi-speaker AI2V/AVC) | `meituan-longcat/LongCat-Video-Avatar-1.5` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/meituan-longcat/LongCat-Video-Avatar-1.5.md) |
 | `BooguImagePipeline` | Boogu-Image | `Boogu/Boogu-Image-0.1-Base`, `Boogu/Boogu-Image-0.1-Edit`, `Boogu/Boogu-Image-0.1-Edit-Turbo` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Boogu/Boogu-Image.md) |
-| `BooguImageTurboPipeline` | Boogu-Image Turbo distilled T2I | `Boogu/Boogu-Image-0.1-Turbo` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Boogu/Boogu-Image.md) |
+| `BooguImageTurboPipeline` | Boogu-Image Turbo distilled T2I | `Boogu/Boogu-Image-0.1-Turbo` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Boogu/Boogu-Image.md) |
 | `StableDiffusionXLPipeline` | Stable-Diffusion-XL | `stabilityai/stable-diffusion-xl-base-1.0` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ | — |
+| `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/StabilityAI/Stable-Diffusion-3.5.md) |
 | `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `AuKForConditionalGeneration` | AuK / AuK-Flash (instruction-driven speech generation and editing) | `tencent/AuK`, `tencent/AuK-Flash` (with `Qwen/Qwen2.5-Omni-3B`; assemble with `tools/prepare_auk_checkpoint.py`) | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/AuK-H100.md) |
 | `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | ✅︎ | — |
 | `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | ✅︎ | — |
-| `BreezeForConditionalGeneration` | Breeze-TTS-2 (TTS, voice clone, voice direction) | `BreezeBlue/Breeze-TTS-2` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/BreezeBlue/Breeze-TTS-2.md) |
-| `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | | — |
+| `BreezeForConditionalGeneration` | Breeze-TTS-2 (voice design, voice clone, voice direction, CFG) | `BreezeBlue/Breeze-TTS-2` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/BreezeBlue/Breeze-TTS-2.md) |
+| `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `MammothModa2ForConditionalGeneration` | MammothModa2-Dev (AR-only image understanding) | `bytedance-research/MammothModa2-Dev` | ✅︎ | | | | — |
-| `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | | — |
+| `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/black-forest-labs/FLUX.2-klein.md) |
+| `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `FluxPipeline` | FLUX.1-dev | `black-forest-labs/FLUX.1-dev` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `FluxPipeline` | FLUX.1-schnell | `black-forest-labs/FLUX.1-schnell` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `OmniGen2Pipeline` | OmniGen2 | `OmniGen2/OmniGen2` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `MiniMaxMusic3ForConditionalGeneration` | MiniMax Music 3 (text-to-music) | `MiniMaxAI/MiniMax-Music3` | ✅︎ | | | | — |
+| `Yue2ForCausalLM` | YuE2-3B (text-to-music with ABC plan) | `m-a-p/YuE2-3B` | ✅︎ | | | | — |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | | | ✅︎ | [Published](https://recipes.vllm.ai/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice) |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
@@ -97,16 +99,18 @@ th {
 | `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | | — |
 | `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | | — |
-| `Flux2Pipeline` | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | ✅︎ | ✅︎ | | | — |
+| `Flux2Pipeline` | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | ✅︎ | — |
-| `Audio8TTSSlowARForConditionalGeneration` | Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | ✅︎ | | | | — |
-| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only), SenseNova-U1.5 | `SenseNova/SenseNova-U1-8B-MoT`, `sensenova/SenseNova-U1.5-8B-MoT` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SenseNova/SenseNova-U1.5.md) |
+| `Audio8TTSSlowARForConditionalGeneration` | Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | ✅︎ | | | ✅︎ | — |
+| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only), SenseNova-U1.5 | `SenseNova/SenseNova-U1-8B-MoT`, `sensenova/SenseNova-U1.5-8B-MoT` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SenseNova/SenseNova-U1.5.md) |
+| `SenseNovaU1Pipeline` | SenseNova-U1-A3B (MoE) | `sensenova/SenseNova-U1-A3B-MoT` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SenseNova/SenseNova-U1.md#a3b-moe-variant) |
 | `LancePipeline` | Lance | `bytedance-research/Lance` | ✅︎ | | | | — |
-| `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | | — |
-| `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | | — |
+| `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/HunyuanVideo-1.5.md) |
+| `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/HunyuanVideo-1.5.md) |
 | `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | | — |
 | `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | | — |
 | `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/OpenBMB/MiniCPM-o-4_5.md) |
+| `PersonaPlexTalkerForConditionalGeneration` | PersonaPlex (full duplex, 24 kHz speech in/out over `/v1/realtime?duplex=1`) | `nvidia/personaplex-7b-v1` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/PersonaPlex.md) |
 | `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `GepardTalkerForConditionalGeneration` | Gepard-1.0 | `nineninesix/gepard-1.0` | ✅︎ | | | ✅︎ | — |
 | `HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | | — |

docs/serving/README.md

@@ -144,23 +144,28 @@ interchangeably.
 | ---------- | ---------- | ------------------- | --------- |
 | Incremental text input for speech synthesis | `WS /v1/audio/speech/stream` | Send text events and receive audio | [Streaming Text to Speech](speech_api.md#streaming-text-input-websocket) |
 | Live video understanding | `WS /v1/video/chat/stream` | Send video frames and receive text/audio | [Streaming Video Input](video_stream_api.md) |
-| Turn-based realtime audio | `WS /v1/realtime` | Stream one audio input and receive transcript/audio events | [Realtime Audio](realtime_api.md) |
+| Turn-based realtime audio | `WS /v1/realtime` | Stream one audio input and receive transcript/audio events | [Realtime Audio](realtime_openai_duplex_api.md) |
 | Continuous speech-to-speech interaction | `WS /v1/realtime?duplex=1` (alias `WS /v1/duplex`) | Listen and speak concurrently with session control | [Full Duplex](full_duplex_api.md) |
 | Generated video chunks | `WS /v1/realtime/video` | Start a diffusion request and receive fragmented MP4 | [Streaming Video Output](streaming_video_output_api.md) |
 | Robot policy inference | `WS /v1/realtime/robot/openpi` | Send MessagePack observations and receive action arrays | [OpenPI Robot Policy](openpi_api.md) |
 
 All six routes are model- or configuration-dependent. In particular,
-`/v1/realtime` is full duplex when the model is a duplex model (its pipeline
-declares a `duplex_plugin` and the deploy configuration sets
-`session_mode: duplex`); a stock Realtime client needs no vendor query
-parameter, and `duplex=0` is the explicit opt-out. Such a server serves the
-websocket route plus `POST /v1/chat/completions`, and no other turn-based
-HTTP route. On a duplex server that chat route is not the turn-based path:
-each request runs on a short-lived duplex session, so it holds one of
-`duplex_session.max_sessions` for its lifetime and answers at the model's
-real-time pace -- see [Full Duplex](full_duplex_api.md). Clients should
-also verify the duplex capability payload because the query-parameter form
-falls back to the ordinary realtime handler when duplex is unavailable.
+`/v1/realtime` is full duplex when the server is a duplex server: the model's
+pipeline declares a `duplex_plugin` and its deploy configuration sets
+`session_mode: duplex` (with `session_mode: turn` the same model boots the
+ordinary turn-based serving stack instead). On a duplex server a stock
+Realtime client needs no vendor query parameter; `duplex=0` selects the
+turn-based Realtime handler, which a duplex server does not mount, so that
+connection is refused with `Realtime API is not available`. Such a server
+serves the websocket route plus `POST /v1/chat/completions`, `/v1/models`
+and `/health`, and no other turn-based HTTP route. The chat route is the
+ordinary chat service running on the duplex engine: a request is a
+turn-based generation on the same stages, served alongside the live
+websocket sessions; it opens no duplex session and holds no
+`duplex_session.max_sessions` slot -- see [Full Duplex](full_duplex_api.md).
+On a server that is not duplex, an explicit `?duplex=1` is refused rather
+than answered by the turn-based handler, so a client that requested duplex
+never silently gets the other protocol.
 
 ## Related Endpoints
 

docs/serving/full_duplex_api.md

@@ -5,7 +5,7 @@ continue receiving speech while producing speech. It adds persistent session
 state, model-specific turn policy, overlap handling, playback acknowledgement,
 and optional session resume.
 
-Full duplex is distinct from the turn-based [Realtime Audio API](realtime_api.md).
+Full duplex is distinct from the turn-based [Realtime Audio API](realtime_openai_duplex_api.md).
 
 This page is the endpoint overview. For the complete wire contract, the
 `vllm_omni.clients.duplex.DuplexClient` Python library, and the per-model
@@ -72,12 +72,21 @@ does not switch engines. The Python `Omni` / `AsyncOmni` APIs are unchanged.
     `session.created.session.capabilities` is present before treating the
     connection as full duplex.
 
-**MiniCPM-o 4.5** (`vllm_omni/deploy/minicpmo_4_5.yaml`) is the only model
-served over this endpoint today. PersonaPlex and Nemotron VoiceChat still carry
-their pre-framework duplex code: their pipelines declare no `duplex_plugin`, so
-they run turn-based until the follow-up PRs port them to the plugin contract
+**MiniCPM-o 4.5** (`vllm_omni/deploy/minicpmo_4_5.yaml`), **Qwen3-Omni**
+(`vllm_omni/deploy/qwen3_omni_duplex.yaml`), and **Nemotron VoiceChat**
+(`vllm_omni/deploy/nemotron_labs_voicechat_duplex.yaml`) are served
+over this endpoint today. PersonaPlex still carries its pre-framework duplex
+code: its pipeline declares no `duplex_plugin`, so it runs turn-based until the
+follow-up PR ports it to the plugin contract
 (RFC [vllm-omni#7181](https://github.com/vllm-project/vllm-omni/issues/7181)).
 
+For Nemotron VoiceChat Python clients, use
+`vllm_omni.clients.nemotron_voicechat.create_duplex_session_config()` with
+`DuplexClient`. The preset selects 16 kHz float32 input, 22.05 kHz PCM16 output,
+and automatic responses. Append 1,280-sample (80 ms) input frames; when using
+`stream_pcm`, set `chunk_ms=80`. Pass `instructions` and optional `tools` to
+the preset before opening the session.
+
 JoyVL is a separate HTTP interaction orchestrator and does not use these
 WebSocket endpoints. See [Standalone Experimental Servers](standalone_servers.md).
 

docs/serving/realtime_duplex_api.md

@@ -8,11 +8,9 @@ history stays honest, and a dropped connection can resume the same session.
 This page covers how to run a duplex deployment, how to drive it from Python
 with `vllm_omni.clients.duplex.DuplexClient`, and the complete wire contract.
 
-The endpoint is served for models that ship a duplex plugin — including
-MiniCPM-o 4.5 and Qwen3-Omni — and just for deploy configurations that declare
-`session_mode: duplex`. PersonaPlex and Nemotron VoiceChat are ported to the
-plugin contract in follow-up PRs and are not served over this endpoint yet. The runtime architecture
-is described in [Full-Duplex Runtime (MiniCPM-o 4.5)](../design/fullduplex.md).
+The endpoint requires a duplex plugin and a deployment with `session_mode: duplex`.
+See [supported models and deployments](full_duplex_api.md#enable-full-duplex)
+and the [runtime architecture](../design/fullduplex.md).
 
 ## Qwen3-Omni conversation history
 
@@ -56,7 +54,7 @@ vllm-omni serve openbmb/MiniCPM-o-4_5 \
 ```
 
 `vllm_omni/deploy/minicpmo_4_5.yaml` declares `session_mode: duplex` and
-`duplex_session.max_sessions: 4`. Because the MiniCPM-o 4.5 pipeline declares a
+`duplex_session.max_sessions: 16`. Because the MiniCPM-o 4.5 pipeline declares a
 `duplex_plugin`, `vllm-omni serve` runs it through `DuplexOmni`: the server
 mounts `ws://<host>:8099/v1/realtime?duplex=1` (this page; `ws://<host>:8099/v1/duplex`
 is an alias of the same route), `POST /v1/chat/completions`, `/v1/models` and
@@ -386,9 +384,9 @@ with no OpenAI counterpart.
 
 The event vocabulary is uniform, but several surfaces are gated by the
 `capabilities` object the server returns in `session.created`; a client must
-branch on those flags rather than on the model name. MiniCPM-o 4.5 is the
-only model on the plugin contract today; the other two columns record what
-their integrations advertise once the follow-up PRs port them:
+branch on those flags rather than on the model name. MiniCPM-o 4.5 and
+PersonaPlex are on the plugin contract today; the Nemotron VoiceChat column
+records what its integration advertises once the follow-up PR ports it:
 
 | Capability | MiniCPM-o 4.5 | PersonaPlex | Nemotron VoiceChat | Gated surface |
 | --- | --- | --- | --- | --- |
@@ -403,7 +401,11 @@ their integrations advertise once the follow-up PRs port them:
 Everything else in the catalogue — session lifecycle, heartbeat and event
 acknowledgement, append/commit/clear, the response envelope, playback
 acknowledgement, and the error envelope — behaves identically for every
-model.
+model. Two PersonaPlex specifics follow from its capabilities rather than from
+special-casing: a model with `supports_client_commit=false` auto-responds
+without `extra_body.auto_response`, and `response.cancel` /
+`output_audio_buffer.clear` restart its conversation context (a new Stage 0
+request replays the voice/persona prefill).
 
 ### Compatibility with the OpenAI Realtime protocol
 
@@ -1340,9 +1342,6 @@ them out into typed events before they reach a client.
 
 ## Known Limitations
 
-- Only MiniCPM-o 4.5 is served over this endpoint today; PersonaPlex and
-  Nemotron VoiceChat arrive with the follow-up PRs that port them to the
-  plugin contract.
 - Several surfaces are capability-gated per model (see *Capability
   negotiation by model* above): PersonaPlex does not support session resume,
   barge-in, or audio truncation; Nemotron VoiceChat does not support barge-in

docs/serving/realtime_openai_duplex_api.md

@@ -1,33 +1,31 @@
 # Realtime Audio WebSocket API
 
-Use `WS /v1/realtime` to stream PCM audio into a compatible omni model and
-receive incremental transcription and synthesized response audio. This is a
-turn-based streaming API: it starts one generation over an audio stream and
-finishes after the client closes that input stream.
-
-For a model that listens while it is already speaking, use the
-[Full-Duplex API](full_duplex_api.md) instead.
+As a companion to the vLLM-Omni-proprietary [fullduplex `/v1/realtime`](full_duplex_api.md),
+supported models also provide a `/v1/realtime` handler supporting
+a substantial subset of the OpenAI `/v1/realtime` API supporting
+audio streaming, client-driven VAD, and tool calling
 
 ## Quick Start
 
-Start a compatible model. The currently documented example uses Qwen3-Omni:
+The endpoint is currently only supported for Qwen3-Omni. To start:
+
+Start vLLM-Omni with port `8000` bound to an address reachable from Docker:
 
 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8000 --host 0.0.0.0
 ```
 
-Run the provided client with a mono, 16-bit PCM, 16 kHz WAV file:
+Then start the docker compose:
 
 ```bash
-python examples/online_serving/qwen3_omni/openai_realtime_client.py \
-  --url ws://localhost:8091/v1/realtime \
-  --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
-  --input-wav input_16k_mono.wav \
-  --output-wav response.wav \
-  --output-text response.txt
+cd examples/online_serving/qwen3_omni/fullduplex-livekit-frontend
+VLLM_OMNI_HOST=$VLLM_OMNI_HOST docker compose up --build
 ```
 
-The client dependency is `websockets`.
+Where `VLLM_OMNI_HOST` is a network address. If vLLM-Omni is running on localhost, use
+`host.docker.internal`
+
+Open [http://localhost:3000](http://localhost:3000) and begin chatting.
 
 ## Protocol
 
@@ -37,10 +35,9 @@ event is base64-encoded raw PCM16.
 | Direction | Event | Purpose |
 | ----------- | ------- | --------- |
 | Server to client | `session.created` | Confirms that the WebSocket connection is ready |
-| Client to server | `session.update` | Selects and validates the served model |
-| Client to server | `input_audio_buffer.commit` with `final: false` | Starts generation over the incoming stream |
+| Client to server | `session.update` | Selects and validates the served model. Configures system prompt and tools |
 | Client to server | `input_audio_buffer.append` | Appends base64 PCM16 audio |
-| Client to server | `input_audio_buffer.commit` with `final: true` | Marks the end of input |
+| Client to server | `input_audio_buffer.commit` | Starts generation over the incoming stream |
 | Server to client | `transcription.delta` | Carries incremental response text |
 | Server to client | `transcription.done` | Carries final text and token usage |
 | Server to client | `response.output_audio.delta` | Carries incremental PCM16 response audio |
@@ -50,15 +47,15 @@ event is base64-encoded raw PCM16.
 A minimal client sends events in this order:
 
 ```json
-{"type":"session.update","model":"Qwen/Qwen3-Omni-30B-A3B-Instruct"}
-{"type":"input_audio_buffer.commit","final":false}
-{"type":"input_audio_buffer.append","audio":"<base64-pcm16>"}
-{"type":"input_audio_buffer.commit","final":true}
+{"type":"input_audio_buffer.append","audio":"<base64-pcm16>"},
+{"type":"input_audio_buffer.commit"}
 ```
 
-The initial non-final commit intentionally starts generation before all audio
-has arrived. Continue sending `input_audio_buffer.append` events while the
-engine is consuming the stream.
+This OpenAI-compatible streaming path is distinct from the vLLM-Omni-proprietary
+lane and as such is reached via different configuration. The vLLM-Omni-proprietary
+path must be specified explicitly via "duplex=1|true|on" as the WS query parameter
+or via an explicit configuration in the `deploy_config`. The OpenAI path simply replaces
+the default `/v1/realtime` handler in all supported models.
 
 ## Audio Handling
 
@@ -67,8 +64,6 @@ engine is consuming the stream.
 - Read `sample_rate_hz` from each audio event instead of assuming an output
   rate. Qwen3-Omni output is typically 24 kHz.
 - Concatenate audio deltas in receive order to construct the output waveform.
-- Set `input_audio_buffer.commit.final` to `true` after the last input chunk so
-  the server can terminate the streaming request.
 
 ## Availability and Limitations
 
@@ -76,7 +71,8 @@ The path is registered on the unified API server, but it is usable only when
 the loaded pipeline implements realtime audio input and produces compatible
 audio output. Unsupported deployments return an `error` event. The endpoint
 does not provide duplex session resume, playback acknowledgement, overlap
-policy, or barge-in controls.
+policy, or barge-in controls internally. Those must be driven by the client application
+via the relevant WS events (e.g. `response.cancel`, `conversation.item.truncate`)
 
 See the [Qwen3-Omni online serving example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/qwen3_omni)
 for concurrency options, chunk pacing, and per-delta audio debugging.

docs/serving/speech_api.md

@@ -1029,3 +1029,64 @@ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
     --enforce-eager \
     --uvicorn-log-level debug
 ```
+
+### Reusing reference audio
+
+The voice upload API and cache settings below are shared serving features.
+For repeated voices, upload the reference once and reuse its name. The example
+uses MOSS-TTS Local: uploading stores the audio, and the first speech request
+encodes it. MOSS-TTS Local can then reuse cached reference codes without resolving
+the waveform while the speaker cache entry remains resident. After a server
+restart or speaker-cache eviction, another speech request warms the codes again.
+Reference processing and cache-hit behavior for other models depend on their
+serving adapters.
+
+```bash
+curl --fail http://localhost:8123/v1/audio/voices \
+  -F 'name=my-speaker' \
+  -F 'consent=user_consent_id' \
+  -F 'audio_sample=@/path/to/reference.wav'
+
+# Run once to warm the reference codes, then reuse the same voice.
+curl --fail http://localhost:8123/v1/audio/speech \
+  -H 'Content-Type: application/json' \
+  -d '{"model":"OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5","voice":"my-speaker","input":"Hello, world.","response_format":"wav"}' \
+  --output speech.wav
+```
+
+If you have the reference transcript, upload it with `-F 'ref_text=...'`.
+MOSS-TTS Local uses a reference transcript for continuation conditioning; omitting
+it uses reference-audio conditioning instead. Keep this choice consistent when
+comparing benchmarks. Supplying inline `ref_audio` overrides the uploaded voice
+and does not use the named-voice cache shortcut.
+
+The separate decoded-waveform resolve cache defaults to **4 GiB and 2048 entries
+per speech server instance**. These are LRU limits, not preallocated memory or a
+limit on total server RSS. Waveforms retain their source sampling rate and are
+stored as owned, contiguous float32 arrays in **CPU memory**. The byte limit
+counts numeric buffers (four bytes per mono sample), not the encoded file size
+or GPU memory.
+Either limit can cause eviction. Configure the API-process caches in the deploy
+YAML passed to `--deploy-config` (alongside `stages`, not inside a stage):
+
+```yaml
+speech_cache:
+  resolve_max_bytes: 4294967296
+  resolve_max_entries: 2048
+  speaker_max_bytes: 536870912
+```
+
+All values must be non-negative integers. Either resolve limit set to `0`
+disables waveform storage; `speaker_max_bytes: 0` disables speaker artifact
+storage. Invalid values fail configuration loading. `base_config` inheritance
+merges this section by field, preserving unspecified values from the base.
+Restart the service to apply changes. The effective settings are logged at startup.
+
+The speaker budget defaults to **512 MiB** and counts cached tensor bytes,
+not Python metadata or total process RSS. It configures the API process's
+speaker cache. MOSS reference codes are cached as **CPU int32 tensors**; this
+budget does not reserve GPU memory or include the reference encoder's GPU
+weights and working memory. The shared cache itself does not move tensors
+between devices; other adapters determine the device of their cached artifacts.
+Separate model-worker caches are not controlled by this section. The process-wide singleton rejects a
+conflicting explicit budget instead of silently ignoring it.

docs/serving/video_stream_api.md

@@ -16,6 +16,10 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \
     --trust-remote-code
 ```
 
+To enable [incremental prefill](#incremental-prefill), set
+`enable_prefix_caching: true` on stage 0 (the thinker) in the deploy config. Otherwise
+the server falls back to rebuilding the prompt per query.
+
 ### Run the Example Client
 
 ```bash
@@ -54,7 +58,7 @@ WebSocket /v1/video/chat/stream
 
 When buffered frames include client-supplied `frame_id` values, the server emits `video.frames.consumed` after the first engine output. Its `frame_ids`, `frames`, and `latest_pts_ms` describe the images included in that query's prompt.
 
-Queries stride-sample the buffered frames, keeping the last frame, then exclude frames already known to have failed decoding. Excluded frames are not replaced with other buffered frames. Frames whose background decoding has not finished remain eligible through the image URL path.
+Queries stride-sample the buffered frames, keeping the last frame, then exclude frames already known to have failed decoding. With incremental prefill active, every buffered frame is included instead of stride-sampling, with the same decode-failure exclusion. Excluded frames are not replaced with other buffered frames. The incremental path waits for the whole window to finish decoding. The legacy path waits only for its sampled positions, so an unselected frame still decoding does not delay the query.
 
 **Bugfix compatibility note:** `video.frames.consumed` now excludes known decode failures that were already excluded from the prompt. Older versions could report those frames and their timestamps as consumed. The event name and fields are unchanged; an empty selection reports empty lists and `latest_pts_ms: null`.
 
@@ -64,10 +68,10 @@ Queries stride-sample the buffered frames, keeping the last frame, then exclude
 | ------- | ------ | --------- | ------------- |
 | `model` | string or null | null | Optional model name. Usually omitted because the server hosts one model. |
 | `modalities` | list[string] | `["text", "audio"]` | Output modalities. Use `["text"]`, `["audio"]`, or both. |
-| `num_frames` | integer, 1-128 | `4` | Number of buffered frames sampled for each query. |
-| `max_frames` | integer, 1-256 | `50` | Maximum retained frame buffer size. Oldest frames are evicted first. |
+| `num_frames` | integer, 1-128 | `4` | Number of buffered frames sampled for each query. Legacy path only: with incremental prefill active, every buffered frame is submitted and frame density is the client's responsibility (clients push discrete frames at their own rate). |
+| `max_frames` | integer, 1-256 | `50` | Maximum retained frame buffer size. Below the cap both paths keep every frame. Legacy evicts the oldest frame once full. Incremental prefill drops the oldest half at once, then appends until full again. |
 | `system_prompt` | string or null | null | Optional custom system prompt. |
-| `use_audio_in_video` | bool | `true` | Include streamed audio chunks in multimodal video understanding when audio is present. |
+| `use_audio_in_video` | bool | `true` | Incremental prefill: forwarded on warmup and every query when this is true, so multimodal hashes stay aligned. Legacy: forwarded only when this query has input audio. |
 | `sampling_params_list` | list or null | null | Optional per-stage parameter dictionaries. Each provided entry replaces that stage's deployment sampling settings. |
 | `enable_frame_filter` | bool | `true` | Enable EVS near-duplicate frame filtering. |
 | `frame_filter_threshold` | float, 0.0-1.0 | `0.95` | EVS similarity threshold. Higher keeps more frames; lower drops more near-duplicates. |
@@ -101,15 +105,48 @@ The server accepts these legacy field names and rewrites them before validation.
 | Variable | Values | Default | Description |
 | -------- | ------ | ------- | ----------- |
 | `VLLM_VIDEO_ASYNC_CHUNK` | `on`, `off` | `on` | Wire-level streaming switch. `off` buffers server-side deltas and emits coalesced outputs at the end of a query. |
-| `VLLM_VIDEO_AUDIO_DELTA_MODE` | `fast`, `slow` | `fast` | Audio delta extraction strategy. `fast` emits only newly produced chunks; `slow` recomputes from accumulated audio and exists for A/B verification. |
+| `VLLM_VIDEO_AUDIO_DELTA_MODE` | `fast`, `slow` | `fast` | Both settings forward every fresh engine audio delta. `slow` is retained as a compatibility alias for `fast`. |
 
 ## EVS Semantics
 
 EVS compares downsampled frames and drops near-duplicate frames before they enter the session frame buffer. `frame_filter_threshold` controls retention: higher values are more permissive and keep more frames; lower values are more aggressive and drop more similar frames.
 
+## Incremental Prefill
+
+Server-side only (stage-0 `enable_prefix_caching: true` and `modalities: ["text"]`).
+There is no `session.config` flag. Audio-output sessions stay on the legacy path.
+
+Input is unchanged: `video.frame` and `audio.chunk` are separate buffers. Warmup
+prefills `history + frames` as they arrive (`max_tokens=1`). When
+`use_audio_in_video` is true, warmup and query both pass that processor kwarg.
+`video.query` is the same prompt plus optional input audio and the question text,
+so the warmed vision prefix cache is reused and the query pays for the audio/text
+suffix. All buffered frames are
+submitted in arrival order (`num_frames` subsample is legacy-only). Once the buffer
+hits `max_frames`, the oldest half is dropped in one step and a warning is logged;
+until then the buffer matches the legacy path. After a turn,
+history compresses to the last two text messages and the next warmup follows the
+new prefix.
+
+### Legacy path (incremental prefill inactive)
+
+Stage-0 prefix caching is off, or `modalities` includes `"audio"`: no warmup,
+frames are re-sampled up to `num_frames` each query, and the query waits only
+for those sampled frames to decode. Prompt shape is the same
+(`image_pil` frames, optional trailing `input_audio`, text). Both paths keep the
+frame buffer across queries (`max_frames`) and compress history the same way.
+
 ## Known Limitations
 
-- Session KV reuse and incremental prefill are not implemented in this PR. Each `video.query` rebuilds the model prompt from the retained frame and audio buffers.
+- Past `max_frames`, incremental prefill keeps a suffix of the newest frames, not a resample of the whole stream. Rebuilding that longer window is left to a follow-up.
+- Incremental prefill applies only to text-only sessions with stage-0 prefix caching enabled
+  on `WS /v1/video/chat/stream`. It does not accelerate `/v1/realtime` camera input, which
+  uses duplex conversation items rather than `video.frame` / `video.query`. Audio-output
+  sessions on this endpoint rebuild the prompt each query.
+- Stage-0 prefix caching is safe for audio-output requests: on a hit, the omni prefix cache
+  restores the thinker hidden states the talker needs. A preempted request that resumes on a
+  hit does not get them back, so size the deployment to avoid preemption; see
+  [Prefix Caching](../design/feature/prefix_caching.md).
 - Back-to-back short replies can still expose an engine-layer scheduler race. The PR notes an observed workaround of at least 200 ms idle between turns when clients repeatedly see idle timeouts.
 - If the audio buffer exceeds the server limit, the server emits `Audio buffer overflow` and clears the currently buffered audio for the session.
 - The API is intended for Qwen3-Omni streaming video understanding; other models may not support the same multimodal processor arguments.

docs/serving/videos_api.md

@@ -120,7 +120,18 @@ request for execution.
 `DELETE /v1/videos/{video_id}` issues a bounded engine abort
 (`VLLM_OMNI_ABORT_TIMEOUT`, default 2s), then cancels the frontend
 task. Cancellation cleanup is also bounded and best-effort: it confirms
-the abort was queued, and the current request batch may still drain.
+the abort was submitted. In request execution mode, MiniMax-H3 checks
+cancellation at model boundaries,
+including before and after input preparation, after each denoising step, and
+before decode, and skips the remaining generation when cancelled. Input
+preparation or decoding already in progress may finish before the next boundary.
+Successful steps do not force device synchronization for cancellation.
+Parallel ranks agree before stopping; independent requests sharing a distributed
+AllGather offload wave can stop that wave early only when all its requests are
+cancelled, so a cancelled request cannot strand its live peers in a collective.
+In step execution mode, cancellation is handled by the scheduler between steps;
+the request-mode component-boundary checks do not apply within an active step.
+Other pipelines may still drain their current request batch.
 The job is then re-read so a completed save is not orphaned.
 
 ### Synchronous Response
@@ -324,6 +335,13 @@ window and output frame count stay unchanged. Wan S2V keeps its existing
 per-clip behavior by default. Zero, negative, fractional, boolean, string, and
 null values are rejected when pre-encoding is enabled.
 
+The transfer ring applies backpressure before another D2H copy when both of
+its two slots are occupied or the combined pending uint8 payload would exceed
+256 MiB. This byte bound keeps large resolutions and uneven final chunks from
+turning a fixed item count into unexpectedly large memory growth. A native VAE
+chunk larger than 256 MiB is admitted only when the ring is otherwise empty,
+so supported chunk shapes cannot deadlock.
+
 `preencode_mp4` applies to the complete-MP4 response paths only. The
 `/v1/realtime/video` WebSocket endpoint rejects it, because that path already
 overlaps encoding through its own incremental fragmented-MP4 encoder. Wan also

docs/user_guide/diffusion/device_side_video_postprocess.md

@@ -70,3 +70,42 @@ as safety checks or audio/video packaging.
 
 See [RFC #6541](https://github.com/vllm-project/vllm-omni/issues/6541) for the
 contract, lifecycle, batching rules, and migration plan.
+
+## Experimental registered-SHM video transport
+
+`video_output_transport.enable_registered_shm` is a separate, default-off option
+for CUDA request-mode workers. It copies eligible video tensors directly into
+CUDA-registered shared memory and returns borrowed CPU views, removing the
+intermediate pinned-to-SHM and SHM-to-consumer host copies. The D2H transfer still
+occurs.
+
+`video_output_transport.enable_borrowed_frames` independently enables borrowed
+RGB AVFrames in the Videos API MP4 response encoder, removing the third host
+copy into PyAV frame buffers. It also defaults to false. Eligible uint8 RGB
+frames are wrapped with `VideoFrame.from_numpy_buffer`; other dtypes/layouts use
+the existing encoder path. Color conversion, compression and muxing still occur.
+
+Enable it through the same CLI, Python or deploy-stage configuration:
+
+```bash
+--video-output-transport '{"enable_registered_shm": true, "enable_borrowed_frames": true}'
+```
+
+Registered SHM applies only to prepared typed video media (currently WAN2.2) and
+explicitly marked legacy video outputs (currently MiniMax-H3). H3 marks the video
+entry of its `(video, audio)` output; audio, trajectory data, images and unmarked
+legacy outputs keep their existing transport. Device-side postprocessing is
+independent and does not need to be enabled for registered SHM.
+Pre-encoded MP4 outputs bypass raw-frame transport and API-side encoding.
+
+The existing SHM routing threshold is unchanged: the tensor or its backing
+storage must exceed 1,000,000 bytes. This is not a measured performance crossover.
+SHM is allocated and registered on every transfer, without pooling; small outputs
+can regress and registration latency can vary. Benchmark representative output
+sizes before opting in. CPU outputs, non-CUDA backends, and synchronous/step-mode
+transfers keep their existing path.
+
+Borrowed views keep the unlinked mapping alive until their last user releases
+it. CUDA registration/copy failures fail the request; if CUDA cleanup cannot be
+confirmed, the worker retains the mapping for safety and must be restarted before
+further registered-SHM transfers.

docs/user_guide/diffusion/lora.md

@@ -27,6 +27,21 @@ The `adapter_config.json` file contains metadata about the LoRA adapter, includi
 - `lora_alpha`: LoRA alpha scaling factor
 - `target_modules`: List of module names to apply LoRA to
 
+Every module supplied by a PEFT adapter must bind successfully when the adapter
+is activated. If any module cannot bind, activation raises an error naming the
+unbound modules and resets the LoRA layers to avoid leaving the adapter partly
+active. An adapter may still target only some projections of a fused layer, such
+as Q and V without K, as long as all supplied modules bind.
+
+Previously, some adapters activated even when unmatched modules were silently
+dropped. Those adapters now fail at activation. For example, in-memory tensor
+adapters for Qwen-Image, Boogu-Image, and Wan can contain checkpoint keys such
+as `to_out.0` where the engine expects `to_out`. PEFT checkpoints with extra
+unbound keys also fail at activation if those keys pass the loader's earlier
+validation; unsupported module suffixes may already fail during loading.
+Checkpoint-to-engine name mapping is tracked in [#8001](https://github.com/vllm-project/vllm-omni/issues/8001);
+until it is available, adapter keys must resolve to supported engine modules.
+
 ## Quick Start
 
 ### Offline Inference
@@ -244,14 +259,22 @@ Notes:
 
 ## MiniMax-H3 adapter-declared schedules
 
-MiniMax-H3 supports two few-step mechanisms that must not be conflated:
+MiniMax-H3 counts `num_inference_steps` as denoising updates: requesting `8`
+produces eight updates using nine sigma boundaries, including the terminal zero.
+Earlier uniform-schedule requests counted boundaries instead; to reproduce an
+old N-point schedule, request N-1 steps. This also changes Turbo requests from
+`5`/`9` to `4`/`8`, preserving their sampling grids. Checkpoint-pinned and native
+LoRA schedules already count updates and are unchanged.
+
+MiniMax-H3 supports the following few-step mechanisms:
 
 - **Checkpoint-pinned schedule**: a merged release writes `base_schedule` into
   `model_index.json`. Requests must pass `num_inference_steps` as the interval
   count (for example `4` for `[1.0, 0.7, 0.4, 0.15, 0.0]`).
 - **Runtime Turbo LoRA**: each LightX2V Turbo artifact carries its own contract,
-  read from its filename. A four-step artifact requests five sigma points and an
-  eight-step one requests nine; the 768p retrains enforce `flow_shift=6` and the
+  read from its filename. A four-step artifact requests `num_inference_steps=4`
+  and an eight-step one requests `8`; the scheduler adds the terminal sigma
+  boundary. The 768p retrains enforce `flow_shift=6` and the
   544p artifacts `flow_shift=12`. `audio_flow_shift=3` across the family. A
   request that does not match the loaded artifact is rejected by name. Alpha
   comes from the artifact's metadata, or 8 when it declares none. `ref2v`

docs/user_guide/diffusion/vae_fast_path.md

@@ -0,0 +1,102 @@
+# Wan VAE Decoder Fast Path
+
+Video pipelines built on the Wan 2.1/2.2 causal VAE (Cosmos3, Wan2.2, LingBot,
+Helios, SANA-Video, LongCat-Video, DreamZero) spend a large share of their
+end-to-end time decoding latents to pixels. The decoder is a chunked 3D
+convolutional network whose reference implementation in diffusers moves every
+activation through memory several times per layer (normalization, activation,
+causal padding, feature-cache bookkeeping, shortcut upsampling). vLLM-Omni
+installs a fast path on every loaded Wan VAE that fuses this data movement into
+a handful of Triton kernels while leaving the convolutions themselves untouched.
+
+## Levels
+
+The fast path is controlled by `--vae-fast-path` (engine argument
+`vae_fast_path`, deploy-config key `vae_fast_path`):
+
+| Level | Default | Output vs. diffusers | What it does |
+|---|---|---|---|
+| `lossless` | yes | bit-identical | Fused RMSNorm epilogue, fused causal-conv input assembly and cache refresh, fused shortcut upsampling and residual adds (with the neighbouring convolution biases folded in), single-pass nearest 2x upsampling, preallocated output assembly. |
+| `channels_last` | no | within tolerance (PSNR typically > 60 dB) | Everything in `lossless`, plus decoder convolution weights converted to channels-last memory format so cuDNN picks its channels-last kernels, and a single-pass channels-last RMSNorm+SiLU kernel that also absorbs the bias of the preceding `conv1`. |
+| `off` | no | bit-identical | Reference diffusers decoder. |
+
+```bash
+# Default: bit-exact fast path
+vllm serve nvidia/Cosmos3-Nano --omni
+
+# Fastest: channels-last decoder (not bit-exact)
+vllm serve nvidia/Cosmos3-Nano --omni --vae-fast-path channels_last
+
+# Reference implementation, e.g. to bisect a quality issue
+vllm serve nvidia/Cosmos3-Nano --omni --vae-fast-path off
+```
+
+The equivalent per-stage deploy configuration is:
+
+```yaml
+stages:
+  - stage_id: 0
+    vae_fast_path: channels_last
+```
+
+## Behavior and limitations
+
+- Only CUDA is supported. On other platforms the reference decoder runs.
+- The fast path is installed once per VAE instance when the pipeline is
+  initialized. It rebinds the forwards of the loaded decoder modules; parameter
+  names, `state_dict` keys and weight loading are unchanged.
+- Installation is skipped, with a logged reason, if a custom forward wrapper
+  or forward hook would be replaced or bypassed by the fused paths. Wrappers
+  and hooks on modules that the fast path still calls normally are preserved.
+- Every fused kernel validates its inputs and falls back to the exact PyTorch
+  expression for anything it does not support (unusual dtypes or layouts, CPU
+  tensors, autograd enabled, `torch.compile` tracing).
+- The SiLU activation is folded into the normalization kernel only after an
+  exhaustive self-test over all bf16/fp16 values proves the fused epilogue
+  bit-identical to `F.silu` on the running toolkit; otherwise SiLU stays a
+  separate operation. The startup log reports `fused_silu=...`.
+- Spatially sharded VAE decode (`--vae-parallel-mode spatial_shard_height` or
+  `spatial_shard_width`) is not combined with the fast path; the installer skips
+  the VAE and logs why. Tiled and tile-parallel decode work unchanged.
+- The `channels_last` level changes the order in which cuDNN accumulates
+  convolutions, so outputs differ from the reference in the last bits. Use
+  `lossless` when bitwise reproducibility against diffusers matters.
+- For VAEs kept in fp32, cuDNN's channels-last convolution algorithms run in
+  TF32 under PyTorch's default `torch.backends.cudnn.allow_tf32 = True`, which
+  dominates the difference to the reference (about 1e-3). Set
+  `torch.backends.cudnn.allow_tf32 = False` if full fp32 convolutions are
+  required.
+
+## Measured
+
+Cosmos3-Nano VAE, 1280x720 x 189 frames, bf16, one GB200 GPU, `bench_wan_vae_decode.py`
+(best of 2 runs after warmup):
+
+| `--vae-fast-path` | Decode time | Speedup | Output vs. `off` |
+|---|-------------|---------|---|
+| `off` | 6.00 s      | 1.00x   | reference |
+| `lossless` | 3.26 s      | 1.84x   | bit-identical |
+| `channels_last` | 2.51 s      | 2.39x   | PSNR 62.6 dB, max abs diff 4.6e-2 |
+
+Peak decode memory was unchanged (about 13 GiB): it is set by the largest
+activations and the cuDNN workspace, not by the output assembly.
+
+## Benchmarking
+
+`benchmarks/diffusion/bench_wan_vae_decode.py` decodes seeded latents with the
+real VAE at each level and reports decode time, peak memory, bitwise equality
+and PSNR against the `off` level. `--profile` prints a per-kernel table with the
+convolution share and any layout-transpose kernels:
+
+```bash
+python benchmarks/diffusion/bench_wan_vae_decode.py --model nvidia/Cosmos3-Nano \
+    --size 1280x720 --frames 189 --fast-path off,lossless,channels_last --profile
+```
+
+The same script benchmarks multi-GPU VAE decode when launched with `torchrun`;
+the decode is timed across all ranks and rank 0 reports:
+
+```bash
+torchrun --nproc-per-node 2 benchmarks/diffusion/bench_wan_vae_decode.py --model nvidia/Cosmos3-Nano \
+    --vae-patch-parallel-size 2 --vae-parallel-mode tile
+```

docs/user_guide/diffusion_features.md

@@ -32,6 +32,30 @@ Cache methods trade minimal quality for significant speedup. Quality loss is typ
 | **[TeaCache](diffusion/cache_acceleration/teacache.md)** | Adaptive caching using modulated inputs | Quick setup, balanced quality/speed on single GPU |
 | **[Cache-DiT](diffusion/cache_acceleration/cache_dit.md)** | Multiple caching techniques: DBCache, TaylorSeer, SCM | Fine-grained control, tunable quality-speed tradeoff |
 
+#### Diffusion KV Prefix Caching
+
+[KV prefix caching](../design/feature/prefix_caching.md#diffusion-kv-prefix-caching)
+reuses stable context KV across requests, rather than approximating denoise
+steps. Enable `diffusion_kv_mode: paged_scheduler` and `enable_prefix_caching: true`
+on the HunyuanImage3 standalone DiT stage. This is separate from the AR stage-output
+cache, TeaCache and Cache-DiT.
+
+| Model / combination | Scope |
+|---------------------|-------|
+| HunyuanImage3 standalone DiT | Stable text/reference-image prefix reuse; dynamic image tokens are excluded |
+| TP4/SP1 + EP, CFGP1 | Validated reference-image prefix hits |
+| TP2/SP2 (Ulysses) + EP, CFGP1 | Validated reference-image prefix hits; CFG guidance does not require CFG parallelism |
+| Other SP/TP combinations, CFGP>1 | Prefix-hit E2E accuracy not yet validated |
+| AR-imported KV / cross-stage missing-page transfer | Not covered by local prefix caching |
+| Sleep mode | Rejected with prefix caching; discarded KV pages would leave stale cache hits |
+| TeaCache / Cache-DiT, offload, quantization | Not validated with prefix hits |
+| Step execution | Not supported with HunyuanImage3 paged KV; use request-level execution |
+| Other diffusion models | No prefix-cache model adapter yet |
+
+This table describes prefix-cache scope; the general parallelism tables below do
+not establish prefix-hit compatibility. Floating-point kernel differences still
+require accuracy validation. `dense_legacy` remains the default.
+
 #### Lossless Acceleration
 
 Parallelism methods distribute computation across GPUs without quality loss (mathematically equivalent to single-GPU).
@@ -139,7 +163,9 @@ The following tables show which models support each feature:
 | **Qwen-Image-Edit**      |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Qwen-Image-Edit-2509** |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |        ✅ (decode)         |          ✅           |       ❌        |        ❌         |
 | **Qwen-Image-Layered**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
-| **SenseNova-U1 / U1.5**  |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **SenseNova-U1**         |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **SenseNova-U1.5**       |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ✅        |        ❌         |
+| **SenseNova-U1-A3B**     |     ❌     |     ❌      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **Stable-Diffusion-XL**  |     ❌     |     ❌      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Stable-Diffusion3.5**  |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Z-Image**              |     ✅     |     ✅      |           ✅           |       ❓        |   ✅ (TP=2 only)   |          ❌          |   ✅    |             ❌             |      ✅ (decode)      |       ✅        |        ❌         |
@@ -155,6 +181,7 @@ The following tables show which models support each feature:
 > 5. HunyuanImage3 supports step execution. Multi-request step execution requires `TORCH_SDPA`; see [Diffusion Execution Modes](diffusion/execution_modes.md#step-execution).
 > 6. BAGEL step execution supports image generation with `bagel.yaml`, `bagel_think.yaml`, and `bagel_single_stage.yaml`; two-stage Thinker execution and explicit single-stage text output remain on their existing complete-request paths. Image requests require `num_inference_steps >= 2`. BAGEL step execution cannot currently be combined with sequence parallelism or a diffusion cache backend; see [Diffusion Execution Modes](diffusion/execution_modes.md#step-execution).
 > 7. MammothModa2 runs its DiT stage on the diffusion runner (`StageExecutionType.DIFFUSION`); Cache-DiT is enabled through the standard diffusion-stage knobs on the stage entry of the deploy YAML (`cache_backend: cache_dit`, plus optional `cache_config` / `enable_cache_dit_summary`). The runner installs the backend at startup and the pipeline adopts it per request. Only the repeated main-layer stack is cached; requests with `text_guidance_scale = 1.0` bypass the cache hooks.
+> 8. SenseNova-U1-A3B: `--cfg-parallel-size 2` and TP=2 both complete. CFG-P covers t2i 2-branch (`cond`+`uncond`), img2img 2-branch (`cond`+`img_cond` when `img_cfg_scale=1`), and img2img 3-branch (`cond`+`img_cond`+`uncond` when `img_cfg_scale` is neither 1 nor equal to `cfg_scale`; 3 branches on 2 ranks dispatch as `[[0, 2], [1]]`). Cache-DiT does not: the adapter is declared on `SenseNovaU1Model`, but discovery walks `_dit_modules` (`language_model` / `SenseNovaU1ForCausalLM`) and raises. Layerwise CPU offload is a no-op for the same reason — that class has no `_layerwise_offload_blocks_attrs`, so the 72.2 GiB expert weights stay resident.
 
 ### VideoGen
 
@@ -252,6 +279,7 @@ The Diffusion Acceleration navigation groups the remaining guides as follows:
 | Parallelism | [Parallelism Overview](diffusion/parallelism/overview.md) |
 | Attention | [Attention Backends](diffusion/attention_backends.md) |
 | Compilation | [Regional Compilation](diffusion/regional_compilation.md) |
+| VAE decode | [Wan VAE Decoder Fast Path](diffusion/vae_fast_path.md) |
 | Video extension | [Frame Interpolation](diffusion/frame_interpolation.md) |
 | Startup | [Startup and Loading](diffusion/startup_and_loading.md) |
 | Adapters | [LoRA](diffusion/lora.md) |

docs/user_guide/examples/offline_inference/text_to_speech.md

@@ -130,7 +130,7 @@ python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
 ### Notes
 
 - Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform.
-- Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config <path>` to override.
+- H200 automatically selects `cosyvoice3_packed_streaming_optimized_standard.yaml` when the CUDA MPS control tool is available; other devices and environments use `cosyvoice3.yaml`. See the [serving defaults and tradeoffs](../online_serving/text_to_speech.md#cosyvoice3). Pass `--deploy-config cosyvoice3.yaml` for the previous profile or `--deploy-config <path>` to override. The example uses the resolved profile's sampling parameters.
 - `async_chunk: true` is the default; pass `--no-async-chunk` to switch to the legacy synchronous path.
 
 ---

docs/user_guide/examples/online_serving/text_to_speech.md

@@ -16,6 +16,7 @@ For the full list of supported architectures across all modalities, see
 
 | Model | HuggingFace repo | Voice cloning | Streaming | Voice presets / upload | Gradio demo |
 | --- | --- | --- | --- | --- | --- |
+| AuK / AuK-Flash | assembled local bundle | ✓ (`ref_audio`) | HTTP audio after full generation | — | — |
 | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | — |
 | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
 | Gepard-1.0 | `nineninesix/gepard-1.0` | — (zero-shot default voice) | ✓ (PCM / WAV stream) | `default` only | — |
@@ -99,6 +100,41 @@ For full request-shape documentation (all parameters, response formats, error co
 
 ---
 
+## AuK / AuK-Flash
+
+AuK serves speech generation and reference-voice synthesis at 24 kHz. Assemble a local bundle before serving:
+
+```bash
+python tools/prepare_auk_checkpoint.py \
+    --auk-dir ckpts/AuK --qwen-dir ckpts/Qwen2.5-Omni-3B --out ckpts/auk-omni
+MODEL=ckpts/auk-omni bash examples/online_serving/text_to_speech/auk/run_server.sh
+```
+
+AuK accepts the complete instruction after you fill in the [local recipe template](../../../../recipes/Tencent/AuK-H100.md#supported-model-contract) or the [official AuK cookbook](https://github.com/Tencent-Hunyuan/AuK/blob/main/docs/COOKBOOK.md). Pass it in `instructions` and leave `input` empty. For instruct TTS, fill in `Generate speech based on the following description: "{voice description}". The content to speak is: "{text}".`; for zero-shot TTS, editing, enhancement and separation, use the corresponding complete instruction from the cookbook. Set `duration_seconds` for text-only generation; with `ref_audio`, omitting it keeps the source length.
+
+For compatibility with clients that can only send `input`, AuK treats an input-only request as the complete instruction and logs a recommendation to move it to `instructions` with `input=""`.
+
+`extra_params` accepts `num_inference_steps`, `guidance_scale`, `sway`, `t_grid`, and `vae_sample`; `seed` is a top-level field.
+
+```bash
+python examples/online_serving/text_to_speech/auk/speech_client.py \
+    --model ckpts/auk-omni \
+    --instructions 'Generate speech based on the following description: "A calm young woman speaking warmly and slowly.". The content to speak is: "Welcome back, how was your day?".' \
+    --duration-seconds 3 --seed 7
+```
+
+Complete-instruction editing also does not need a separate text argument:
+
+```bash
+python examples/online_serving/text_to_speech/auk/speech_client.py \
+    --model ckpts/auk-omni --instructions "Keep pure speech voice, remove noise and reverberation." \
+    --ref-audio noisy.wav --seed 7 --output clean.wav
+```
+
+The client accepts local reference files, URLs, and data URLs. `--stream` writes signed 16-bit 24 kHz PCM after the complete waveform is ready. See `speech_client.py --help` for editing and sampling options.
+
+---
+
 ## CosyVoice3
 
 2-stage TTS (`talker` + flow-matching `code2wav`) at 24 kHz. Voice cloning only — every request needs `ref_audio` + `ref_text`; there are no built-in voice presets.
@@ -119,16 +155,22 @@ vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 --omni --port 8091 --trust-remot
 ./cosyvoice3/run_server.sh
 ```
 
-Streaming is on by default via `async_chunk: true` in `vllm_omni/deploy/cosyvoice3.yaml`. Pass `--no-async-chunk` (or `NO_ASYNC_CHUNK=1 ./cosyvoice3/run_server.sh`) for the legacy synchronous path.
+Streaming is on by default. On CUDA Hopper GPUs with at least 140 GiB of device memory (H200), serving automatically selects `cosyvoice3_packed_streaming_optimized_standard.yaml` when `nvidia-cuda-mps-control` is available on `PATH`: AR32 / codec32, standard sampling, packed Flow, cached ISTFT, HiFT decode graphs, and local CUDA MPS. No `--hf-overrides` is needed. Other devices and environments without the MPS control tool use `cosyvoice3.yaml` with RAS sampling and AR8 / codec8.
 
-Cross-request Stage-1 flow batching is opt-in. Enable it in the server environment when concurrent requests should share a flow-estimator call:
+The optimized profile is not a universal hardware configuration. Its packed kernels are Hopper-only; the talker's Model Runner V2 is unsupported on NPU/XPU, and CUDA MPS requires a local NVIDIA GPU and its control tool. Larger batches, 25-frame chunks, and graph allocations also require per-device validation on smaller or slower GPUs. Automatic selection retains the generic profile outside the validated device class; an explicit deploy configuration takes precedence.
+
+The H200 default targets streaming throughput at C32/C64. It changes sampling from RAS to standard (temperature 0.7, top-p 0.8, top-k 20, repetition penalty 1.21); output lengths and quality can differ. HiFT captures a recurring exact shape on its third use, which adds latency to that request, and its graph pool needs memory headroom beyond the engine reservations. This profile is not a claim of optimal first-audio latency at low concurrency. Pass `--deploy-config cosyvoice3.yaml` to retain the previous configuration, or `--deploy-config <path>` to select another profile.
+
+Pass `--no-async-chunk` (or `NO_ASYNC_CHUNK=1 ./cosyvoice3/run_server.sh`) for the legacy synchronous path; use `--deploy-config cosyvoice3.yaml` alongside it when selecting that path on H200.
+
+Cross-request Stage-1 flow batching is enabled in the H200 profile. For the generic configuration, enable it in the server environment when concurrent requests should share a flow-estimator call:
 
 ```bash
 export COSYVOICE3_BATCH_FLOW=1
 vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 --omni --port 8091 --trust-remote-code
 ```
 
-Batching preserves output lengths and streaming cache alignment, but the different GEMM shapes can produce small waveform differences compared with processing each request separately. Leave `COSYVOICE3_BATCH_FLOW` unset (or set it to `0`) when request-independent numerical behavior is required. Set `COSYVOICE3_BATCH_FLOW_DEBUG=1` to log the observed group-size distribution and enable detailed Stage-1 profiler scopes; diagnostics are disabled by default to avoid per-step profiling overhead.
+Batching preserves output lengths and streaming cache alignment, but the different GEMM shapes can produce small waveform differences compared with processing each request separately. Use `--deploy-config cosyvoice3.yaml` and leave `COSYVOICE3_BATCH_FLOW` unset (or set it to `0`) when request-independent numerical behavior is required. Set `COSYVOICE3_BATCH_FLOW_DEBUG=1` to log the observed group-size distribution and enable detailed Stage-1 profiler scopes; diagnostics are disabled by default to avoid per-step profiling overhead.
 
 ### CLI client
 
@@ -158,7 +200,7 @@ The client supports `--api-base`, `--model`, `--text`, `--ref-audio`, `--ref-tex
 ### Notes
 
 - Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform.
-- Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config <path>` to override.
+- Deploy config auto-loads the device-specific CosyVoice3 default described above. Pass `--deploy-config <path>` to override.
 - For offline inference and the end-to-end script, see the [offline CosyVoice3 section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#cosyvoice3).
 
 ---

docs/user_guide/quantization/fp8.md

@@ -65,7 +65,17 @@ vLLM-Omni points that cache at `~/.cache/vllm_omni/quack` (override with
 `QUACK_CACHE_DIR`) instead of quack's default under `/tmp`, so it survives restarts.
 In containers, set `QUACK_CACHE_DIR` to a mounted/persistent path — or bake it into
 the image — so the first cold start does not recompile. The engine's startup dummy
-run already exercises the kernels, so with a warm cache the first real request is fast.
+run exercises the kernels, but new shapes, layouts, dtypes, or bias settings may
+still need compilation or tuning. The warmup helper uses inference mode and
+transposed weights without bias. Daemon workers compile candidates in-process
+while retaining autotuning and caching.
+
+Scale validation and Quack/FlashInfer dispatch run inside a PyTorch custom op.
+This keeps layer-specific scale addresses and validation-cache updates out of
+Dynamo tracing without introducing a graph break. Unpopulated scales and Quack
+failures still fall back to FlashInfer at runtime. CUDA graph capture additionally
+requires warming the dispatch with populated scales; Python validation and
+dispatch do not rerun during CUDA graph replay.
 
 To pre-warm specific shapes (e.g. at image build time):
 
@@ -96,6 +106,7 @@ warmup_quack_fp8([(14040, 2048, 6144), (14040, 2048, 2048)])
 | HunyuanVideo-1.5 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `720p_t2v`, `480p_i2v` | Yes | Yes | All layers | None | |
 | Cosmos3 | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | Yes | Not validated | All layers | None | |
 | MiniMax-H3 | `MiniMaxAI/MiniMax-H3` (`FL2VA` / `Ref2VA`) | Yes | Not validated | `quantization="fp8"` quantizes eligible DiT and text-encoder linears; mixed-precision input/output heads stay FP32 | None | ✅︎ |
+| SenseNova-U1.5 | `sensenova/SenseNova-U1.5-8B-MoT` | Yes | Not validated | UND/GEN language-model linears only | None | |
 
 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
 

examples/offline_inference/text_to_image/README.md

@@ -222,6 +222,8 @@ Anima uses the official model dimensions by default. A checkpoint may provide a
 
 To load Anima, point `--model` to the single-file checkpoint path, pass the native pipeline class name using `--model-class-name`, and supply the converted components directory using `--custom-pipeline-args`:
 
+Download the checkpoint and components using the [Anima recipe](../../../recipes/circlestone-labs/Anima.md#command---prepare-assets), then replace the paths below with their local locations.
+
 ```bash
 python examples/offline_inference/text_to_image/text_to_image.py \
   --model /path/to/models/anima-official/split_files/diffusion_models/anima-base-v1.0.safetensors \

examples/offline_inference/text_to_speech/breeze_tts_2/end2end.py

@@ -14,12 +14,10 @@ from pathlib import Path
 
 import soundfile as sf
 import torch
-from transformers import AutoConfig
+from transformers import AutoTokenizer
 
 from vllm_omni import Omni
-from vllm_omni.model_executor.models.breeze_tts_2.prompt_builder import (
-    BreezeTTS2PromptBuilder,
-)
+from vllm_omni.model_executor.models.breeze_tts_2.prompt import DEFAULT_INSTRUCTION, build_breeze_prompt
 from vllm_omni.utils.tracking_parser import TrackingArgumentParser
 
 REPO_ROOT = Path(__file__).resolve().parents[4]
@@ -95,8 +93,7 @@ def resolve_template(args) -> str:
 def reference_payload(ref_audio_path: str | None) -> dict:
     """Load the reference clip into the payload keys the prompt builder reads.
 
-    ``BreezeTTS2PromptBuilder`` consumes ``ref_audio`` (a waveform) plus
-    ``ref_audio_sample_rate``; a bare path field is not read.
+    The prompt helper consumes a waveform and its sample rate.
     """
     if ref_audio_path is None:
         return {}
@@ -107,36 +104,17 @@ def reference_payload(ref_audio_path: str | None) -> dict:
 
 
 def build_prompt(args) -> dict:
-    config = AutoConfig.from_pretrained(args.model, trust_remote_code=True)
-    reference_encoder = None
-    if args.ref_audio is not None:
-        from vllm_omni.model_executor.models.breeze_tts_2.audio_tokenizer import (
-            BreezeReferenceAudioTokenizer,
-        )
-
-        reference_encoder = BreezeReferenceAudioTokenizer.from_pretrained(
-            args.model,
-            num_codebooks=int(getattr(config, "num_codebooks", 16)),
-            device_map="cpu",
-        )
-    builder = BreezeTTS2PromptBuilder.from_pretrained(
-        args.model,
-        config,
-        reference_audio_encoder=reference_encoder,
+    tokenizer = AutoTokenizer.from_pretrained(args.model)
+    payload = reference_payload(args.ref_audio)
+    reference = (payload["ref_audio"], payload["ref_audio_sample_rate"]) if payload else None
+    return build_breeze_prompt(
+        tokenizer,
+        args.text,
+        DEFAULT_INSTRUCTION if args.instruction is None else args.instruction,
+        speaker=args.voice,
+        ref_audio=reference,
+        ref_text=args.ref_text,
     )
-    payload = {
-        "text": args.text,
-        "speaker": args.voice,
-        **reference_payload(args.ref_audio),
-    }
-    if args.instruction:
-        payload["instruction"] = args.instruction
-    if args.ref_text:
-        payload["ref_text"] = args.ref_text
-    prompt = builder.build(payload, template=resolve_template(args))
-    if args.max_new_tokens is not None:
-        prompt["additional_information"]["breeze_max_new_frames"] = int(args.max_new_tokens)
-    return prompt
 
 
 def extract_audio(multimodal_output: dict) -> torch.Tensor:
@@ -180,7 +158,12 @@ def main():
     print(f"Prompt len  : {len(prompt['prompt_token_ids'])} tokens")
 
     t_start = time.perf_counter()
-    outputs = engine.generate([prompt])
+    if args.max_new_tokens is not None:
+        sampling = [params.clone() for params in engine.resolve_sampling_params_list(None)]
+        sampling[0].max_tokens = args.max_new_tokens
+        outputs = engine.generate([prompt], sampling_params_list=sampling)
+    else:
+        outputs = engine.generate([prompt])
     elapsed = time.perf_counter() - t_start
 
     request_output = outputs[0]

examples/offline_inference/text_to_speech/cosyvoice3/end2end.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 import argparse
+import copy
 import functools
 import json
 import os
@@ -10,13 +11,11 @@ from typing import Any
 
 import numpy as np
 import soundfile as sf
-from vllm import SamplingParams
 from vllm.multimodal.media.audio import load_audio
 
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
 from vllm_omni.model_executor.models.cosyvoice3.utils import extract_text_token
-from vllm_omni.transformers_utils.configs.cosyvoice3 import CosyVoice3Config
 
 # Upstream zero-shot reference clip
 ZERO_SHOT_PROMPT_URL = "https://raw.githubusercontent.com/FunAudioLLM/CosyVoice/main/asset/zero_shot_prompt.wav"
@@ -59,8 +58,7 @@ def run_e2e():
         "--deploy-config",
         type=str,
         default=None,
-        help="Override the deploy config path. If unset, auto-loads "
-        "vllm_omni/deploy/cosyvoice3.yaml based on the HF model_type.",
+        help="Override the deploy config path. If unset, selects the bundled CosyVoice3 profile for the device.",
     )
     parser.add_argument("--text", type=str, default="Hello, this is a test of the CosyVoice system capability.")
     parser.add_argument(
@@ -105,8 +103,6 @@ def run_e2e():
         profiler_config=args.profiler_config,
     )
 
-    sampling_cfg = {"top_p": 0.8, "top_k": 25, "eos_token_id": 6561 + 1}
-
     print("Model initialized. Preparing inputs...")
     ref_audio_path = args.ref_audio or _default_ref_audio()
     if not os.path.exists(ref_audio_path):
@@ -138,7 +134,7 @@ def run_e2e():
 
     print(f"Generating for prompt: {args.text}")
 
-    config = CosyVoice3Config()
+    config = omni.engine.stage_vllm_configs[0].model_config.hf_config
     tokenizer = get_qwen_tokenizer(
         token_path=args.tokenizer,
         skip_special_tokens=config.skip_special_tokens,
@@ -149,29 +145,10 @@ def run_e2e():
     min_len = int(base_len * config.min_token_text_ratio)
     max_len = int(base_len * config.max_token_text_ratio)
 
-    # Build SamplingParams for each stage (GPT, S2Mel, Vocoder)
-    gpt_sampling = SamplingParams(
-        temperature=1.0,
-        top_p=sampling_cfg["top_p"],
-        top_k=sampling_cfg["top_k"],
-        repetition_penalty=2.0,
-        min_tokens=min_len,
-        max_tokens=max_len,
-        stop_token_ids=[sampling_cfg["eos_token_id"]],
-        # allowed_token_ids=list(range(6561+3)),
-        detokenize=False,
-    )
-    # Not used
-    s2mel_sampling = SamplingParams(
-        temperature=1.0,
-        top_p=1.0,
-        top_k=-1,
-        repetition_penalty=2.0,
-        max_tokens=256,
-        detokenize=False,
-    )
-
-    sampling_params_list = [gpt_sampling, s2mel_sampling]
+    # Keep the deploy profile's sampling policy, penalties and required stops.
+    sampling_params_list = copy.deepcopy(omni.default_sampling_params_list)
+    sampling_params_list[0].max_tokens = max(1, min(2048, max_len))
+    sampling_params_list[0].min_tokens = min(max(1, min_len), sampling_params_list[0].max_tokens)
 
     profiler_enabled = args.profiler_config is not None
     if profiler_enabled:

examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """Offline inference example for MOSS-TTS-Nano via vLLM-Omni.
 
 Single-stage pipeline: the 0.1B AR LM and MOSS-Audio-Tokenizer-Nano codec
@@ -136,21 +136,20 @@ def main(args) -> None:
     )
     params_list = sampling_params
 
-    for stage_outputs in omni.generate(inputs, params_list):
-        for i, req_output in enumerate(stage_outputs):
-            for j, out in enumerate(req_output.outputs):
-                mm = out.multimodal_output
-                if mm is None:
-                    print(f"  [req {i}] No audio output.")
-                    continue
-                audio = mm.get("audio")
-                sr_tensor = mm.get("sr")
-                if audio is None:
-                    print(f"  [req {i}] No waveform in multimodal_output.")
-                    continue
-                sr = int(sr_tensor.item()) if sr_tensor is not None else 48000
-                out_path = str(output_dir / f"output_{i}_{j}.wav")
-                save_audio(audio.cpu(), out_path, sr)
+    for i, req_output in enumerate(omni.generate(inputs, params_list)):
+        for j, out in enumerate(req_output.outputs):
+            mm = out.multimodal_output
+            if mm is None:
+                print(f"  [req {i}] No audio output.")
+                continue
+            audio = mm.get("audio")
+            sr_tensor = mm.get("sr")
+            if audio is None:
+                print(f"  [req {i}] No waveform in multimodal_output.")
+                continue
+            sr = int(sr_tensor.item()) if sr_tensor is not None else 48000
+            out_path = str(output_dir / f"output_{i}_{j}.wav")
+            save_audio(audio.cpu(), out_path, sr)
 
     print("Done.")
 

examples/offline_inference/text_to_speech/qwen3_tts/word_timestamps.py

@@ -1,170 +0,0 @@
-"""Offline Qwen3-TTS example with word-level timestamps.
-
-This demo runs Qwen3-TTS offline, aligns the synthesized audio in memory with
-the same shared forced aligner used by the streaming server path, then writes
-the WAV and JSON sidecar:
-
-    python examples/offline_inference/text_to_speech/qwen3_tts/word_timestamps.py \
-        --model Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
-        --forced-aligner Qwen/Qwen3-ForcedAligner-0.6B
-
-On machines without a local CUDA toolkit, set ``VLLM_USE_FLASHINFER_SAMPLER=0``
-to avoid FlashInfer sampler JIT during warmup.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import json
-import os
-from pathlib import Path
-from typing import Any
-
-os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
-# The in-process offline aligner consumes token_classify outputs. Keeping V1
-# multiprocessing off avoids msgspec serialization issues for token-wise logits.
-os.environ.setdefault("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
-
-import numpy as np
-import soundfile as sf
-import torch
-from end2end import _estimate_prompt_len
-from vllm.utils.argparse_utils import FlexibleArgumentParser
-
-from vllm_omni import Omni
-from vllm_omni.utils.forced_aligner import align, build_forced_aligner_config
-
-
-def _default_deploy_config() -> str:
-    repo_root = Path(__file__).resolve().parents[4]
-    return str(repo_root / "vllm_omni" / "deploy" / "qwen3_tts.yaml")
-
-
-def _build_custom_voice_input(args: Any) -> dict[str, Any]:
-    additional_information = {
-        "task_type": ["CustomVoice"],
-        "text": [args.text],
-        "language": [args.language],
-        "speaker": [args.speaker],
-        "instruct": [args.instructions],
-        "max_new_tokens": [args.max_new_tokens],
-    }
-    return {
-        "prompt_token_ids": [0] * _estimate_prompt_len(additional_information, args.model),
-        "additional_information": additional_information,
-    }
-
-
-def _audio_tensor_and_sample_rate(mm: dict[str, Any]) -> tuple[torch.Tensor, int]:
-    audio_data = mm["audio"]
-    sr_raw = mm["sr"]
-    sr_val = sr_raw[-1] if isinstance(sr_raw, list) and sr_raw else sr_raw
-    sample_rate = int(sr_val.item()) if hasattr(sr_val, "item") else int(sr_val)
-    audio_tensor = torch.cat(audio_data, dim=-1) if isinstance(audio_data, list) else audio_data
-    return audio_tensor.float().detach().cpu().flatten(), sample_rate
-
-
-def _float_audio_to_pcm16_bytes(audio: torch.Tensor) -> bytes:
-    samples = audio.numpy()
-    pcm = (np.clip(samples, -1.0, 1.0) * 32767.0).astype("<i2")
-    return pcm.tobytes()
-
-
-async def _run_alignment(args: Any, audio: torch.Tensor, sample_rate: int) -> list[dict[str, Any]] | None:
-    aligner_config = build_forced_aligner_config(args)
-    if aligner_config is None:
-        raise ValueError("--forced-aligner or --forced-aligner-config is required")
-    timestamps = await align(
-        audio=_float_audio_to_pcm16_bytes(audio),
-        text=args.text,
-        sample_rate=sample_rate,
-        config=aligner_config,
-        language=args.language,
-    )
-    if timestamps is None:
-        return None
-    return [
-        {
-            "word": item.word,
-            "start_ms": item.start_ms,
-            "end_ms": item.end_ms,
-        }
-        for item in timestamps
-    ]
-
-
-def main(args: Any) -> None:
-    output_dir = Path(args.output_dir)
-    output_dir.mkdir(parents=True, exist_ok=True)
-
-    omni_kwargs = vars(args).copy()
-    for key in (
-        "forced_aligner",
-        "text",
-        "language",
-        "speaker",
-        "instructions",
-        "max_new_tokens",
-        "output_dir",
-    ):
-        omni_kwargs.pop(key, None)
-    omni_kwargs["deploy_config"] = args.deploy_config or _default_deploy_config()
-    omni_kwargs["log_stats"] = args.log_stats
-    omni = Omni(**omni_kwargs)
-
-    prompt = _build_custom_voice_input(args)
-    final_output = None
-    for stage_outputs in omni.generate([prompt]):
-        final_output = stage_outputs
-    if final_output is None:
-        raise RuntimeError("Qwen3-TTS did not produce an output.")
-
-    mm = final_output.outputs[0].multimodal_output
-    audio, sample_rate = _audio_tensor_and_sample_rate(mm)
-
-    wav_path = output_dir / "qwen3_tts_word_timestamps.wav"
-    # Align the in-memory tensor, not the saved WAV. The WAV is only a demo
-    # artifact so users can listen to the same audio referenced by the sidecar.
-    timestamps = asyncio.run(_run_alignment(args, audio, sample_rate))
-    sf.write(wav_path, audio.numpy(), samplerate=sample_rate, format="WAV")
-
-    sidecar = {
-        "text": args.text,
-        "sample_rate": sample_rate,
-        "audio_path": str(wav_path),
-        "timestamps": timestamps,
-    }
-    json_path = output_dir / "qwen3_tts_word_timestamps.json"
-    json_path.write_text(json.dumps(sidecar, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
-
-    print(f"Saved audio: {wav_path}")
-    print(f"Saved timestamps: {json_path}")
-    print(json.dumps(sidecar["timestamps"], ensure_ascii=False, indent=2))
-
-
-def parse_args() -> Any:
-    parser = FlexibleArgumentParser(description="Offline Qwen3-TTS word timestamp example")
-    parser.add_argument("--model", default="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", help="Qwen3-TTS model path/name")
-    parser.add_argument(
-        "--forced-aligner",
-        default=None,
-        help="Qwen3 forced aligner model path/name",
-    )
-    parser.add_argument(
-        "--forced-aligner-config",
-        default=None,
-        help="Optional YAML file for forced aligner settings (incl. gpu_memory_utilization)",
-    )
-    parser.add_argument("--deploy-config", default=None, help="Qwen3-TTS deploy YAML")
-    parser.add_argument("--text", default="Hello world.", help="Text to synthesize and align")
-    parser.add_argument("--language", default="English", help="Qwen3-TTS language field")
-    parser.add_argument("--speaker", default="Vivian", help="CustomVoice speaker name")
-    parser.add_argument("--instructions", default="", help="Optional speaking style instruction")
-    parser.add_argument("--max-new-tokens", type=int, default=2048, help="TTS max_new_tokens")
-    parser.add_argument("--output-dir", default="output_audio", help="Directory for WAV and JSON sidecar")
-    parser.add_argument("--log-stats", action="store_true", default=False, help="Enable vLLM-Omni stats logging")
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    main(parse_args())

examples/offline_inference/yue2/end2end.py

@@ -0,0 +1,190 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""YuE2-3B offline end-to-end text-to-music.
+
+One song is one or two engine requests against the single-stage pipeline:
+``cot=full|melody`` first asks the model to write an ABC score (the abc phase,
+tokens only), then the semantic phase generates codec tokens and the model
+solves the acoustic ODE + decodes 48 kHz stereo as the request finishes.
+``cot=off`` and a user-supplied ``--abc-file`` skip the first request.
+Prefix caching lets the second request reuse the first one's blocks.
+
+Example:
+    python end2end.py --model /models/YuE2-3B --vae /models/YuE2-Vae \
+        --style "gentle lo-fi hip-hop, 85 BPM, F minor" \
+        --lyrics "[Verse]\nHumming on a quiet street\n[Chorus]\nStay a while" \
+        --cot full --seed 831001 --max-frames 200 --output song.wav
+"""
+
+from __future__ import annotations
+
+import argparse
+import copy
+from pathlib import Path
+
+import soundfile as sf
+import torch
+
+from vllm_omni import Omni
+from vllm_omni.model_executor.models.yue2.yue2 import (
+    ABC_SAMPLING,
+    CONTEXT,
+    KEY_MAX_AUDIO_FRAMES,
+    KEY_MAX_HOLD_STEPS,
+    KEY_MIN_TOKENS,
+    KEY_PENALTY_WINDOW,
+    KEY_PHASE,
+    KEY_PREFIX_IDS,
+    KEY_REPETITION_PENALTY,
+    KEY_SEED,
+    KEY_SKIP_SYNTHESIS,
+    KEY_TEMPERATURE,
+    KEY_TOP_K,
+    KEY_TOP_P,
+    SEMANTIC_SAMPLING,
+    STOP_TOKEN_IDS,
+    SYNTHESIS_HOLD_STEPS,
+)
+from vllm_omni.tokenizers.yue2_prompt import (
+    abc_ids_from_generated,
+    abc_prefix_ids,
+    semantic_frames,
+    semantic_prefix_ids,
+)
+from vllm_omni.tokenizers.yue2_tokenizer import YuE2TextTokenizer
+
+
+def sampling_params(engine, *, phase, seed, max_frames, prompt_ids):
+    """Deep-copy the stage defaults, then set the yue2_* keys for one phase."""
+    params = copy.deepcopy(engine.resolve_sampling_params_list(None))[0]
+    preset = ABC_SAMPLING if phase == "abc" else SEMANTIC_SAMPLING
+    params.extra_args = {
+        **(params.extra_args or {}),
+        KEY_PHASE: phase,
+        KEY_SEED: seed,
+        KEY_TEMPERATURE: preset["temperature"],
+        KEY_TOP_P: preset["top_p"],
+        KEY_TOP_K: preset["top_k"],
+        KEY_REPETITION_PENALTY: preset["repetition_penalty"],
+        KEY_PENALTY_WINDOW: preset["penalty_window"],
+        KEY_MIN_TOKENS: preset["min_tokens"],
+        KEY_MAX_AUDIO_FRAMES: max_frames,
+        KEY_SKIP_SYNTHESIS: phase == "abc",
+        # Full prompt ids: under a KV prefix-cache hit the engine schedules
+        # only the uncached tail, so the model cannot rebuild its NAR
+        # conditioning prefix from the scheduled tokens alone.
+        KEY_PREFIX_IDS: list(prompt_ids),
+    }
+    if phase == "abc":
+        params.max_tokens = preset["max_tokens"]
+    else:
+        # The song is synthesized on a side stream while the request is held;
+        # leave max_tokens headroom for the hold tokens.
+        params.max_tokens = min(max_frames + 1 + SYNTHESIS_HOLD_STEPS, CONTEXT - len(prompt_ids))
+        params.extra_args[KEY_MAX_HOLD_STEPS] = max(0, params.max_tokens - max_frames - 1)
+    params.stop_token_ids = list(STOP_TOKEN_IDS)
+    params.detokenize = False
+    return params
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--model", required=True, help="m-a-p/YuE2-3B checkpoint dir or repo id")
+    parser.add_argument("--vae", default=None, help="m-a-p/YuE2-Vae dir or repo id (or $YUE2_VAE)")
+    parser.add_argument("--style", required=True)
+    parser.add_argument("--lyrics", required=True)
+    parser.add_argument("--cot", choices=["off", "melody", "full"], default="full")
+    parser.add_argument("--seed", type=int, default=831001)
+    parser.add_argument("--max-frames", type=int, default=200, help="semantic frame budget (25 frames = 1 s)")
+    parser.add_argument("--abc-file", default=None, help="external ABC score (requires cot=melody/full)")
+    parser.add_argument("--output", default="yue2_song.wav")
+    parser.add_argument(
+        "--dump-tokens",
+        default=None,
+        metavar="PATH",
+        help="write prompt + generated token ids to this json (default: <output>.tokens.json)",
+    )
+    parser.add_argument(
+        "--gpu-memory-utilization",
+        type=float,
+        default=None,
+        help="override the deploy yaml's per-stage gpu_memory_utilization "
+        "(necessary on cards shared with other tenants: the yaml fraction is "
+        "of the WHOLE card, not of the free remainder)",
+    )
+    args = parser.parse_args()
+
+    if args.vae:
+        import os
+
+        os.environ["YUE2_VAE"] = args.vae
+
+    tokenizer = YuE2TextTokenizer(Path(args.model) / "qwen.tiktoken")
+    engine_kwargs: dict = {}
+    if args.gpu_memory_utilization is not None:
+        engine_kwargs["gpu_memory_utilization"] = args.gpu_memory_utilization
+    engine = Omni(model=args.model, **engine_kwargs)
+
+    abc_ids = None
+    if args.abc_file is not None:
+        if args.cot == "off":
+            raise SystemExit("--abc-file requires --cot melody|full")
+        abc_ids = tokenizer.encode(Path(args.abc_file).read_text())
+        print(f"Using supplied ABC score ({len(abc_ids)} tokens)")
+    elif args.cot != "off":
+        prompt_ids = abc_prefix_ids(tokenizer.encode, args.style, args.lyrics, args.cot)
+        prompt = {"prompt_token_ids": prompt_ids}
+        params = sampling_params(engine, phase="abc", seed=args.seed, max_frames=args.max_frames, prompt_ids=prompt_ids)
+        outputs = engine.generate([prompt], [params])
+        generated = list(outputs[0].outputs[0].token_ids)
+        abc_ids = abc_ids_from_generated(generated)
+        print(f"Generated ABC score ({len(abc_ids)} tokens):\n{tokenizer.decode(abc_ids)}\n")
+
+    prompt_ids = semantic_prefix_ids(tokenizer.encode, args.style, args.lyrics, args.cot, abc_ids=abc_ids)
+    prompt = {"prompt_token_ids": prompt_ids}
+    params = sampling_params(
+        engine, phase="semantic", seed=args.seed, max_frames=args.max_frames, prompt_ids=prompt_ids
+    )
+    outputs = engine.generate([prompt], [params])
+    output = outputs[0].outputs[0]
+    generated_ids = list(output.token_ids)
+    if args.dump_tokens:
+        import json
+
+        payload = {
+            "prompt_token_ids": prompt_ids,
+            "generated_token_ids": generated_ids,
+            # Codec frames only: without the end token and the hold tokens
+            # the request emits while its song is synthesized.
+            "semantic_frames": semantic_frames(generated_ids),
+            "seed": args.seed,
+            "cot": args.cot,
+            "max_frames": args.max_frames,
+        }
+        dump_path = args.dump_tokens if args.dump_tokens.endswith(".json") else args.output + ".tokens.json"
+        Path(dump_path).write_text(json.dumps(payload))
+        print(f"Dumped {len(generated_ids)} generated token ids to {dump_path}")
+    mm = output.multimodal_output or {}
+    audio = mm.get("audio")
+    if audio is None and "model_outputs" in mm:
+        audio = mm["model_outputs"]
+    if audio is None:
+        raise SystemExit("No audio in the model output; check server logs")
+    sr = int(mm.get("sr").item()) if mm.get("sr") is not None else 48000
+    truncated = False
+    meta = mm.get("meta") or {}
+    if "truncated" in meta:
+        truncated = bool(torch.as_tensor(meta["truncated"]).reshape(-1)[-1].item())
+
+    waveform = audio.float()
+    # soundfile wants [frames, channels] float in [-1, 1]; torchaudio has no
+    # wheel matching vllm 0.29.0's torch pin, so the driver uses libsndfile.
+    sf.write(args.output, waveform.T.contiguous(), sr)
+    print(
+        f"Saved {args.output}: {waveform.shape[-1] / sr:.1f}s @ {sr} Hz stereo, "
+        f"truncated={truncated}, generated {len(output.token_ids)} semantic tokens"
+    )
+
+
+if __name__ == "__main__":
+    main()

examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 import base64
 import concurrent.futures
 import os
@@ -457,7 +460,8 @@ def run_multimodal_generation(args, client: OpenAI) -> None:
                         f.write(audio_data)
                     print(f"Audio saved to {audio_file_path}")
                     count += 1
-                elif choice.message.content:
+                # A choice can contain both audio and text.
+                if choice.message.content:
                     print("Chat completion output from text:", choice.message.content)
     else:
         printed_content = False

examples/online_serving/personaplex/README.md

@@ -2,15 +2,15 @@
 
 Serve [`nvidia/personaplex-7b-v1`](https://huggingface.co/nvidia/personaplex-7b-v1)
 (a Moshi-based full-duplex speech-to-speech model) with the native vLLM-Omni engine
-through the unified duplex serving stack (`/v1/duplex` and `/v1/realtime?duplex=1`).
+through the unified full-duplex framework (`/v1/realtime?duplex=1`, alias `/v1/duplex`).
 
 > Requires a GPU and Hugging Face access to the gated repo
 > (`HF_TOKEN` with access to `nvidia/personaplex-7b-v1`).
 
 ## Start the server
 
-The default `vllm_omni/deploy/personaplex.yaml` enables the engine-owned
-full-duplex control plane (`session_mode: duplex`):
+The default `vllm_omni/deploy/personaplex.yaml` is a duplex deployment
+(`session_mode: duplex`, two sessions per replica):
 
 ```bash
 HF_TOKEN=... CUDA_VISIBLE_DEVICES=0 python -m vllm_omni.entrypoints.cli.main serve \
@@ -19,18 +19,32 @@ HF_TOKEN=... CUDA_VISIBLE_DEVICES=0 python -m vllm_omni.entrypoints.cli.main ser
   --deploy-config vllm_omni/deploy/personaplex.yaml
 ```
 
-This exposes:
+This exposes `WS /v1/realtime?duplex=1` (alias `WS /v1/duplex`): the OpenAI
+Realtime session protocol projected onto vLLM-Omni duplex sessions (client API and
+wire protocol: [`docs/serving/realtime_duplex_api.md`](../../../docs/serving/realtime_duplex_api.md)).
+There is no `/v1/chat/completions` route: PersonaPlex answers speech only.
 
-- `WS /v1/duplex` — the native duplex session dialect
-  (`session.create` / `input_audio_buffer.append` / `response.output_audio.delta` ...);
-- `WS /v1/realtime?duplex=1` — the same sessions projected onto the
-  OpenAI Realtime event vocabulary (client API and wire protocol:
-  [`docs/serving/realtime_duplex_api.md`](../../../docs/serving/realtime_duplex_api.md)).
+PersonaPlex is a pure-lockstep model: every session is native duplex, audio flows
+continuously in both directions in 80 ms frames, the model decides when to speak,
+and there are no client commits or external turn signals
+(`supports_client_commit=false`, `supports_external_turn_signal=false`). A session
+therefore auto-responds without any vendor flag.
 
-PersonaPlex is a pure-lockstep model: every session on a PersonaPlex deployment is
-native duplex (`is_enabled()` is unconditionally true), audio flows continuously in
-both directions, and there are no client commits or external turn signals
-(`supports_client_commit=false`, `supports_external_turn_signal=false`).
+## Talk to it
+
+With the client library and the PersonaPlex preset (24 kHz `pcm_f32le` in, bundled
+voice prompt, persona text):
+
+```python
+from vllm_omni.clients.duplex import DuplexClient
+from vllm_omni.clients.personaplex import create_duplex_session_config
+
+cfg = create_duplex_session_config(voice="NATF2.pt", persona="You are a concise assistant.")
+async with DuplexClient("ws://127.0.0.1:8000/v1/realtime?duplex=1", model="/path/to/personaplex-7b-v1", config=cfg) as c:
+    await c.stream_pcm(pcm_f32le_24k)          # keep streaming; the model speaks while it listens
+```
+
+Voice and persona are fixed for the session (`session.update` cannot change them).
 
 ## Validate the serving path
 
@@ -45,9 +59,11 @@ python tests/e2e/online_serving/personaplex_realtime_duplex.py \
   --output-dir /tmp/personaplex-realtime-duplex
 ```
 
-The unified endpoint advertises `supports_barge_in=false`: overlapping speech
-is native model behavior, but destructive output interruption and model-state
-rewind have not been validated for PersonaPlex.
+The endpoint advertises `supports_barge_in=false`: overlapping speech is native
+model behaviour, but destructive output interruption and model-state rewind have
+not been validated for PersonaPlex. `response.cancel` and
+`output_audio_buffer.clear` restart the model's conversation context (a fresh
+Stage 0 request replays the voice/persona prefill).
 
 ## Notes
 
@@ -56,8 +72,8 @@ rewind have not been validated for PersonaPlex.
   regardless of engine speed. On localhost it is smooth.
 - The earlier standalone Moshi-web compatibility server (browser client at `/`,
   binary WS protocol at `/api/chat`, raw-PCM `/v1/audio/duplex`) was demo-only
-  and has been removed; use the unified endpoints above.
-- Session config (voice / persona / sampling) is passed per session via
-  `extra_body`; see
-  `vllm_omni/model_executor/models/personaplex/duplex/serving_adapter.py`.
+  and has been removed; use the unified endpoint above.
+- The model plugin, worker-side lockstep runtime and input framing live in
+  `vllm_omni/model_executor/models/personaplex/duplex/`; design notes in
+  [`docs/design/fullduplex-personaplex.md`](../../../docs/design/fullduplex-personaplex.md).
 - Full runbook: [`recipes/NVIDIA/PersonaPlex.md`](../../../recipes/NVIDIA/PersonaPlex.md).

examples/online_serving/qwen3_omni/fullduplex-livekit-frontend/.dockerignore

@@ -0,0 +1,6 @@
+agent-starter-react/
+.git/
+.claude/
+__pycache__/
+*.py[cod]
+*.log

examples/online_serving/qwen3_omni/fullduplex-livekit-frontend/Dockerfile.agent

@@ -0,0 +1,25 @@
+# syntax=docker/dockerfile:1
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+FROM python:3.11-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y ca-certificates ffmpeg libsndfile1 \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN pip install \
+    "livekit-agents[openai,turn-detector,silero]>=1.6.10"
+
+RUN useradd --create-home --uid 10001 agent
+
+WORKDIR /app
+COPY --chown=agent:agent agent.py ./agent.py
+
+USER agent
+
+ENTRYPOINT ["python", "agent.py", "start"]

examples/online_serving/qwen3_omni/fullduplex-livekit-frontend/Dockerfile.frontend

@@ -0,0 +1,36 @@
+# syntax=docker/dockerfile:1
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+FROM node:24-bookworm-slim AS dependencies
+
+ARG FRONTEND_REF=main
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y ca-certificates git \
+    && rm -rf /var/lib/apt/lists/* \
+    && npm install --global pnpm@9.15.9
+
+WORKDIR /app
+RUN git clone --depth 1 --branch "${FRONTEND_REF}" \
+    https://github.com/livekit-examples/agent-starter-react.git . \
+    && pnpm install --frozen-lockfile \
+    && rm -rf .git
+
+FROM node:24-bookworm-slim
+
+ENV NODE_ENV=development \
+    NEXT_TELEMETRY_DISABLED=1
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y ca-certificates \
+    && rm -rf /var/lib/apt/lists/* \
+    && npm install --global pnpm@9.15.9
+
+WORKDIR /app
+COPY --from=dependencies --chown=node:node /app /app
+
+USER node
+
+EXPOSE 3000
+CMD ["pnpm", "dev", "--hostname", "0.0.0.0"]

examples/online_serving/qwen3_omni/fullduplex-livekit-frontend/Dockerfile.livekit

@@ -0,0 +1,28 @@
+# syntax=docker/dockerfile:1
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+FROM golang:1.26-alpine AS builder
+
+ARG LIVEKIT_REF=v1.10.1
+ARG TARGETARCH
+
+RUN apk add --no-cache git ca-certificates
+
+WORKDIR /src
+RUN git clone --depth 1 --branch "${LIVEKIT_REF}" \
+    https://github.com/livekit/livekit.git . \
+    && go mod download \
+    && CGO_ENABLED=0 GOOS=linux GOARCH="${TARGETARCH}" \
+        go build -trimpath -ldflags="-s -w" -o /out/livekit-server ./cmd/server
+
+FROM alpine:3.22
+
+RUN apk add --no-cache ca-certificates tzdata \
+    && addgroup -S livekit \
+    && adduser -S -G livekit livekit
+
+COPY --from=builder /out/livekit-server /usr/local/bin/livekit-server
+
+USER livekit
+ENTRYPOINT ["/usr/local/bin/livekit-server"]

examples/online_serving/qwen3_omni/fullduplex-livekit-frontend/README.md

@@ -0,0 +1,32 @@
+# Qwen3-Omni LiveKit demo
+
+This directory contains a Docker Compose version of the local LiveKit demo.
+It builds the LiveKit server, the Next.js frontend, and the Python agent as
+separate images. The vLLM-Omni server runs outside Compose and is configured
+with `VLLM_OMNI_HOST`.
+
+Start vLLM-Omni with port `8000` bound to an address reachable from Docker,
+for example with `--host 0.0.0.0`, then run:
+
+```bash
+cd examples/online_serving/qwen3_omni/fullduplex-livekit-frontend
+VLLM_OMNI_HOST=host.docker.internal docker compose up --build
+```
+
+Open [http://localhost:3000](http://localhost:3000). The LiveKit server is
+available at `ws://localhost:7880`; its development credentials are `devkey`
+and `secret`.
+
+For a machine other than the local host, set both the vLLM address and the
+address LiveKit advertises to WebRTC clients:
+
+```bash
+VLLM_OMNI_HOST=192.168.1.20 \
+LIVEKIT_NODE_IP=192.168.1.10 \
+LIVEKIT_PUBLIC_URL=ws://192.168.1.10:7880 \
+docker compose up --build
+```
+
+Use `docker compose up --build --abort-on-container-exit` when the Compose
+process should stop the remaining services after one component exits, matching
+the behavior of `run-livekit-stack.sh`.

examples/online_serving/qwen3_omni/fullduplex-livekit-frontend/agent.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env -S uv run --script
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "livekit-agents[openai,turn-detector,silero]>=1.6.10",
+# ]
+# ///
+import logging
+import os
+
+from livekit.agents import (
+    Agent,
+    AgentServer,
+    AgentSession,
+    JobContext,
+    RunContext,
+    TurnHandlingOptions,
+    cli,
+    function_tool,
+    inference,
+)
+from livekit.plugins.openai.realtime import RealtimeModel
+
+logger = logging.getLogger("voice-assistant")
+
+VLLM_BASE_URL = f"http://{os.environ['VLLM_OMNI_HOST']}:8000/v1"
+LIVEKIT_URL = os.environ.get("LIVEKIT_URL", "ws://localhost:7880")
+
+server = AgentServer(
+    ws_url=LIVEKIT_URL,
+    api_key=os.environ.get("LIVEKIT_API_KEY", "devkey"),
+    api_secret=os.environ.get("LIVEKIT_API_SECRET", "secret"),
+)
+
+
+class VoiceAssistant(Agent):
+    @function_tool()
+    async def lookup_weather(
+        self,
+        context: RunContext,
+        location: str,
+    ) -> dict[str, str | int]:
+        """Return deterministic sample data for the demo tool."""
+        return {
+            "location": location,
+            "weather": "clear skies",
+            "temperature_f": 72,
+        }
+
+    def __init__(self) -> None:
+        super().__init__(
+            instructions="You are a helpful voice assistant. Respond naturally and concisely.",
+        )
+
+
+@server.rtc_session()
+async def entrypoint(ctx: JobContext):
+    # Qwen3-Omni only supports turn_detection=null (manual mode) -- it does
+    # not perform voice activity detection itself, so turn-taking must be
+    # decided client-side. semantic_vad (MiniCPM-o only) is rejected by
+    # vLLM-Omni's OpenAI-compatible realtime endpoint for Qwen3-Omni.
+    # Omni serves one model, so an empty model lets the server select it.
+    model = RealtimeModel(
+        base_url=VLLM_BASE_URL,
+        model="",
+        api_key="unused",
+    )
+
+    # turn_handling's client-side TurnDetector means the server isn't asked
+    # to decide turns, so turn_detection is left NOT_GIVEN on RealtimeModel.
+    session = AgentSession(
+        llm=model,
+        turn_handling=TurnHandlingOptions(turn_detection=inference.TurnDetector()),
+    )
+
+    await session.start(
+        agent=VoiceAssistant(),
+        room=ctx.room,
+    )
+
+    logger.info(
+        "Voice assistant started, connected to vLLM-Omni at %s",
+        VLLM_BASE_URL,
+    )
+
+
+if __name__ == "__main__":
+    cli.run_app(server)