[Bugfix] Restrict OmniRunner cleanup to owned EngineCore workers

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
[CI/Build] Fix MOSS CPU tests on ROCm workers (#8447 )
2026-10-04 03:19:00 +08:00 · 2026-10-03 09:13:51 +08:00 · 2026-10-02 18:03:14 +00:00 · 2026-10-02 09:14:39 -07:00 · 2026-10-02 08:49:50 -07:00 · 2026-10-02 15:43:55 +00:00
690 changed files with 76800 additions and 6171 deletions
@@ -41,7 +41,33 @@ steps:
        - export VLLM_ROCM_USE_AITER=0
        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
        # that is not supported on CPU
-        - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+        # Keep multi-GPU tests out of this one-GPU pool even when their module
+        # also carries the cpu marker for unit tests.
+        - "pytest -sv tests/diffusion -m 'core_model and cpu and not (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)' --ignore=tests/diffusion/cache/test_teacache_extractors.py --ignore=tests/diffusion/models/pi05/test_pi05_units.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+    # This module takes about 22 minutes on MI300 and was consistently assigned to shard 4,
+    # pushing that shard to the 45-minute timeout. Keep its module-scoped fixture in one lane.
+    - label: "Simple · Pi0.5 CPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 45
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/diffusion/models/pi05/test_pi05_units.py -m 'core_model and cpu'"
+
+    - label: "Diffusion · AuK VAE GPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 40
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - >-
+          timeout --signal=TERM --kill-after=2m 35m
+          pytest -sv tests/diffusion/models/auk/test_auk_vae_fastpath.py
+          -m 'core_model and rocm and cards_1' --run-level core_model

    - label: "Simple · Engine&Entrypoints Test"
      agent_pool: mi300_1
@@ -131,6 +157,11 @@ steps:
      grade: Blocking
      commands:
        - pytest -s -v tests/diffusion/distributed/test_tensor_parallel.py
+        - >-
+          timeout 20m pytest -s -v
+          tests/diffusion/attention/test_ulysses_uaa.py::test_ulysses_uaa_2d_mask_layout_matches_baseline
+          -m 'core_model and cards_2'
+          --run-level "core_model"

    - label: "Diffusion Model Test"
      agent_pool: mi300_1
@@ -138,7 +169,9 @@ steps:
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
-        - timeout 30m pytest -s -v tests/e2e/offline_inference/test_z_image.py
+        # A cold Z-Image run spends about six minutes loading weights before
+        # the first-request AITER compilation, which can itself exceed 23m.
+        - timeout 55m pytest -s -v tests/e2e/offline_inference/test_z_image.py

    - label: "Diffusion GPU Worker Test"
      agent_pool: mi300_2
@@ -41,7 +41,33 @@ steps:
        - export VLLM_ROCM_USE_AITER=0
        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
        # that is not supported on CPU
-        - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+        # Keep multi-GPU tests out of this one-GPU pool even when their module
+        # also carries the cpu marker for unit tests.
+        - "pytest -sv tests/diffusion -m 'core_model and cpu and not (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)' --ignore=tests/diffusion/cache/test_teacache_extractors.py --ignore=tests/diffusion/models/pi05/test_pi05_units.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+    # This module takes about 22 minutes on MI300 and was consistently assigned to shard 4,
+    # pushing that shard to the 45-minute timeout. Keep its module-scoped fixture in one lane.
+    - label: "Simple · Pi0.5 CPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 45
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/diffusion/models/pi05/test_pi05_units.py -m 'core_model and cpu'"
+
+    - label: "Diffusion · AuK VAE GPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 40
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - >-
+          timeout --signal=TERM --kill-after=2m 35m
+          pytest -sv tests/diffusion/models/auk/test_auk_vae_fastpath.py
+          -m 'core_model and rocm and cards_1' --run-level core_model

    - label: "Simple · Engine&Entrypoints Test"
      agent_pool: mi300_1
@@ -141,6 +167,11 @@ steps:
        - export VLLM_LOGGING_LEVEL=DEBUG
        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
        - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
+        - >-
+          timeout 20m pytest -s -v
+          tests/diffusion/attention/test_ulysses_uaa.py::test_ulysses_uaa_2d_mask_layout_matches_baseline
+          -m 'core_model and cards_2'
+          --run-level "core_model"
        - >-
          timeout 20m pytest -s -v
          tests/diffusion/models/ltx2/test_ltx2_transformer_ulysses.py
@@ -165,6 +196,7 @@ steps:
        # registered multi-GPU marker from this one-GPU job.
        - >-
          timeout 15m pytest -sv tests/diffusion/models/
+          --ignore=tests/diffusion/models/auk/test_auk_vae_fastpath.py
          -m 'core_model and cuda and not
          (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
          --run-level "core_model"
@@ -16,10 +16,14 @@ x-qwen3-tts: &qwen3_tts
  - vllm_omni/worker/gpu_model_runner.py
  - vllm_omni/model_executor/models/qwen3_tts/
  - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+  - vllm_omni/model_executor/models/common/fused_sampling.py
+  - vllm_omni/model_executor/models/common/short_kv_attention.py
+  - vllm_omni/model_executor/models/common/talker_first_audio.py
  - vllm_omni/model_executor/models/common/snake_activation.py
  - vllm_omni/model_executor/models/common/alias_free_activation.py
  - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
  - vllm_omni/deploy/qwen3_tts.yaml
+  - vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2_single_gpu.yaml
  - vllm_omni/entrypoints/openai/serving_speech.py
  - vllm_omni/entrypoints/openai/tts_adapters/
  - vllm_omni/engine/stage_pool.py
@@ -30,6 +34,9 @@ x-qwen3-tts: &qwen3_tts
  - vllm_omni/config/omni_config.py
  - vllm_omni/engine/orchestrator.py
  - vllm_omni/engine/stage_engine_core_proc.py
+  - vllm_omni/engine/cuda_mps.py
+  - vllm_omni/engine/stage_runtime.py
+  - vllm_omni/engine/stage_init_utils.py
  - vllm_omni/deploy/qwen3_tts_mrv2.yaml
  - vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2.yaml
  - vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2_b4.yaml
@@ -81,11 +88,16 @@ x-qwen3-omni: &qwen3_omni
  - vllm_omni/entrypoints/openai/serving_chat.py
  - vllm_omni/model_executor/models/qwen3_omni/
  - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+  - vllm_omni/model_executor/models/common/fused_sampling.py
+  - vllm_omni/model_executor/models/common/short_kv_attention.py
+  - vllm_omni/model_executor/models/common/talker_first_audio.py
  - vllm_omni/model_executor/models/common/snake_activation.py
  - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
  - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
  - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
  - vllm_omni/deploy/qwen3_omni_moe.yaml
+  - vllm_omni/deploy/qwen3_omni_moe_mrv2.yaml
+  - vllm_omni/deploy/qwen3_omni_moe_mrv2_h200.yaml
  - vllm_omni/deploy/qwen3_omni_moe_thinking.yaml

 x-wan22: &wan22
@@ -418,6 +430,7 @@ source_file_dependencies:

  diffusion_minimax_h3_function:
    - *minimax_h3
+    - tests/helpers/assertions.py
    - tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py
    - tests/e2e/online_serving/minimax_h3/
    - tests/diffusion/models/minimax_h3/
@@ -124,6 +124,30 @@ steps:
          - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py -m 'advanced_model and cuda' --run-level 'advanced_model'
        mirror_hardwares: h100_2

+      - label: "Omni · Qwen3-Omni Async-Chunk SHM Leak"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_omni_shm_leak.py
+          - tests/helpers/runtime.py
+          - tests/helpers/stage_config.py
+          - vllm_omni/core/sched/omni_ar_scheduler.py
+          - vllm_omni/distributed/omni_connectors/connectors/shm_connector.py
+          - vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py
+          - vllm_omni/engine/orchestrator.py
+          - vllm_omni/engine/stage_engine_core_proc.py
+          - vllm_omni/engine/stage_pool.py
+          - vllm_omni/entrypoints/openai/serving_chat.py
+          - vllm_omni/model_executor/models/qwen3_omni/
+          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+          - vllm_omni/model_executor/models/common/snake_activation.py
+          - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
+          - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
+          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
+          - vllm_omni/deploy/qwen3_omni_moe.yaml
+        timeout_in_minutes: 40
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_shm_leak.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+        mirror_hardwares: h100_2
+
      - label: "Omni · MiniCPM-o 4.5 Test"
        source_file_dependencies: omni_minicpmo_4_5_function
        timeout_in_minutes: 50
@@ -208,3 +232,14 @@ steps:
            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_turbo_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_fasth3_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
        mirror_hardwares: h100_2
+
+      - label: "Diffusion · MiniMax-H3 Latent Upscale and Refine"
+        source_file_dependencies: diffusion_minimax_h3_function
+        timeout_in_minutes: 40
+        commands:
+          - |
+            set -eu
+            VLLM_TEST_MINIMAX_H3_LATENT_UPSCALER=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id="LBH-123-AI/Minimax_h3_latent_Upscaler", filename="minimax_h3_latent_upscaler_3d_conv_v1/minimax_h3_latent_upscaler_3d_conv_v1_bf16.safetensors"))')
+            export VLLM_TEST_MINIMAX_H3_LATENT_UPSCALER
+            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+        mirror_hardwares: h100_2
@@ -21,7 +21,7 @@ steps:
          - omni_minicpmo_4_5_function
        timeout_in_minutes: 120
        commands:
-          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py --ignore=tests/e2e/online_serving/test_minicpmo_4_5_window.py

      - label: ":full_moon: Omni · Function Test with H100 · 2-GPU"
        source_file_dependencies:
@@ -33,9 +33,10 @@ steps:

      - label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
        source_file_dependencies: omni_minicpmo_4_5_duplex_function
-        timeout_in_minutes: 50
+        timeout_in_minutes: 120
        commands:
          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and B200 and omni and cards_1" --run-level "full_model"
+          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_window.py -m "full_model and cuda and H100 and B200 and omni and cards_1" --run-level "full_model"

      - label: ":full_moon: Omni · Doc Test with L4 · 4-GPU"
        source_file_dependencies: omni_qwen2_5_omni_doc
@@ -229,17 +230,6 @@ steps:
            -m "full_model and diffusion and H100 and B200 and cards_1"
            --run-level "full_model"

-      - label: ":robot_face: Robot Policy OpenPI · π0.5 · H100 · Single-GPU"
-        timeout_in_minutes: 120
-        commands:
-          - pip install --no-deps "openpi-client @ git+https://github.com/Physical-Intelligence/openpi.git@215abfb217dbac7d5f1273282331b9b1866c0479#subdirectory=packages/openpi-client"
-          - >-
-            pytest -sv
-            tests/e2e/online_serving/test_pi05_expansion.py
-            -m "full_model and diffusion and H100 and cards_1"
-            --run-level "full_model"
-        mirror_hardwares: h100_1
-
      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU"
        source_file_dependencies: diffusion_qwen_image_function
        timeout_in_minutes: 120
@@ -16,6 +16,25 @@ steps:
          - "pytest -sv tests/model_executor -m 'core_model and cpu'"
        mirror_hardwares: l4_1

+      - label: "Simple · CosyVoice3 Packed Flow Test"
+        timeout_in_minutes: 15
+        source_file_dependencies:
+          - tests/model_executor/models/cosyvoice3/test_cosyvoice3_full_batch.py
+          - vllm_omni/model_executor/models/cosyvoice3/
+        commands:
+          - "pytest -sv tests/model_executor/models/cosyvoice3/test_cosyvoice3_full_batch.py -k packed -m 'core_model and cuda' --run-level=core_model"
+        mirror_hardwares: h100_1
+
+      - label: "Simple · MRV2 FA3 Graph Bounds Test"
+        timeout_in_minutes: 15
+        source_file_dependencies:
+          - tests/worker_v2/test_capture_query_bound_gpu.py
+          - tests/worker_v2/test_decode_query_bound_gpu.py
+          - vllm_omni/worker_v2/model_states/omni_model_state.py
+        commands:
+          - "pytest -sv tests/worker_v2/test_capture_query_bound_gpu.py tests/worker_v2/test_decode_query_bound_gpu.py -m 'core_model and cuda and H100' --run-level=core_model"
+        mirror_hardwares: h100_1
+
      - label: "Simple · Diffusion Test"
        timeout_in_minutes: 30
        commands:
@@ -196,6 +215,16 @@ steps:
          - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda'
        mirror_hardwares: h100_1

+      - label: "TTS · Higgs v3 Full Response Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_higgs_audio_v3.py
+          - vllm_omni/model_executor/models/higgs_audio_v3/
+          - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py
+          - vllm_omni/deploy/higgs_multimodal_qwen3.yaml
+        commands:
+          - timeout 20m pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py::test_full_response_v1_preserves_audio -m 'core_model and cuda' --run-level 'core_model'
+        mirror_hardwares: h100_1
+
      - label: "TTS · Qwen3-TTS CustomVoice Test"
        source_file_dependencies: tts_qwen3_tts_function
        commands:
@@ -152,15 +152,122 @@ steps:
            --run-level "full_model"
        mirror_hardwares: h100_4

-      - label: "TTS · L4"
-        timeout_in_minutes: 120
+      - label: "TTS · Moss · L4"
+        timeout_in_minutes: 90
        commands:
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - export VLLM_USE_DEEP_GEMM="0"
          - export VLLM_MOE_USE_DEEP_GEMM="0"
          - |
            failed=0
-            for f in $(pytest --collect-only -q tests/e2e/ -m "slow and L4 and tts" --run-level "full_model" | grep '\.py::' | sed -E 's|::.*||' | sort -u); do
+            for f in \
+              tests/e2e/offline_inference/test_moss_tts_expansion.py \
+              tests/e2e/offline_inference/test_moss_tts_nano_expansion.py \
+              tests/e2e/offline_inference/test_moss_tts_realtime.py \
+              tests/e2e/offline_inference/test_moss_tts_realtime_expansion.py \
+              tests/e2e/online_serving/test_moss_tts_expansion.py \
+              tests/e2e/online_serving/test_moss_tts_nano_expansion.py
+            do
+              pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
+            done
+            exit $$failed
+        mirror_hardwares: l4_1
+
+      - label: "TTS · VoxCPM2 · Voxtral · L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - |
+            failed=0
+            for f in \
+              tests/e2e/offline_inference/test_voxcpm2_tts.py \
+              tests/e2e/online_serving/test_voxcpm2_tts.py \
+              tests/e2e/online_serving/test_voxcpm2_tts_expansion.py \
+              tests/e2e/online_serving/test_voxcpm2_tts_lora.py \
+              tests/e2e/offline_inference/test_voxtral_tts_expansion.py \
+              tests/e2e/online_serving/test_voxtral_tts_expansion.py
+            do
+              pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
+            done
+            exit $$failed
+        mirror_hardwares: l4_1
+
+      - label: "TTS · Higgs · L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - |
+            failed=0
+            for f in \
+              tests/e2e/offline_inference/test_higgs_audio_v2_expansion.py \
+              tests/e2e/online_serving/test_higgs_audio_v2_expansion.py \
+              tests/e2e/online_serving/test_higgs_audio_v3.py
+            do
+              pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
+            done
+            exit $$failed
+        mirror_hardwares: l4_1
+
+      - label: "TTS · OmniVoice · Step-Audio · L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - |
+            failed=0
+            for f in \
+              tests/e2e/offline_inference/test_omnivoice_expansion.py \
+              tests/e2e/online_serving/test_omnivoice_expansion.py \
+              tests/e2e/online_serving/test_omnivoice_parity.py \
+              tests/e2e/offline_inference/test_step_audio2_expansion.py
+            do
+              pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
+            done
+            exit $$failed
+        mirror_hardwares: l4_1
+
+      - label: "TTS · Ming · GLM · IndexTTS · Breeze · L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - |
+            failed=0
+            for f in \
+              tests/e2e/offline_inference/test_ming_tts_expansion.py \
+              tests/e2e/online_serving/test_ming_tts_expansion.py \
+              tests/e2e/offline_inference/test_glm_tts_expansion.py \
+              tests/e2e/online_serving/test_glm_tts_expansion.py \
+              tests/e2e/offline_inference/test_indextts2_expansion.py \
+              tests/e2e/online_serving/test_indextts2_expansion.py \
+              tests/e2e/online_serving/test_breeze_tts_2.py
+            do
+              pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
+            done
+            exit $$failed
+        mirror_hardwares: l4_1
+
+      - label: "TTS · Audex · CosyVoice · Qwen3-TTS · L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - |
+            failed=0
+            for f in \
+              tests/e2e/offline_inference/test_audex_tts.py \
+              tests/e2e/offline_inference/test_audex_s2t.py \
+              tests/e2e/offline_inference/test_audex_tta.py \
+              tests/e2e/offline_inference/test_cosyvoice3_expansion.py \
+              tests/e2e/online_serving/test_qwen3_tts_word_timestamps.py
+            do
              pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
            done
            exit $$failed
@@ -224,6 +331,17 @@ steps:

      # Split the old shared H100 Single-GPU sweep: one pytest process collected
      # every slow+cards_1 diffusion file and exceeded 150 minutes.
+      - label: ":robot_face: Robot Policy OpenPI · π0.5 · H100 · Single-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - pip install --no-deps "openpi-client @ git+https://github.com/Physical-Intelligence/openpi.git@215abfb217dbac7d5f1273282331b9b1866c0479#subdirectory=packages/openpi-client"
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_pi05_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
      - label: "Diffusion · Qwen-Image · H100 · Single-GPU"
        timeout_in_minutes: 120
        commands:
@@ -64,6 +64,11 @@ steps:
            tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
            -m "npu and A3 and omni"
            --run-level full_model
+          - >-
+            pytest -s -v
+            tests/e2e/online_serving/test_minicpmo_4_5_window.py
+            -m "full_model and npu and A3 and omni"
+            --run-level full_model
      - label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
        source_file_dependencies: omni_minicpmo_4_5_accuracy
        timeout_in_minutes: 180
@@ -56,6 +56,35 @@ jobs:
        run: |
          bash scripts/build_wheel.sh --python python

+      - name: Validate runtime package data from source archive
+        run: |
+          set -euo pipefail
+          archive_dir="$(mktemp -d)"
+          install_dir="$(mktemp -d)"
+          vcs_wheels=(dist/*.whl)
+          if [[ ${#vcs_wheels[@]} -ne 1 || ! -f "${vcs_wheels[0]}" ]]; then
+            echo "Expected exactly one VCS-built wheel" >&2
+            exit 1
+          fi
+          git archive HEAD | tar -x -C "${archive_dir}"
+          (
+            cd "${archive_dir}"
+            VLLM_OMNI_TARGET_DEVICE=cuda \
+              VLLM_OMNI_VERSION_OVERRIDE=0.0.0 \
+              python -m build --wheel --outdir archive-dist
+          )
+          wheels=("${archive_dir}"/archive-dist/*.whl)
+          if [[ ${#wheels[@]} -ne 1 || ! -f "${wheels[0]}" ]]; then
+            echo "Expected exactly one archive-built wheel" >&2
+            exit 1
+          fi
+          python -m pip install --no-deps --target "${install_dir}" "${wheels[0]}"
+          python tools/check_wheel_package_data.py \
+            --source-root "${archive_dir}" \
+            --wheel "${wheels[0]}" \
+            --vcs-wheel "${vcs_wheels[0]}" \
+            --installed-root "${install_dir}"
+
      - name: Upload Wheel Artifact
        uses: actions/upload-artifact@v7
        with:
@@ -243,6 +243,7 @@ datasets/
 *.csv
 *.json
 !recipes/cosmos3/negative_prompt.json
+!benchmarks/mammoth_moda2/reference_results.json
 !tests/dfx/perf/tests/*.json
 !apps/ComfyUI-vLLM-Omni/example_workflows/*.json
 # Omni-DuplexEval CI config JSON (e.g. accuracy/omni_duplex_eval)
@@ -250,6 +251,7 @@ datasets/
 *.jsonl
 # Shared Hunyuan IT2I input for accuracy tests and vllm bench.
 !tests/assets/hunyuan_image3/it2i.jsonl
+!tests/assets/livekit/client_vad_replay.jsonl
 *.parquet

 # Output files
@@ -16,6 +16,7 @@ Easy, fast, and cheap omni-modality model serving for everyone

 *Latest News* 🔥

+- [2026/09] We released [0.30.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.30.0), rebased onto vLLM 0.30.0, featuring a unified full-duplex serving framework around engine-owned sessions for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md) and AURA, native cross-stage KV and multimodal payload transfer (Mooncake AR-to-DiT handoff and NIXL connectors), interactive world-model serving with [LingBot World](recipes/Robbyant/LingBot-World-2.0.md), and realtime [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) Turbo inference on NVIDIA Blackwell and Ascend 950.
 - [2026/08] We released [0.28.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.28.0), featuring production-ready [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) serving on GPU and NPU, a unified AR/DiT paged KV cache runtime, and enhanced realtime full-duplex serving for the [MiniCPM-o series](recipes/OpenBMB/MiniCPM-o-4_5.md).
 - [2026/08] [VeRL-Omni](https://github.com/verl-project/verl-omni) `v0.2.0` is released: faster diffusion RL powered by vLLM-Omni (request-level/step-wise batching with FA3), rebuilt Qwen3-Omni multimodal training (DPO & GSPO), plus LTX-2.3, Qwen-Image-Edit support and more. See the [release notes](https://github.com/verl-project/verl-omni/releases/tag/v0.2.0).
 - [2026/08] We released [0.26.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.26.0) - aligned with the vLLM 0.26 release line, featuring [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) joint video/audio generation, an experimental full-duplex realtime runtime for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md), distributed layerwise diffusion offload, and broader model, hardware, streaming, TTS, and quantization support.
@@ -58,9 +59,9 @@ vLLM-Omni is flexible and easy to use with:
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:

 - **Omni-modality models** (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
- **TTS models** (e.g. Qwen3-TTS, IndexTTS 2.5, CosyVoice3)
- **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
- **Robot-policy and action models** (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
+- **TTS models** (e.g. Qwen3-TTS, Tencent AuK, Breeze-TTS-2, CosyVoice3)
+- **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, LingBot World, MAGI-2, LTX-2.5, Wan2.2)
+- **Robot-policy and action models** (e.g. π0.5, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)

 ## Getting Started

@@ -66,6 +66,13 @@ Accuracy benchmarks for image generation/editing models, adapting external suite
 - **Layout**: `accuracy/text_to_image/` (GEBench), `accuracy/image_to_image/` (GEdit-Bench)
 - **Method**: generation and judge scoring both run through local `vllm-omni serve` endpoints

+### [MammothModa2](mammoth_moda2/README.md) — Startup and Weight Loading
+
+Startup / model-loading benchmark for the two-stage MammothModa2 (AR → DiT) deployment, plus a raw safetensors loading micro-benchmark.
+
+- **Layout**: `mammoth_moda2/bench_startup.py` (single-process startup and first/steady request timing), `mammoth_moda2/parse_startup_log.py` (per-stage breakdown from engine logs), `mammoth_moda2/bench_storage_scenarios.sh` (cold/warm page cache on local disk vs. network storage), `mammoth_moda2/raw_load_bench.py` (safetensors → GPU without vLLM)
+- **Key metrics**: time to engine ready, per-stage spawn/init/weight-load/profile time, first vs. steady-state request latency
+
 ### Common serving metrics framework

 `vllm_omni/benchmarks/` extends `vllm bench serve --omni` with Omni-specific datasets, backends, and multimodal metrics. Key metrics include:
@@ -107,6 +107,8 @@ async def async_request_image_edits(

    if extra_body.get("guidance_scale") is not None:
        form.add_field("guidance_scale", str(extra_body["guidance_scale"]))
+    if extra_body.get("guidance_scale_2") is not None:
+        form.add_field("guidance_scale_2", str(extra_body["guidance_scale_2"]))
    if extra_body.get("negative_prompt") is not None:
        form.add_field("negative_prompt", str(extra_body["negative_prompt"]))
    if extra_body.get("true_cfg_scale") is not None:
@@ -1107,6 +1107,11 @@ def calculate_metrics(
    error_outputs = [o for o in outputs if not o.success]

    num_success = len(success_outputs)
+    error_log_path = os.environ.get("BENCH_ERROR_LOG")
+    if error_outputs and error_log_path:
+        with open(error_log_path, "a") as f:
+            for output in error_outputs:
+                f.write(f"ERROR latency={output.latency:.3f} detail={output.error!r}\n")
    latencies = [o.latency for o in success_outputs]
    peak_memories = [o.peak_memory_mb for o in success_outputs if o.peak_memory_mb > 0]

@@ -0,0 +1,194 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Cold-cache torch.cuda timing and correctness gates for MAGI-2 microbenchmarks."""
+
+import argparse
+import importlib
+import importlib.metadata
+import json
+import logging
+import math
+import os
+import subprocess
+import sys
+from collections.abc import Callable
+from pathlib import Path
+from types import ModuleType
+from typing import Any
+
+LOGGER = logging.getLogger(__name__)
+
+
+def emit(**fields: Any) -> None:
+    LOGGER.info(json.dumps(fields, sort_keys=True))
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--tokens", type=int, nargs="+", default=[1, 4, 16, 64, 129, 4096])
+    parser.add_argument("--dtype", choices=["bfloat16", "float16", "float32"], default="bfloat16")
+    parser.add_argument("--num-tests", type=int, default=30, help="Measured samples per implementation.")
+    parser.add_argument("--warmup", type=int, default=10, help="Interleaved warmup calls before sampling.")
+    parser.add_argument("--seed", type=int, default=42)
+
+
+def load_musa(args: argparse.Namespace, parser: argparse.ArgumentParser) -> Any:
+    if min(*args.tokens, args.num_tests) <= 0 or args.warmup < 0:
+        parser.error("tokens and num-tests must be positive; warmup must be nonnegative")
+    logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout)
+    torch = importlib.import_module("torch")
+    cuda = getattr(torch, "cuda")
+    # torchada is needed only on MUSA; after it is loaded, use torch.cuda for
+    # both CUDA and MUSA as documented by torchada.
+    if hasattr(torch.version, "musa") and torch.version.musa is not None:
+        try:
+            importlib.import_module("torchada")
+        except ModuleNotFoundError as error:
+            if error.name != "torchada":
+                raise
+    has_gpu = cuda.is_available() or (
+        hasattr(torch.version, "musa") and torch.version.musa is not None and torch.musa.is_available()
+    )
+    if not has_gpu:
+        emit(status="skip", reason="CUDA/MUSA device is unavailable", torch=torch.__version__)
+        return None
+    torch.manual_seed(args.seed)
+    cuda.manual_seed_all(args.seed)
+    return torch
+
+
+def report_environment(torch: Any, args: argparse.Namespace, module: ModuleType) -> None:
+    root = Path(__file__).resolve().parents[2]
+    packages = {}
+    for name in ("torch", "torch_musa", "torchada", "triton", "vllm", "vllm-omni"):
+        try:
+            packages[name] = importlib.metadata.version(name)
+        except importlib.metadata.PackageNotFoundError:
+            packages[name] = None
+    cuda = getattr(torch, "cuda")
+    properties = cuda.get_device_properties(cuda.current_device())
+    try:
+        source_sha = subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True).strip()
+        source_dirty = bool(subprocess.check_output(["git", "-C", str(root), "status", "--porcelain"], text=True))
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        source_sha = os.environ.get("MAGI2_BENCH_SOURCE_SHA", "unknown")
+        source_dirty = None
+    emit(
+        status="environment",
+        command=sys.argv,
+        source_root=str(root),
+        source_sha=source_sha,
+        source_dirty=source_dirty,
+        kernel_source=module.__file__,
+        device=cuda.get_device_name(cuda.current_device()),
+        sm_count=getattr(properties, "multi_processor_count", None),
+        packages=packages,
+        env={name: os.environ.get(name) for name in ("MUSA_VISIBLE_DEVICES", "TORCHDYNAMO_DISABLE")},
+        config=vars(args),
+        mode="eager operator diagnostic; no compile or graph capture",
+        comparison="forward_native versus the default CustomOp dispatch",
+        timing="torch.cuda.Event; alternating native/fused calls; vanilla 512 MiB flush outside timed regions",
+        timing_scope="whole callable, including device launch gaps; not isolated kernel duration",
+        l2_flush=True,
+        l2_flush_bytes=512 * 1024 * 1024,
+    )
+
+
+class _LaunchProbe:
+    def __init__(self, kernel: Any) -> None:
+        self.kernel = kernel
+        self.count = 0
+
+    def __getitem__(self, grid: Any) -> Any:
+        self.count += 1
+        return self.kernel[grid]
+
+
+def check_pair(
+    torch: Any,
+    reference: Callable,
+    fused: Callable,
+    module: ModuleType,
+    kernels: tuple[str, ...],
+    *,
+    rtol: float,
+    atol: float,
+) -> dict:
+    cuda = getattr(torch, "cuda")
+    expected = reference()
+    probes = {name: _LaunchProbe(getattr(module, name)) for name in kernels}
+    try:
+        for name, probe in probes.items():
+            setattr(module, name, probe)
+        actual = fused()
+        cuda.synchronize()
+    finally:
+        for name, probe in probes.items():
+            setattr(module, name, probe.kernel)
+    if any(probe.count != 1 for probe in probes.values()):
+        raise RuntimeError(f"Expected one launch per fused kernel; got {[(k, p.count) for k, p in probes.items()]}")
+    expected = expected if isinstance(expected, tuple) else (expected,)
+    actual = actual if isinstance(actual, tuple) else (actual,)
+    errors = []
+    for result, baseline in zip(actual, expected, strict=True):
+        for value in (baseline, result):
+            if not torch.isfinite(value).all().item():
+                raise AssertionError("Nonfinite benchmark output")
+            if value.numel() > 1 and (value == value.flatten()[0]).all().item():
+                raise AssertionError("Constant benchmark output")
+        torch.testing.assert_close(result, baseline, rtol=rtol, atol=atol)
+        difference = (result.float() - baseline.float()).abs()
+        errors.append(
+            {
+                "max_abs": difference.max().item(),
+                "max_rel": (difference / baseline.float().abs().clamp_min(1e-6)).max().item(),
+            }
+        )
+    return {"parity": "pass", "errors": errors, "rtol": rtol, "atol": atol, "kernels": list(kernels)}
+
+
+def _percentile(samples: list[float], quantile: float) -> float:
+    ordered = sorted(samples)
+    position = (len(ordered) - 1) * quantile
+    lower, upper = math.floor(position), math.ceil(position)
+    return ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower)
+
+
+def measure_pair(torch: Any, reference: Callable, fused: Callable, args: argparse.Namespace) -> dict:
+    cuda = getattr(torch, "cuda")
+    functions = (reference, fused)
+    order: list[int] = []
+
+    def alternate() -> Any:
+        mode = len(order) % 2
+        order.append(mode)
+        return functions[mode]()
+
+    times_ms = []
+    for _ in range(args.warmup):
+        alternate()
+    cuda.synchronize()
+    order.clear()
+    flush = torch.empty(512 * 1024 * 1024, dtype=torch.uint8, device="cuda")
+    for _ in range(2 * args.num_tests):
+        flush.zero_()
+        cuda.synchronize()
+        start = cuda.Event(enable_timing=True)
+        end = cuda.Event(enable_timing=True)
+        start.record()
+        alternate()
+        end.record()
+        end.synchronize()
+        times_ms.append(start.elapsed_time(end))
+    samples: tuple[list[float], list[float]] = ([], [])
+    for mode, milliseconds in zip(order[-len(times_ms) :], times_ms, strict=True):
+        if not math.isfinite(milliseconds) or milliseconds <= 0:
+            raise RuntimeError(f"Invalid GPU event sample: {milliseconds}")
+        samples[mode].append(milliseconds * 1000)
+    stats = {}
+    for label, values in zip(("native", "fused"), samples, strict=True):
+        if len(values) != args.num_tests:
+            raise RuntimeError("GPU timing harness returned an unexpected sample count")
+        p50 = _percentile(values, 0.5)
+        stats[label] = {"median_us": p50, "p50_us": p50, "p90_us": _percentile(values, 0.9), "samples": len(values)}
+    stats["speedup_p50"] = stats["native"]["p50_us"] / stats["fused"]["p50_us"]
+    return stats
@@ -0,0 +1,310 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Compare MAGI-2 routed MoE with equal preprocessing boundaries.
+
+python -m benchmarks.kernels.benchmark_magi2_bf16_moe --tokens 2 4096 10000
+
+Default (`--mode dispatch`): the same MAGI-2 layer is timed with its native
+reference and default BF16 fast path, including routing and all preprocessing.
+With `--mode routed`, routing probabilities/IDs are inputs to BOTH timed paths. Sorting,
+padding, packing, allocations and reduction remain inside their respective
+calls. The reference uses deterministic routed storage in diagnostic mode.
+Use --mode dispatch to measure the
+actual layer with the native and default fast paths, including the router. Unsupported
+dispatch fails explicitly; it is never silently replaced by another baseline.
+
+Stage timings are standalone diagnostics, not additive or env speedup claims.
+These synthetic local shapes are not full-model performance measurements.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib
+import json
+import math
+import os
+import statistics
+import subprocess
+import time
+from contextlib import contextmanager
+from pathlib import Path
+from unittest.mock import patch
+
+
+def parse_args(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--tokens", type=int, nargs="+", default=[2, 4096])
+    parser.add_argument("--heads", type=int, default=3, help="EP4 local head count for MAGI-2 Preview.")
+    parser.add_argument("--experts", type=int, default=256, help="Released experts per head.")
+    parser.add_argument("--top-k", type=int, default=6)
+    parser.add_argument("--warmup", type=int, default=10)
+    parser.add_argument("--iters", type=int, default=50)
+    parser.add_argument("--seed", type=int, default=419)
+    parser.add_argument("--mode", choices=["routed", "dispatch"], default="dispatch")
+    parser.add_argument(
+        "--native-deterministic",
+        action="store_true",
+        help="Use the portable deterministic native reference (needed by Triton-MUSA 3.2).",
+    )
+    parser.add_argument("--stages", action="store_true")
+    parser.add_argument("--output", type=Path)
+    args = parser.parse_args(argv)
+    if min(*args.tokens, args.heads, args.experts, args.top_k, args.iters) < 1 or args.warmup < 0:
+        parser.error("Shapes/iters must be positive; warmup must be nonnegative")
+    if args.top_k > args.experts:
+        parser.error("top-k must not exceed experts per head")
+    return args
+
+
+@contextmanager
+def env_switch(enabled):
+    # Keep the legacy benchmark context API; dispatch is no longer opt-in.
+    with patch.dict(os.environ, MAGI2_DETERMINISTIC="0"):
+        yield
+
+
+def stats(values):
+    if not values or any(not math.isfinite(x) or x <= 0 for x in values):
+        raise ValueError("Missing/non-positive/non-finite timing sample")
+    ordered = sorted(values)
+    return {
+        "mean_us": statistics.mean(values),
+        "p50_us": statistics.median(values),
+        "p90_us": ordered[math.ceil(0.9 * len(ordered)) - 1],
+        "samples_us": values,
+    }
+
+
+def measure(torch, calls, args):
+    """Alternating AB/BA; cold-cache flush is outside each timed region."""
+    cuda = getattr(torch, "cuda")
+    names = list(calls)
+    sequence = names + names[::-1]
+    wall_samples = []
+    labels = []
+    counter = 0
+
+    def invoke():
+        nonlocal counter
+        name = sequence[counter % len(sequence)]
+        counter += 1
+        cuda.synchronize()
+        start = time.perf_counter_ns()
+        output = calls[name]()
+        cuda.synchronize()
+        elapsed = (time.perf_counter_ns() - start) / 1000
+        wall_samples.append(elapsed)
+        labels.append(name)
+        return output
+
+    count = len(names) * args.iters
+    cache = torch.empty(512 * 1024 * 1024, device="cuda", dtype=torch.uint8)
+    gpu_ms = []
+    for index in range(len(names) * args.warmup + count):
+        cache.zero_()
+        cuda.synchronize()
+        begin, end = cuda.Event(enable_timing=True), cuda.Event(enable_timing=True)
+        begin.record()
+        invoke()
+        end.record()
+        end.synchronize()
+        if index >= len(names) * args.warmup:
+            gpu_ms.append(begin.elapsed_time(end))
+    labels, wall_samples = labels[-count:], wall_samples[-count:]
+    result = {}
+    for name in names:
+        wall = [x for label, x in zip(labels, wall_samples) if label == name]
+        event = [float(x) * 1000 for label, x in zip(labels, gpu_ms) if label == name]
+        if len(wall) != args.iters or len(event) != args.iters:
+            raise RuntimeError("Timing harness did not return the requested sample counts")
+        result[name] = {"synchronized_wall": stats(wall), "gpu_event": stats(event)}
+    return result
+
+
+def make_inputs(torch, tokens, args):
+    # Fan-in-scaled synthetic weights. Unlike the old script, W_up and W_gate
+    # have the same distribution and top-k IDs are unique within each token.
+    torch.manual_seed(args.seed)
+    x = torch.randn(tokens, args.heads, 256, device="cuda").to(torch.bfloat16)
+    shape = (args.heads * args.experts, 256, 1280)
+    gate, up = [(torch.randn(shape, device="cuda") / 16).to(torch.bfloat16) for _ in range(2)]
+    down = (torch.randn(shape[0], 1280, 256, device="cuda") / math.sqrt(1280)).to(torch.bfloat16)
+    logits = torch.randn(args.heads, tokens, args.experts, device="cuda")
+    selected = logits.topk(args.top_k, dim=-1)
+    probabilities = selected.values.softmax(-1)
+    return x, probabilities, selected.indices, gate, up, down
+
+
+def check_and_capture(torch, moe, kernels, data, args):
+    x, probabilities, indices, gate, up, down = data
+    gather, weights, offsets = moe.global_sort_routes(probabilities, indices, args.experts)
+    sorted_routes, launches = [], []
+    scatter, invoke = kernels._deterministic_scatter, moe.invoke_fused_moe_bf16
+
+    def capture_scatter(routes, *params):
+        sorted_routes.append(routes.detach().clone())
+        return scatter(routes, *params)
+
+    def capture_launch(*params, **kwargs):
+        launches.append((params, kwargs))
+        return invoke(*params, **kwargs)
+
+    with patch.object(kernels, "_deterministic_scatter", capture_scatter):
+        moe.triton_mh_moe_forward(x, gather, weights, offsets, gate, up, down, deterministic=True)
+    packed_w13 = moe._pack_bf16_w13(gate, up)
+    route_buffers = moe._allocate_bf16_route_buffers(
+        x.shape[0] * args.heads * args.top_k,
+        args.heads * args.experts,
+        128,
+        x.device,
+    )
+    with patch.object(moe, "invoke_fused_moe_bf16", capture_launch):
+        actual = moe._bf16_fused_moe_forward(x, probabilities, indices, packed_w13, down, route_buffers)
+    if len(sorted_routes) != 1 or len(launches) != 2:
+        raise RuntimeError("Unexpected kernel dispatch: benchmark contract needs updating")
+    flat_experts = (indices + torch.arange(args.heads, device=x.device)[:, None, None] * args.experts).flatten()
+    order = flat_experts.argsort(stable=True)
+    new_routes = launches[1][0][2].reshape(-1, 256).index_select(0, order)
+    scatter_ids = gather.long() * args.heads + flat_experts[order] // args.experts
+    expected = torch.zeros(x.shape[0] * args.heads, 256, device=x.device, dtype=torch.float32)
+    expected.index_add_(0, scatter_ids, sorted_routes[0].float())
+    expected = expected.to(x.dtype).view_as(x)
+    errors = []
+    # Same expert/FP32 reduction contract as test_bf16_moe_wiring.py, rather
+    # than relaxing tolerances on the inherited BF16 scatter reduction.
+    for result, reference in ((new_routes, sorted_routes[0]), (actual, expected)):
+        if not (torch.isfinite(result).all() and torch.isfinite(reference).all()):
+            raise RuntimeError("Non-finite MoE output")
+        torch.testing.assert_close(result, reference, rtol=2e-2, atol=0.5)
+        relative_l2 = (result.float() - reference.float()).norm() / reference.float().norm().clamp_min(1e-6)
+        if relative_l2.item() >= 1e-3:
+            raise AssertionError(f"MoE relative L2 {relative_l2.item()} >= 1e-3")
+        errors.append(
+            {"max_abs": (result.float() - reference.float()).abs().max().item(), "relative_l2": relative_l2.item()}
+        )
+    return launches, errors
+
+
+def benchmark(torch, moe, kernels, tokens, args):
+    data = make_inputs(torch, tokens, args)
+    x, probabilities, indices, gate, up, down = data
+    launches, errors = check_and_capture(torch, moe, kernels, data, args)
+    packed_w13 = moe._pack_bf16_w13(gate, up)
+    route_count = tokens * args.heads * args.top_k
+    route_buffers = moe._allocate_bf16_route_buffers(route_count, args.heads * args.experts, 128, x.device)
+
+    def reference():
+        gather, weights, offsets = moe.global_sort_routes(probabilities, indices, args.experts)
+        return moe.triton_mh_moe_forward(x, gather, weights, offsets, gate, up, down, deterministic=True)
+
+    calls = {
+        "reference_deterministic_routed": reference,
+        "candidate_routed": lambda: moe._bf16_fused_moe_forward(
+            x, probabilities, indices, packed_w13, down, route_buffers
+        ),
+    }
+    if args.mode == "dispatch":
+        config = moe.Magi2MultiHeadMoEConfig(
+            args.heads * 256, args.heads, args.experts, args.top_k, 1280, torch.bfloat16
+        )
+        layer = moe.Magi2MultiHeadMoE(config).to(x.device)
+        for parameter, value in ((layer.W_gate, gate), (layer.W_up, up), (layer.W_down, down)):
+            parameter.copy_(value)
+        layer.gate.normal_(std=1 / 16)
+
+        flat_x = x.reshape(tokens, -1)
+
+        def native_dispatch():
+            x_heads = flat_x.view(tokens, args.heads, 256)
+            probs, idx = layer._route(x_heads)
+            gather, weights, offsets = moe.global_sort_routes(probs, idx, args.experts)
+            return moe.triton_mh_moe_forward(
+                x_heads,
+                gather,
+                weights,
+                offsets,
+                layer.W_gate,
+                layer.W_up,
+                layer.W_down,
+                deterministic=args.native_deterministic,
+            ).reshape(tokens, -1)
+
+        calls = {"native_dispatch": native_dispatch, "fused_dispatch": lambda: layer(flat_x)}
+        torch.testing.assert_close(calls["fused_dispatch"](), calls["native_dispatch"](), rtol=2e-2, atol=0.5)
+    timings = measure(torch, calls, args)
+    names = list(calls)
+    result = {"status": "pass", "tokens": tokens, "mode": args.mode, "correctness": errors, "timings": timings}
+    result["speedup_p50_wall"] = (
+        timings[names[0]]["synchronized_wall"]["p50_us"] / timings[names[1]]["synchronized_wall"]["p50_us"]
+    )
+    if args.stages:
+        route_ids = (indices.int() + torch.arange(args.heads, device=x.device)[:, None, None] * args.experts).reshape(
+            -1, args.top_k
+        )
+
+        def gems():
+            for params, kwargs in launches:
+                moe.invoke_fused_moe_bf16(*params, **kwargs)
+
+        result["standalone_stages_not_additive"] = measure(
+            torch,
+            {
+                "reference_sort": lambda: moe.global_sort_routes(probabilities, indices, args.experts),
+                "candidate_align": lambda: moe._align_bf16_routes(route_ids, args.heads * args.experts, 128),
+                "candidate_pack_w13": lambda: torch.stack((gate.transpose(1, 2), up.transpose(1, 2)), dim=2).reshape(
+                    args.heads * args.experts, 2560, 256
+                ),
+                "candidate_two_gemms_prepared": gems,
+            },
+            args,
+        )
+    return result
+
+
+def main():
+    args = parse_args()
+    import torch
+
+    if hasattr(torch.version, "musa") and torch.version.musa is not None:
+        importlib.import_module("torchada")  # before importing vLLM / capturing CUDA APIs
+    cuda = getattr(torch, "cuda")
+    has_gpu = cuda.is_available() or (
+        hasattr(torch.version, "musa") and torch.version.musa is not None and torch.musa.is_available()
+    )
+    if not has_gpu:
+        print(json.dumps({"status": "skip", "reason": "No visible CUDA/MUSA device"}))
+        return
+    import vllm_omni.diffusion.models.magi2.fused_moe_kernels as kernels
+    import vllm_omni.diffusion.models.magi2.mh_moe as moe
+
+    root = Path(__file__).resolve().parents[2]
+    try:
+        sha = subprocess.check_output(
+            ["git", "-C", str(root), "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL
+        ).strip()
+    except (OSError, subprocess.CalledProcessError):
+        sha = os.environ.get("MAGI2_BENCH_SOURCE_SHA", "unknown")
+    report = {
+        "device": cuda.get_device_name(),
+        "sm_count": cuda.get_device_properties(0).multi_processor_count,
+        "source_sha": sha,
+        "module_path": moe.__file__,
+        "torch": torch.__version__,
+        "args": {**vars(args), "output": str(args.output)},
+        "cache": "512 MiB flush outside each region",
+        "capture": "eager; JIT warmed; no graph/compile",
+        "results": [],
+    }
+    with torch.inference_mode():
+        for tokens in args.tokens:
+            result = benchmark(torch, moe, kernels, tokens, args)
+            report["results"].append(result)
+            print(json.dumps(result), flush=True)
+    if args.output:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(json.dumps(report, indent=2) + "\n")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,87 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Compare native and fused MAGI-2 SwiGLU7 operators on CUDA or MUSA.
+
+Run from the checkout root in an environment with PyTorch and the selected
+backend dependencies.  torchada is loaded automatically only on MUSA::
+
+    python -m benchmarks.kernels.benchmark_magi2_swiglu7 \
+        --tokens 1 4 16 64 129 4096 --intermediate-size 1280 --num-tests 30
+
+The released MAGI-2 config supplies the 1280 intermediate width. Token counts
+are a synthetic sweep; pass per-rank sizes from the target trace for production
+comparisons. Input is contiguous interleaved gate/up [tokens, 2 * intermediate].
+Both paths evaluate FP32 arithmetic with one cast to the requested output dtype.
+This measures eager operator latency, not compiled serving performance.
+No CUDA/MUSA device (or no PyTorch) produces a JSON skip record and exit status zero.
+"""
+
+import argparse
+from functools import partial
+
+from benchmarks.kernels._magi2_musa_bench import (
+    add_arguments,
+    check_pair,
+    emit,
+    load_musa,
+    measure_pair,
+    report_environment,
+)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    add_arguments(parser)
+    parser.add_argument("--intermediate-size", type=int, default=1280)
+    parser.add_argument("--out-dtype", choices=["bfloat16", "float16", "float32"], default=None)
+    args = parser.parse_args()
+    if args.intermediate_size <= 0:
+        parser.error("intermediate-size must be positive")
+    torch = load_musa(args, parser)
+    if torch is None:
+        return
+    cuda = getattr(torch, "cuda")
+    from vllm_omni.diffusion.layers import swiglu7
+
+    report_environment(torch, args, swiglu7)
+    dtype = getattr(torch, args.dtype)
+    out_dtype = getattr(torch, args.out_dtype or args.dtype)
+    device = torch.device("cuda")
+    rtol = {torch.bfloat16: 1.6e-2, torch.float16: 2e-3, torch.float32: 1e-5}[out_dtype]
+    atol = 1e-5 if out_dtype == torch.float32 else 2e-3
+    op = swiglu7.SwiGLU7()
+    with torch.inference_mode():
+        for tokens in args.tokens:
+            # Values outside both clamp boundaries exercise the released formula.
+            x = (torch.randn(tokens, 2 * args.intermediate_size, device=device) * 10).to(dtype)
+            reference = partial(op.forward_native, x, out_dtype=out_dtype)
+            fused = partial(op, x, out_dtype=out_dtype)
+            parity = check_pair(torch, reference, fused, swiglu7, ("_swiglu7_kernel",), rtol=rtol, atol=atol)
+            timing = measure_pair(torch, reference, fused, args)
+            output_elements = tokens * args.intermediate_size
+            minimum_io_bytes = (
+                x.numel() * x.element_size() + output_elements * torch.empty((), dtype=out_dtype).element_size()
+            )
+            emit(
+                status="pass",
+                op="swiglu7",
+                input_shape=list(x.shape),
+                input_stride=list(x.stride()),
+                dtype=args.dtype,
+                out_dtype=args.out_dtype or args.dtype,
+                alpha=1.702,
+                limit=7.0,
+                minimum_io_bytes=minimum_io_bytes,
+                # Six scalar arithmetic operations and one exp per output;
+                # clamps and transcendental costs are not modeled as FLOPs.
+                scalar_flops_excluding_exp_and_clamps=6 * output_elements,
+                effective_fused_gbps=minimum_io_bytes / timing["fused"]["p50_us"] / 1000,
+                **parity,
+                **timing,
+            )
+            del x, reference, fused
+            cuda.empty_cache()
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,124 @@
+# MammothModa2 startup benchmark
+
+Measures `Omni()` initialization and first/subsequent image requests for
+MammothModa2-Preview. This is a measurement baseline for
+[#7075](https://github.com/vllm-project/vllm-omni/issues/7075); it changes no runtime code.
+
+## Run
+
+From the repository root, with the model downloaded and vLLM-Omni installed:
+
+```bash
+set -o pipefail
+python benchmarks/mammoth_moda2/bench_startup.py \
+    --model /path/MammothModa2-Preview \
+    --deploy-config vllm_omni/deploy/mammoth_moda2.yaml \
+    --height 512 --width 512 --seed 42 \
+    --extra-body '{"text_guidance_scale": 4.0, "cfg_range": [0.0, 1.0], "num_inference_steps": 8}' \
+    --repeat 1 --label baseline --output-json baseline.json --save-image baseline.png \
+    2>&1 | tee baseline.log
+python benchmarks/mammoth_moda2/parse_startup_log.py baseline.log --require-complete --markdown
+```
+
+Use `OMP_NUM_THREADS=16` to set CPU threads and `--parallel-stage-init` to
+initialize stages concurrently. For first/subsequent requests, use
+`--height 1024 --width 1024 --repeat 3` and 50 inference steps. Memory polling
+is opt-in (`--sample-memory`); collect it in separate runs to avoid perturbing
+the timing comparison. `--save-image` saves only the first request's image.
+Compare that image across runs; a fixed seed alone does not guarantee equality
+across execution modes or software versions.
+
+## Measurement boundaries
+
+| Metric | Boundary / limitation |
+| --- | --- |
+| Imports | Before torch import → benchmark imports complete; excludes interpreter startup |
+| Time to ready | `Omni()` call → return; excludes client imports and requests |
+| Time to first image | Benchmark clock start → first `generate()` return, including request setup |
+| First / subsequent request | Each `generate()` call → return; same prompt and settings |
+| Stage spawn / device init | Log timestamps: launch → `world_size=` → `Starting to load model`; 1 s resolution |
+| Weight load | Engine-reported `Loading weights took`; read/cast/copy are merged |
+| Model setup estimate | Reported model-load time minus weight-load time; not independently instrumented |
+| Profile + KV + warmup | One merged interval after loading; compilation/capture may be nested within it |
+| Compile / graph capture | Explicit engine log durations when present; N/A for eager execution |
+| Memory | Optional 1 s samples: sum of process-tree RSS and maximum device memory used across GPUs reported by `nvidia-smi`; includes other GPU users and can miss transient peaks |
+
+Stage timelines overlap with parallel initialization; do not sum them to
+calculate time to ready. Missing log fields stay unmeasured. The parser marks
+logs without `BENCH_JSON` as incomplete; `--require-complete` makes this an error.
+
+## Reference results
+
+A800-SXM4-80GB; Ubuntu 22.04; 16-CPU cgroup quota on a 128-core host;
+vLLM 0.28.0+cu129, torch 2.13.0+cu129, vLLM-Omni `34aa1d2a`.
+Checkpoint: 34.5 GiB, 8 shards, mostly fp32, on local NVMe.
+Committed eager deploy config, GPU memory fractions 0.5 / 0.3.
+
+The 2026-09-10 rerun used the corrected benchmark: one warm-up process,
+then three rounds of four configurations with order rotated between rounds.
+The checkpoint page cache was warmed before the matrix; no cache eviction
+was requested during it. Each fresh process sent one 512×512, 8-step request
+(guidance 4.0, seed 42). Memory polling was disabled. All 12 measured runs
+are retained in [the results](reference_results.json).
+Values below are `Omni()` seconds, mean ± sample standard deviation, n=3;
+client imports took approximately another 10 s.
+
+| CPU threads | Serial init | Parallel init |
+| --- | --- | --- |
+| Default (64) | 60.1 ± 0.2 | 41.1 ± 1.0 |
+| 16 | 56.5 ± 0.9 | 36.7 ± 0.2 |
+
+On this configuration, parallel initialization reduced time to ready by about
+32%; combining it with 16 threads reduced it by about 39%. This supports
+trying these settings on similar deployments, not changing defaults for all
+models or hardware. Fixed 512×512 samples matched across all 12 measured runs;
+this is a smoke check, not a broad quality evaluation.
+
+For context, in the earlier serial baseline AR / DiT spent approximately 11 / 11 s in
+spawn and imports, 6 / 6 s in device setup, 6 / 4 s loading weights, and
+5 / 2 s in profile + KV + warmup. About 6 s followed the last stage becoming
+ready. These coarse log intervals locate work; they are not independent
+operator timings.
+
+Separate observations from the earlier experiment (not rerun in this matrix):
+
+- Requested page-cache eviction: 80.8 s; FlashInfer JIT cache removed:
+  129.1 s (means, n=3 each). These are separate cache conditions, not guaranteed
+  cold-machine startup or isolated disk/JIT costs.
+- Three 1024×1024, 50-step eager requests in one process took
+  95.2 / 94.9 / 94.6 s. No large first-request penalty was observed in that run.
+- With AR `enforce_eager: false` and serial init, time to ready was
+  62.5 ± 0.1 s (warm caches, n=3); logged graph capture was 4 s / 0.20 GiB.
+  The model reported torch.compile unsupported. KV capacity fell from about
+  15.7 to 2.0 GiB, and one parallel-init trial failed. The reported non-KV
+  profiling difference is not a measurement of graph allocation. Graph-mode
+  samples differed from eager. This needs separate memory and quality analysis
+  before recommending the configuration.
+
+## Loading diagnostic and storage scenarios
+
+```bash
+OMP_NUM_THREADS=16 python benchmarks/mammoth_moda2/raw_load_bench.py \
+    --model /path/MammothModa2-Preview --shards 6,7,8 --mode cpu_cast
+# Compare --mode gpu_cast in a fresh process with the same cache conditions.
+
+MODEL_LOCAL=/local/MammothModa2-Preview OUT_DIR=./startup-bench \
+    bash benchmarks/mammoth_moda2/bench_storage_scenarios.sh
+# Optional MODEL_NFS=/nfs/MammothModa2-Preview; SCENARIOS="local-cold local-warm".
+```
+
+The diagnostic clones each mmap-backed tensor into owned CPU memory before
+casting. `materialize_s` includes page faults, CPU allocation and copying;
+`cast_s` measures dtype conversion; `h2d_s` includes GPU allocation and copy.
+CUDA context/operation warmup is outside the timer. Total also includes file
+opening and bookkeeping. This extra CPU copy differs from the engine loader,
+so these numbers do not decompose its loading time or measure disk bandwidth.
+The revised diagnostic was run three times each for CPU cast with 64 / 16
+threads and GPU cast with 16 threads; per-run values are in the results file.
+Both CPU materialization and conversion were faster with 16 threads here; this
+does not isolate how much of the engine-level improvement comes from either.
+
+Storage scenarios explicitly read shards before warm runs and request
+`posix_fadvise(DONTNEED)` before cold runs. Eviction is not guaranteed, and
+server-side NFS caches are uncontrolled. The script stops on failure and
+summarizes only logs from the current invocation.
@@ -1,4 +1,2 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
-
-"""Qwen3-Omni turn-based browser call UI."""
@@ -0,0 +1,338 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Startup / loading benchmark for multi-stage omni models (MammothModa2 first).
+
+Measures, from a single fresh process:
+  * python import time (torch + vllm_omni)
+  * Omni() construction wall time (all stages: weight load, profiling, KV cache, warmup)
+  * first request latency (includes JIT / lazy init that only happens on first run)
+  * steady-state request latency (repeat N-1 more requests)
+  * optional host RSS / device memory samples (--sample-memory, once a second)
+
+Per-stage breakdown (weight loading, model-load memory, engine init) is emitted by the
+stage engine-core subprocesses into the log; use parse_startup_log.py on the captured
+stdout/stderr to merge them with the JSON written here.
+
+Usage (keep the log: the per-stage breakdown is parsed from it afterwards):
+  python benchmarks/mammoth_moda2/bench_startup.py --model /path/MammothModa2-Preview \
+      --deploy-config vllm_omni/deploy/mammoth_moda2.yaml \
+      --height 1024 --width 1024 --seed 42 \
+      --extra-body '{"text_guidance_scale": 4.0, "cfg_range": [0.0, 1.0], "num_inference_steps": 50}' \
+      --repeat 2 --label nfs-warm --output-json out.json 2>&1 | tee run.log
+  python benchmarks/mammoth_moda2/parse_startup_log.py run.log --markdown
+"""
+
+import argparse
+import json
+import os
+import platform
+import subprocess
+import threading
+import time
+
+T_PROCESS_START = time.perf_counter()
+
+import torch  # noqa: E402
+
+from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args  # noqa: E402
+from vllm_omni.entrypoints.omni import Omni  # noqa: E402
+from vllm_omni.entrypoints.openai.stage_params import clone_sampling_params  # noqa: E402
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams  # noqa: E402
+from vllm_omni.model_extras import (  # noqa: E402
+    build_text_to_image_prompt as build_model_text_to_image_prompt,
+)
+from vllm_omni.model_extras import (  # noqa: E402
+    get_extra_body_params,
+    get_model_class_name,
+    should_init_extra_args_for_non_diffusion_stages,
+)
+from vllm_omni.platforms import current_omni_platform  # noqa: E402
+
+T_IMPORTS_DONE = time.perf_counter()
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--model", required=True)
+    p.add_argument("--deploy-config", default=None)
+    p.add_argument("--prompt", default="A stylish woman riding a motorcycle in NYC, movie poster style")
+    p.add_argument("--negative-prompt", default=None)
+    p.add_argument("--height", type=int, default=1024)
+    p.add_argument("--width", type=int, default=1024)
+    p.add_argument("--seed", type=int, default=42)
+    p.add_argument("--extra-body", type=json.loads, default=None, help="JSON object forwarded as model extra_body")
+    p.add_argument(
+        "--repeat", type=int, default=2, help="number of generate() calls; 1st = first-request, rest = steady"
+    )
+    p.add_argument("--label", default="run", help="free-form label, e.g. nfs-cold / nfs-warm / local-nvme")
+    p.add_argument("--output-json", default=None)
+    p.add_argument("--save-image", default=None, help="optional path to save the first request's image")
+    p.add_argument("--sample-memory", action="store_true", help="sample memory once a second (adds overhead)")
+    p.add_argument(
+        "--parallel-stage-init",
+        action="store_true",
+        help="initialize stages that share a GPU concurrently (VllmOmniOrchestratorConfig.parallel_stage_init)",
+    )
+    args = p.parse_args()
+    if args.repeat < 1:
+        p.error("--repeat must be at least 1")
+    return args
+
+
+def gpu_info() -> dict:
+    info = {"torch": torch.__version__, "cuda": torch.version.cuda, "python": platform.python_version()}
+    try:
+        import vllm
+
+        info["vllm"] = vllm.__version__
+    except Exception:  # pragma: no cover
+        pass
+    try:
+        import vllm_omni
+
+        info["vllm_omni"] = getattr(vllm_omni, "__version__", "src")
+    except Exception:  # pragma: no cover
+        pass
+    try:
+        out = subprocess.run(
+            ["nvidia-smi", "--query-gpu=name,driver_version,memory.total", "--format=csv,noheader"],
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+        info["gpu"] = out.stdout.strip()
+    except Exception:
+        info["gpu"] = "n/a"
+    return info
+
+
+class PeakSampler:
+    """Poll host RSS of this process tree and GPU memory in use once a second (benchmark-side only)."""
+
+    def __init__(self, interval_s: float = 1.0):
+        self.interval_s = interval_s
+        self.host_rss_peak_gib = 0.0
+        self.gpu_used_peak_gib = 0.0
+        self._stop = threading.Event()
+        self._thread = threading.Thread(target=self._run, daemon=True)
+
+    def _sample(self) -> None:
+        try:
+            import psutil
+
+            root = psutil.Process()
+            rss = root.memory_info().rss + sum(
+                c.memory_info().rss for c in root.children(recursive=True) if c.is_running()
+            )
+            self.host_rss_peak_gib = max(self.host_rss_peak_gib, rss / 2**30)
+        except Exception:
+            pass
+        try:
+            out = subprocess.run(
+                ["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"],
+                capture_output=True,
+                text=True,
+                timeout=5,
+            )
+            used = max(float(x) for x in out.stdout.split() if x.strip())
+            self.gpu_used_peak_gib = max(self.gpu_used_peak_gib, used / 1024)
+        except Exception:
+            pass
+
+    def _run(self) -> None:
+        while not self._stop.is_set():
+            self._sample()
+            self._stop.wait(self.interval_s)
+
+    def start(self) -> "PeakSampler":
+        self._thread.start()
+        return self
+
+    def stop(self) -> dict:
+        self._stop.set()
+        self._thread.join(timeout=10)
+        self._sample()
+        return {
+            "host_rss_peak_gib": round(self.host_rss_peak_gib, 2),
+            "gpu_used_peak_gib": round(self.gpu_used_peak_gib, 2),
+        }
+
+
+def fs_type(path: str) -> str:
+    try:
+        out = subprocess.run(["stat", "-f", "-c", "%T", path], capture_output=True, text=True, timeout=10)
+        return out.stdout.strip()
+    except Exception:
+        return "unknown"
+
+
+def build_request(omni: Omni, args: argparse.Namespace, generator: torch.Generator):
+    model_class_name = get_model_class_name(omni)
+    declared = get_extra_body_params(model_class_name)
+
+    prompt_dict = {"prompt": args.prompt}
+    if args.negative_prompt:
+        prompt_dict["negative_prompt"] = args.negative_prompt
+    prompt_dict = build_model_text_to_image_prompt(
+        model_class_name=model_class_name, prompt=prompt_dict, height=args.height, width=args.width
+    )
+
+    diffusion_params = OmniDiffusionSamplingParams(
+        height=args.height,
+        width=args.width,
+        seed=args.seed,
+        generator=generator,
+    )
+    user_extra = dict(args.extra_body or {})
+    if declared:
+        apply_declared_extra_args(diffusion_params, declared, user_extra)
+    else:
+        diffusion_params.extra_args.update({k: v for k, v in user_extra.items() if v is not None})
+
+    init_non_diffusion = should_init_extra_args_for_non_diffusion_stages(model_class_name)
+    sampling_params_list = [clone_sampling_params(p) for p in (omni.default_sampling_params_list or [])]
+    if not sampling_params_list:
+        sampling_params_list = [diffusion_params]
+
+    replaced = False
+    for idx, params in enumerate(sampling_params_list):
+        if isinstance(params, OmniDiffusionSamplingParams):
+            sampling_params_list[idx] = diffusion_params
+            replaced = True
+        elif init_non_diffusion and hasattr(params, "extra_args"):
+            params.extra_args = {**(params.extra_args or {}), **(diffusion_params.extra_args or {})}
+            if hasattr(params, "seed"):
+                params.seed = args.seed
+            # MammothModa2 AR stage: one visual token per grid cell + one EOL per row + 1 look-ahead.
+            info = prompt_dict.get("additional_information", {})
+            if idx == 0 and info.get("omni_task") == ["t2i"]:
+                ar_w = int(info.get("ar_width", [0])[0])
+                ar_h = int(info.get("ar_height", [0])[0])
+                if ar_w > 0 and ar_h > 0:
+                    params.max_tokens = ar_h * (ar_w + 1) + 1
+    if not replaced and len(sampling_params_list) == 1:
+        sampling_params_list = [diffusion_params]
+    return prompt_dict, sampling_params_list
+
+
+def stage_metrics(outputs) -> dict:
+    """Pull vllm-omni's per-stage timing out of the request output when present."""
+    res = {}
+    for out in outputs or []:
+        metrics = getattr(out, "metrics", None)
+        if isinstance(metrics, dict) and "stage_metrics" in metrics:
+            for sid, m in metrics["stage_metrics"].items():
+                if not isinstance(m, dict):
+                    continue
+                res[str(sid)] = {
+                    k: m.get(k)
+                    for k in ("stage_gen_time_ms", "num_tokens_in", "num_tokens_out", "vllm_ttft_ms", "vllm_tpot_ms")
+                    if k in m
+                }
+    return res
+
+
+def main() -> None:
+    args = parse_args()
+    result = {
+        "label": args.label,
+        "model": args.model,
+        "prompt": args.prompt,
+        "negative_prompt": args.negative_prompt,
+        "sample_memory": args.sample_memory,
+        "model_fs": fs_type(args.model),
+        "deploy_config": args.deploy_config,
+        "height": args.height,
+        "width": args.width,
+        "seed": args.seed,
+        "extra_body": args.extra_body,
+        "parallel_stage_init": args.parallel_stage_init,
+        "omp_num_threads": os.environ.get("OMP_NUM_THREADS"),
+        "env": gpu_info(),
+        "phases_s": {"imports": round(T_IMPORTS_DONE - T_PROCESS_START, 3)},
+        "requests": [],
+    }
+
+    omni_kwargs = {"model": args.model, "mode": "text-to-image", "log_stats": True}
+    if args.deploy_config:
+        omni_kwargs["deploy_config"] = args.deploy_config
+    if args.parallel_stage_init:
+        omni_kwargs["parallel_stage_init"] = True
+
+    sampler = PeakSampler().start() if args.sample_memory else None
+    t0 = time.perf_counter()
+    omni = None
+    try:
+        omni = Omni(**omni_kwargs)
+        t1 = time.perf_counter()
+        result["peak_mem_startup"] = sampler.stop() if sampler else {}
+        sampler = None
+        result["phases_s"]["engine_init"] = round(t1 - t0, 3)
+        result["phases_s"]["process_to_engine_ready"] = round(t1 - T_PROCESS_START, 3)
+        print(
+            f"BENCH engine_init={t1 - t0:.2f}s process_to_engine_ready={t1 - T_PROCESS_START:.2f}s "
+            f"peak_mem_startup={result['peak_mem_startup']}",
+            flush=True,
+        )
+        sampler = PeakSampler().start() if args.sample_memory else None
+
+        generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
+        outputs = None
+        first_outputs = None
+        first_image_elapsed = 0.0
+        for i in range(args.repeat):
+            prompt_dict, sampling_params_list = build_request(omni, args, generator)
+            tr0 = time.perf_counter()
+            outputs = omni.generate(prompt_dict, sampling_params_list=sampling_params_list)
+            tr1 = time.perf_counter()
+            if first_outputs is None:
+                first_outputs = outputs
+                first_image_elapsed = tr1 - T_PROCESS_START
+            rec = {
+                "index": i,
+                "kind": "first" if i == 0 else "steady",
+                "latency_s": round(tr1 - tr0, 3),
+                "stages": stage_metrics(outputs),
+            }
+            result["requests"].append(rec)
+            print(f"BENCH request[{i}] {rec['kind']} latency={tr1 - tr0:.2f}s stages={rec['stages']}", flush=True)
+
+        result["peak_mem_requests"] = sampler.stop() if sampler else {}
+        sampler = None
+        steady = [r["latency_s"] for r in result["requests"] if r["kind"] == "steady"]
+        result["summary_s"] = {
+            "imports": result["phases_s"]["imports"],
+            "engine_init": result["phases_s"]["engine_init"],
+            "first_request": result["requests"][0]["latency_s"],
+            "steady_request_avg": round(sum(steady) / len(steady), 3) if steady else None,
+            "time_to_first_image_from_process_start": round(first_image_elapsed, 3),
+        }
+
+        for key in ("host_rss_peak_gib", "gpu_used_peak_gib"):
+            result["summary_s"][key] = (
+                max(result["peak_mem_startup"][key], result["peak_mem_requests"][key]) if args.sample_memory else None
+            )
+
+        from vllm_omni.diffusion.utils.image_output import extract_images_from_outputs
+
+        images = extract_images_from_outputs(first_outputs)
+        if not images:
+            raise RuntimeError("First request returned no image")
+        if args.save_image:
+            os.makedirs(os.path.dirname(os.path.abspath(args.save_image)), exist_ok=True)
+            images[0].save(args.save_image)
+
+        print("BENCH_JSON " + json.dumps(result), flush=True)
+        if args.output_json:
+            with open(args.output_json, "w") as f:
+                json.dump(result, f, indent=2)
+            print(f"BENCH wrote {args.output_json}", flush=True)
+    finally:
+        if sampler:
+            sampler.stop()
+        if omni is not None:
+            omni.close()
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,67 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+# MODEL_LOCAL=/local/model MODEL_NFS=/nfs/model OUT_DIR=./startup-bench bash "$0"
+# Optional SCENARIOS: local-cold local-warm nfs-cold nfs-warm.
+set -euo pipefail
+HERE=$(cd "$(dirname "$0")" && pwd)
+OUT_DIR=${OUT_DIR:-./startup-bench}
+DEPLOY_CONFIG=${DEPLOY_CONFIG:-vllm_omni/deploy/mammoth_moda2.yaml}
+STEPS=${STEPS:-50}
+SIZE=${SIZE:-1024}
+REPEAT=${REPEAT:-2}
+SEED=${SEED:-42}
+EXTRA=${EXTRA:-'{"text_guidance_scale": 4.0, "cfg_range": [0.0, 1.0], "num_inference_steps": '$STEPS'}'}
+mkdir -p "$OUT_DIR/logs" "$OUT_DIR/json" "$OUT_DIR/images"
+rm -f "$OUT_DIR/summary.json" "$OUT_DIR/summary.md"
+
+prepare_cache() {
+  python - "$1" "$2" <<'PY'
+import os
+import sys
+from pathlib import Path
+
+files = sorted(Path(sys.argv[1]).glob("*.safetensors"))
+if not files:
+    raise SystemExit("no checkpoint shards found")
+for path in files:
+    with path.open("rb") as f:
+        if sys.argv[2] == "cold":
+            os.posix_fadvise(f.fileno(), 0, 0, os.POSIX_FADV_DONTNEED)
+        else:
+            while f.read(8 * 1024 * 1024):
+                pass
+print("[bench]", "requested eviction of" if sys.argv[2] == "cold" else "read", len(files), "shards")
+PY
+}
+
+default_scenarios=""
+if [ -n "${MODEL_LOCAL:-}" ]; then default_scenarios="local-cold local-warm"; fi
+if [ -n "${MODEL_NFS:-}" ]; then default_scenarios="$default_scenarios nfs-cold nfs-warm"; fi
+read -r -a scenarios <<< "${SCENARIOS:-$default_scenarios}"
+if [ "${#scenarios[@]}" -eq 0 ]; then echo "Set MODEL_LOCAL or MODEL_NFS" >&2; exit 1; fi
+logs=()
+for label in "${scenarios[@]}"; do
+  case "$label" in
+    local-cold|local-warm) model=${MODEL_LOCAL:?Set MODEL_LOCAL} ;;
+    nfs-cold|nfs-warm) model=${MODEL_NFS:?Set MODEL_NFS} ;;
+    *) echo "Unknown scenario: $label" >&2; exit 1 ;;
+  esac
+  prepare_cache "$model" "${label##*-}"
+  log="$OUT_DIR/logs/$label.log"
+  # Remove previous outputs for this label so a failed rerun cannot leave stale results.
+  rm -f "$OUT_DIR/json/$label.json" "$OUT_DIR/images/$label.png"
+  if python "$HERE/bench_startup.py" --model "$model" --deploy-config "$DEPLOY_CONFIG" \
+    --height "$SIZE" --width "$SIZE" --seed "$SEED" --extra-body "$EXTRA" --repeat "$REPEAT" \
+    --label "$label" --output-json "$OUT_DIR/json/$label.json" --save-image "$OUT_DIR/images/$label.png" \
+    > "$log" 2>&1; then
+    logs+=("$log")
+  else
+    rc=$?
+    echo "[bench] $label failed (exit $rc); see $log" >&2
+    exit "$rc"
+  fi
+done
+python "$HERE/parse_startup_log.py" "${logs[@]}" --require-complete --markdown \
+  --json-out "$OUT_DIR/summary.json" | tee "$OUT_DIR/summary.md"
+echo BENCH_ALL_DONE
@@ -0,0 +1,245 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Merge per-stage startup numbers from a bench_startup.py log with its BENCH_JSON line.
+
+The stage engine cores run in subprocesses and report weight-loading / init timings
+only through log lines such as:
+
+  (StageEngineCoreProc_stage0_replica0 pid=1) INFO ... Loading weights took 39.66 seconds
+  (StageEngineCoreProc_stage0_replica0 pid=1) INFO ... Model loading took 21.45 GiB memory and 40.59 seconds
+  (StageEngineCoreProc_stage0_replica0 pid=1) INFO ... init engine (profile, create kv cache, warmup model) took 7.00 s
+  (StageEngineCoreProc_stage0_replica0 pid=1) INFO ... Filesystem type for checkpoints: NFS4.
+      Checkpoint size: 34.52 GiB. Available RAM: 22.76 GiB.
+  INFO ... [Omni] AsyncOmniEngine initialized in 159.74 seconds
+
+Usage:
+  python benchmarks/mammoth_moda2/parse_startup_log.py run.log [more.log ...] [--markdown] [--json-out summary.json]
+"""
+
+import argparse
+import json
+import re
+import sys
+from datetime import datetime
+from typing import Any
+
+STAGE = r"\(StageEngineCoreProc_stage(?P<stage>\d+)_replica\d+ pid=\d+\)"
+PATTERNS = {
+    "weights_load_s": re.compile(STAGE + r".*Loading weights took (?P<v>[\d.]+) seconds"),
+    "model_load_s": re.compile(STAGE + r".*Model loading took (?P<mem>[\d.]+) GiB memory and (?P<v>[\d.]+) seconds"),
+    "engine_init_s": re.compile(
+        STAGE + r".*init engine \(profile, create kv cache, warmup model\) took (?P<v>[\d.]+) s"
+    ),
+    # Only present when the stage does not run with enforce_eager.
+    "graph_capture_s": re.compile(
+        STAGE + r".*Graph capturing finished in (?P<v>[\d.]+) secs, took (?P<mem>[\d.]+) GiB"
+    ),
+    "compile_s": re.compile(STAGE + r".*torch\.compile takes (?P<v>[\d.]+) s in total"),
+    "ckpt_fs": re.compile(
+        STAGE
+        + r".*Filesystem type for checkpoints: (?P<fs>\w+)\. Checkpoint size: (?P<size>[\d.]+) GiB\."
+        + r" Available RAM: (?P<ram>[\d.]+) GiB"
+    ),
+}
+ENGINE_READY = re.compile(r"AsyncOmniEngine initialized in (?P<v>[\d.]+) seconds")
+PREFETCH_SKIP = re.compile(STAGE + r".*exceeds 90% of available RAM")
+COMPILE_UNSUPPORTED = re.compile(STAGE + r".*`torch\.compile` is turned on, but the model .* does not support it")
+
+# Startup timeline: (milestone key, regex). Timestamps in vllm logs are "MM-DD HH:MM:SS" (1 s resolution).
+TS = re.compile(r"(?:INFO|WARNING|ERROR) (?P<ts>\d\d-\d\d \d\d:\d\d:\d\d)")
+MILESTONES = [
+    ("omni_init_start", re.compile(r"\[Omni\] Initializing with model")),
+    ("stage{stage}_launch", re.compile(r"Stage-(?P<stage>\d+) set runtime devices")),
+    ("stage{stage}_proc_up", re.compile(STAGE + r".*world_size=\d+ rank=\d+")),
+    ("stage{stage}_load_start", re.compile(STAGE + r".*Starting to load model")),
+    ("stage{stage}_load_done", re.compile(STAGE + r".*Loading weights took")),
+    ("stage{stage}_init_done", re.compile(STAGE + r".*init engine \(profile, create kv cache, warmup model\) took")),
+    ("stage{stage}_ready", re.compile(r"\[StageRuntime\] Stage (?P<stage>\d+) initialized")),
+    ("engine_ready", re.compile(r"AsyncOmniEngine initialized in")),
+]
+
+
+def _ts_seconds(ts: str) -> int:
+    # Use a leap year because engine timestamps do not include the year.
+    value = datetime.strptime("2000-" + ts, "%Y-%m-%d %H:%M:%S")
+    return int((value - datetime(2000, 1, 1)).total_seconds())
+
+
+def timeline(lines: list[str]) -> dict:
+    """Return milestone timestamps (s, relative to omni_init_start) and derived per-stage phase durations."""
+    marks: dict[str, int] = {}
+    for line in lines:
+        tsm = TS.search(line)
+        if not tsm:
+            continue
+        for key, pat in MILESTONES:
+            m = pat.search(line)
+            if m:
+                k = key.format(stage=m.group("stage")) if "{stage}" in key else key
+                marks.setdefault(k, _ts_seconds(tsm.group("ts")))
+                break
+    if "omni_init_start" not in marks:
+        return {}
+    t0 = marks["omni_init_start"]
+    rel = {k: (v - t0) % (366 * 86400) for k, v in marks.items()}
+    phases: dict[str, Any] = {}
+    names = ("launch", "proc_up", "load_start", "load_done", "init_done", "ready")
+    for s in sorted({k.split("_")[0] for k in rel if k.startswith("stage")}):
+        found = {name: rel.get(f"{s}_{name}") for name in names}
+        if any(v is None for v in found.values()):
+            continue
+        ts = {name: int(v) for name, v in found.items() if v is not None}
+        phases[s] = {
+            "spawn_import_config_s": ts["proc_up"] - ts["launch"],
+            "device_dist_init_s": ts["load_start"] - ts["proc_up"],
+            "model_and_weights_load_s": ts["load_done"] - ts["load_start"],
+            "profile_kv_warmup_s": ts["init_done"] - ts["load_done"],
+            "ready_handoff_s": ts["ready"] - ts["init_done"],
+            "total_s": ts["ready"] - ts["launch"],
+        }
+    if "engine_ready" in rel:
+        last_ready = max((v for k, v in rel.items() if k.endswith("_ready") and k != "engine_ready"), default=0)
+        phases["post_stages_wiring_s"] = rel["engine_ready"] - last_ready
+        phases["engine_ready_s"] = rel["engine_ready"]
+    return {"marks_rel_s": rel, "phases": phases}
+
+
+def parse(path: str) -> dict:
+    stages: dict[str, dict] = {}
+    rec: dict[str, Any] = {"log": path, "stages": stages}
+    with open(path, errors="replace") as f:
+        lines = f.readlines()
+    rec["timeline"] = timeline(lines)
+    for line in lines:
+        if line.startswith("BENCH_JSON "):
+            rec["bench"] = json.loads(line[len("BENCH_JSON ") :])
+            continue
+        m = ENGINE_READY.search(line)
+        if m:
+            rec["engine_ready_s_reported"] = float(m.group("v"))
+            continue
+        m = PREFETCH_SKIP.search(line)
+        if m:
+            stages.setdefault(m.group("stage"), {})["page_cache_prefetch_skipped"] = True
+            continue
+        m = COMPILE_UNSUPPORTED.search(line)
+        if m:
+            stages.setdefault(m.group("stage"), {})["torch_compile_unsupported"] = True
+            continue
+        for key, pat in PATTERNS.items():
+            m = pat.search(line)
+            if not m:
+                continue
+            st = stages.setdefault(m.group("stage"), {})
+            if key == "ckpt_fs":
+                st["checkpoint_fs"] = m.group("fs")
+                st["checkpoint_size_gib"] = float(m.group("size"))
+                st["available_ram_gib"] = float(m.group("ram"))
+            elif key == "graph_capture_s":
+                st["graph_capture_s"] = float(m.group("v"))
+                st["graph_capture_mem_gib"] = float(m.group("mem"))
+            elif key == "model_load_s":
+                st["model_load_s"] = float(m.group("v"))
+                st["model_load_mem_gib"] = float(m.group("mem"))
+                # "Model loading took" covers model construction + weight loading; "Loading weights took" only
+                # the weights. The difference also includes loader setup/teardown overhead.
+                if "weights_load_s" in st:
+                    st["model_setup_estimate_s"] = round(st["model_load_s"] - st["weights_load_s"], 2)
+            else:
+                st[key] = float(m.group("v"))
+            break
+    rec["status"] = "complete" if "bench" in rec else "incomplete"
+    return rec
+
+
+def _table(columns: dict[str, str], records: list[dict]) -> str:
+    def row(values):
+        return "| " + " | ".join("" if v is None else str(v) for v in values) + " |"
+
+    lines = [row(columns.values()), row(["---"] * len(columns))]
+    lines.extend(row(record.get(key) for key in columns) for record in records)
+    return "\n".join(lines)
+
+
+def to_markdown(recs: list[dict]) -> str:
+    summary, stages, phases, warnings = [], [], [], []
+    for r in recs:
+        b = r.get("bench", {})
+        label = b.get("label", r["log"])
+        if r.get("status") == "incomplete":
+            warnings.append(f"Incomplete log: {r['log']} (no BENCH_JSON).")
+        summary.append({"label": label, **b.get("summary_s", {})})
+        for sid, st in sorted(r["stages"].items()):
+            stages.append({"label": label, "stage": sid, **st})
+            if st.get("torch_compile_unsupported"):
+                stages[-1]["compile_s"] = "unsupported"
+        ph = r.get("timeline", {}).get("phases", {})
+        for sid, values in sorted(ph.items()):
+            if sid.startswith("stage"):
+                phases.append({"label": label, "stage": sid, **values})
+        if "post_stages_wiring_s" in ph:
+            phases.append({"label": label, "stage": "after last stage", "total_s": ph["post_stages_wiring_s"]})
+    tables = [
+        (
+            {
+                "label": "label",
+                "imports": "imports (s)",
+                "engine_init": "engine ready (s)",
+                "first_request": "first request (s)",
+                "steady_request_avg": "subsequent avg (s)",
+                "time_to_first_image_from_process_start": "process→first image (s)",
+                "host_rss_peak_gib": "sampled host RSS (GiB)",
+                "gpu_used_peak_gib": "sampled GPU used (GiB)",
+            },
+            summary,
+        ),
+        (
+            {
+                "label": "label",
+                "stage": "stage",
+                "model_setup_estimate_s": "model setup estimate (s)",
+                "weights_load_s": "weights load (s)",
+                "compile_s": "compile (s)",
+                "graph_capture_s": "graph capture (s)",
+                "engine_init_s": "profile/KV/warmup reported (s)",
+            },
+            stages,
+        ),
+        (
+            {
+                "label": "label",
+                "stage": "stage",
+                "spawn_import_config_s": "spawn/import (s)",
+                "device_dist_init_s": "device init (s)",
+                "model_and_weights_load_s": "model+weights load (s)",
+                "profile_kv_warmup_s": "profile/KV/warmup interval (s)",
+                "total_s": "total (s)",
+            },
+            phases,
+        ),
+    ]
+    return "\n\n".join(warnings + [_table(columns, rows) for columns, rows in tables])
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("logs", nargs="+")
+    ap.add_argument("--markdown", action="store_true")
+    ap.add_argument("--json-out", default=None)
+    ap.add_argument("--require-complete", action="store_true", help="fail if any log has no BENCH_JSON")
+    args = ap.parse_args()
+    recs = [parse(p) for p in args.logs]
+    if args.require_complete and any(r["status"] != "complete" for r in recs):
+        ap.error("incomplete benchmark log: missing BENCH_JSON")
+    if args.json_out:
+        with open(args.json_out, "w") as f:
+            json.dump(recs, f, indent=2)
+    if args.markdown:
+        print(to_markdown(recs))
+    else:
+        json.dump(recs, sys.stdout, indent=2)
+        print()
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,127 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Raw safetensors -> GPU micro-benchmark, without vLLM's loader.
+
+Compares CPU and GPU dtype conversion after materializing each mmap-backed tensor.
+Materialization includes page faults, CPU allocation and copying; it is not a disk
+bandwidth measurement. CUDA setup is excluded; transfer includes GPU allocation.
+
+  --mode cpu_cast         materialize -> cast to bf16 on the CPU -> copy to GPU
+  --mode gpu_cast         materialize -> copy source dtype to GPU -> cast on the GPU
+
+  python benchmarks/mammoth_moda2/raw_load_bench.py --model /path/MammothModa2-Preview --shards 6,7,8
+  OMP_NUM_THREADS=4 python benchmarks/mammoth_moda2/raw_load_bench.py --model /path --shards 6,7,8 --mode cpu_cast
+"""
+
+import argparse
+import json
+import os
+import re
+import time
+
+import torch
+from safetensors import safe_open
+
+SHARD_NO = re.compile(r"-(\d+)-of-\d+\.safetensors$")
+
+
+def shard_files(model: str, shards: str) -> list[str]:
+    """Resolve ``--shards`` to files, refusing empty or partial matches so a typo cannot pass as a fast load."""
+    files = sorted(f for f in os.listdir(model) if f.endswith(".safetensors"))
+    if not files:
+        raise SystemExit(f"no .safetensors files under {model}")
+    if shards != "all":
+        want = {int(x) for x in shards.split(",")}
+        found = {int(m.group(1)): f for f in files if (m := SHARD_NO.search(f))}
+        missing = sorted(want - found.keys())
+        if missing:
+            raise SystemExit(f"shard(s) {missing} not found under {model}; available: {sorted(found)}")
+        files = [found[n] for n in sorted(want)]
+    return [os.path.join(model, f) for f in files]
+
+
+def sync() -> None:
+    if torch.cuda.is_available():
+        torch.accelerator.synchronize()
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--model", required=True)
+    ap.add_argument("--shards", default="all", help="comma list of 1-based shard numbers, or 'all'")
+    ap.add_argument("--mode", choices=("cpu_cast", "gpu_cast"), default="cpu_cast")
+    ap.add_argument("--label", default="raw")
+    ap.add_argument("--no-cuda", action="store_true", help="read + CPU cast only (mode must be cpu_cast)")
+    a = ap.parse_args()
+    if a.no_cuda and a.mode != "cpu_cast":
+        ap.error("--no-cuda only makes sense with --mode cpu_cast")
+
+    paths = shard_files(a.model, a.shards)
+    total_bytes = sum(os.path.getsize(p) for p in paths)
+
+    if not a.no_cuda:
+        # Initialize the context and exercise both operations outside the timer.
+        warmup = torch.zeros(1).to("cuda")
+        warmup = warmup.to(torch.bfloat16)
+        sync()
+        del warmup
+
+    t0 = time.perf_counter()
+    n_tensors, cpu_bytes = 0, 0
+    dtypes: dict[str, int] = {}
+    t_materialize = t_cast = t_h2d = 0.0
+    for p in paths:
+        with safe_open(p, framework="pt", device="cpu") as f:
+            for k in f.keys():
+                r0 = time.perf_counter()
+                t = f.get_tensor(k).clone()  # touch every page and own the CPU storage
+                t_materialize += time.perf_counter() - r0
+                dtypes[str(t.dtype)] = dtypes.get(str(t.dtype), 0) + 1
+                cpu_bytes += t.numel() * t.element_size()
+                needs_cast = t.is_floating_point() and t.dtype != torch.bfloat16
+                if a.mode == "cpu_cast":
+                    c0 = time.perf_counter()
+                    if needs_cast:
+                        t = t.to(torch.bfloat16)
+                    t_cast += time.perf_counter() - c0
+                    if not a.no_cuda:
+                        h0 = time.perf_counter()
+                        g = t.to("cuda")
+                        sync()
+                        t_h2d += time.perf_counter() - h0
+                        del g
+                else:
+                    h0 = time.perf_counter()
+                    g = t.to("cuda")
+                    sync()
+                    t_h2d += time.perf_counter() - h0
+                    c0 = time.perf_counter()
+                    if needs_cast:
+                        g = g.to(torch.bfloat16)
+                    sync()
+                    t_cast += time.perf_counter() - c0
+                    del g
+                n_tensors += 1
+                del t
+    total = time.perf_counter() - t0
+    res = {
+        "label": a.label,
+        "mode": a.mode,
+        "files": [os.path.basename(p) for p in paths],
+        "file_bytes_gib": round(total_bytes / 2**30, 2),
+        "tensor_bytes_gib": round(cpu_bytes / 2**30, 2),
+        "n_tensors": n_tensors,
+        "dtypes": dtypes,
+        "torch_threads": torch.get_num_threads(),
+        "cpu_affinity": len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else None,
+        "materialize_s": round(t_materialize, 3),
+        "cast_s": round(t_cast, 3),
+        "h2d_s": round(t_h2d, 3),
+        "total_s": round(total, 3),
+        "cuda_setup_included": False,
+    }
+    print("RAWLOAD_JSON " + json.dumps(res))
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,39 @@
+{
+  "date": "2026-09-10",
+  "runtime_commit": "34aa1d2afa59547945fc59f88ab6f4f25f52a777",
+  "environment": {"torch": "2.13.0+cu129", "cuda": "12.9", "python": "3.12.14", "vllm": "0.28.0", "vllm_omni": "0.28.1.dev81+g34aa1d2af.d20260908", "gpu": "NVIDIA A800-SXM4-80GB, 580.126.09, 81920 MiB"},
+  "method": "One excluded warm-up/canary; warm page cache; three rounds of the 2x2 eager configurations with rotated order; fresh process for every run; memory polling disabled.",
+  "request": {"height": 512, "width": 512, "seed": 42, "steps": 8, "guidance": 4.0, "prompt": "A stylish woman riding a motorcycle in NYC, movie poster style"},
+  "gpu_memory_utilization": [0.5, 0.3],
+  "cpu_quota": 16,
+  "startup_columns": ["configuration", "round", "engine_init_s", "imports_s", "first_request_s", "time_to_first_image_s", "image_md5"],
+  "startup_runs": [
+    ["default-serial", 1, 60.022, 10.158, 20.827, 91.089, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["default-serial", 2, 60.015, 10.082, 20.766, 90.909, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["default-serial", 3, 60.34, 9.911, 21.089, 91.381, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["omp-serial", 1, 55.45, 9.965, 20.711, 86.169, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["omp-serial", 2, 57.01, 9.667, 20.697, 87.418, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["omp-serial", 3, 57.044, 9.707, 20.415, 87.211, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["default-parallel", 1, 40.258, 10.112, 20.392, 70.829, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["default-parallel", 2, 42.177, 10.135, 20.692, 73.093, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["default-parallel", 3, 40.848, 9.997, 20.995, 71.887, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["omp-parallel", 1, 36.88, 9.666, 20.319, 66.932, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["omp-parallel", 2, 36.534, 9.6, 20.314, 66.517, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["omp-parallel", 3, 36.689, 9.582, 20.467, 66.802, "60c1fe4d464fba8c6ffd1b752fbbd40c"]
+  ],
+  "raw_loading_columns": ["configuration", "round", "torch_threads", "n_tensors", "materialize_s", "cast_s", "h2d_s", "total_s"],
+  "raw_loading_runs": [
+    ["cpu64", 1, 64, 620, 3.174, 2.309, 1.217, 7.796],
+    ["cpu64", 2, 64, 620, 2.957, 2.443, 1.433, 7.846],
+    ["cpu64", 3, 64, 620, 2.916, 2.613, 1.067, 7.942],
+    ["cpu16", 1, 16, 620, 0.857, 1.137, 0.713, 3.643],
+    ["cpu16", 2, 16, 620, 0.871, 1.174, 0.746, 3.608],
+    ["cpu16", 3, 16, 620, 0.814, 1.142, 0.724, 3.487],
+    ["gpu16", 1, 16, 620, 0.938, 0.028, 1.41, 3.74],
+    ["gpu16", 2, 16, 620, 0.926, 0.029, 1.467, 3.83],
+    ["gpu16", 3, 16, 620, 0.87, 0.029, 1.395, 3.661]
+  ],
+  "raw_loading_workload": {"shards": [6, 7, 8], "source_dtype": "float32", "tensor_gib": 13.3, "warmup": "CUDA context, copy, cast and synchronize before timer", "order_per_round": ["cpu64", "cpu16", "gpu16"]},
+  "executed_script_sha256": {"parse_startup_log.py": "aa260de2cfcaec2fc4a678b245bb7c16fa464543aaec72b947fd4f769d547011", "raw_load_bench.py": "1f70f6b3c205402b2c2c7c52b224e1be3c04a12d12d11044a86f1b9b6c7a1d0f", "bench_startup.py": "04b1e8b9398b4d9885fbb6fe341e8a9ee98792d0be200c15c57e1fc5183c6ff0", "__init__.py": "b1d617f45f34dbe6f5f58d792d90c84af8d8d0a7e7def77c48170ca93495ca0d"},
+  "notes": ["Standard deviations use ddof=1.", "Only first-request ONGs are compared.", "Raw materialization includes CPU allocation/copy; H2D includes GPU allocation; not engine-loader timings."]
+}
@@ -1,4 +1,7 @@
 #!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 """Universal TTS benchmark CLI for vllm-omni.

 Runs ``vllm bench serve --omni`` with model-aware defaults loaded from
@@ -87,6 +90,7 @@ def build_bench_args(
    served_model_name: str | None = None,
    num_warmups: int = 2,
    request_seed: int | None = None,
+    duration_seconds: float | None = None,
 ) -> list[str]:
    """Build the ``vllm bench serve --omni`` command for one (task, concurrency) run."""
    dataset_name = _TASK_TO_DATASET[task]
@@ -95,7 +99,10 @@ def build_bench_args(
    task_extra_body: dict[str, Any] = dict((model_cfg.get("task_extra_body") or {}).get(task) or {})
    if request_seed is not None:
        task_extra_body["seed"] = request_seed
-
+    if duration_seconds is not None:
+        if not math.isfinite(duration_seconds) or duration_seconds <= 0:
+            raise ValueError("duration_seconds must be finite and positive")
+        task_extra_body["duration_seconds"] = duration_seconds
    # Resolve dataset path
    if dataset_path:
        resolved_dataset_path = dataset_path
@@ -194,7 +201,7 @@ def print_summary_table(results: list[dict[str, Any]]) -> None:
    if not results:
        return
    header = (
-        f"{'Task':<16} {'Concurrency':>11} {'RTF mean':>10} "
+        f"{'Task':<16} {'Task Type':<12} {'Concurrency':>11} {'RTF mean':>10} "
        f"{'TTFP (ms)':>10} {'Throughput':>12} {'WER':>7} {'SIM':>7} {'UTMOS':>7}"
    )
    print(f"\n{'=' * len(header)}")
@@ -204,6 +211,7 @@ def print_summary_table(results: list[dict[str, Any]]) -> None:
    print("-" * len(header))
    for r in results:
        task = r.get("_task", "?")
+        task_type = r.get("_task_type", "-")
        conc = r.get("_concurrency", "?")
        rtf = r.get("mean_audio_rtf", float("nan"))
        ttfp = r.get("mean_audio_ttfp_ms", float("nan"))
@@ -216,7 +224,7 @@ def print_summary_table(results: list[dict[str, Any]]) -> None:
            return f"{v:.{digits}f}" if not math.isnan(v) else "  n/a"

        print(
-            f"{task:<16} {str(conc):>11} {fmt(rtf):>10} {fmt(ttfp, 0):>10} "
+            f"{task:<16} {task_type:<12} {str(conc):>11} {fmt(rtf):>10} {fmt(ttfp, 0):>10} "
            f"{fmt(throughput):>12} {fmt(wer):>7} {fmt(sim):>7} {fmt(utmos):>7}"
        )
    print("=" * len(header))
@@ -249,6 +257,12 @@ def main() -> None:
        help="Number of prompts per run. If one value, applied to all concurrency levels.",
    )
    parser.add_argument("--num-warmups", type=int, default=2, help="Warmup requests before each measured run")
+    parser.add_argument(
+        "--duration-seconds",
+        type=float,
+        default=None,
+        help="Target audio duration per request (AuK defaults to 5 seconds). Independent of --output-len.",
+    )
    parser.add_argument(
        "--request-seed",
        type=int,
@@ -277,6 +291,8 @@ def main() -> None:

    if args.num_warmups < 0:
        parser.error("--num-warmups cannot be negative")
+    if args.duration_seconds is not None and (not math.isfinite(args.duration_seconds) or args.duration_seconds <= 0):
+        parser.error("--duration-seconds must be finite and positive")

    model_configs = load_model_configs(Path(args.model_configs))
    if args.model not in model_configs:
@@ -333,11 +349,16 @@ def main() -> None:
                served_model_name=args.served_model_name,
                num_warmups=args.num_warmups,
                request_seed=args.request_seed,
+                duration_seconds=args.duration_seconds,
            )
            result = run_one_benchmark(cmd)
            if result is not None:
                result["_task"] = task
                result["_concurrency"] = concurrency
+                task_body = model_cfg.get("task_extra_body", {}).get(task, {})
+                task_type = task_body.get("task_type")
+                if task_type is not None:
+                    result["_task_type"] = task_type
                all_results.append(result)
                # Persist the metadata so plot_results.py can pick it up.
                if args.output_dir and result_filename:
@@ -7,6 +7,22 @@
 # is tracked here.

 models:
+  tencent/AuK: &auk
+    supported_tasks: [default_voice, voice_clone]
+    backend: openai-audio-speech
+    endpoint: /v1/audio/speech
+    trust_remote_code: true
+    task_extra_body:
+      default_voice:
+        duration_seconds: 5.0
+        voice: default
+        task_type: CustomVoice
+      voice_clone:
+        duration_seconds: 5.0
+        voice: default
+        task_type: Base
+  tencent/AuK-Flash: *auk
+
  BreezeBlue/Breeze-TTS-2:
    supported_tasks: [default_voice, voice_design, voice_clone]
    backend: openai-audio-speech
@@ -15,7 +15,7 @@ nav:
        - Image Edit: serving/image_edit_api.md
        - Videos: serving/videos_api.md
      - Streaming and Realtime:
-        - Realtime Audio: serving/realtime_api.md
+        - Full Duplex OpenAI: serving/realtime_openai_duplex_api.md
        - Full Duplex: serving/full_duplex_api.md
        - Realtime Duplex Reference: serving/realtime_duplex_api.md
        - Streaming Video Input: serving/video_stream_api.md
@@ -47,6 +47,7 @@ nav:
    - configuration/*
  - Models:
    - models/supported_models.md
+    - SeedVR2: models/seedvr2.md
  - Features:
    - Overview: features/README.md
    - Runtime and Stage Execution:
@@ -151,6 +152,7 @@ nav:
          - Hybrid Sharded Data Parallel (HSDP): design/feature/hsdp.md
          - Pipeline Parallel: design/feature/pipeline_parallel.md
          - Sequence Parallel: design/feature/sequence_parallel.md
+          - Window Sequence Parallel: design/feature/window_sequence_parallel.md
          - Tensor Parallel: design/feature/tensor_parallel.md
          - VAE Patch Parallelism: design/feature/vae_parallel.md
        - Attention Optimization:
@@ -56,9 +56,9 @@ vLLM-Omni is flexible and easy to use with:
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:

 - Omni-modality models (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
- TTS models (e.g. Qwen3-TTS, IndexTTS 2.5, CosyVoice3)
- Diffusion models — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
- Robot-policy and action models (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
+- TTS models (e.g. Qwen3-TTS, Tencent AuK, Breeze-TTS-2, CosyVoice3)
+- Diffusion models — image, video, and audio generation (e.g. MiniMax H3, LingBot World, MAGI-2, LTX-2.5, Wan2.2)
+- Robot-policy and action models (e.g. π0.5, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)

 For more information, checkout the following:

@@ -2,7 +2,7 @@

 This section lists the most common options for running vLLM-Omni.

-For options within a vLLM Engine, please refer to the [vLLM 0.29 configuration guide](https://docs.vllm.ai/en/v0.29.0/configuration/index.html).
+For options within a vLLM Engine, please refer to the [vLLM 0.30 configuration guide](https://docs.vllm.ai/en/v0.30.0/configuration/index.html).

 Each model defines fixed topology in a registered `PipelineConfig` and runtime
 overrides in a deploy YAML.
@@ -64,6 +64,7 @@ The lifecycle labels used below are:
 | Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
 | --- | --- | --- | --- | --- |
 | `DIFFUSION_ATTENTION_BACKEND` | Backend name or `auto`; default `auto` (platform selection) | Diffusion stages; read when `OmniDiffusionConfig` is constructed | `diffusion_attention_config.default` wins. An unknown backend fails during backend resolution. | Stable fallback |
+| `DIFFUSION_ATTENTION_QUANT` | `<dtype_qk>:<dtype_vo>[:<q_block_size>:<k_block_size>]`; default unset | Diffusion stages; read when `OmniDiffusionConfig` is constructed | Used only when `DIFFUSION_ATTENTION_BACKEND` sets a non-`auto` backend and no explicit attention default exists. Quantization support varies by backend. Malformed fields raise `ValueError`. | Stable fallback |
 | `DIFFUSION_CACHE_BACKEND` | `none`, `cache_dit`, `tea_cache`, `mag_cache`, `step_cache`, `stepcache`, or `step_cache_dit`; default `none` | Diffusion runner startup | Explicit `cache_backend` in config wins. Otherwise this name wins over the deprecated alias. Unsupported values raise `ValueError` during runner setup. | Stable fallback |
 | `DIFFUSION_CACHE_ADAPTER` | Same values as `DIFFUSION_CACHE_BACKEND`; default `none` | Diffusion runner startup | Used only when neither explicit `cache_backend` nor `DIFFUSION_CACHE_BACKEND` is set. Unsupported values raise `ValueError`. | Deprecated; use `DIFFUSION_CACHE_BACKEND` |
 | `OMNI_DIFFUSION_PROMPT_EMBED_CACHE` | Boolean: `1`, `true`, `yes`, `on`, `0`, `false`, `no`, or `off`; default disabled | Each diffusion runner; resolved during model setup | A recognized environment value overrides the explicit enable setting. An unrecognized value is ignored. | Experimental |
@@ -77,6 +78,13 @@ Backend names for `DIFFUSION_ATTENTION_BACKEND` are the members of
 `SAGE_ATTN`, `FLASHINFER_ATTN`, and `TRTLLM_ATTN`. Platform support still
 depends on the installed kernels and model path.

+For `DIFFUSION_ATTENTION_QUANT`, the dtype fields accept `float16`,
+`bfloat16`, `int8`, or `fp8_e4m3`; the selected backend may support only a
+subset. For example, `DIFFUSION_ATTENTION_BACKEND=TRTLLM_ATTN` with
+`DIFFUSION_ATTENTION_QUANT=int8:bfloat16:1:16` selects INT8 Q/K SageAttention.
+Explicit `diffusion_attention_config.default` takes precedence over both
+environment variables.
+
 ### Serving and runtime

 | Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
@@ -130,6 +138,22 @@ settings.
 Omni-owned variable, even though vLLM-Omni supplies a persistent default when
 it is unset.

+### Torch compilation
+
+| Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
+| --- | --- | --- | --- | --- |
+| `VLLM_OMNI_TORCH_DYNAMO_RECOMPILE_LIMIT` | Positive integer; unset preserves the current Torch setting | Process default for Torch Dynamo; read when `vllm_omni` is imported | Sets `torch._dynamo.config.recompile_limit` to the requested value. A non-positive or non-integer value raises `ValueError`. Later backend-specific overrides still take precedence. | Diagnostic |
+
+Set this variable before launching Omni, for example:
+
+```bash
+export VLLM_OMNI_TORCH_DYNAMO_RECOMPILE_LIMIT=64
+```
+
+The setting applies across model families and platforms. vLLM compilation
+contexts and backends can subsequently apply their own limits; this variable
+does not replace those backend-specific policies.
+
 ## Per-stage environment

 Deploy configurations can set arbitrary environment keys for one stage:
@@ -162,7 +186,7 @@ their keys only.
 ## Inherited vLLM variables

 vLLM-Omni also reads variables through its aligned vLLM dependency. Refer to
-the [vLLM 0.29 environment-variable reference](https://docs.vllm.ai/en/v0.29.0/configuration/env_vars.html)
+the [vLLM 0.30 environment-variable reference](https://docs.vllm.ai/en/v0.30.0/configuration/env_vars.html)
 for their definitions. This includes vLLM launch, cache, logging, plugin, ROCm,
 XPU, ModelScope, and FlashInfer workspace settings.

@@ -190,7 +214,7 @@ collection, examples, bug reports, or logs.

 ## Model-specific variables

-The audit found 58 variables read by a single model or pipeline family. They are
+The inventory includes 93 variables read by a single model or pipeline family. They are
 not listed as public usage options here because doing so would turn implementation
 escape hatches into an accidental compatibility contract.

@@ -199,11 +223,11 @@ Every audited model-specific name has a migration disposition in the

 | Disposition | Count | Required outcome |
 | --- | ---: | --- |
-| Promote | 35 | Move a stable setting into typed stage or model configuration. |
-| Request scope | 6 | Move request-varying behavior into a declared request-option schema. |
+| Promote | 64 | Move a stable setting into typed stage or model configuration. |
+| Request scope | 5 | Move request-varying behavior into a declared request-option schema. |
 | External | 0 | Retain only when a supported third-party library owns the contract. |
-| Internalize | 11 | Keep a debug or diagnostic switch out of public documentation and configuration. |
-| Deprecate/remove | 6 | Remove a compatibility escape hatch that has no continuing contract. |
+| Internalize | 17 | Keep a debug or diagnostic switch out of public documentation and configuration. |
+| Deprecate/remove | 7 | Remove a compatibility escape hatch that has no continuing contract. |

 The disposition is a migration target, not a statement that the existing
 environment switch is stable. Promote or request-scope work should land in
@@ -0,0 +1,22 @@
+# Pre-sharded HSDP loading
+
+Use `--use-hsdp --hsdp-shard-size 8 --hsdp-weight-load-strategy pre_sharded`
+to shard transformer parameters before loading checkpoint values. Set the shard
+size to the available GPU count (or configure replication explicitly).
+The default strategy, `full`, retains the existing load-then-shard behavior.
+The same `hsdp_weight_load_strategy` option is available through Python and stage
+engine configuration.
+
+The pre-sharded strategy reads rank-local slices from ordinary Hugging Face
+safetensors checkpoints into FSDP-owned storage. No converted checkpoint is
+required. Cosmos 3 and Cosmos 3 Edge transformer blocks release their initialized
+parameter storage to meta as they are constructed; nonpersistent buffers retain
+their values.
+Encoders and VAEs follow their existing loading paths.
+
+This strategy requires the default diffusion pipeline loader, complete dedicated
+transformer checkpoint sources, and supported checkpoint key mappings. Tensor
+layouts must match the runtime layout; tensor transforms are unsupported.
+Checkpoint tensor dtypes must match runtime tensor dtypes at preflight; dtype conversion is unsupported.
+Quantization and LoRA are unsupported. Every worker needs checkpoint access.
+`num_weight_load_threads` controls the safetensors reader's threads per rank.
@@ -80,6 +80,19 @@ Note: for the diffusion path, an omitted `distributed_executor_backend` selects
 segments) and `mp` when `num_gpus > 1`. Set `mp` explicitly to keep a worker
 subprocess on one GPU. `ray` / `external_launcher` are not fully supported yet.

+### Stage-level runner selection
+
+`model_runner: v1` or `v2` at the deploy level sets the default runner.
+A `model_runner` on an individual stage overrides that default, allowing
+one stage to migrate or roll back independently. This does not change
+`PipelineConfig` topology, input adapters, or the connector payload contract.
+An omitted stage override preserves the deploy-level selection.
+
+MRv2 native downstream receivers currently support turn-based requests only;
+streaming sessions and resumable input require the V1 prompt-replacement path.
+Selecting a runner does not add the session capabilities it lacks. Platform
+fallbacks and stage overrides are validated after configuration resolution.
+
 ### Stage fields

 Each entry under `stages:` accepts any `StageDeployConfig` field directly (no nested `engine_args:`). Only fields whose value legitimately varies across stages live here; pipeline-wide settings (trust_remote_code, distributed_executor_backend, dtype, quantization, prefix/chunked prefill, DP/PP sizes) are declared at the top level and applied to every stage. Unknown keys fall through to `engine_extras:` and are forwarded to the engine. Frequently used fields are listed below; the source-of-truth schema is `StageDeployConfig` in `vllm_omni/config/stage_config.py`.
@@ -306,7 +319,9 @@ ROCm and MUSA. Select one of these deployment profiles:
 | `qwen3_tts_mrv2.yaml` | V2 | B1 | Explicit MRV2 profile (same runner selection as the default) |
 | `qwen3_tts_high_concurrency_mrv2.yaml` | V2 | B1, B2 | Opt-in throughput tuning |
 | `qwen3_tts_high_concurrency_mrv2_b4.yaml` | V2 | B1, B2, B3, B4 | Experimental throughput / buffered playback |
+| `qwen3_tts_high_concurrency_mrv2_single_gpu.yaml` | V2 | B1–B8 | Experimental two-stage deployment on one GPU, with MPS |
 | `qwen3_tts_high_concurrency.yaml` | V1 | Existing defaults | V1 high-concurrency control |
+| `qwen3_tts_fused_single_gpu.yaml` | V2 | N/A (in-Talker decoder) | Opt-in single-stage CUDA pipeline on one GPU |

 ```bash
 # MRV2 is the default; pass a copy with `model_runner: v1` to force V1.
@@ -337,6 +352,102 @@ supply `mtp_sampling_params` and `get_mtp_seed(sampling_params)` for model-local
 (with `mtp_sample_steps` and `mtp_sample_vocab_size`) as needed. Qwen3-TTS retains
 its existing `talker_mtp` entry point for V1.

+### Selecting one or two Qwen3-TTS stages
+
+Choose the deployment profile explicitly with `--deploy-config`. The default
+`qwen3_tts` pipeline retains two stages: the Talker publishes codec chunks to
+Code2Wav, which produces audio. Its stage devices can be placed on the same
+GPU or on separate GPUs. `qwen3_tts_fused` is a separate, opt-in pipeline for
+one CUDA GPU: the Talker owns the stateful codec decoder and emits PCM directly,
+without a Code2Wav engine or stage connector.
+
+```bash
+# Two stages on one GPU; the two-stage pipeline remains independently selectable.
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni \
+  --deploy-config vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2_single_gpu.yaml
+
+# One stage on one GPU; no inter-stage transport or Code2Wav scheduling.
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni \
+  --deploy-config vllm_omni/deploy/qwen3_tts_fused_single_gpu.yaml
+```
+
+The single-stage profile requires CUDA, V2, asynchronous chunks, TP=PP=1,
+an in-process worker and disabled prefix caching. Select the full profile;
+adding `talker_stream_decode: true` to a two-stage Talker is rejected because
+its latent output belongs to the Code2Wav input contract. Single-stage options
+live in `stages[].additional_config`; two-stage transport and first-frame
+options live in `connectors.*.extra`.
+
+The single-stage profile defaults `talker_stream_first_audio` to `false` for
+throughput. To opt into a separate first-frame delivery queue, save this
+overlay beside the bundled deployment files and pass its path to
+`--deploy-config`:
+
+```yaml
+base_config: qwen3_tts_fused_single_gpu.yaml
+stages:
+  - stage_id: 0
+    additional_config:
+      talker_stream_decode: true
+      talker_stream_first_audio: true
+      ref_code_context_frames: 72
+      code_predictor_kv_cache: true
+      code_predictor_fused_sampling: true
+      code_predictor_fused: true
+```
+
+This queue uses PCM from the single-stage decoder. The two-stage
+`talker_first_audio` option below uses its own first-frame decoder and does
+not require single-stage execution. The two options cannot be enabled
+together. Single-stage prefix-cache reset with
+`reset_running_requests=True` returns `False` while requests are active;
+wait for completion or abort them before retrying.
+
+Reference-cloning context is selected independently by each profile. The
+single-stage profile explicitly uses 72 reference frames, matching the
+`qwen3_tts.yaml` context boundary. A shorter
+`additional_config.ref_code_context_frames` (for example 25) can change
+timbre continuity even when WER is similar;
+the throughput measurements for CustomVoice do not qualify Base voice
+cloning quality. Reference layout, grouping and priming are model hooks;
+the shared worker does not select Qwen's context length or codebook layout.
+
+### Qwen3-TTS first-frame delivery and rollback
+
+The standard `qwen3_tts.yaml`, `qwen3_tts_high_concurrency.yaml`,
+`qwen3_tts_mrv2.yaml` and `qwen3_tts_high_concurrency_mrv2.yaml` profiles
+enable the `talker_first_audio` connector option, as does the experimental
+two-stage `qwen3_tts_high_concurrency_mrv2_single_gpu.yaml` profile.
+This changes the default CUDA MRV2 streaming path:
+residual prediction runs eagerly after the Talker sample, the Talker loads
+an additional first-frame decoder with its weights and CUDA graphs, and the
+orchestrator orders its audio before subsequent Code2Wav chunks.
+
+Code2Wav retains full-audio prefix graphs alongside the optional state-only
+graphs. Each request's delivery marker selects the graph and audio trimming;
+enabling the option alone never suppresses audio or disables prefix batching
+for requests that retain regular codec delivery.
+
+The path requires asynchronous chunks, TP/PP 1, an in-process executor and
+disabled prefix caching. Requests with reference codes and unsupported
+runners or platforms retain regular Code2Wav delivery.
+
+To restore regular codec delivery and avoid loading the Talker's additional
+decoder, use a deploy overlay with the relevant base profile:
+
+```yaml
+base_config: qwen3_tts.yaml
+connectors:
+  connector_of_shared_memory:
+    extra:
+      talker_first_audio: false
+```
+
+First-packet latency measures when PCM starts arriving. Time to first audible
+audio (TTFA) also includes any leading silence in the generated audio. An
+earlier first packet therefore does not necessarily improve TTFA; measure
+both for the intended voice and workload.
+
 ### Included performance work

 - Request snapshots have a fast path for immutable scalar leaves, including
@@ -372,14 +483,81 @@ speaker similarity before adopting either batching preset for a production
 workload. Floating-point decoder outputs can differ across batch sizes; this PR
 does not claim bitwise or quality equivalence.

-### Optional MPS deployment
+### Experimental MPS deployment

-NVIDIA MPS is an optional operator setting for colocated CUDA processes, not a
-YAML option or a library default. This PR does not establish a throughput or
-first-packet latency benefit from MPS. Measure the exact deployment with and
-without MPS before enabling it.
+NVIDIA MPS lets colocated CUDA stage processes share GPU execution resources.
+It is disabled by default. The experimental
+`qwen3_tts_high_concurrency_mrv2_single_gpu.yaml` profile sets `cuda_mps: true`,
+alongside cached residual prediction, fused sampling, time-major codec
+convolutions, first-frame delivery, and larger graph batches.

-Use only assigned GPUs and an independent MPS pipe directory. A private MPS
-server does not provide exclusive GPU ownership or MIG isolation. For a
-single-GPU deployment, explicitly place both stages on that GPU; the supplied
-high-concurrency profile places its two stages on different GPUs by default.
+The runtime requires `nvidia-cuda-mps-control` on `PATH` and one explicit CUDA
+GPU per local EngineCore stage, with `parallel_stage_init: false`.
+Use numeric GPU ordinals for stage placement and visibility so initialization
+locks identify the physical GPU before MPS remaps it. It starts a private MPS daemon for each selected
+GPU and stops its own daemon after the stages exit. If
+`CUDA_MPS_PIPE_DIRECTORY` already names an operator-managed daemon, the runtime
+reuses it without stopping it. Diffusion and remote stages are unsupported.
+Set `cuda_mps: false` in a deploy overlay to disable automatic MPS management.
+
+Stage `env.CUDA_MPS_PIPE_DIRECTORY` overrides the parent setting when selecting
+the daemon; an explicit empty string selects a private daemon. Stages sharing
+a GPU reuse the first stage's daemon. Later stages may omit the setting or
+explicitly match that policy; a conflicting explicit setting fails before
+that stage starts. The parent environment is unchanged.
+
+MPS can improve throughput under concurrent load while increasing first-packet
+or first-audible-audio latency, particularly at low request rates. For a
+latency-sensitive workload, compare the same profile with `cuda_mps: false`.
+An inherited operator-managed MPS daemon must also be disabled by its owner
+for that comparison; this switch only controls automatic MPS management.
+
+MPS does not reserve a GPU. Use only assigned GPUs, explicitly place stages on
+the intended GPU, and warm the complete pipeline before measuring performance.
+
+## MOSS-TTS Local 1.5 with Model Runner V2
+
+`OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5` can opt into CUDA MRV2 using
+the shared runtime introduced for Qwen3-TTS:
+
+```bash
+vllm serve OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 --omni \
+  --deploy-config vllm_omni/deploy/moss_tts_local_mrv2.yaml
+```
+
+This profile inherits the batching, codec graph buckets and 1-frame/15-frame
+chunk geometry from `moss_tts_local.yaml`, and selects V2 for both the Local
+Talker and codec stages. The Local depth predictor exposes the MRV2 `mtp`
+capabilities while retaining its V1 `talker_mtp` implementation, sampling
+defaults and explicit request-seed handling. Codec chunks use the native data
+plane; the internal Talker retains its final-only orchestrator output policy.
+
+On CUDA, the profile bounds stage-0 prefill work to 512 tokens per iteration.
+The codec retains its own CUDA Graph capture/replay, including batch buckets
+through 64, but disables Inductor compilation (`compilation_config.mode: 0`)
+of the stateful decoder to avoid lengthy compilation at startup. This is
+independent of `enforce_eager`; the profile keeps `enforce_eager: false`.
+It explicitly selects `cudagraph_mode: FULL`, which works without Inductor;
+the default `FULL_AND_PIECEWISE` would otherwise normalize to `NONE` and clear
+the codec's capture buckets when compilation is disabled.
+
+For sustained C128 serving on a large-memory CUDA GPU, select the
+separate `moss_tts_local_mrv2_high_concurrency.yaml` profile. It uses 128
+stream slots per stage, a 32 GiB Talker KV budget, and the codec's
+`triton_slot` backend with Inductor compilation and codec-owned CUDA graphs.
+The bounded KV budget leaves room for codec state and graphs; it does not
+guarantee that 128 maximum-length prompts fit simultaneously. See the
+[MOSS recipe](gh-file:recipes/OpenMOSS/MOSS-TTS.md#local-15-mrv2-and-slot-attention)
+for activation, backend comparisons, memory requirements and benchmark commands.
+
+Omitting `--deploy-config`, or selecting `moss_tts_local.yaml`, retains V1.
+NPU, XPU, ROCm and MUSA overrides also retain V1. This profile does not enable
+MRV2 for MOSS Delay, Realtime or Nano. Local 1.5 outputs 48 kHz stereo audio;
+set `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=48000` and
+`VLLM_OMNI_BENCH_AUDIO_CHANNELS=2` when benchmarking raw PCM.
+
+Event-driven orchestration remains independently selectable with
+`VLLM_OMNI_EVENT_DRIVEN_ORCH=0` or `1`. Keep the runner and deployment identical
+when comparing these modes. Model-runner selection does not change the
+orchestration default or enable experimental reference encoding, chunk ramps,
+generation-output draining or MPS.
@@ -4,6 +4,9 @@ This document defines the selection and extension contract for diffusion
 attention backends. User-facing backend choices, installation, and tuning are
 in the [attention backend guides](../../user_guide/diffusion/attention_backends.md).

+The [execution contract PoC](attention_execution_contract_poc.md) describes the
+experimental post-selection capability and lifecycle contract from RFC #7226.
+
 ## Scope

 The contract applies to DiT and other diffusion attention layers. It is
@@ -0,0 +1,133 @@
+# Diffusion attention execution contract PoC
+
+This prototype accompanies [RFC #7226](https://github.com/vllm-project/vllm-omni/issues/7226)
+and extends the [attention selection design](attention_backend_selection.md).
+It demonstrates path-specific capabilities with dense BF16 FA4, NPU/ROCm
+routing examples, and tensor-state lifetime with a test-only attention module.
+
+## Execution contract
+
+`ExecutionContext` describes the requested execution path. `ExecutionPathResult`
+reports its identity, support status and reason, and compilation mode.
+`AttentionBackend.resolve_capabilities()` provides conservative pre-construction
+results. After initialization, `Attention.resolve_execution_path()` supplies the
+active parallel, paged-KV, and HSDP context; the backend combines it with the
+selected kernel and normalized metadata.
+
+Resolution runs outside compiled execution. FA4 reports `SUPPORTED` and
+`CUSTOM_OP` for dense, noncausal BF16 without parallel or HSDP boundaries when
+its kernel accepts the head dimensions. Dimension validation delegates to FA4's
+architecture-specific rules through `backends/utils/fa.py`. Kernel rejections
+report `UNSUPPORTED` with an actionable reason. Missing private validators and
+other unmigrated paths report `UNMIGRATED` with advisory `EAGER_ONLY` defaults.
+`requested_support()` checks a fullgraph request; it does not enforce selection
+or change existing execution.
+
+FA4 execution uses an opaque custom op. Its fake output preserves Q's batch,
+sequence, and head count and uses V's head dimension. Output is contiguous even
+when Q is noncontiguous. Metadata normalization is
+shared with dispatch. Producers must update published mask semantics when masks
+change; unpublished masks remain runtime-dependent to avoid synchronization.
+Callers resolve again when execution metadata changes.
+
+## NPU and ROCm worked examples
+
+The same `FLASH_ATTN` backend resolves MindIE paths on NPU and AITER paths on
+ROCm. Both examples report `UNMIGRATED`: their route is inspected, while hardware
+correctness and compilation remain unvalidated. `EAGER_ONLY` is an advisory
+default, not a measured compiler limitation. Existing execution is unchanged.
+
+| Input | NPU / MindIE | ROCm / AITER |
+| --- | --- | --- |
+| Dense | `npu_dense` | `rocm_dense` |
+| Padding mask | `npu_masked`: expand the key mask to `[B, 1, Q, K]` | `rocm_masked_varlen`: unpad, call varlen attention, then restore rows |
+| Packed inputs | Opt-in `[real, pad]` contract; choose varlen or prefix-KV slicing from the environment | Forward complete packed-document boundaries to varlen attention |
+| Incomplete packing | Rebuild a mask when possible; otherwise reject before the kernel | Shared CUDA-like normalization rejects incomplete metadata |
+| Unpublished mask semantics | Route to the masked wrapper | Keep resolution runtime-dependent without reading mask values |
+
+NPU resolution reuses `_resolve_packed_seq_npu`; ROCm reuses the metadata
+normalization used by dispatch. AITER identity comes from initialization rather
+than caller-provided context. Quantized, paged, piecewise, and parallel paths
+remain outside these worked examples.
+
+The new L1 tests use CPU tensors and replace only vendor kernel calls. They
+check path identity, forwarded masks/boundaries, fallback rejection, and
+conservative fullgraph-request handling. They do not test vendor numerics or
+compiler support. Existing L1 CI already collects this test file.
+
+```bash
+python -m pytest tests/diffusion/attention/test_flash_attn.py \
+  -k 'npu_contract or rocm_contract' -m 'core_model and cpu' -q
+```
+
+### Hardware validation required before migration
+
+On an Ascend/MindIE or ROCm/AITER runner, record device, driver, PyTorch, vendor
+library, and compiler versions, then validate the corresponding rows above:
+
+1. Compare eager outputs with FP32 SDPA for dense and padding-mask inputs,
+   including unequal Q/K lengths. Compare valid query rows; do not assume both
+   platforms define padded-query outputs identically.
+2. Compare packed outputs with separate per-document references. On NPU, cover
+   both varlen and laser prefix-slicing modes, plus mask reconstruction and the
+   missing-mask error. A padding fallback does not establish multi-document
+   isolation. On ROCm, cover multiple real documents.
+3. Check output shape, dtype, device, finite values, and unchanged inputs.
+   Start BF16 comparisons at `atol=rtol=1e-2` and have the platform maintainer
+   confirm the tolerance against its reference tests.
+4. Run `torch.compile(fullgraph=True, dynamic=True)` with the platform's supported
+   compiler over multiple sequence lengths. Record graph breaks and recompiles;
+   choose `TRACEABLE`, `CUSTOM_OP`, or verified `EAGER_ONLY` from that evidence.
+   A custom-op path additionally needs schema/fake checks and repeated replay.
+
+Promote only the paths validated on that platform. These hardware checks have
+not been run locally; ROCm/NPU fullgraph support is not claimed.
+
+## State lifetime example
+
+`test_attention_state_lifetime.py` prepares an owned query-scale tensor eagerly
+and passes it explicitly to an opaque attention op. The compiled module retains
+its state after the caller drops its reference and releases ownership when the
+compiled callable is deleted. Equivalent instances reuse a graph with their own
+state values.
+
+This example tests ownership and release, dynamic replay across instances,
+custom-op schema/fake behavior, and Inductor execution. It adds no production
+backend or state registry. Planning keys, caching, fallback-policy declarations,
+opaque handles, CUDA graph capture, and TRTLLM integration are deferred.
+
+## Validation
+
+```bash
+python -m pytest tests/diffusion/attention/test_flash_attn_compile.py \
+  tests/diffusion/attention/test_flash_attn.py \
+  tests/diffusion/attention/test_attention_capabilities.py \
+  tests/diffusion/attention/test_attention_state_lifetime.py -q -rs
+```
+
+The focused suite covers capability decisions and metadata changes, existing
+padding/mask regressions, tensor-state lifetime, and real FA4 fullgraph execution
+against FP32 SDPA and eager FA4. Representative Q/K and V dimensions are
+(32, 32), (64, 64), (80, 48), (192, 128), and (256, 256), with batches 1 and 2
+and sequence lengths up to 1,024. Invalid dimensions are checked against actual
+kernel errors. These samples exercise the contract rather than define support.
+BF16 comparisons use `atol=rtol=1e-2`; `torch.library.opcheck` checks schema and
+fake-output correctness.
+
+Single-graph reuse is asserted in the tensor-state test. The FA4 numerical tests
+retain compiler caches across shapes within each case but do not require one
+graph across different batch sizes, Q/K length equality, or head dimensions.
+
+Tested environment: GB300, PyTorch `2.13.0+cu130`, FA4 `4.0.0b18`, CUTLASS DSL
+`4.6.2` with CUDA 13 libraries, Quack `0.6.4`, and TVM FFI `0.1.11`.
+Real-kernel tests require CUDA and CuTe FA4; their availability is checked at runtime.
+
+Before the rebase, the focused suite passed all 97 tests without skips, including
+11 CPU NPU/ROCm routing tests. After rebasing onto upstream `e3be42e05`, the
+expanded suite reports 99 passed and 11 failed in the local vLLM 0.28 environment.
+All 11 failures are import errors: upstream now requires `compute_layout_strides`
+from vLLM, which this environment lacks. Both noncontiguous FA4 regression cases
+and real-kernel schema/fake checks pass. The full suite must be rerun with the
+vLLM version required by upstream. NPU/ROCm device numerics and compilation remain
+unvalidated. Pre-commit passes with the CI hook skips. Upstream PyTorch/CUTLASS
+warnings remain.
@@ -0,0 +1,186 @@
+# Window-Aligned Sequence Parallelism (SeedVR2)
+
+## Motivation
+
+SeedVR2's diffusion transformer (`NaDiT`) has no global attention: every layer
+partitions the video tokens into 3D windows and attends only inside a window.
+The window size is normalised against a fixed 720p reference area, so raising the
+output resolution grows the *number* of windows, never their size.
+
+Ulysses redistributes the sequence and head dimensions around attention.
+For SeedVR2, window-aligned SP offers a different communication tradeoff: one
+hidden-state exchange per layout boundary versus Q/K/V and output exchanges
+for Ulysses. The actual performance crossover requires measurement.
+Window-aligned SP instead assigns **whole windows** to ranks:
+
+* every rank keeps **all** attention heads, so no head-count divisibility is
+  required,
+* attention inside a layer needs **zero** communication,
+* regular and shifted layouts alternate at every layer, so the released
+  32-layer model has 31 inter-layer transitions, plus one text reduction per
+  layer and a final video gather. A transition can be a local permutation
+  when ownership does not change.
+
+## Window layouts
+
+`nadit.py` ports the reference windowing verbatim (Apache-2.0,
+ByteDance SeedVR2). For a post-patch token grid `(T, H, W)`:
+
+```text
+scale   = sqrt(45 * 80 / (H * W))          # 720p reference area
+wh, ww  = ceil(round(H * scale) / nh), ceil(round(W * scale) / nw)
+wt      = ceil(min(T, 30) / nt)            # BYTEDANCE_MAX_TEMPORAL_WINDOW
+```
+
+The regular method tiles with `slice(it * wt, min((it + 1) * wt, T))`; the
+shifted method moves every cut by half a window and **clips at the boundary**
+(`max(..., 0)` / `min(..., size)`) -- it is not a cyclic `torch.roll`. Both
+methods produce an exact partition of the grid; the shifted layout is genuinely
+different, so an `A -> B -> A -> B` layer schedule needs both directions of
+re-sharding.
+
+Windows are enumerated `for iw ... for ih ... for it ...` (w outermost, t
+innermost) and tokens inside a window are row-major in `(t, h, w)`. The
+canonical token id is `(t * H + h) * W + w`, which is exactly what the reference
+obtains by windowing `torch.arange`.
+
+Consequences that matter (all verified against the reference):
+
+* Window token counts are **ragged** (edge windows are clipped), so a shard must
+  be balanced by token count, not by window count.
+* The number of layout transitions follows the actual layer schedule. For the
+  released 3B checkpoint (`num_layers=32`, alternating methods) there are 31
+  inter-layer transitions, not 16; `ensure_layout(current, required)` derives
+  them instead of assuming `layer % 2`.
+
+## Planner
+
+`nadit.py` implements a deterministic LPT assignment:
+
+```text
+windows sorted by (-video_token_count, original_window_id)
+each window is placed on the rank with the smallest (assigned_tokens, rank_id)
+each rank then lists its windows in ascending original order
+```
+
+The cache key covers the post-patch token grid, the full window method and
+parameters (via a SHA-1 geometry fingerprint), the geometry version, the **SP
+world size** and the planner version, so a different degree can never reuse a
+stale assignment. Caches are LRU-bounded.
+
+## Plan A redistribution
+
+At a layout boundary the activations must move from the source assignment to the
+destination assignment. With `Z_src[r]` / `Z_dst[d]` the canonical token ids held
+by each rank and `C[r, d]` the number of rows rank `d` needs from rank `r`:
+
+* `input_split_sizes` of rank `r` is row `r` of `C`, `output_split_sizes` is
+  column `r`,
+* each peer segment is packed in **destination-local order**, so the receiver can
+  reorder with a single `argsort` of the concatenated destination positions,
+* `network_exchange_required` is computed from the global count matrix, so every
+  rank takes the same branch; a rank that happens to exchange nothing never
+  skips the collective alone.
+
+The runtime is a variable-split `torch.distributed.all_to_all_single` over an
+regular SP process group (`get_sp_group().device_group`), plus a local-permutation fast
+path when the whole group stays in place (which is the SP=1 case). The first
+version is deliberately synchronous: no extra streams, no per-layer
+`cuda.synchronize()` or barrier "fixups", and no hidden all-gather.
+
+## Text stream
+
+Text is replicated on every rank and stays identical: each layer's attention
+produces one text output per window, ranks sum their local window outputs,
+one SUM all-reduce runs on the window-SP group, and every rank divides by the
+**global** window count of the current layout.
+
+```text
+T_next = (sum_r sum_{w in Omega_r} T_w) / W_global
+```
+
+This is neither a token-weighted mean nor a mean of rank means; a rank with no
+window contributes a zero tensor (with the *same dtype as its peers* -- a dtype
+mismatch changes the collective and deadlocks NCCL, which is why the empty-rank
+path is covered by a GPU test).
+
+## Existing SP configuration
+
+Configure `ulysses_degree=N` to create the existing N-rank SP group. Pass
+`DiffusionParallelConfig` to `SeedVR2NaDiT.build_runtime(parallel_config=...)`;
+the model validates the configuration and obtains `get_sp_group().device_group`.
+This adds no framework parallel option or group-construction branch.
+
+For SeedVR2 this selects **window-aligned Plan A**, not head-sharded Ulysses.
+Every rank retains all 20 attention heads, so SP=8 does not require the head
+count to divide by eight. The model has no `_sp_plan`, and its attention uses
+`skip_sequence_parallel=True`; generic sequence hooks and Ulysses attention
+exchanges are not installed. Reports name the actual execution path explicitly.
+
+SeedVR2 rejects ring, allgather, `advanced_uaa`, and `ulysses_a2a_permute`.
+Other diffusion models keep their existing Ulysses/Ring/AllGather behavior.
+
+## SeedVR2 integration
+
+`SeedVR2WindowRuntime` owns, per request: the layer schedule, the entry
+distribution, `ensure_layout` transitions, the text reduction and the final
+reconstruction back to canonical token order. The DiT forward keeps only this
+rank's video rows from the patch embedding onwards; the patch projection,
+norms, ADA modulation, residual and MLP are all tokenwise, so they run locally.
+
+The block never calls a parallel attention strategy: the shared `Attention`
+layer is created with `skip_sequence_parallel=True`, and window attention is a
+packed-varlen call over `[video window, replicated text]` segments (with a
+grouped-SDPA fallback for backends without a packed-varlen entry point).
+
+## Reference quirks reproduced on purpose
+
+* `vid_out_ada` is declared with `layers=["out"]`, whose embedding regrouping is
+  arithmetically incompatible with its 1-D parameters. The reference never uses
+  that value: it asks the forward-wide cache for `emb_repeat_0_vid`, a key the
+  first block already wrote, and therefore applies the *block* attention
+  modulation together with the learned `out_scale` / `out_shift`. The port
+  applies that effective modulation directly; changing it would break parity.
+* `mm_layers=10` controls weight sharing only. Text is consumed and produced by
+  all 32 blocks; only the last block freezes its text MLP path.
+* RoPE positions are window-local: a video token at window-local `(dt, dh, dw)`
+  uses axial indices `(text_len + dt, dh, dw)`, and every window restarts at 0.
+* The rotary span is `3 * (rope_dim // 3)` (126 of 128 channels for the 3B), so
+  the last two channels pass through unrotated.
+
+## Checkpoint compatibility
+
+The parameter layout of the port mirrors the reference, with one explicit
+exception: the released checkpoint stores the per-block RoPE table as
+`blocks.<i>.attn.rope.rope.freqs`, while this port registers it as
+`blocks.<i>.attn.rope.freqs`. The validation loader normalizes exactly that
+suffix (and only that suffix); every other key must already match.
+
+> The validation loader explicitly normalizes the reference RoPE buffer suffix
+> from `.rope.rope.freqs` to `.rope.freqs`. Checkpoint validation fails on any
+> remaining missing or unexpected key after normalization.
+
+Normalization is restricted and collision-checked: a key that is already
+normalized is left untouched, and two source keys mapping onto the same target
+key raise instead of silently overwriting a tensor. Loading then fails on any
+remaining missing, unexpected or shape-mismatched key, so a checkpoint that does
+not match the port cannot reach inference. Full 32-layer loading is the
+acceptance path; truncating the block list is a development fixture and requires
+an explicit flag.
+
+The native serving loader applies the same suffix normalization and strict
+checkpoint check.
+
+## Integration limits
+
+The native serving pipeline passes its parallel configuration to the DiT runtime
+and returns restored video through the standard media path. The causal VAE can
+run replicated, tiled, or height-sharded across the window-SP group.
+
+## Limitations
+
+* Plan A re-shards the full activation at every layout boundary. Plan B halo
+  exchange is a follow-up with a separate crossover evaluation.
+* The packed-varlen attention kernel needs a backend that accepts
+  `cu_seqlens`; the grouped-SDPA path is the portable fallback and is what the
+  pinned L20 environment exercises.
@@ -1,142 +1,137 @@
-# PersonaPlex Unified Full-Duplex Design
+# PersonaPlex on the Unified Full-Duplex Framework

 ## Status and target

-This design adapts PersonaPlex PR #4771 to the unified full-duplex runtime on
-top of:
-
- vLLM-Omni `origin/main`: `67c54777bb22e9e7e08fdf7c47a64f06b566fc47`
- PersonaPlex PR head: `477fb7c225f0c06991bc8aa55eadbd908ba282e4`
-
-The target is the engine-native path:
+PersonaPlex (`nvidia/personaplex-7b-v1`, a Moshi finetune) is served by the
+[Unified Full-Duplex Framework](fullduplex.md) through one model plugin,
+`PersonaPlexDuplexPlugin` (`vllm_omni/model_executor/models/personaplex/duplex/plugin.py`),
+selected by `PipelineConfig.duplex_plugin`. The serving path is:

 ```text
 /v1/realtime?duplex=1
-  -> OpenAI Realtime session actor
-  -> DuplexRequestClient
-  -> AsyncOmni correlated RPC
-  -> DuplexControlPlane
-  -> resumable Stage 0 request
-  -> PersonaPlex Talker
-  -> streaming PersonaPlex Code2Wav
-  -> PersonaPlex data-plane projector
+  -> OmniDuplexSessionHandler (thin: websocket I/O, Realtime <-> DuplexCommand/DuplexEvent)
+  -> DuplexOmni / DuplexOmniEngine
+  -> DuplexSessionManager -> DuplexSessionRunner (one per session)
+       plugin.plan_append: one 80 ms frame -> one resumable Stage 0 append
+  -> PersonaPlex Talker (Stage 0, lockstep temporal transformer + depformer)
+  -> streaming PersonaPlex Code2Wav (Stage 1, Mimi)
+  -> plugin.data_plane: cumulative audio/text -> deltas
  -> response.output_audio.delta + response.output_audio_transcript.delta
 ```

-The standalone `/api/chat` and `/v1/audio/duplex` server that accompanied the
-original PR was demo-only and has been removed from the tree; the unified
-engine path above is the only serving surface. It was never evidence that the
-unified engine path works.
-
-## Why configuration-only enablement was invalid
-
-The staged pipeline as shipped by the original PR was explicitly turn based.
-Its Talker read `pplex_user_codes`, `pplex_prefill_text`, and
-`pplex_silence_codes`, but no production staged input path wrote those fields.
-The voice prompt, persona prefill, and streaming Mimi state lived only in the
-standalone `PersonaPlexEngine` (demo-only, since removed).
-
-Setting only the following fields would therefore advertise an endpoint whose
-model never receives the live microphone stream:
-
-```python
-duplex_control_enabled = True
-duplex_runtime_extension = "..."
-duplex_serving_adapter = "..."
-```
-
-The adapter must supply a real scheduler data plane, not just endpoint
-capabilities.
+The pre-framework pair (engine runtime extension plus serving adapter) and
+the standalone `/api/chat` / `/v1/audio/duplex` server are gone; the plugin is
+the only integration surface, and this document describes it.

 ## Supported scope

-The unified implementation supports:
+The integration supports:

- up to two engine-owned sessions on one replica;
- 24 kHz mono float PCM input;
- one 1920-sample, 80 ms model frame per physical append unit;
- continuous user input while assistant audio is generated or played;
- bundled `.pt` voice prompts and a session persona;
- greedy text and depformer sampling, matching the current PersonaPlex port;
- `/v1/realtime?duplex=1` (the wire vocabulary is catalogued in the
-  [Realtime Duplex API](../serving/realtime_duplex_api.md) serving guide);
+- up to `duplex_session.max_sessions` engine-owned sessions on one replica
+  (the shipped deploy sets two);
+- 24 kHz mono float PCM input, one 1920-sample (80 ms) model frame per
+  physical append unit; clients may send any chunking, the session buffers
+  whole frames;
+- continuous user input while assistant audio is generated or played
+  (pure lockstep: the model listens while it speaks);
+- bundled `.pt` voice prompts (`voice`) and a session persona (`instructions`);
+- greedy text and depformer sampling (one temporal token per frame);
 - the public client preset
  `vllm_omni.clients.personaplex.create_duplex_session_config()` (24 kHz
-  `pcm_f32le` input format, voice prompt, persona) as the canonical
-  session-config source for `DuplexClient` consumers;
- engine lease close, disconnect cleanup, reconnect after cleanup, and explicit
-  response cancellation without cross-session state reuse.
+  `pcm_f32le` input, voice, persona) as the canonical session-config source
+  for `DuplexClient` / `InlineDuplexClient` consumers;
+- engine lease close, disconnect cleanup and explicit response cancellation
+  without cross-session state reuse.

-The implementation does not claim:
+It does not claim:

- more than two simultaneous PersonaPlex sessions on one replica;
- arbitrary WAV voice cloning;
- turn-based `response.create` semantics for an otherwise continuous model;
- scheduler migration of live codec state between replicas;
- exact output equality with the standalone engine after different scheduling
-  boundaries.
+- arbitrary WAV voice cloning (voices are bundled basenames, resolved by the
+  worker under the checkpoint);
+- turn-based `response.create` semantics, client commits or external turn
+  signals (`supports_client_commit=false`, `supports_external_turn_signal=false`);
+- text seeding, hence no `/v1/chat/completions` route
+  (`supports_chat_completions=false`);
+- session resume across a transport drop (`supports_session_resume=false`);
+- destructive output interruption or model-state rewind at a playback cursor
+  (`supports_barge_in=false`, `supports_audio_truncate=false`): overlapping
+  speech is native model behaviour, not a barge-in contract.

-The capability payload derives multi-session support from the configured
-session limit. The shipped two-session deployment reports
-`supports_multi_session=true` and `supports_multi_session_same_replica=true`.
 `duplex_session.max_sessions` is the only capacity source: config resolution
-propagates it to every stage as `duplex_max_sessions`, and both Mimi pools read
-that model-config value. Connector extras do not carry a second model-specific
-capacity knob that could drift from engine admission.
-It still reports `supports_barge_in=false`: the generic epoch fence can suppress
-stale transport output, but neither PersonaPlex nor the current MiniCPM-o 4.5
-adapter proves that model-owned streaming state can be destructively rewound or
-restarted at a playback cursor. Continuous overlapping speech is model-native
-duplex behavior, not by itself a barge-in contract.
+propagates it to every stage as `duplex_max_sessions`, and both Mimi pools
+(Stage 0 encoders, Stage 1 decoders) read that model-config value. The
+capability payload derives `supports_multi_session` /
+`supports_multi_session_same_replica` from it.

 ## Components and ownership

-### Serving adapter
+### The plugin (engine side)

-`PersonaPlexServingRuntimeAdapter` owns only serving-side state:
+`PersonaPlexDuplexPlugin` owns both halves of the contract.

- a transactional PCM append buffer;
- validation of `voice_prompt` and `instructions`;
- public capabilities;
- PersonaPlex data-plane output projection.
+Engine policy:

-It accepts 24 kHz `pcm_f32le`, groups client packets into whole 1920-sample
-frames, zero-pads only the final residual, and rolls a reservation back when an
-engine append fails. It never loads CUDA weights and never encodes user audio.
+- `configure_sampling_params`: Stage 0 greedy (`temperature=0`, `top_k=1`,
+  `max_tokens=1`), other stages untouched.
+- `plan_append`: validates exactly one 1920-sample 24 kHz `pcm_f32le` frame
+  (through `model_executor/common/duplex/payload.py`) and reserves one
+  scheduler slot per frame plus, on the first append of an epoch (`seq == 1`),
+  the `personaplex_prefill_slots` of the voice/persona prefill. The prompt
+  carries the fence, `seq`, the payload and the runtime config under
+  `model_intermediate_buffer["duplex"]`.
+- `decide_output`: never decides. PersonaPlex is always-clocked; what the
+  client hears comes from the final-stage data plane.
+- `silence_unit_payload`: one frame of zeros at 24 kHz
+  (`silence_continuation_samples=1920`,
+  `silence_continuation_sample_rate_hz=24000`). The runner uses it to keep a
+  model turn clocked when the client pauses; the startup warmup sends it.

-Client input cannot provide local filesystem paths. A voice is a bundled
-basename such as `NATF2.pt`; the worker resolves it under the local model
-checkpoint. `instructions` is the persona string.
+Session policy:

-### Runtime extension
+- `capabilities`: `personaplex_capabilities(max_sessions)` (80 ms units,
+  append-only, no commits, no barge-in, no resume, no chat route).
+- `prepare_runtime_config`: `voice` must be a bundled `.pt` basename,
+  `instructions` defaults to the shipped persona; the prefill slot count is
+  computed once per `(model, voice, persona)` off the orchestrator loop
+  (`personaplex_prefill_slots`, itself caching the tokenizer and the voice
+  bundle row count). The result is server-owned runtime config
+  (`personaplex_model_path`, `personaplex_voice_prompt`,
+  `personaplex_persona`, `personaplex_prefill_slots`); the same keys are
+  refused in a client's `extra_body`.
+- `runtime_config_for_update`: persona and voice are immutable for the
+  session (`persona_update_unsupported`, `voice_update_unsupported`).
+- session state, extra-body validation and data-plane context are the
+  framework defaults (`DefaultDuplexModelSessionState` with the 80 ms
+  `PersonaPlexPcmAppendBuffer`, `DuplexDataPlaneContext`).

-`PersonaPlexDuplexRuntimeExtension` is pure model policy. It:
+Because `supports_client_commit` is off, the session auto-responds without
+`extra_body.auto_response`: a stock Realtime client streams audio and hears
+the model without any vendor flag.

- configures greedy Stage 0 sampling and bounded segment lengths;
- maps each accepted PCM append to a scheduler prompt;
- places immutable session identity, append sequence, PCM payload, voice, and
-  persona under `model_intermediate_buffer["duplex"]`;
- reserves exactly one scheduler prompt slot per encoded Mimi frame, plus the
-  first-append voice/persona prefill length;
- never performs model inference or owns session state.
+### Input framing

-The extension returns no turn/listen decision. PersonaPlex is an always-clocked
-model, so visible audio/text comes from the final stage data plane.
+`PersonaPlexPcmAppendBuffer` is the `FixedFramePcmAppendBuffer` of
+`model_executor/common/duplex/pcm_buffer.py` at 24 kHz / 1920 samples /
+80 ms. It accepts 24 kHz `pcm_f32le` only, groups client packets into whole
+frames, takes one frame out per append as a reservation (committed when the
+stage accepted the append, rolled back to the front of the buffer when it did
+not), zero-pads only a final residual on commit, and never encodes audio.

-### Stage 0 streaming runtime
+### Stage 0 streaming runtime (worker side)

-The Talker owns a `PersonaPlexStage0DuplexRuntime` helper, analogous to
-MiniCPM-o's Stage 0 helper but with PersonaPlex lockstep semantics.
+The Talker owns a `PersonaPlexStage0DuplexRuntime`
+(`model_executor/models/personaplex/duplex/stage0.py`), created lazily in
+`_duplex_stage0_runtime()` and released through `on_requests_finished`.

-For each admitted session it owns:
+For each live `(session_id, epoch)` it owns:

- streaming Mimi encoder convolution and transformer state;
- the selected voice embedding bundle;
- persona tokenization and prefill embeddings;
- the prior user code frame needed by the one-frame acoustic delay;
- append identity used to make a retried scheduler update idempotent.
+- the streaming Mimi encoder convolution and transformer state;
+- the selected voice embedding bundle and the persona prefill embeddings;
+- the prior user code frames needed by the one- and two-frame acoustic
+  delays;
+- the append identity `(epoch, seq)` that makes a retried scheduler update
+  idempotent.

-The first append builds this ordered prefill:
+The first append of an epoch builds this ordered prefill:

 ```text
 voice embeddings
@@ -146,45 +141,47 @@ voice embeddings
  -> first live user frame
 ```

-Later appends encode only new 1920-sample frames. The helper returns the
+Later appends encode only new 1920-sample frames. The runtime returns the
 per-frame user codes and prompt embeddings through the request's
-`model_intermediate_buffer`. The normal vLLM runner remains authoritative for
-attention metadata, block tables, KV allocation, scheduling, and sampling.
+`model_intermediate_buffer`; the normal vLLM runner remains authoritative for
+attention metadata, block tables, KV allocation, scheduling and sampling.

-The Talker must distinguish resumable prompt prefill from decode positions.
-Prompt rows consume the exact prepared embeddings; sampled decode rows continue
-to use the existing delayed agent/user frame construction. No code path may
-fall back to an all-initial user stream for a duplex request.
+**Epochs.** The framework identifies a Stage 0 request by
+`(session_id, epoch)`; `response.cancel` and `output_audio_buffer.clear`
+advance the epoch, abort the current request and start the next append at
+`seq == 1` on a fresh request with fresh KV. The Stage 0 runtime therefore
+keys its state by `(session_id, epoch)`: when an append of a newer epoch
+arrives, any older-epoch state of the same session is closed first (its Mimi
+encoder returns to the pool, so the codec budget never counts a superseded
+epoch), and the voice/persona prefill is replayed because the plan reserved
+the slots again. The user-visible consequence is that a cancel restarts the
+model's conversation context. A late `on_requests_finished` for the aborted
+request finds nothing to close.

-Cleanup is keyed by the full `(session_id, incarnation)` identity. Every live
-session has an independent Mimi encoder instance; encoder convolution/KV state
-is never shared between asynchronously scheduled sessions. A finished or
+Every live session has an independent Mimi encoder instance; encoder state is
+never shared between asynchronously scheduled sessions. A finished or
 aborted scheduler request resets and returns only that session's encoder.

 ### Stage 1 streaming decoder

-The current `PersonaPlexCode2Wav` calls one-shot `MimiModel.decode` and is not
-CUDA-graph safe. Unified duplex uses eager Stage 1 and maintains an independent
-streaming Mimi decoder for every active request. Decoder ownership is keyed by
-the stable Stage 1 request id and released by `on_requests_finished`; a mixed
-batch must never advance another request's convolution or transformer state.
-
-Each Stage 0 segment emits de-delayed agent codebooks. Stage 1 decodes only the
-new code frames, emits only the new PCM suffix, and resets state when the
-request is closed. Connector chunk boundaries retain the final raw code frame
-needed to de-delay the next chunk.
-
-The deploy default sets Stage 1 `enforce_eager: true`; a default configuration
-that fails during CUDA graph capture is not an acceptable deployment profile.
+`PersonaPlexCode2Wav` runs eager (`enforce_eager: true` in the deploy) and
+maintains an independent streaming Mimi decoder for every active request,
+keyed by the Stage 1 request id and released by `on_requests_finished`. Each
+Stage 0 segment emits de-delayed agent codebooks; Stage 1 decodes only the new
+code frames and emits only the new PCM suffix. Connector chunk boundaries
+retain the final raw code frame needed to de-delay the next chunk.

 ### Data-plane projector

-`PersonaPlexDataPlaneSession` converts cumulative or delta Stage 1 output into
-model-neutral native results:
+`PersonaPlexDataPlaneSession` is the `CumulativeAudioTextDataPlane` of
+`model_executor/common/duplex/data_plane.py` with a 24 kHz default rate. It
+keeps one audio/text cursor per request so cumulative Stage 1 output cannot
+replay old audio, and yields one internal result per new suffix:

 ```python
 {
    "stage_role": "tts",
+    "is_listen": False,
    "data_plane_request_id": request_id,
    "text": text_delta,
    "audio_data": encoded_audio_delta,
@@ -192,82 +189,74 @@ model-neutral native results:
    "sample_rate_hz": 24000,
    "audio_duration_ms": delta_duration,
    "end_of_turn": False,
+    "runtime_impl": "scheduler_data_plane",
+    ...
 }
 ```

-It owns per-request audio and text cursors so a cumulative output cannot replay
-old audio. The generic projector turns each native result into the Realtime
-`response.output_audio.delta` / `response.output_audio_transcript.delta` pair (and
-`response.output_text.delta` for text) under one `response_id`; the full
-mapping is the name map in the
-[Realtime Duplex API](../serving/realtime_duplex_api.md) serving guide.
-PersonaPlex keeps one visible response open while continuous output arrives.
-Session close or cancellation terminates that response through the generic
-Realtime lifecycle; a codec segment finishing is not a conversational turn
-boundary. PersonaPlex advertises `supports_barge_in=false` and
-`supports_session_resume=false`, so `barge_in`, `turn_detection.server_vad`,
-and `session.resume` are rejected on this model.
+The runner turns each result into the Realtime
+`response.output_audio.delta` / `response.output_audio_transcript.delta` pair
+under one `response_id`. PersonaPlex keeps one visible response open while
+continuous output arrives; session close or cancellation terminates it through
+the generic Realtime lifecycle. A codec segment finishing is not a
+conversational turn boundary.

 ## Error and lifecycle contracts

- Unsupported sample rate, malformed base64, non-finite PCM, invalid voice
-  basename, or changed format fails before scheduler submission.
+- Unsupported sample rate, malformed base64, non-finite PCM, a partial frame
+  at the plan, an invalid voice basename or a changed format fails before
+  scheduler submission, as a typed `error` event.
 - Append is prepare/submit/commit. Failure rolls back the exact reserved PCM
  bytes and does not advance the model frame cursor.
- A repeated `operation_id` must not encode or submit the same frame twice.
- Input iterator exceptions execute the same cleanup as explicit close.
- Cancellation does not release the engine lease until the stage request and
-  any in-flight codec operation are actually finished.
- A bounded cleanup timeout returns a cleanup error and keeps the session in
-  the closing admission set; it must not make the slot available while work
-  still mutates shared state.
- New sessions cannot observe the previous voice, persona, PCM tail, Mimi
-  convolution state, or Talker delayed code frame.
-
-(The legacy `PersonaPlexDuplexRuntime.run()` and standalone server drain path,
-which followed the same exception-safe rule, have been removed from the tree.)
+- A repeated `(epoch, seq)` identity does not encode or submit the same frame
+  twice.
+- Cancellation advances the epoch; the aborted request's Stage 0 state is
+  released by the next-epoch append or by `on_requests_finished`, whichever
+  comes first.
+- Close releases the admission slot only once stage cleanup succeeded; the
+  reaper retries a failed cleanup while the slot stays held.
+- New sessions cannot observe a previous voice, persona, PCM tail, Mimi
+  convolution state or Talker delayed code frame.

 ## Testing and acceptance

-### Contract tests
+### CPU contract tests

-Tests first cover:
+- `tests/model_executor/models/personaplex/duplex/test_plugin.py`: pipeline
+  binding and plugin load, honest capabilities, private keys, voice/persona
+  resolution and caching, immutable updates, greedy sampling, one-slot-plus-
+  prefill planning, frame validation, the 24 kHz silence unit, and the
+  CPU-checked helpers of the E2E driver.
+- `tests/model_executor/models/personaplex/duplex/test_stage0_runtime.py`:
+  first-append prefill, depformer teacher forcing, causal user-frame delays,
+  idempotent retries, independent encoders per session, capacity, epoch
+  restart and late finish of a superseded request.
+- `tests/engine/duplex/test_session_runner_personaplex.py`: the session
+  runner with the real plugin -- one submission per frame, prefill on the
+  first append only, half frames buffered, wrong rate refused, cumulative
+  Code2Wav output projected as 24 kHz deltas, cancel restarting the epoch,
+  close aborting the request.
+- `tests/model_executor/executor_common/`: the shared toolbox (PCM helpers, payload
+  validation, fixed-frame buffer, cumulative data plane, request-output
+  readers).

- pipeline registration enables the control plane and selects both PersonaPlex
-  adapters;
- the capability payload is two-session, 80 ms, append-only, and honest;
- PCM reservation commit/rollback, partial-frame flush, invalid input, and
-  operation idempotency;
- runtime prompt fields and exact token budgeting;
- first-append voice/persona prefill followed by live user codes;
- later appends retain per-session Mimi state and do not replay prefill;
- interleaved Stage 0 and Stage 1 work preserves independent codec histories;
- output projection emits only audio/text deltas;
- close, exception, timeout, and reconnect cleanup;
- ordinary non-duplex imports do not load PersonaPlex modules.
+### GPU validation

-### Remote H20 validation
+`tests/e2e/online_serving/personaplex_realtime_duplex.py` (wrapped by
+`tests/e2e/online_serving/test_personaplex_duplex.py` when
+`PERSONAPLEX_MODEL_PATH` is set) drives the default `personaplex.yaml`
+deployment and requires:

-Validation runs in an isolated remote worktree using the ModelScope
-`nv-community/personaplex-7b-v1` checkpoint and its Mimi dependency.
-
-The required evidence is:
-
-1. default `personaplex.yaml` reaches ready without a local eager override;
-2. `/health` returns 200;
-3. `/v1/realtime?duplex=1` reports `model_native_duplex`, `chunk_period_ms=80`,
-   and two-session admission;
-4. paced 24 kHz PCM appends produce finite, non-silent 24 kHz audio deltas and
-   text deltas;
-5. microphone input continues during assistant output without cancelling the
-   scheduler request;
-6. two paced sessions simultaneously produce independent non-empty audio, and
-   a third session is rejected with `resource_exhausted`;
-7. closing either session frees only its scheduler request and GPU codec state,
-   after which a replacement session can use a different persona without state
-   leakage;
-8. malformed input returns one typed error and does not poison the next append;
-9. all focused unit tests and `git diff --check` pass.
+1. the server reaches ready without a local eager override; `/health` is 200;
+2. `/v1/realtime?duplex=1` reports `model_native_duplex`, `chunk_period_ms=80`
+   and two-session admission, with server-allocated session ids;
+3. paced 24 kHz PCM appends produce finite, non-silent, whole-frame 24 kHz
+   audio deltas and text deltas;
+4. two paced sessions simultaneously produce independent non-empty audio, and
+   a third session is refused with `resource_exhausted`;
+5. closing one session frees only its scheduler request and codec state, after
+   which a replacement session with a different persona is admitted with a
+   fresh id and no state leakage.

 Audio that is empty, all zero, non-finite, or only a protocol `listen` event is
 not a successful end-to-end result.
@@ -68,6 +68,7 @@ guides list selectable backends, platform defaults, installation, and tuning.
 The design contracts separate selection mechanics from backend algorithms:

 - [Attention Backend Selection](feature/attention_backend_selection.md)
+- [Attention Execution Contract PoC](feature/attention_execution_contract_poc.md)
 - [Skip-Softmax](feature/skip_softmax.md)

 #### CPU offloading
@@ -115,3 +116,6 @@ The design contracts separate selection mechanics from backend algorithms:
 The pre-#5137 pages are preserved in the
 [legacy module archive](module/archive/README.md) for historical reference and
 are not active design contracts.
+
+- [Qwen3-Omni MRv2 performance](qwen3_omni_mrv2_performance.md)
+- [MiniCPM-o 4.5 turn-mode MRv2 performance](minicpm_o45_mrv2_performance.md)
@@ -0,0 +1,22 @@
+# MiniCPM-o 4.5 MRv2 profiles
+
+Turn mode uses the stage-level MRv2 contracts from #8184. Thinker emits live
+latent metadata outside graph replay; Talker keeps codec history and EOS
+control on device. Code2Wav reuses mainline Whole-Euler Flow graphs and shared
+prompt state. The previous experimental block compilation, tiled attention,
+channels-last and merged-CFM implementations are removed.
+
+`minicpmo_4_5_turn_mrv2_h200.yaml` selects Talker capacity 16 and 4 GiB KV;
+the generic MRv2 profile selects capacity 8 and 2 GiB KV. These configurations
+require new end-to-end performance measurements after the mainline integration.
+Previously reported numbers do not describe this revised codec backend.
+
+The generic MRv2 profile and the opt-in native V1 duplex H200 profile enable
+TF32 matmuls within Code2Wav forward/capture only; the previous process policy
+is restored afterwards. cuDNN's TF32 policy is independent. TF32 changes rounding.
+Duplex keeps the mainline V1 session path. Turn results do not establish duplex
+performance or interruption correctness.
+
+The shared asynchronous output snapshot and batched Talker preprocessing also
+apply to V1 when async chunking/scheduling is enabled. Default V1 behavior must
+therefore be included in end-to-end regression validation.
@@ -0,0 +1,22 @@
+# Qwen3-Omni MRv2 performance
+
+This model adaptation builds on #8184's existing stage runtime and output
+contracts. It adds Talker CPU metadata parsing, sampled-token embedding
+handoff, an optional first-frame decoder, length-grouped Code2Wav graph
+buckets, and opt-in incremental short-KV code prediction. The H200 profile
+also enables conservative fused sampling and bounded cuDNN algorithm search.
+The shared predictor keeps the upstream re-prefill helper and its default
+sampling behavior; short KV is explicitly selected by the deployment profile.
+
+Use `vllm_omni/deploy/qwen3_omni_moe_mrv2_h200.yaml` for the experimental H200
+profile. Preserve a control with `code_predictor_fused_sampling=false` and
+`codec_cudnn_benchmark=false` when C64 P99 is the primary requirement.
+cuDNN search restores process settings on exit and has startup cost. Kernel
+arithmetic is not bitwise-equivalent; the first-frame decoder uses additional
+model memory. Full-duplex performance is not established by turn-mode tests.
+
+First-frame delivery requires `talker_first_audio: true` in connector extra,
+a one-frame initial codec chunk, CUDA, a local single-rank Talker and prefix caching disabled.
+Other deployments retain the regular Code2Wav path.
+`codec_fused_snake` explicitly opts CUDA profiles into fused decoder blocks;
+the base V1 profile retains its original blocks and indexed text cache.
@@ -93,13 +93,13 @@ Each optimization stacks on the previous one. The summary plots below show the c

 **Benchmark environment:**

-| | Qwen3-Omni                  | Qwen3-TTS |
-| --- |-----------------------------| --- |
-| **GPU** | A100                        | H200 |
+| | Qwen3-Omni | Qwen3-TTS |
+| --- | ----------------------------- | --- |
+| **GPU** | A100 | H200 |
 | **Model** | Qwen3-Omni-30B-A3B-Instruct | Qwen3-TTS-12Hz-1.7B-CustomVoice |
-| **vLLM** | v0.17.0                     | v0.18.0 |
-| **vllm-omni** | commit 199f7832             | v0.18.0rc2 |
-| **CUDA** | 12.9                        | 12.8 |
+| **vLLM** | v0.17.0 | v0.18.0 |
+| **vllm-omni** | commit 199f7832 | v0.18.0rc2 |
+| **CUDA** | 12.9 | 12.8 |

 This post walks through each optimization in the same order they are typically enabled in practice, then ends with deployment playbooks for both models.

@@ -310,7 +310,68 @@ self._compiled_model_fwd = torch.compile(

 `mode="default"` is used instead of `mode="reduce-overhead"` to avoid conflicts with vLLM's own CUDA graph capture on the main Talker model. `dynamic=True` handles the growing sequence length without recompilation.

-These optimizations are always-on in the current codebase - all Qwen3-TTS benchmark results in this post include them.
+The historical benchmark results above use re-prefill. It remains the default
+for Qwen3-TTS and the shared Qwen3-Omni predictor.
+
+### Experimental Qwen3-TTS single-GPU throughput profile
+
+The opt-in `qwen3_tts_high_concurrency_mrv2_single_gpu.yaml` profile uses a
+Qwen3-TTS-specific execution path on CUDA with BF16 weights:
+
+- `code_predictor_kv_cache` reuses each layer's keys and values across the 15
+  residual-codebook predictions within one audio frame. The first step writes
+  two positions; subsequent steps write one. Scratch storage is bounded by the
+  configured batch capacity, overwritten for every frame, and never shared
+  between requests across frames. The worker's serialized CUDA stream owns it;
+  it is separate from the Talker's paged KV cache.
+- A short causal/GQA Triton kernel masks unwritten K/V loads. Dynamic-batch
+  `torch.compile` warms the predictor before the runner captures whole-MTP
+  CUDA graphs. This path creates no inner CUDA graphs. Unsupported devices and
+  dtypes retain re-prefill; FP16 retains its existing precision safeguards.
+- `code_predictor_fused_sampling` fuses temperature scaling, top-k filtering,
+  and Gumbel argmax when the runner supplies FP32 uniforms. It preserves top-k
+  ties and uses the supplied randomness. Explicit generator paths retain the
+  existing sampler. Qwen3-Omni does not read either TTS option.
+- `decode_cudnn_benchmark` selects convolution algorithms during Code2Wav
+  warmup and graph capture. The process's cuDNN flags are restored afterwards,
+  including on capture failure. Captured graphs retain the selected algorithms.
+
+- `decode_time_major_conv` keeps codec convolution activations in time-major
+  layout to reduce transposes and copies between layers.
+- `talker_first_audio` completes residual prediction immediately after the
+  first codebook sample and decodes the first frame in the Talker process.
+  The codec advances its state without repeating the delivered frame; the
+  orchestrator orders that frame before subsequent codec chunks. This path
+  requires CUDA, MRV2, asynchronous chunks, a single-process executor, TP/PP 1,
+  and disabled prefix caching. Reference-code requests retain codec delivery.
+  Supported Qwen3-TTS profiles enable it explicitly; other models do not opt in.
+
+The predictor, sampling, convolution-layout and autotuning options default to
+false outside this throughput profile. It uses fixed 1/25-frame chunks,
+72 frames of left context, Talker/codec capacities of 128/64, codec batches
+through 8, and synchronous codec scheduling. Client concurrency is independent
+of those capacities. This profile also sets `cuda_mps: true`; the runtime owns
+a private MPS daemon unless an operator supplies an existing MPS pipe directory.
+Ordinary profiles leave MPS disabled. See
+[experimental MPS deployment](../configuration/stage_configs.md#experimental-mps-deployment)
+for requirements and ownership. Startup includes compilation and graph capture.
+
+```bash
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni \
+  --deploy-config vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2_single_gpu.yaml \
+  --api-server-count 4 --stage-init-timeout 1800 --init-timeout 2400
+```
+
+KV reuse changes GEMM shapes and attention reduction order; cuDNN autotuning
+can also change waveform numerics. This path is not bitwise equivalent to
+re-prefill, and aggregate WER alone cannot establish perceptual or speaker
+similarity equivalence. Validate quality on the intended workload before
+adopting it. Qwen3-Omni keeps its existing predictor and audio-delivery behavior.
+
+For a controlled ablation, copy the profile and disable individual options while keeping capacities, chunking, hardware, client concurrency,
+and warmup identical. Report repeated runs, first-audio latency and failures
+alongside throughput. Warm requests measure a ready service; they do not measure
+cold-start latency.

 ---

@@ -251,8 +251,8 @@ curl -X POST http://localhost:8000/v1/images/generations \

 | Method | Arguments | Return Type | Description |
 | :--- | :--- | :--- | :--- |
-| **sleep** | `stage_ids: List[int], level: int` | `List[OmniACK]` | Triggers hibernation for specified stages. |
-| **wake_up** | `stage_ids: List[int]` | `List[OmniACK]` | Reloads weights and re-maps memory. |
+| **sleep** | `stage_ids: List[int], level: int` | `List[OmniACK]` | Triggers hibernation for specified stages. Raises `RuntimeError` if a stage fails; the stages stay marked as sleeping, so `wake_up` brings them back, unless a level-2 sleep already discarded weights on some replica. |
+| **wake_up** | `stage_ids: List[int]` | `List[OmniACK]` | Reloads weights and re-maps memory. Raises `RuntimeError` if a stage fails; that stage stays sleeping and the call can be retried. |
 | **pause_generation** | `mode: str, stage_ids: List[int]` | `None` | Stops admission; with `mode="keep"` also stops diffusion schedulers and returns after their ACK. |
 | **resume_generation** | `stage_ids: List[int]` | `None` | Reopens paused schedulers, then admission. |

@@ -261,7 +261,7 @@ curl -X POST http://localhost:8000/v1/images/generations \
 | Field | Type | Description |
 | :--- | :--- | :--- |
 | **task_id** | `str` | Unique identifier for the operation. |
-| **status** | `str` | `SUCCESS` or `ERROR`. |
+| **status** | `str` | `SUCCESS` or `ERROR`. `sleep` and `wake_up` raise instead of returning an `ERROR` ACK, and `/v1/omni/sleep` and `/v1/omni/wakeup` return HTTP 500 with the error. |
 | **stage_id** | `int` | The ID of the stage that responded. |
 | **rank** | `int` | The rank ID within the Tensor Parallel group. |
 | **freed_bytes** | `int` | Actual amount of physical VRAM reclaimed. |
@@ -1,7 +1,7 @@
 # Installation

 !!! important
-    vLLM-Omni is released against the matching upstream vLLM major/minor version. The 0.30 development line requires vLLM 0.30.x; the source-install instructions pin vLLM 0.30.0. Published vLLM-Omni 0.28.0 wheels and images still require vLLM 0.28.x.
+    vLLM-Omni is released against the matching upstream vLLM major/minor version. The vLLM-Omni 0.30.x release line uses vLLM 0.30.x; the source-install instructions pin vLLM 0.30.0.

 vLLM-Omni supports the following hardware platforms:

@@ -34,9 +34,7 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr

 ### Pre-built wheels

-The source-install instructions target vLLM 0.30.0. Until vLLM-Omni 0.30.0 artifacts are published, the CUDA/ROCm pre-built examples below remain on the matching 0.28.0 release pair.
-
-Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, 0.25.0, 0.26.0, and 0.28.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
+Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, 0.25.0, 0.26.0, 0.28.0, and 0.30.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).

 === "NVIDIA CUDA"

@@ -5,7 +5,7 @@
 # --8<-- [end:requirements]
 # --8<-- [start:set-up-using-python]

-vLLM-Omni depends on the matching major/minor release of vLLM. The 0.30 development line uses vLLM 0.30.x. Published 0.28.0 wheels and images use vLLM 0.28.x.
+vLLM-Omni depends on the matching major/minor release of vLLM. The vLLM-Omni 0.30.x release line uses vLLM 0.30.x.

 !!! note
    PyTorch installed via `conda` will statically link `NCCL` library, which can cause issues when vLLM tries to use `NCCL`. See <gh-issue:8420> for more details.
@@ -18,22 +18,22 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new**

 #### Installation of vLLM

-These pre-built wheel instructions install the published vLLM-Omni 0.28.0 release. For the 0.30 development line, use the source-install instructions below.
+These pre-built wheel instructions install the published vLLM-Omni 0.30.0 release.

 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm==0.28.0 --torch-backend=auto
+uv pip install vllm==0.30.0 --torch-backend=auto
 ```

 #### Installation of vLLM-Omni

 ```bash
-uv pip install vllm-omni==0.28.0
+uv pip install vllm-omni==0.30.0
 ```

 To run Gradio demos, also install the optional extras:
 ```bash
-uv pip install 'vllm-omni[demo]==0.28.0'
+uv pip install 'vllm-omni[demo]==0.30.0'
 ```

 # --8<-- [end:pre-built-wheels]
@@ -111,7 +111,7 @@ docker run --runtime nvidia --gpus 2 \
    --env "HF_TOKEN=$HF_TOKEN" \
    -p 8091:8091 \
    --ipc=host \
-    vllm/vllm-omni:v0.28.0 \
+    vllm/vllm-omni:v0.30.0 \
    vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```

@@ -10,9 +10,8 @@

 For ROCm, vLLM-Omni currently recommends the setup steps through Docker Images.

-vLLM-Omni depends on the matching major/minor release of vLLM. The 0.30
-development line uses vLLM 0.30.x. Published 0.28.0 wheels and images use vLLM
-0.28.x.
+vLLM-Omni depends on the matching major/minor release of vLLM. The vLLM-Omni
+0.30.x release line uses vLLM 0.30.x.

 The Dockerfile's `BASE_IMAGE` pin applies only to Docker builds. The
 `vllm-omni` package does not install vLLM as a dependency, so non-Docker source
@@ -23,12 +22,12 @@ installing vLLM-Omni, as shown below.

 #### Installation of vLLM

-These pre-built wheel instructions install the published vLLM-Omni 0.28.0 release. For the 0.30 development line, use the source-install instructions below.
+These pre-built wheel instructions install the published vLLM-Omni 0.30.0 release.

 vLLM-Omni is built based on vLLM. Please install it with command below.

 ```bash
-uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.28.0/rocm723
+uv pip install vllm==0.30.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.30.0/rocm723
 ```

 #### Installation of vLLM-Omni
@@ -37,7 +36,7 @@ uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/roc
 # we need to add --no-build-isolation as the torch
 # is not obtained from pypi, we have to install using the
 # torch installed in our environment
-uv pip install vllm-omni==0.28.0
+uv pip install vllm-omni==0.30.0

 # Optional if want to run Qwen3 TTS
 uv pip uninstall onnxruntime # should be removed before we can install onnxruntime-rocm
@@ -150,6 +149,8 @@ vllm-omni-rocm

 vLLM-Omni offers an official docker image for deployment. These images are built on top of vLLM docker images and available on Docker Hub as [vllm/vllm-omni-rocm](https://hub.docker.com/r/vllm/vllm-omni-rocm/tags). The version of vLLM-Omni indicates which release of vLLM it is based on.

+The prebuilt ROCm image is published separately from the release pipeline: `v0.28.0` is the latest tag currently available on Docker Hub, and newer-tag availability is tracked in [#7405](https://github.com/vllm-project/vllm-omni/issues/7405).
+
 #### Launch vLLM-Omni Server

 Here's an example deployment command that has been verified on 2 x MI300's:
@@ -53,7 +53,7 @@ docker run --rm \

 # Inside the container, install vLLM-Omni from source
 cd /vllm-workspace
-git clone -b v0.30.0rc1 https://github.com/vllm-project/vllm-omni.git
+git clone -b v0.30.0 https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
 pip install -v -e . --no-build-isolation
 # or VLLM_OMNI_TARGET_DEVICE=npu pip install -v -e .
@@ -0,0 +1,280 @@
+# SeedVR2 3B video restoration
+
+The native `SeedVR2Pipeline` restores an input video using the released 3B FP16
+NaDiT and s8/c16/t4 causal VAE. It uses fixed checkpoint conditioning, CFG=1,
+and one Euler step. Text prompts do not change conditioning.
+
+## Code ownership
+
+The model package groups each computation stage in one file: `nadit.py` owns
+window geometry, routing, RoPE and the DiT; `vae.py` owns the causal VAE and its
+temporal/spatial tiling; `pipeline_seedvr2.py` owns input validation, decoding,
+conditioning, the Euler step and colour transfer. `config.py` owns model limits,
+and `long_video.py` owns the optional HTTP job/window lifecycle. Framework code
+only registers the model, its pre/postprocessors, metadata and HTTP routes.
+
+`color_correction_method` is model-specific: online requests use the existing
+`extra_params` JSON object, and offline sampling uses
+`extra_args={"color_correction_method": "wavelet"}`. The default remains `lab`;
+invalid values are rejected by model-side validation before inference. It is
+not a top-level field in the common video API.
+
+## Model directory
+
+Place these files together, retaining their names:
+
+| File | SHA-256 of the validated release |
+| --- | --- |
+| `seedvr2_ema_3b_fp16.safetensors` | `2fd0e03a3dad24e07086750360727ca437de4ecd456f769856e960ae93e2b304` |
+| `ema_vae_fp16.safetensors` | `20678548f420d98d26f11442d3528f8b8c94e57ee046ef93dbb7633da8612ca1` |
+| `pos_emb.pt` | `fa07a14844314772266b66c3b95deb0027696d8fe7065721263db5176f45d799` |
+
+Add `model_index.json` containing:
+
+```json
+{"_class_name": "SeedVR2Pipeline"}
+```
+
+The loader checks every checkpoint tensor, including RoPE buffers, and rejects
+missing, duplicate, unexpected, or incompatible tensors. Install PyAV for video
+decoding. The validated environment uses PyAV 18.1, PyTorch 2.13/CUDA 13, and L20 GPUs.
+
+## Serve and restore a video
+
+```bash
+vllm serve "$MODEL_DIR" --omni \
+  --model-class-name SeedVR2Pipeline --dtype float16 --enforce-eager \
+  --num-gpus 1 --host 127.0.0.1 --port 8098
+
+curl --fail-with-body http://127.0.0.1:8098/v1/videos/sync \
+  -F 'prompt= ' \
+  -F 'input_references=@input.mp4;type=video/mp4' \
+  -F 'size=224x128' -F 'num_inference_steps=1' \
+  -F 'guidance_scale=1' -F 'seed=7723' \
+  --output restored.mp4
+```
+
+For two-rank window sequence parallelism, expose two GPUs and replace `--num-gpus 1` with:
+
+```bash
+--num-gpus 2 --distributed-executor-backend mp \
+  --stage-overrides '{"0":{"ulysses_degree":2}}'
+```
+
+Use degree 4 and four visible GPUs for SP4. Window-SP alone replicates the VAE.
+To shard VAE activations across the same ranks, use:
+
+```bash
+--vae-use-tiling --num-gpus 2 --distributed-executor-backend mp \
+  --stage-overrides '{"0":{"ulysses_degree":2,"vae_patch_parallel_size":2,"vae_parallel_mode":"spatial_shard_height"}}'
+```
+
+The VAE patch degree must match the window-SP degree.
+For a 16-frame 1280×720 input restored at its original size, use four GPUs,
+VAE tiling, and height sharding with both degrees set to four:
+
+```bash
+CUDA_VISIBLE_DEVICES=0,1,2,3 vllm serve "$MODEL_DIR" --omni \
+  --model-class-name SeedVR2Pipeline --dtype float16 --enforce-eager \
+  --vae-use-tiling --num-gpus 4 --distributed-executor-backend mp \
+  --stage-overrides '{"0":{"ulysses_degree":4,"vae_patch_parallel_size":4,"vae_parallel_mode":"spatial_shard_height"}}' \
+  --host 127.0.0.1 --port 8098
+```
+
+Request `size=1280x720` for this landscape input or `size=720x1280` for a
+portrait input of the same dimensions. Keep the requested output size equal to
+the input size when testing restoration without upscaling.
+
+The multipart API requires a prompt field; a single space supplies a blank
+prompt. Nonblank text is rejected. Omit `fps` to retain the source frame rate;
+an explicit rate must match the source. Output dimensions are explicit multiples
+of 16. Input frames are resized with antialiased bicubic interpolation. The final
+frame is repeated internally to reach 4n+1, and the decoded output is cropped back
+to the original frame count. Five frames remain five; six frames are internally
+padded to nine and return six.
+
+## Colour correction
+
+Restoration reproduces detail faithfully but shifts global colour, so the
+resized input is used to transfer colour back onto the restored frames. This is
+on by default; pass `extra_params={"color_correction_method":"wavelet"}` to choose how:
+
+| Value | Behavior |
+| --- | --- |
+| `lab` (default) | Swaps the lowest frequency band, then histogram-matches CIELAB chroma. Most faithful colour. |
+| `wavelet` | Swaps only the lowest frequency band, keeping every higher band from the restoration. |
+| `adain` | Matches per-channel mean and standard deviation. Cheapest, and corrects only a global tint. |
+| `none` | Skips the transfer and returns raw restored colour. |
+
+```bash
+curl --fail-with-body http://127.0.0.1:8098/v1/videos/sync \
+  -F 'prompt= ' -F 'input_references=@input.mp4;type=video/mp4' \
+  -F 'size=224x128' -F 'num_inference_steps=1' -F 'guidance_scale=1' \
+  -F 'seed=7723' -F 'extra_params={"color_correction_method":"wavelet"}' \
+  --output restored.mp4
+```
+
+`lab` and `wavelet` retain the restored high-frequency detail; `adain` measurably
+softens it because it rescales every frequency. The transfer runs in FP32 after
+the VAE decode and costs a fraction of a second per frame. Unrecognized values
+are rejected with 400 before the clip is admitted. Correction cannot recover
+colour lost to 4:2:0 chroma subsampling in the returned MP4, which dominates the
+remaining deviation once the transfer is applied.
+
+## Request admission
+
+The default 3B limit is 848×480 pixels per input or output frame and 2,035,200
+pixels across the requested output clip. With VAE tiling and window SP of at
+least four ranks whose `vae_patch_parallel_size` matches `ulysses_degree`, the
+per-frame limit rises to 2560×1472 and the clip budget to 18,841,600 pixels.
+Raising the degree never lowers the budget. Both budgets count temporal
+padding to 4n+1 frames: the default admits up to five 848×480 frames or 93
+192×112 frames; the sharded profile admits up to 45 848×480 frames or five
+2560×1472 frames. A 16-frame 1280×720 or 720×1280 clip pads to 17 frames and
+uses 15,667,200 of the sharded clip's 18,841,600 pixels. Both orientations passed
+original-size HTTP restoration on four RTX 5090 GPUs with VAE tiling and height
+sharding. An independent 257-frame cap bounds per-frame decoder work for tiny
+inputs. The other longer combinations above are admission bounds, not completed
+GPU validation. The decoder checks
+declared duration, frame count, and input dimensions when available, then
+enforces the limits as frames arrive. Requests outside these budgets return 400
+before building the resized whole-clip tensor.
+
+The per-frame cap is calibrated for the smallest qualified device. The clip cap
+scales with the smallest visible device's memory: it stays at the calibrated
+five 2560×1472 frames on a 32 GB device and grows on larger ones from a
+per-rank memory model measured at SP4 (see `sharded_budget` in
+`vllm_omni/diffusion/models/seedvr2/pipeline_seedvr2.py`), which keeps 15% of the device
+free and leaves room for allocator fragmentation. It stops at 2,118,057,984
+padded pixels (513 frames at 1536×2688), the largest clip validated on four
+B300 ranks; 80 GB devices get about 1.08 billion and 141 GB or larger devices
+reach the cap. Clips at the cap in both memory regimes (561 frames at 2560×1472
+and 2,049 frames at 768×1344) peaked at 54 GiB of activations per rank. Set
+these before starting the server to override the caps; each must be a positive
+integer, and validating an override on the target hardware is the operator's
+responsibility:
+
+| Variable | Default | Bounds |
+| --- | --- | --- |
+| `VLLM_OMNI_SEEDVR2_SHARDED_FRAME_PIXELS` | 3,768,320 | Pixels per frame on the sharded profile |
+| `VLLM_OMNI_SEEDVR2_SHARDED_CLIP_PIXELS` | 18,841,600, scaled up with device memory | Padded pixels per clip on the sharded profile |
+| `VLLM_OMNI_SEEDVR2_MAX_FRAMES` | 257 | Decoder-work frame cap, all profiles |
+
+The sharded profile applies to any `ulysses_degree` of four or more, so eight
+ranks inherit a budget calibrated on four. Raising these caps trades a 400 for a
+CUDA OOM: the admission check passes and the request fails later inside the DiT
+forward or the VAE decode, which can abort every in-flight request on the engine
+rather than only the oversized one. Before serving with raised caps, restore the
+largest clip they admit once and confirm it completes.
+
+A 362-frame 1536×2688 2x upscale was restored on eight ranks with
+`ulysses_degree=8`, VAE tiling and height sharding, using 4,300,000 for the
+per-frame cap, the memory-scaled clip cap and 2,000 for the frame cap. The long-video
+route sizes its windows from these caps, so the clip cap sets how many frames
+each window covers.
+
+## Long-video restoration
+
+`POST /v1/seedvr2/restore-long` accepts one uploaded 24 FPS video, a blank
+prompt, `size`, `num_frames` (up to 7,200), and optional `loop_input=true` and
+`extra_params={"color_correction_method":"wavelet"}`, which it applies to every window.
+An output frame must fit the sharded per-frame pixel cap. It returns a job ID; poll
+`GET /v1/seedvr2/restore-long/{id}` and download the completed MP4 from
+`GET /v1/seedvr2/restore-long/{id}/content`. `DELETE
+/v1/seedvr2/restore-long/{id}` asks a running job to stop; it settles as
+`cancelled` at the next window boundary. A job whose owning process is gone,
+such as after a server restart, reports `failed` on the next poll.
+
+Jobs live on the API server's local disk, so the route requires
+`--api-server-count 1` and runs one job at a time. Storage is bounded by an
+upload cap and by deleting settled jobs once they age out:
+
+| Variable | Default | Bounds |
+| --- | --- | --- |
+| `VLLM_OMNI_SEEDVR2_LONG_OUTPUT_DIR` | system temp dir | Parent of the job directories |
+| `VLLM_OMNI_SEEDVR2_LONG_MAX_UPLOAD_BYTES` | 8 GiB | Largest accepted upload |
+| `VLLM_OMNI_SEEDVR2_LONG_JOB_TTL_SECONDS` | 3,600 | Age at which a settled job is deleted |
+| `VLLM_OMNI_SEEDVR2_LONG_MAX_WINDOW` | 121 | Longest model window, in frames |
+
+Download a completed job before its TTL expires; the sweep runs on each new
+submission and removes the output with the job directory.
+
+The service sends windows through the existing SeedVR2 endpoint, blends four
+frames at each boundary, and writes one continuous MP4 encoder. Each window is
+the longest 4n+1-frame clip that the sharded clip-pixel cap, the frame cap and
+`VLLM_OMNI_SEEDVR2_LONG_MAX_WINDOW` admit at the output size. Every window pays
+a fixed round trip through the serving stack and recomputes its overlap, and a
+window of four latent frames gets no temporal attention in the DiT, so raise
+the clip cap as far as the device allows. Windows carry source pixels, which
+the model resizes on the device, and cross the endpoint losslessly; only the
+final MP4 is compressed. The next window is restored while the previous one is
+blended and encoded. It repeats source frames and audio only when
+`loop_input=true`. The sharded serving profile above (VAE tiling, VAE height
+sharding, Ulysses degree of at least four) is required. This route keeps a
+window within the whole-clip pixel budget; it does not raise the
+`/v1/videos/sync` 257-frame or pixel limits. The long route
+uses fixed one-step, CFG=1 conditioning and requires `imageio[ffmpeg]` for
+audio muxing. The 7,200-frame 768×1344 case is still undergoing full GPU E2E.
+
+## Temporal and spatial VAE tiling
+
+Add `--vae-use-tiling` to bound VAE intermediate activations along time. The
+encoder processes nine frames first, then eight per chunk; the decoder processes
+two latent frames per chunk. Each causal convolution carries its past inputs
+within that encode/decode call, with temporal stride and first-frame upsampling
+alignment preserved. GroupNorm and attention still see the entire spatial frame.
+There is no overlap blend or independent restart at chunk boundaries.
+
+Five-frame clips use one temporal chunk. On a single GPU, convolutions with
+more than 256 input rows additionally process tiles of 128 output rows with
+exact halos. GroupNorm and bottleneck attention retain full-frame statistics.
+This bounds convolution workspace; full-frame activations and the DiT's
+whole-clip activations still consume memory.
+
+With VAE patch parallelism, each rank owns a band of latent rows and the
+corresponding encoder/decoder rows. Neighbor exchanges supply convolution halos;
+all-reduced centered statistics preserve full-frame GroupNorm. Bottleneck
+attention gathers the low-resolution frame before splitting it again. Encoder
+parameters are gathered before latent sampling, and decoded pixels are gathered
+before returning the video. Unequal bands are supported; fewer latent rows than
+ranks fall back to replicated execution. Width sharding and batch slicing are
+unsupported. FP16 reduction and convolution order can change rounding, so
+parallel outputs are numerically close rather than bitwise identical.
+
+## Current integration scope
+
+| Property | Contract |
+| --- | --- |
+| Weights / dtype | Released 3B DiT and VAE, FP16 |
+| Reference semantics | C0 whole clip, colour correction on by default |
+| Video input | One uploaded file, or offline TCHW RGB floats / PIL frames |
+| Timing | Constant frame rate, increasing PTS; normalize the video origin to zero |
+| Audio | First mono/stereo track, aligned by source PTS, cropped to the video interval, re-encoded as AAC |
+| Randomness | Per-request generator; preserve reference latent strides when sampling noise |
+| Sequence parallelism | Native engine SP1/2/4 correctness verified on five-frame L20 cases |
+| VAE placement | Replicated by default; optional height sharding on the window-SP group |
+| Unsupported | VFR, multichannel audio, 7B, other sampling schedules, quantization, cache acceleration, VAE width sharding / batch slicing, CPU offload, CFG/TP/PP parallelism, compiled execution, LoRA |
+
+Unsupported engine modes are rejected before process hooks and worker creation.
+`ulysses_degree` selects the model-owned SP group. This branch assigns whole
+windows to ranks; the specialized head-sharded path is a dependent change.
+Ring and AllGather-KV are unsupported.
+
+The practical P0 reference's five-frame batching, overlap, and CPU swapping are
+separate execution semantics. Temporal tiling and VAE patch
+parallelism reduce activation peaks but do not bound the whole-clip DiT memory.
+Large output frames may still exceed device capacity.
+
+Invalid inputs return 400. A single-GPU model OOM fails that request; a
+subsequent short request succeeds. With VAE patch parallelism, restart the
+service after an OOM: another rank may still be waiting in a collective. Cancelling an in-progress video job removes it through the
+existing DELETE endpoint; in-flight GPU work may drain before memory is reusable.
+A lost SP worker fails the request and makes health return 503. Restart the
+service to restore availability; automatic rank recovery is not provided.
+
+## Reproducible deployment
+
+See the [RTX 5090 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/ByteDance/SeedVR2-RTX-5090.md) for the
+input/output contract, complete serving command, and media checks. The local
+tests in `tests/diffusion/models/seedvr2/test_seedvr2_e2e.py` exercise the full
+3B pipeline, all colour modes, and HTTP restoration with USP 1 and 8.
@@ -19,9 +19,9 @@ th {

 | Architecture | Models | Example HF Models | NVIDIA GPU | AMD GPU | Ascend NPU | Intel GPU | Recipe |
 | -------------- | -------- | ------------------- | ------------ | --------- | ----- | ----------- | -------- |
-| `Qwen3OmniMoeForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen3-Omni.md) |
+| `Qwen3OmniMoeForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen3-Omni.md) |
 | `Qwen2_5OmniForConditionalGeneration` | Qwen2.5-Omni | `Qwen/Qwen2.5-Omni-7B`, `Qwen/Qwen2.5-Omni-3B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `MingFlashOmniForConditionalGeneration` + `MingImagePipeline` | Ming-flash-omni-2.0 (omni-speech + imagegen<sup>1</sup>) | `Jonathan1909/Ming-flash-omni-2.0` | ✅︎ | | | | — |
+| `MingFlashOmniForConditionalGeneration` + `MingImagePipeline` | Ming-flash-omni-2.0 (omni-speech + imagegen<sup>1</sup>) | `Jonathan1909/Ming-flash-omni-2.0` | ✅︎ | | | ✅︎ | — |
 | `BagelForConditionalGeneration` | BAGEL (DiT-only) | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `InternVLAA1Pipeline` | InternVLA-A1 | `InternRobotics/InternVLA-A1-3B` | ✅︎ | ✅︎ | | | — |
 | `Gr00tN1d7Pipeline` | GR00T N1.7 | `nvidia/GR00T-N1.7-3B` | ✅︎ | | | | — |
@@ -33,9 +33,9 @@ th {
 | `QwenImageLayeredPipeline` | Qwen-Image-Layered | `Qwen/Qwen-Image-Layered` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2511 | `Qwen/Qwen-Image-Edit-2511` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `GlmImagePipeline` | GLM-Image | `zai-org/GLM-Image` | ✅︎ | ✅︎ | | | — |
-| `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `Krea2Pipeline` | Krea 2 (Raw + Turbo) | `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | ✅︎ | | | ✅︎ | — |
-| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tongyi-MAI/Z-Image-Turbo.md) |
+| `Krea2Pipeline` | Krea 2 (Raw + Turbo) | `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/krea/Krea-2.md) |
+| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Wan-AI/Wan2.2-T2V.md) |
 | `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `LingBotWorldCausalDMDPipeline` | LingBot-World 2.0 v1 (experimental) | `robbyant/lingbot-world-v2-14b-causal-fast-diffusers` | Experimental | | | | — |
 | `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V, V2V, T2V with sound, action policy | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
@@ -43,17 +43,18 @@ th {
 | `Wan22S2VPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ | | | ✅︎ | — |
-| `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | — |
-| `LTX2TwoStagePipeline` | LTX-2 / LTX-2.3 ordinary two-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` + matching Lightricks LoRA and upsampler | ✅︎ | ✅︎ | | | — |
-| `LTX2DistilledOneStagePipeline` | LTX-2 / LTX-2.3 merged-distilled one-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` | ✅︎ | ✅︎ | | | — |
-| `LTX2DistilledTwoStagePipeline` | LTX-2 / LTX-2.3 merged-distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` + matching Lightricks upsampler | ✅︎ | ✅︎ | | | — |
-| `LTX2Pipeline` | LTX-2.5 Full/SFT one-stage T2V and I2V | `Lightricks/LTX-2.5-Diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.5.md) |
+| `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.md) |
+| `LTX2TwoStagePipeline` | LTX-2 / LTX-2.3 ordinary two-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` + matching Lightricks LoRA and upsampler | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.md) |
+| `LTX2DistilledOneStagePipeline` | LTX-2 / LTX-2.3 merged-distilled one-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.md) |
+| `LTX2DistilledTwoStagePipeline` | LTX-2 / LTX-2.3 merged-distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` + matching Lightricks upsampler | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.md) |
+| `LTX2Pipeline` | LTX-2.5 Full/SFT one-stage T2V and I2V | `Lightricks/LTX-2.5-Diffusers` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.5.md) |
 | `LTX2TwoStagePipeline` | LTX-2.5 Full/SFT two-stage T2V and I2V | `Lightricks/LTX-2.5-Diffusers` + `Lightricks/LTX-2.5` LoRA | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.5.md) |
 | `LTX2DistilledOneStagePipeline` | LTX-2.5 merged-distilled one-stage T2V and I2V | `Lightricks/LTX-2.5-Diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.5.md) |
 | `LTX2DistilledTwoStagePipeline` | LTX-2.5 merged-distilled two-stage T2V and I2V | `Lightricks/LTX-2.5-Diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.5.md) |
 | `LingBotVideoPipeline` | LingBot-Video dense and MoE T2I, T2V, TI2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | | — |
-| `SanaVideoPipeline` | SANA-Video-2B T2V (native 480p/720p; Diffusers adapter validated at both resolutions) | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers`, `Efficient-Large-Model/SANA-Video_2B_720p_diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/SANA-Video-2B.md) |
+| `SanaVideoPipeline` | SANA-Video-2B T2V (native 480p/720p; Diffusers adapter validated at both resolutions) | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers`, `Efficient-Large-Model/SANA-Video_2B_720p_diffusers` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/SANA-Video-2B.md) |
 | `SanaImageToVideoPipeline` | SANA-Video-2B I2V (native and Diffusers adapter validated at 480p/720p) | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers`, `Efficient-Large-Model/SANA-Video_2B_720p_diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/SANA-Video-2B.md) |
+| `SeedVR2Pipeline` | [SeedVR2 3B](seedvr2.md) video restoration | `ByteDance-Seed/SeedVR2-3B` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/ByteDance/SeedVR2-RTX-5090.md) |
 | `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | ✅︎ | | | [Published](https://recipes.vllm.ai/MiniMaxAI/MiniMax-H3) |
 | `Magi2Pipeline` | MAGI-2 native Preview-stage T2VA and I2VA (272p/540p) | `sand-ai/MAGI-2-preview` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SandAI/MAGI-2-preview-L20X.md) |
 | `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
@@ -65,23 +66,24 @@ th {
 | `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `LongCatVideoAvatarPipeline` | LongCat-Video-Avatar-1.5 A2V/AI2V (native single-speaker and multi-speaker AI2V/AVC) | `meituan-longcat/LongCat-Video-Avatar-1.5` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/meituan-longcat/LongCat-Video-Avatar-1.5.md) |
 | `BooguImagePipeline` | Boogu-Image | `Boogu/Boogu-Image-0.1-Base`, `Boogu/Boogu-Image-0.1-Edit`, `Boogu/Boogu-Image-0.1-Edit-Turbo` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Boogu/Boogu-Image.md) |
-| `BooguImageTurboPipeline` | Boogu-Image Turbo distilled T2I | `Boogu/Boogu-Image-0.1-Turbo` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Boogu/Boogu-Image.md) |
+| `BooguImageTurboPipeline` | Boogu-Image Turbo distilled T2I | `Boogu/Boogu-Image-0.1-Turbo` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Boogu/Boogu-Image.md) |
 | `StableDiffusionXLPipeline` | Stable-Diffusion-XL | `stabilityai/stable-diffusion-xl-base-1.0` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ | — |
+| `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/StabilityAI/Stable-Diffusion-3.5.md) |
 | `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `AuKForConditionalGeneration` | AuK / AuK-Flash (instruction-driven speech generation and editing) | `tencent/AuK`, `tencent/AuK-Flash` (with `Qwen/Qwen2.5-Omni-3B`; assemble with `tools/prepare_auk_checkpoint.py`) | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/AuK-H100.md) |
 | `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | ✅︎ | — |
 | `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | ✅︎ | — |
 | `BreezeForConditionalGeneration` | Breeze-TTS-2 (voice design, voice clone, voice direction, CFG) | `BreezeBlue/Breeze-TTS-2` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/BreezeBlue/Breeze-TTS-2.md) |
-| `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | | — |
+| `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `MammothModa2ForConditionalGeneration` | MammothModa2-Dev (AR-only image understanding) | `bytedance-research/MammothModa2-Dev` | ✅︎ | | | | — |
-| `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | | — |
+| `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/black-forest-labs/FLUX.2-klein.md) |
+| `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `FluxPipeline` | FLUX.1-dev | `black-forest-labs/FLUX.1-dev` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `FluxPipeline` | FLUX.1-schnell | `black-forest-labs/FLUX.1-schnell` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `OmniGen2Pipeline` | OmniGen2 | `OmniGen2/OmniGen2` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `MiniMaxMusic3ForConditionalGeneration` | MiniMax Music 3 (text-to-music) | `MiniMaxAI/MiniMax-Music3` | ✅︎ | | | | — |
+| `Yue2ForCausalLM` | YuE2-3B (text-to-music with ABC plan) | `m-a-p/YuE2-3B` | ✅︎ | | | | — |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | | | ✅︎ | [Published](https://recipes.vllm.ai/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice) |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
@@ -97,16 +99,18 @@ th {
 | `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | | — |
 | `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | | — |
-| `Flux2Pipeline` | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | ✅︎ | ✅︎ | | | — |
+| `Flux2Pipeline` | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | ✅︎ | — |
-| `Audio8TTSSlowARForConditionalGeneration` | Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | ✅︎ | | | | — |
-| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only), SenseNova-U1.5 | `SenseNova/SenseNova-U1-8B-MoT`, `sensenova/SenseNova-U1.5-8B-MoT` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SenseNova/SenseNova-U1.5.md) |
+| `Audio8TTSSlowARForConditionalGeneration` | Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | ✅︎ | | | ✅︎ | — |
+| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only), SenseNova-U1.5 | `SenseNova/SenseNova-U1-8B-MoT`, `sensenova/SenseNova-U1.5-8B-MoT` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SenseNova/SenseNova-U1.5.md) |
+| `SenseNovaU1Pipeline` | SenseNova-U1-A3B (MoE) | `sensenova/SenseNova-U1-A3B-MoT` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SenseNova/SenseNova-U1.md#a3b-moe-variant) |
 | `LancePipeline` | Lance | `bytedance-research/Lance` | ✅︎ | | | | — |
-| `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | | — |
-| `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | | — |
+| `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/HunyuanVideo-1.5.md) |
+| `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/HunyuanVideo-1.5.md) |
 | `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | | — |
 | `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | | — |
 | `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/OpenBMB/MiniCPM-o-4_5.md) |
+| `PersonaPlexTalkerForConditionalGeneration` | PersonaPlex (full duplex, 24 kHz speech in/out over `/v1/realtime?duplex=1`) | `nvidia/personaplex-7b-v1` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/PersonaPlex.md) |
 | `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `GepardTalkerForConditionalGeneration` | Gepard-1.0 | `nineninesix/gepard-1.0` | ✅︎ | | | ✅︎ | — |
 | `HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | | — |
@@ -144,7 +144,7 @@ interchangeably.
 | ---------- | ---------- | ------------------- | --------- |
 | Incremental text input for speech synthesis | `WS /v1/audio/speech/stream` | Send text events and receive audio | [Streaming Text to Speech](speech_api.md#streaming-text-input-websocket) |
 | Live video understanding | `WS /v1/video/chat/stream` | Send video frames and receive text/audio | [Streaming Video Input](video_stream_api.md) |
-| Turn-based realtime audio | `WS /v1/realtime` | Stream one audio input and receive transcript/audio events | [Realtime Audio](realtime_api.md) |
+| Turn-based realtime audio | `WS /v1/realtime` | Stream one audio input and receive transcript/audio events | [Realtime Audio](realtime_openai_duplex_api.md) |
 | Continuous speech-to-speech interaction | `WS /v1/realtime?duplex=1` (alias `WS /v1/duplex`) | Listen and speak concurrently with session control | [Full Duplex](full_duplex_api.md) |
 | Generated video chunks | `WS /v1/realtime/video` | Start a diffusion request and receive fragmented MP4 | [Streaming Video Output](streaming_video_output_api.md) |
 | Robot policy inference | `WS /v1/realtime/robot/openpi` | Send MessagePack observations and receive action arrays | [OpenPI Robot Policy](openpi_api.md) |
@@ -5,7 +5,7 @@ continue receiving speech while producing speech. It adds persistent session
 state, model-specific turn policy, overlap handling, playback acknowledgement,
 and optional session resume.

-Full duplex is distinct from the turn-based [Realtime Audio API](realtime_api.md).
+Full duplex is distinct from the turn-based [Realtime Audio API](realtime_openai_duplex_api.md).

 This page is the endpoint overview. For the complete wire contract, the
 `vllm_omni.clients.duplex.DuplexClient` Python library, and the per-model
@@ -54,7 +54,7 @@ vllm-omni serve openbmb/MiniCPM-o-4_5 \
 ```

 `vllm_omni/deploy/minicpmo_4_5.yaml` declares `session_mode: duplex` and
-`duplex_session.max_sessions: 4`. Because the MiniCPM-o 4.5 pipeline declares a
+`duplex_session.max_sessions: 16`. Because the MiniCPM-o 4.5 pipeline declares a
 `duplex_plugin`, `vllm-omni serve` runs it through `DuplexOmni`: the server
 mounts `ws://<host>:8099/v1/realtime?duplex=1` (this page; `ws://<host>:8099/v1/duplex`
 is an alias of the same route), `POST /v1/chat/completions`, `/v1/models` and
@@ -384,9 +384,9 @@ with no OpenAI counterpart.

 The event vocabulary is uniform, but several surfaces are gated by the
 `capabilities` object the server returns in `session.created`; a client must
-branch on those flags rather than on the model name. MiniCPM-o 4.5 is the
-only model on the plugin contract today; the other two columns record what
-their integrations advertise once the follow-up PRs port them:
+branch on those flags rather than on the model name. MiniCPM-o 4.5 and
+PersonaPlex are on the plugin contract today; the Nemotron VoiceChat column
+records what its integration advertises once the follow-up PR ports it:

 | Capability | MiniCPM-o 4.5 | PersonaPlex | Nemotron VoiceChat | Gated surface |
 | --- | --- | --- | --- | --- |
@@ -401,7 +401,11 @@ their integrations advertise once the follow-up PRs port them:
 Everything else in the catalogue — session lifecycle, heartbeat and event
 acknowledgement, append/commit/clear, the response envelope, playback
 acknowledgement, and the error envelope — behaves identically for every
-model.
+model. Two PersonaPlex specifics follow from its capabilities rather than from
+special-casing: a model with `supports_client_commit=false` auto-responds
+without `extra_body.auto_response`, and `response.cancel` /
+`output_audio_buffer.clear` restart its conversation context (a new Stage 0
+request replays the voice/persona prefill).

 ### Compatibility with the OpenAI Realtime protocol

@@ -1,33 +1,31 @@
 # Realtime Audio WebSocket API

-Use `WS /v1/realtime` to stream PCM audio into a compatible omni model and
-receive incremental transcription and synthesized response audio. This is a
-turn-based streaming API: it starts one generation over an audio stream and
-finishes after the client closes that input stream.
-
-For a model that listens while it is already speaking, use the
-[Full-Duplex API](full_duplex_api.md) instead.
+As a companion to the vLLM-Omni-proprietary [fullduplex `/v1/realtime`](full_duplex_api.md),
+supported models also provide a `/v1/realtime` handler supporting
+a substantial subset of the OpenAI `/v1/realtime` API supporting
+audio streaming, client-driven VAD, and tool calling

 ## Quick Start

-Start a compatible model. The currently documented example uses Qwen3-Omni:
+The endpoint is currently only supported for Qwen3-Omni. To start:
+
+Start vLLM-Omni with port `8000` bound to an address reachable from Docker:

 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8000 --host 0.0.0.0
 ```

-Run the provided client with a mono, 16-bit PCM, 16 kHz WAV file:
+Then start the docker compose:

 ```bash
-python examples/online_serving/qwen3_omni/openai_realtime_client.py \
-  --url ws://localhost:8091/v1/realtime \
-  --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
-  --input-wav input_16k_mono.wav \
-  --output-wav response.wav \
-  --output-text response.txt
+cd examples/online_serving/qwen3_omni/fullduplex-livekit-frontend
+VLLM_OMNI_HOST=$VLLM_OMNI_HOST docker compose up --build
 ```

-The client dependency is `websockets`.
+Where `VLLM_OMNI_HOST` is a network address. If vLLM-Omni is running on localhost, use
+`host.docker.internal`
+
+Open [http://localhost:3000](http://localhost:3000) and begin chatting.

 ## Protocol

@@ -37,10 +35,9 @@ event is base64-encoded raw PCM16.
 | Direction | Event | Purpose |
 | ----------- | ------- | --------- |
 | Server to client | `session.created` | Confirms that the WebSocket connection is ready |
-| Client to server | `session.update` | Selects and validates the served model |
-| Client to server | `input_audio_buffer.commit` with `final: false` | Starts generation over the incoming stream |
+| Client to server | `session.update` | Selects and validates the served model. Configures system prompt and tools |
 | Client to server | `input_audio_buffer.append` | Appends base64 PCM16 audio |
-| Client to server | `input_audio_buffer.commit` with `final: true` | Marks the end of input |
+| Client to server | `input_audio_buffer.commit` | Starts generation over the incoming stream |
 | Server to client | `transcription.delta` | Carries incremental response text |
 | Server to client | `transcription.done` | Carries final text and token usage |
 | Server to client | `response.output_audio.delta` | Carries incremental PCM16 response audio |
@@ -50,15 +47,15 @@ event is base64-encoded raw PCM16.
 A minimal client sends events in this order:

 ```json
-{"type":"session.update","model":"Qwen/Qwen3-Omni-30B-A3B-Instruct"}
-{"type":"input_audio_buffer.commit","final":false}
-{"type":"input_audio_buffer.append","audio":"<base64-pcm16>"}
-{"type":"input_audio_buffer.commit","final":true}
+{"type":"input_audio_buffer.append","audio":"<base64-pcm16>"},
+{"type":"input_audio_buffer.commit"}
 ```

-The initial non-final commit intentionally starts generation before all audio
-has arrived. Continue sending `input_audio_buffer.append` events while the
-engine is consuming the stream.
+This OpenAI-compatible streaming path is distinct from the vLLM-Omni-proprietary
+lane and as such is reached via different configuration. The vLLM-Omni-proprietary
+path must be specified explicitly via "duplex=1|true|on" as the WS query parameter
+or via an explicit configuration in the `deploy_config`. The OpenAI path simply replaces
+the default `/v1/realtime` handler in all supported models.

 ## Audio Handling

@@ -67,8 +64,6 @@ engine is consuming the stream.
 - Read `sample_rate_hz` from each audio event instead of assuming an output
  rate. Qwen3-Omni output is typically 24 kHz.
 - Concatenate audio deltas in receive order to construct the output waveform.
- Set `input_audio_buffer.commit.final` to `true` after the last input chunk so
-  the server can terminate the streaming request.

 ## Availability and Limitations

@@ -76,7 +71,8 @@ The path is registered on the unified API server, but it is usable only when
 the loaded pipeline implements realtime audio input and produces compatible
 audio output. Unsupported deployments return an `error` event. The endpoint
 does not provide duplex session resume, playback acknowledgement, overlap
-policy, or barge-in controls.
+policy, or barge-in controls internally. Those must be driven by the client application
+via the relevant WS events (e.g. `response.cancel`, `conversation.item.truncate`)

 See the [Qwen3-Omni online serving example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/qwen3_omni)
 for concurrency options, chunk pacing, and per-delta audio debugging.
@@ -1029,3 +1029,64 @@ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
    --enforce-eager \
    --uvicorn-log-level debug
 ```
+
+### Reusing reference audio
+
+The voice upload API and cache settings below are shared serving features.
+For repeated voices, upload the reference once and reuse its name. The example
+uses MOSS-TTS Local: uploading stores the audio, and the first speech request
+encodes it. MOSS-TTS Local can then reuse cached reference codes without resolving
+the waveform while the speaker cache entry remains resident. After a server
+restart or speaker-cache eviction, another speech request warms the codes again.
+Reference processing and cache-hit behavior for other models depend on their
+serving adapters.
+
+```bash
+curl --fail http://localhost:8123/v1/audio/voices \
+  -F 'name=my-speaker' \
+  -F 'consent=user_consent_id' \
+  -F 'audio_sample=@/path/to/reference.wav'
+
+# Run once to warm the reference codes, then reuse the same voice.
+curl --fail http://localhost:8123/v1/audio/speech \
+  -H 'Content-Type: application/json' \
+  -d '{"model":"OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5","voice":"my-speaker","input":"Hello, world.","response_format":"wav"}' \
+  --output speech.wav
+```
+
+If you have the reference transcript, upload it with `-F 'ref_text=...'`.
+MOSS-TTS Local uses a reference transcript for continuation conditioning; omitting
+it uses reference-audio conditioning instead. Keep this choice consistent when
+comparing benchmarks. Supplying inline `ref_audio` overrides the uploaded voice
+and does not use the named-voice cache shortcut.
+
+The separate decoded-waveform resolve cache defaults to **4 GiB and 2048 entries
+per speech server instance**. These are LRU limits, not preallocated memory or a
+limit on total server RSS. Waveforms retain their source sampling rate and are
+stored as owned, contiguous float32 arrays in **CPU memory**. The byte limit
+counts numeric buffers (four bytes per mono sample), not the encoded file size
+or GPU memory.
+Either limit can cause eviction. Configure the API-process caches in the deploy
+YAML passed to `--deploy-config` (alongside `stages`, not inside a stage):
+
+```yaml
+speech_cache:
+  resolve_max_bytes: 4294967296
+  resolve_max_entries: 2048
+  speaker_max_bytes: 536870912
+```
+
+All values must be non-negative integers. Either resolve limit set to `0`
+disables waveform storage; `speaker_max_bytes: 0` disables speaker artifact
+storage. Invalid values fail configuration loading. `base_config` inheritance
+merges this section by field, preserving unspecified values from the base.
+Restart the service to apply changes. The effective settings are logged at startup.
+
+The speaker budget defaults to **512 MiB** and counts cached tensor bytes,
+not Python metadata or total process RSS. It configures the API process's
+speaker cache. MOSS reference codes are cached as **CPU int32 tensors**; this
+budget does not reserve GPU memory or include the reference encoder's GPU
+weights and working memory. The shared cache itself does not move tensors
+between devices; other adapters determine the device of their cached artifacts.
+Separate model-worker caches are not controlled by this section. The process-wide singleton rejects a
+conflicting explicit budget instead of silently ignoring it.
@@ -16,6 +16,10 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \
    --trust-remote-code
 ```

+To enable [incremental prefill](#incremental-prefill), set
+`enable_prefix_caching: true` on stage 0 (the thinker) in the deploy config. Otherwise
+the server falls back to rebuilding the prompt per query.
+
 ### Run the Example Client

 ```bash
@@ -54,7 +58,7 @@ WebSocket /v1/video/chat/stream

 When buffered frames include client-supplied `frame_id` values, the server emits `video.frames.consumed` after the first engine output. Its `frame_ids`, `frames`, and `latest_pts_ms` describe the images included in that query's prompt.

-Queries stride-sample the buffered frames, keeping the last frame, then exclude frames already known to have failed decoding. Excluded frames are not replaced with other buffered frames. Frames whose background decoding has not finished remain eligible through the image URL path.
+Queries stride-sample the buffered frames, keeping the last frame, then exclude frames already known to have failed decoding. With incremental prefill active, every buffered frame is included instead of stride-sampling, with the same decode-failure exclusion. Excluded frames are not replaced with other buffered frames. The incremental path waits for the whole window to finish decoding. The legacy path waits only for its sampled positions, so an unselected frame still decoding does not delay the query.

 **Bugfix compatibility note:** `video.frames.consumed` now excludes known decode failures that were already excluded from the prompt. Older versions could report those frames and their timestamps as consumed. The event name and fields are unchanged; an empty selection reports empty lists and `latest_pts_ms: null`.

@@ -64,10 +68,10 @@ Queries stride-sample the buffered frames, keeping the last frame, then exclude
 | ------- | ------ | --------- | ------------- |
 | `model` | string or null | null | Optional model name. Usually omitted because the server hosts one model. |
 | `modalities` | list[string] | `["text", "audio"]` | Output modalities. Use `["text"]`, `["audio"]`, or both. |
-| `num_frames` | integer, 1-128 | `4` | Number of buffered frames sampled for each query. |
-| `max_frames` | integer, 1-256 | `50` | Maximum retained frame buffer size. Oldest frames are evicted first. |
+| `num_frames` | integer, 1-128 | `4` | Number of buffered frames sampled for each query. Legacy path only: with incremental prefill active, every buffered frame is submitted and frame density is the client's responsibility (clients push discrete frames at their own rate). |
+| `max_frames` | integer, 1-256 | `50` | Maximum retained frame buffer size. Below the cap both paths keep every frame. Legacy evicts the oldest frame once full. Incremental prefill drops the oldest half at once, then appends until full again. |
 | `system_prompt` | string or null | null | Optional custom system prompt. |
-| `use_audio_in_video` | bool | `true` | Include streamed audio chunks in multimodal video understanding when audio is present. |
+| `use_audio_in_video` | bool | `true` | Incremental prefill: forwarded on warmup and every query when this is true, so multimodal hashes stay aligned. Legacy: forwarded only when this query has input audio. |
 | `sampling_params_list` | list or null | null | Optional per-stage parameter dictionaries. Each provided entry replaces that stage's deployment sampling settings. |
 | `enable_frame_filter` | bool | `true` | Enable EVS near-duplicate frame filtering. |
 | `frame_filter_threshold` | float, 0.0-1.0 | `0.95` | EVS similarity threshold. Higher keeps more frames; lower drops more near-duplicates. |
@@ -107,9 +111,42 @@ The server accepts these legacy field names and rewrites them before validation.

 EVS compares downsampled frames and drops near-duplicate frames before they enter the session frame buffer. `frame_filter_threshold` controls retention: higher values are more permissive and keep more frames; lower values are more aggressive and drop more similar frames.

+## Incremental Prefill
+
+Server-side only (stage-0 `enable_prefix_caching: true` and `modalities: ["text"]`).
+There is no `session.config` flag. Audio-output sessions stay on the legacy path.
+
+Input is unchanged: `video.frame` and `audio.chunk` are separate buffers. Warmup
+prefills `history + frames` as they arrive (`max_tokens=1`). When
+`use_audio_in_video` is true, warmup and query both pass that processor kwarg.
+`video.query` is the same prompt plus optional input audio and the question text,
+so the warmed vision prefix cache is reused and the query pays for the audio/text
+suffix. All buffered frames are
+submitted in arrival order (`num_frames` subsample is legacy-only). Once the buffer
+hits `max_frames`, the oldest half is dropped in one step and a warning is logged;
+until then the buffer matches the legacy path. After a turn,
+history compresses to the last two text messages and the next warmup follows the
+new prefix.
+
+### Legacy path (incremental prefill inactive)
+
+Stage-0 prefix caching is off, or `modalities` includes `"audio"`: no warmup,
+frames are re-sampled up to `num_frames` each query, and the query waits only
+for those sampled frames to decode. Prompt shape is the same
+(`image_pil` frames, optional trailing `input_audio`, text). Both paths keep the
+frame buffer across queries (`max_frames`) and compress history the same way.
+
 ## Known Limitations

- Session KV reuse and incremental prefill are not implemented in this PR. Each `video.query` rebuilds the model prompt from the retained frame and audio buffers.
+- Past `max_frames`, incremental prefill keeps a suffix of the newest frames, not a resample of the whole stream. Rebuilding that longer window is left to a follow-up.
+- Incremental prefill applies only to text-only sessions with stage-0 prefix caching enabled
+  on `WS /v1/video/chat/stream`. It does not accelerate `/v1/realtime` camera input, which
+  uses duplex conversation items rather than `video.frame` / `video.query`. Audio-output
+  sessions on this endpoint rebuild the prompt each query.
+- Stage-0 prefix caching is safe for audio-output requests: on a hit, the omni prefix cache
+  restores the thinker hidden states the talker needs. A preempted request that resumes on a
+  hit does not get them back, so size the deployment to avoid preemption; see
+  [Prefix Caching](../design/feature/prefix_caching.md).
 - Back-to-back short replies can still expose an engine-layer scheduler race. The PR notes an observed workaround of at least 200 ms idle between turns when clients repeatedly see idle timeouts.
 - If the audio buffer exceeds the server limit, the server emits `Audio buffer overflow` and clears the currently buffered audio for the session.
 - The API is intended for Qwen3-Omni streaming video understanding; other models may not support the same multimodal processor arguments.
@@ -335,6 +335,13 @@ window and output frame count stay unchanged. Wan S2V keeps its existing
 per-clip behavior by default. Zero, negative, fractional, boolean, string, and
 null values are rejected when pre-encoding is enabled.

+The transfer ring applies backpressure before another D2H copy when both of
+its two slots are occupied or the combined pending uint8 payload would exceed
+256 MiB. This byte bound keeps large resolutions and uneven final chunks from
+turning a fixed item count into unexpectedly large memory growth. A native VAE
+chunk larger than 256 MiB is admitted only when the ring is otherwise empty,
+so supported chunk shapes cannot deadlock.
+
 `preencode_mp4` applies to the complete-MP4 response paths only. The
 `/v1/realtime/video` WebSocket endpoint rejects it, because that path already
 overlaps encoding through its own incremental fragmented-MP4 encoder. Wan also
@@ -70,3 +70,42 @@ as safety checks or audio/video packaging.

 See [RFC #6541](https://github.com/vllm-project/vllm-omni/issues/6541) for the
 contract, lifecycle, batching rules, and migration plan.
+
+## Experimental registered-SHM video transport
+
+`video_output_transport.enable_registered_shm` is a separate, default-off option
+for CUDA request-mode workers. It copies eligible video tensors directly into
+CUDA-registered shared memory and returns borrowed CPU views, removing the
+intermediate pinned-to-SHM and SHM-to-consumer host copies. The D2H transfer still
+occurs.
+
+`video_output_transport.enable_borrowed_frames` independently enables borrowed
+RGB AVFrames in the Videos API MP4 response encoder, removing the third host
+copy into PyAV frame buffers. It also defaults to false. Eligible uint8 RGB
+frames are wrapped with `VideoFrame.from_numpy_buffer`; other dtypes/layouts use
+the existing encoder path. Color conversion, compression and muxing still occur.
+
+Enable it through the same CLI, Python or deploy-stage configuration:
+
+```bash
+--video-output-transport '{"enable_registered_shm": true, "enable_borrowed_frames": true}'
+```
+
+Registered SHM applies only to prepared typed video media (currently WAN2.2) and
+explicitly marked legacy video outputs (currently MiniMax-H3). H3 marks the video
+entry of its `(video, audio)` output; audio, trajectory data, images and unmarked
+legacy outputs keep their existing transport. Device-side postprocessing is
+independent and does not need to be enabled for registered SHM.
+Pre-encoded MP4 outputs bypass raw-frame transport and API-side encoding.
+
+The existing SHM routing threshold is unchanged: the tensor or its backing
+storage must exceed 1,000,000 bytes. This is not a measured performance crossover.
+SHM is allocated and registered on every transfer, without pooling; small outputs
+can regress and registration latency can vary. Benchmark representative output
+sizes before opting in. CPU outputs, non-CUDA backends, and synchronous/step-mode
+transfers keep their existing path.
+
+Borrowed views keep the unlinked mapping alive until their last user releases
+it. CUDA registration/copy failures fail the request; if CUDA cleanup cannot be
+confirmed, the worker retains the mapping for safety and must be restarted before
+further registered-SHM transfers.
@@ -163,7 +163,9 @@ The following tables show which models support each feature:
 | **Qwen-Image-Edit**      |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Qwen-Image-Edit-2509** |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |        ✅ (decode)         |          ✅           |       ❌        |        ❌         |
 | **Qwen-Image-Layered**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
-| **SenseNova-U1 / U1.5**  |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **SenseNova-U1**         |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **SenseNova-U1.5**       |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ✅        |        ❌         |
+| **SenseNova-U1-A3B**     |     ❌     |     ❌      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **Stable-Diffusion-XL**  |     ❌     |     ❌      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Stable-Diffusion3.5**  |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Z-Image**              |     ✅     |     ✅      |           ✅           |       ❓        |   ✅ (TP=2 only)   |          ❌          |   ✅    |             ❌             |      ✅ (decode)      |       ✅        |        ❌         |
@@ -179,6 +181,7 @@ The following tables show which models support each feature:
 > 5. HunyuanImage3 supports step execution. Multi-request step execution requires `TORCH_SDPA`; see [Diffusion Execution Modes](diffusion/execution_modes.md#step-execution).
 > 6. BAGEL step execution supports image generation with `bagel.yaml`, `bagel_think.yaml`, and `bagel_single_stage.yaml`; two-stage Thinker execution and explicit single-stage text output remain on their existing complete-request paths. Image requests require `num_inference_steps >= 2`. BAGEL step execution cannot currently be combined with sequence parallelism or a diffusion cache backend; see [Diffusion Execution Modes](diffusion/execution_modes.md#step-execution).
 > 7. MammothModa2 runs its DiT stage on the diffusion runner (`StageExecutionType.DIFFUSION`); Cache-DiT is enabled through the standard diffusion-stage knobs on the stage entry of the deploy YAML (`cache_backend: cache_dit`, plus optional `cache_config` / `enable_cache_dit_summary`). The runner installs the backend at startup and the pipeline adopts it per request. Only the repeated main-layer stack is cached; requests with `text_guidance_scale = 1.0` bypass the cache hooks.
+> 8. SenseNova-U1-A3B: `--cfg-parallel-size 2` and TP=2 both complete. CFG-P covers t2i 2-branch (`cond`+`uncond`), img2img 2-branch (`cond`+`img_cond` when `img_cfg_scale=1`), and img2img 3-branch (`cond`+`img_cond`+`uncond` when `img_cfg_scale` is neither 1 nor equal to `cfg_scale`; 3 branches on 2 ranks dispatch as `[[0, 2], [1]]`). Cache-DiT does not: the adapter is declared on `SenseNovaU1Model`, but discovery walks `_dit_modules` (`language_model` / `SenseNovaU1ForCausalLM`) and raises. Layerwise CPU offload is a no-op for the same reason — that class has no `_layerwise_offload_blocks_attrs`, so the 72.2 GiB expert weights stay resident.

 ### VideoGen

@@ -16,6 +16,7 @@ For the full list of supported architectures across all modalities, see

 | Model | HuggingFace repo | Voice cloning | Streaming | Voice presets / upload | Gradio demo |
 | --- | --- | --- | --- | --- | --- |
+| AuK / AuK-Flash | assembled local bundle | ✓ (`ref_audio`) | HTTP audio after full generation | — | — |
 | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | — |
 | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
 | Gepard-1.0 | `nineninesix/gepard-1.0` | — (zero-shot default voice) | ✓ (PCM / WAV stream) | `default` only | — |
@@ -99,6 +100,41 @@ For full request-shape documentation (all parameters, response formats, error co

 ---

+## AuK / AuK-Flash
+
+AuK serves speech generation and reference-voice synthesis at 24 kHz. Assemble a local bundle before serving:
+
+```bash
+python tools/prepare_auk_checkpoint.py \
+    --auk-dir ckpts/AuK --qwen-dir ckpts/Qwen2.5-Omni-3B --out ckpts/auk-omni
+MODEL=ckpts/auk-omni bash examples/online_serving/text_to_speech/auk/run_server.sh
+```
+
+AuK accepts the complete instruction after you fill in the [local recipe template](../../../../recipes/Tencent/AuK-H100.md#supported-model-contract) or the [official AuK cookbook](https://github.com/Tencent-Hunyuan/AuK/blob/main/docs/COOKBOOK.md). Pass it in `instructions` and leave `input` empty. For instruct TTS, fill in `Generate speech based on the following description: "{voice description}". The content to speak is: "{text}".`; for zero-shot TTS, editing, enhancement and separation, use the corresponding complete instruction from the cookbook. Set `duration_seconds` for text-only generation; with `ref_audio`, omitting it keeps the source length.
+
+For compatibility with clients that can only send `input`, AuK treats an input-only request as the complete instruction and logs a recommendation to move it to `instructions` with `input=""`.
+
+`extra_params` accepts `num_inference_steps`, `guidance_scale`, `sway`, `t_grid`, and `vae_sample`; `seed` is a top-level field.
+
+```bash
+python examples/online_serving/text_to_speech/auk/speech_client.py \
+    --model ckpts/auk-omni \
+    --instructions 'Generate speech based on the following description: "A calm young woman speaking warmly and slowly.". The content to speak is: "Welcome back, how was your day?".' \
+    --duration-seconds 3 --seed 7
+```
+
+Complete-instruction editing also does not need a separate text argument:
+
+```bash
+python examples/online_serving/text_to_speech/auk/speech_client.py \
+    --model ckpts/auk-omni --instructions "Keep pure speech voice, remove noise and reverberation." \
+    --ref-audio noisy.wav --seed 7 --output clean.wav
+```
+
+The client accepts local reference files, URLs, and data URLs. `--stream` writes signed 16-bit 24 kHz PCM after the complete waveform is ready. See `speech_client.py --help` for editing and sampling options.
+
+---
+
 ## CosyVoice3

 2-stage TTS (`talker` + flow-matching `code2wav`) at 24 kHz. Voice cloning only — every request needs `ref_audio` + `ref_text`; there are no built-in voice presets.
@@ -106,6 +106,7 @@ warmup_quack_fp8([(14040, 2048, 6144), (14040, 2048, 2048)])
 | HunyuanVideo-1.5 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `720p_t2v`, `480p_i2v` | Yes | Yes | All layers | None | |
 | Cosmos3 | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | Yes | Not validated | All layers | None | |
 | MiniMax-H3 | `MiniMaxAI/MiniMax-H3` (`FL2VA` / `Ref2VA`) | Yes | Not validated | `quantization="fp8"` quantizes eligible DiT and text-encoder linears; mixed-precision input/output heads stay FP32 | None | ✅︎ |
+| SenseNova-U1.5 | `sensenova/SenseNova-U1.5-8B-MoT` | Yes | Not validated | UND/GEN language-model linears only | None | |

 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)

@@ -1,170 +0,0 @@
-"""Offline Qwen3-TTS example with word-level timestamps.
-
-This demo runs Qwen3-TTS offline, aligns the synthesized audio in memory with
-the same shared forced aligner used by the streaming server path, then writes
-the WAV and JSON sidecar:
-
-    python examples/offline_inference/text_to_speech/qwen3_tts/word_timestamps.py \
-        --model Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
-        --forced-aligner Qwen/Qwen3-ForcedAligner-0.6B
-
-On machines without a local CUDA toolkit, set ``VLLM_USE_FLASHINFER_SAMPLER=0``
-to avoid FlashInfer sampler JIT during warmup.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import json
-import os
-from pathlib import Path
-from typing import Any
-
-os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
-# The in-process offline aligner consumes token_classify outputs. Keeping V1
-# multiprocessing off avoids msgspec serialization issues for token-wise logits.
-os.environ.setdefault("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
-
-import numpy as np
-import soundfile as sf
-import torch
-from end2end import _estimate_prompt_len
-from vllm.utils.argparse_utils import FlexibleArgumentParser
-
-from vllm_omni import Omni
-from vllm_omni.utils.forced_aligner import align, build_forced_aligner_config
-
-
-def _default_deploy_config() -> str:
-    repo_root = Path(__file__).resolve().parents[4]
-    return str(repo_root / "vllm_omni" / "deploy" / "qwen3_tts.yaml")
-
-
-def _build_custom_voice_input(args: Any) -> dict[str, Any]:
-    additional_information = {
-        "task_type": ["CustomVoice"],
-        "text": [args.text],
-        "language": [args.language],
-        "speaker": [args.speaker],
-        "instruct": [args.instructions],
-        "max_new_tokens": [args.max_new_tokens],
-    }
-    return {
-        "prompt_token_ids": [0] * _estimate_prompt_len(additional_information, args.model),
-        "additional_information": additional_information,
-    }
-
-
-def _audio_tensor_and_sample_rate(mm: dict[str, Any]) -> tuple[torch.Tensor, int]:
-    audio_data = mm["audio"]
-    sr_raw = mm["sr"]
-    sr_val = sr_raw[-1] if isinstance(sr_raw, list) and sr_raw else sr_raw
-    sample_rate = int(sr_val.item()) if hasattr(sr_val, "item") else int(sr_val)
-    audio_tensor = torch.cat(audio_data, dim=-1) if isinstance(audio_data, list) else audio_data
-    return audio_tensor.float().detach().cpu().flatten(), sample_rate
-
-
-def _float_audio_to_pcm16_bytes(audio: torch.Tensor) -> bytes:
-    samples = audio.numpy()
-    pcm = (np.clip(samples, -1.0, 1.0) * 32767.0).astype("<i2")
-    return pcm.tobytes()
-
-
-async def _run_alignment(args: Any, audio: torch.Tensor, sample_rate: int) -> list[dict[str, Any]] | None:
-    aligner_config = build_forced_aligner_config(args)
-    if aligner_config is None:
-        raise ValueError("--forced-aligner or --forced-aligner-config is required")
-    timestamps = await align(
-        audio=_float_audio_to_pcm16_bytes(audio),
-        text=args.text,
-        sample_rate=sample_rate,
-        config=aligner_config,
-        language=args.language,
-    )
-    if timestamps is None:
-        return None
-    return [
-        {
-            "word": item.word,
-            "start_ms": item.start_ms,
-            "end_ms": item.end_ms,
-        }
-        for item in timestamps
-    ]
-
-
-def main(args: Any) -> None:
-    output_dir = Path(args.output_dir)
-    output_dir.mkdir(parents=True, exist_ok=True)
-
-    omni_kwargs = vars(args).copy()
-    for key in (
-        "forced_aligner",
-        "text",
-        "language",
-        "speaker",
-        "instructions",
-        "max_new_tokens",
-        "output_dir",
-    ):
-        omni_kwargs.pop(key, None)
-    omni_kwargs["deploy_config"] = args.deploy_config or _default_deploy_config()
-    omni_kwargs["log_stats"] = args.log_stats
-    omni = Omni(**omni_kwargs)
-
-    prompt = _build_custom_voice_input(args)
-    final_output = None
-    for stage_outputs in omni.generate([prompt]):
-        final_output = stage_outputs
-    if final_output is None:
-        raise RuntimeError("Qwen3-TTS did not produce an output.")
-
-    mm = final_output.outputs[0].multimodal_output
-    audio, sample_rate = _audio_tensor_and_sample_rate(mm)
-
-    wav_path = output_dir / "qwen3_tts_word_timestamps.wav"
-    # Align the in-memory tensor, not the saved WAV. The WAV is only a demo
-    # artifact so users can listen to the same audio referenced by the sidecar.
-    timestamps = asyncio.run(_run_alignment(args, audio, sample_rate))
-    sf.write(wav_path, audio.numpy(), samplerate=sample_rate, format="WAV")
-
-    sidecar = {
-        "text": args.text,
-        "sample_rate": sample_rate,
-        "audio_path": str(wav_path),
-        "timestamps": timestamps,
-    }
-    json_path = output_dir / "qwen3_tts_word_timestamps.json"
-    json_path.write_text(json.dumps(sidecar, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
-
-    print(f"Saved audio: {wav_path}")
-    print(f"Saved timestamps: {json_path}")
-    print(json.dumps(sidecar["timestamps"], ensure_ascii=False, indent=2))
-
-
-def parse_args() -> Any:
-    parser = FlexibleArgumentParser(description="Offline Qwen3-TTS word timestamp example")
-    parser.add_argument("--model", default="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", help="Qwen3-TTS model path/name")
-    parser.add_argument(
-        "--forced-aligner",
-        default=None,
-        help="Qwen3 forced aligner model path/name",
-    )
-    parser.add_argument(
-        "--forced-aligner-config",
-        default=None,
-        help="Optional YAML file for forced aligner settings (incl. gpu_memory_utilization)",
-    )
-    parser.add_argument("--deploy-config", default=None, help="Qwen3-TTS deploy YAML")
-    parser.add_argument("--text", default="Hello world.", help="Text to synthesize and align")
-    parser.add_argument("--language", default="English", help="Qwen3-TTS language field")
-    parser.add_argument("--speaker", default="Vivian", help="CustomVoice speaker name")
-    parser.add_argument("--instructions", default="", help="Optional speaking style instruction")
-    parser.add_argument("--max-new-tokens", type=int, default=2048, help="TTS max_new_tokens")
-    parser.add_argument("--output-dir", default="output_audio", help="Directory for WAV and JSON sidecar")
-    parser.add_argument("--log-stats", action="store_true", default=False, help="Enable vLLM-Omni stats logging")
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    main(parse_args())
@@ -0,0 +1,190 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""YuE2-3B offline end-to-end text-to-music.
+
+One song is one or two engine requests against the single-stage pipeline:
+``cot=full|melody`` first asks the model to write an ABC score (the abc phase,
+tokens only), then the semantic phase generates codec tokens and the model
+solves the acoustic ODE + decodes 48 kHz stereo as the request finishes.
+``cot=off`` and a user-supplied ``--abc-file`` skip the first request.
+Prefix caching lets the second request reuse the first one's blocks.
+
+Example:
+    python end2end.py --model /models/YuE2-3B --vae /models/YuE2-Vae \
+        --style "gentle lo-fi hip-hop, 85 BPM, F minor" \
+        --lyrics "[Verse]\nHumming on a quiet street\n[Chorus]\nStay a while" \
+        --cot full --seed 831001 --max-frames 200 --output song.wav
+"""
+
+from __future__ import annotations
+
+import argparse
+import copy
+from pathlib import Path
+
+import soundfile as sf
+import torch
+
+from vllm_omni import Omni
+from vllm_omni.model_executor.models.yue2.yue2 import (
+    ABC_SAMPLING,
+    CONTEXT,
+    KEY_MAX_AUDIO_FRAMES,
+    KEY_MAX_HOLD_STEPS,
+    KEY_MIN_TOKENS,
+    KEY_PENALTY_WINDOW,
+    KEY_PHASE,
+    KEY_PREFIX_IDS,
+    KEY_REPETITION_PENALTY,
+    KEY_SEED,
+    KEY_SKIP_SYNTHESIS,
+    KEY_TEMPERATURE,
+    KEY_TOP_K,
+    KEY_TOP_P,
+    SEMANTIC_SAMPLING,
+    STOP_TOKEN_IDS,
+    SYNTHESIS_HOLD_STEPS,
+)
+from vllm_omni.tokenizers.yue2_prompt import (
+    abc_ids_from_generated,
+    abc_prefix_ids,
+    semantic_frames,
+    semantic_prefix_ids,
+)
+from vllm_omni.tokenizers.yue2_tokenizer import YuE2TextTokenizer
+
+
+def sampling_params(engine, *, phase, seed, max_frames, prompt_ids):
+    """Deep-copy the stage defaults, then set the yue2_* keys for one phase."""
+    params = copy.deepcopy(engine.resolve_sampling_params_list(None))[0]
+    preset = ABC_SAMPLING if phase == "abc" else SEMANTIC_SAMPLING
+    params.extra_args = {
+        **(params.extra_args or {}),
+        KEY_PHASE: phase,
+        KEY_SEED: seed,
+        KEY_TEMPERATURE: preset["temperature"],
+        KEY_TOP_P: preset["top_p"],
+        KEY_TOP_K: preset["top_k"],
+        KEY_REPETITION_PENALTY: preset["repetition_penalty"],
+        KEY_PENALTY_WINDOW: preset["penalty_window"],
+        KEY_MIN_TOKENS: preset["min_tokens"],
+        KEY_MAX_AUDIO_FRAMES: max_frames,
+        KEY_SKIP_SYNTHESIS: phase == "abc",
+        # Full prompt ids: under a KV prefix-cache hit the engine schedules
+        # only the uncached tail, so the model cannot rebuild its NAR
+        # conditioning prefix from the scheduled tokens alone.
+        KEY_PREFIX_IDS: list(prompt_ids),
+    }
+    if phase == "abc":
+        params.max_tokens = preset["max_tokens"]
+    else:
+        # The song is synthesized on a side stream while the request is held;
+        # leave max_tokens headroom for the hold tokens.
+        params.max_tokens = min(max_frames + 1 + SYNTHESIS_HOLD_STEPS, CONTEXT - len(prompt_ids))
+        params.extra_args[KEY_MAX_HOLD_STEPS] = max(0, params.max_tokens - max_frames - 1)
+    params.stop_token_ids = list(STOP_TOKEN_IDS)
+    params.detokenize = False
+    return params
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--model", required=True, help="m-a-p/YuE2-3B checkpoint dir or repo id")
+    parser.add_argument("--vae", default=None, help="m-a-p/YuE2-Vae dir or repo id (or $YUE2_VAE)")
+    parser.add_argument("--style", required=True)
+    parser.add_argument("--lyrics", required=True)
+    parser.add_argument("--cot", choices=["off", "melody", "full"], default="full")
+    parser.add_argument("--seed", type=int, default=831001)
+    parser.add_argument("--max-frames", type=int, default=200, help="semantic frame budget (25 frames = 1 s)")
+    parser.add_argument("--abc-file", default=None, help="external ABC score (requires cot=melody/full)")
+    parser.add_argument("--output", default="yue2_song.wav")
+    parser.add_argument(
+        "--dump-tokens",
+        default=None,
+        metavar="PATH",
+        help="write prompt + generated token ids to this json (default: <output>.tokens.json)",
+    )
+    parser.add_argument(
+        "--gpu-memory-utilization",
+        type=float,
+        default=None,
+        help="override the deploy yaml's per-stage gpu_memory_utilization "
+        "(necessary on cards shared with other tenants: the yaml fraction is "
+        "of the WHOLE card, not of the free remainder)",
+    )
+    args = parser.parse_args()
+
+    if args.vae:
+        import os
+
+        os.environ["YUE2_VAE"] = args.vae
+
+    tokenizer = YuE2TextTokenizer(Path(args.model) / "qwen.tiktoken")
+    engine_kwargs: dict = {}
+    if args.gpu_memory_utilization is not None:
+        engine_kwargs["gpu_memory_utilization"] = args.gpu_memory_utilization
+    engine = Omni(model=args.model, **engine_kwargs)
+
+    abc_ids = None
+    if args.abc_file is not None:
+        if args.cot == "off":
+            raise SystemExit("--abc-file requires --cot melody|full")
+        abc_ids = tokenizer.encode(Path(args.abc_file).read_text())
+        print(f"Using supplied ABC score ({len(abc_ids)} tokens)")
+    elif args.cot != "off":
+        prompt_ids = abc_prefix_ids(tokenizer.encode, args.style, args.lyrics, args.cot)
+        prompt = {"prompt_token_ids": prompt_ids}
+        params = sampling_params(engine, phase="abc", seed=args.seed, max_frames=args.max_frames, prompt_ids=prompt_ids)
+        outputs = engine.generate([prompt], [params])
+        generated = list(outputs[0].outputs[0].token_ids)
+        abc_ids = abc_ids_from_generated(generated)
+        print(f"Generated ABC score ({len(abc_ids)} tokens):\n{tokenizer.decode(abc_ids)}\n")
+
+    prompt_ids = semantic_prefix_ids(tokenizer.encode, args.style, args.lyrics, args.cot, abc_ids=abc_ids)
+    prompt = {"prompt_token_ids": prompt_ids}
+    params = sampling_params(
+        engine, phase="semantic", seed=args.seed, max_frames=args.max_frames, prompt_ids=prompt_ids
+    )
+    outputs = engine.generate([prompt], [params])
+    output = outputs[0].outputs[0]
+    generated_ids = list(output.token_ids)
+    if args.dump_tokens:
+        import json
+
+        payload = {
+            "prompt_token_ids": prompt_ids,
+            "generated_token_ids": generated_ids,
+            # Codec frames only: without the end token and the hold tokens
+            # the request emits while its song is synthesized.
+            "semantic_frames": semantic_frames(generated_ids),
+            "seed": args.seed,
+            "cot": args.cot,
+            "max_frames": args.max_frames,
+        }
+        dump_path = args.dump_tokens if args.dump_tokens.endswith(".json") else args.output + ".tokens.json"
+        Path(dump_path).write_text(json.dumps(payload))
+        print(f"Dumped {len(generated_ids)} generated token ids to {dump_path}")
+    mm = output.multimodal_output or {}
+    audio = mm.get("audio")
+    if audio is None and "model_outputs" in mm:
+        audio = mm["model_outputs"]
+    if audio is None:
+        raise SystemExit("No audio in the model output; check server logs")
+    sr = int(mm.get("sr").item()) if mm.get("sr") is not None else 48000
+    truncated = False
+    meta = mm.get("meta") or {}
+    if "truncated" in meta:
+        truncated = bool(torch.as_tensor(meta["truncated"]).reshape(-1)[-1].item())
+
+    waveform = audio.float()
+    # soundfile wants [frames, channels] float in [-1, 1]; torchaudio has no
+    # wheel matching vllm 0.29.0's torch pin, so the driver uses libsndfile.
+    sf.write(args.output, waveform.T.contiguous(), sr)
+    print(
+        f"Saved {args.output}: {waveform.shape[-1] / sr:.1f}s @ {sr} Hz stereo, "
+        f"truncated={truncated}, generated {len(output.token_ids)} semantic tokens"
+    )
+
+
+if __name__ == "__main__":
+    main()
@@ -2,15 +2,15 @@

 Serve [`nvidia/personaplex-7b-v1`](https://huggingface.co/nvidia/personaplex-7b-v1)
 (a Moshi-based full-duplex speech-to-speech model) with the native vLLM-Omni engine
-through the unified duplex serving stack (`/v1/duplex` and `/v1/realtime?duplex=1`).
+through the unified full-duplex framework (`/v1/realtime?duplex=1`, alias `/v1/duplex`).

 > Requires a GPU and Hugging Face access to the gated repo
 > (`HF_TOKEN` with access to `nvidia/personaplex-7b-v1`).

 ## Start the server

-The default `vllm_omni/deploy/personaplex.yaml` enables the engine-owned
-full-duplex control plane (`session_mode: duplex`):
+The default `vllm_omni/deploy/personaplex.yaml` is a duplex deployment
+(`session_mode: duplex`, two sessions per replica):

 ```bash
 HF_TOKEN=... CUDA_VISIBLE_DEVICES=0 python -m vllm_omni.entrypoints.cli.main serve \
@@ -19,18 +19,32 @@ HF_TOKEN=... CUDA_VISIBLE_DEVICES=0 python -m vllm_omni.entrypoints.cli.main ser
  --deploy-config vllm_omni/deploy/personaplex.yaml
 ```

-This exposes:
+This exposes `WS /v1/realtime?duplex=1` (alias `WS /v1/duplex`): the OpenAI
+Realtime session protocol projected onto vLLM-Omni duplex sessions (client API and
+wire protocol: [`docs/serving/realtime_duplex_api.md`](../../../docs/serving/realtime_duplex_api.md)).
+There is no `/v1/chat/completions` route: PersonaPlex answers speech only.

- `WS /v1/duplex` — the native duplex session dialect
-  (`session.create` / `input_audio_buffer.append` / `response.output_audio.delta` ...);
- `WS /v1/realtime?duplex=1` — the same sessions projected onto the
-  OpenAI Realtime event vocabulary (client API and wire protocol:
-  [`docs/serving/realtime_duplex_api.md`](../../../docs/serving/realtime_duplex_api.md)).
+PersonaPlex is a pure-lockstep model: every session is native duplex, audio flows
+continuously in both directions in 80 ms frames, the model decides when to speak,
+and there are no client commits or external turn signals
+(`supports_client_commit=false`, `supports_external_turn_signal=false`). A session
+therefore auto-responds without any vendor flag.

-PersonaPlex is a pure-lockstep model: every session on a PersonaPlex deployment is
-native duplex (`is_enabled()` is unconditionally true), audio flows continuously in
-both directions, and there are no client commits or external turn signals
-(`supports_client_commit=false`, `supports_external_turn_signal=false`).
+## Talk to it
+
+With the client library and the PersonaPlex preset (24 kHz `pcm_f32le` in, bundled
+voice prompt, persona text):
+
+```python
+from vllm_omni.clients.duplex import DuplexClient
+from vllm_omni.clients.personaplex import create_duplex_session_config
+
+cfg = create_duplex_session_config(voice="NATF2.pt", persona="You are a concise assistant.")
+async with DuplexClient("ws://127.0.0.1:8000/v1/realtime?duplex=1", model="/path/to/personaplex-7b-v1", config=cfg) as c:
+    await c.stream_pcm(pcm_f32le_24k)          # keep streaming; the model speaks while it listens
+```
+
+Voice and persona are fixed for the session (`session.update` cannot change them).

 ## Validate the serving path

@@ -45,9 +59,11 @@ python tests/e2e/online_serving/personaplex_realtime_duplex.py \
  --output-dir /tmp/personaplex-realtime-duplex
 ```

-The unified endpoint advertises `supports_barge_in=false`: overlapping speech
-is native model behavior, but destructive output interruption and model-state
-rewind have not been validated for PersonaPlex.
+The endpoint advertises `supports_barge_in=false`: overlapping speech is native
+model behaviour, but destructive output interruption and model-state rewind have
+not been validated for PersonaPlex. `response.cancel` and
+`output_audio_buffer.clear` restart the model's conversation context (a fresh
+Stage 0 request replays the voice/persona prefill).

 ## Notes

@@ -56,8 +72,8 @@ rewind have not been validated for PersonaPlex.
  regardless of engine speed. On localhost it is smooth.
 - The earlier standalone Moshi-web compatibility server (browser client at `/`,
  binary WS protocol at `/api/chat`, raw-PCM `/v1/audio/duplex`) was demo-only
-  and has been removed; use the unified endpoints above.
- Session config (voice / persona / sampling) is passed per session via
-  `extra_body`; see
-  `vllm_omni/model_executor/models/personaplex/duplex/serving_adapter.py`.
+  and has been removed; use the unified endpoint above.
+- The model plugin, worker-side lockstep runtime and input framing live in
+  `vllm_omni/model_executor/models/personaplex/duplex/`; design notes in
+  [`docs/design/fullduplex-personaplex.md`](../../../docs/design/fullduplex-personaplex.md).
 - Full runbook: [`recipes/NVIDIA/PersonaPlex.md`](../../../recipes/NVIDIA/PersonaPlex.md).
@@ -0,0 +1,6 @@
+agent-starter-react/
+.git/
+.claude/
+__pycache__/
+*.py[cod]
+*.log
@@ -0,0 +1,25 @@
+# syntax=docker/dockerfile:1
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+FROM python:3.11-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y ca-certificates ffmpeg libsndfile1 \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN pip install \
+    "livekit-agents[openai,turn-detector,silero]>=1.6.10"
+
+RUN useradd --create-home --uid 10001 agent
+
+WORKDIR /app
+COPY --chown=agent:agent agent.py ./agent.py
+
+USER agent
+
+ENTRYPOINT ["python", "agent.py", "start"]
@@ -0,0 +1,36 @@
+# syntax=docker/dockerfile:1
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+FROM node:24-bookworm-slim AS dependencies
+
+ARG FRONTEND_REF=main
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y ca-certificates git \
+    && rm -rf /var/lib/apt/lists/* \
+    && npm install --global pnpm@9.15.9
+
+WORKDIR /app
+RUN git clone --depth 1 --branch "${FRONTEND_REF}" \
+    https://github.com/livekit-examples/agent-starter-react.git . \
+    && pnpm install --frozen-lockfile \
+    && rm -rf .git
+
+FROM node:24-bookworm-slim
+
+ENV NODE_ENV=development \
+    NEXT_TELEMETRY_DISABLED=1
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y ca-certificates \
+    && rm -rf /var/lib/apt/lists/* \
+    && npm install --global pnpm@9.15.9
+
+WORKDIR /app
+COPY --from=dependencies --chown=node:node /app /app
+
+USER node
+
+EXPOSE 3000
+CMD ["pnpm", "dev", "--hostname", "0.0.0.0"]
@@ -0,0 +1,28 @@
+# syntax=docker/dockerfile:1
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+FROM golang:1.26-alpine AS builder
+
+ARG LIVEKIT_REF=v1.10.1
+ARG TARGETARCH
+
+RUN apk add --no-cache git ca-certificates
+
+WORKDIR /src
+RUN git clone --depth 1 --branch "${LIVEKIT_REF}" \
+    https://github.com/livekit/livekit.git . \
+    && go mod download \
+    && CGO_ENABLED=0 GOOS=linux GOARCH="${TARGETARCH}" \
+        go build -trimpath -ldflags="-s -w" -o /out/livekit-server ./cmd/server
+
+FROM alpine:3.22
+
+RUN apk add --no-cache ca-certificates tzdata \
+    && addgroup -S livekit \
+    && adduser -S -G livekit livekit
+
+COPY --from=builder /out/livekit-server /usr/local/bin/livekit-server
+
+USER livekit
+ENTRYPOINT ["/usr/local/bin/livekit-server"]
@@ -0,0 +1,32 @@
+# Qwen3-Omni LiveKit demo
+
+This directory contains a Docker Compose version of the local LiveKit demo.
+It builds the LiveKit server, the Next.js frontend, and the Python agent as
+separate images. The vLLM-Omni server runs outside Compose and is configured
+with `VLLM_OMNI_HOST`.
+
+Start vLLM-Omni with port `8000` bound to an address reachable from Docker,
+for example with `--host 0.0.0.0`, then run:
+
+```bash
+cd examples/online_serving/qwen3_omni/fullduplex-livekit-frontend
+VLLM_OMNI_HOST=host.docker.internal docker compose up --build
+```
+
+Open [http://localhost:3000](http://localhost:3000). The LiveKit server is
+available at `ws://localhost:7880`; its development credentials are `devkey`
+and `secret`.
+
+For a machine other than the local host, set both the vLLM address and the
+address LiveKit advertises to WebRTC clients:
+
+```bash
+VLLM_OMNI_HOST=192.168.1.20 \
+LIVEKIT_NODE_IP=192.168.1.10 \
+LIVEKIT_PUBLIC_URL=ws://192.168.1.10:7880 \
+docker compose up --build
+```
+
+Use `docker compose up --build --abort-on-container-exit` when the Compose
+process should stop the remaining services after one component exits, matching
+the behavior of `run-livekit-stack.sh`.
@@ -0,0 +1,91 @@
+#!/usr/bin/env -S uv run --script
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "livekit-agents[openai,turn-detector,silero]>=1.6.10",
+# ]
+# ///
+import logging
+import os
+
+from livekit.agents import (
+    Agent,
+    AgentServer,
+    AgentSession,
+    JobContext,
+    RunContext,
+    TurnHandlingOptions,
+    cli,
+    function_tool,
+    inference,
+)
+from livekit.plugins.openai.realtime import RealtimeModel
+
+logger = logging.getLogger("voice-assistant")
+
+VLLM_BASE_URL = f"http://{os.environ['VLLM_OMNI_HOST']}:8000/v1"
+LIVEKIT_URL = os.environ.get("LIVEKIT_URL", "ws://localhost:7880")
+
+server = AgentServer(
+    ws_url=LIVEKIT_URL,
+    api_key=os.environ.get("LIVEKIT_API_KEY", "devkey"),
+    api_secret=os.environ.get("LIVEKIT_API_SECRET", "secret"),
+)
+
+
+class VoiceAssistant(Agent):
+    @function_tool()
+    async def lookup_weather(
+        self,
+        context: RunContext,
+        location: str,
+    ) -> dict[str, str | int]:
+        """Return deterministic sample data for the demo tool."""
+        return {
+            "location": location,
+            "weather": "clear skies",
+            "temperature_f": 72,
+        }
+
+    def __init__(self) -> None:
+        super().__init__(
+            instructions="You are a helpful voice assistant. Respond naturally and concisely.",
+        )
+
+
+@server.rtc_session()
+async def entrypoint(ctx: JobContext):
+    # Qwen3-Omni only supports turn_detection=null (manual mode) -- it does
+    # not perform voice activity detection itself, so turn-taking must be
+    # decided client-side. semantic_vad (MiniCPM-o only) is rejected by
+    # vLLM-Omni's OpenAI-compatible realtime endpoint for Qwen3-Omni.
+    # Omni serves one model, so an empty model lets the server select it.
+    model = RealtimeModel(
+        base_url=VLLM_BASE_URL,
+        model="",
+        api_key="unused",
+    )
+
+    # turn_handling's client-side TurnDetector means the server isn't asked
+    # to decide turns, so turn_detection is left NOT_GIVEN on RealtimeModel.
+    session = AgentSession(
+        llm=model,
+        turn_handling=TurnHandlingOptions(turn_detection=inference.TurnDetector()),
+    )
+
+    await session.start(
+        agent=VoiceAssistant(),
+        room=ctx.room,
+    )
+
+    logger.info(
+        "Voice assistant started, connected to vLLM-Omni at %s",
+        VLLM_BASE_URL,
+    )
+
+
+if __name__ == "__main__":
+    cli.run_app(server)
@@ -0,0 +1,55 @@
+name: vllm-omni-livekit
+
+services:
+  livekit:
+    build:
+      context: .
+      dockerfile: Dockerfile.livekit
+      args:
+        LIVEKIT_REF: ${LIVEKIT_REF:-v1.10.1}
+    command:
+      - --config
+      - /etc/livekit.yaml
+      - --dev
+      - --node-ip
+      - ${LIVEKIT_NODE_IP:-127.0.0.1}
+    volumes:
+      - ./livekit.yaml:/etc/livekit.yaml:ro
+    ports:
+      - "${LIVEKIT_HTTP_PORT:-7880}:7880"
+      - "${LIVEKIT_TCP_PORT:-7881}:7881/tcp"
+      - "${LIVEKIT_UDP_PORT:-7882}:7882/udp"
+    restart: unless-stopped
+
+  frontend:
+    build:
+      context: .
+      dockerfile: Dockerfile.frontend
+      args:
+        FRONTEND_REF: ${FRONTEND_REF:-main}
+    environment:
+      NODE_ENV: development
+      LIVEKIT_API_KEY: ${LIVEKIT_API_KEY:-devkey}
+      LIVEKIT_API_SECRET: ${LIVEKIT_API_SECRET:-secret}
+      LIVEKIT_URL: ${LIVEKIT_PUBLIC_URL:-ws://localhost:7880}
+      AGENT_NAME: ${AGENT_NAME:-}
+    ports:
+      - "${FRONTEND_PORT:-3000}:3000"
+    depends_on:
+      - livekit
+    restart: unless-stopped
+
+  agent:
+    build:
+      context: .
+      dockerfile: Dockerfile.agent
+    environment:
+      VLLM_OMNI_HOST: ${VLLM_OMNI_HOST:-host.docker.internal}
+      LIVEKIT_URL: ws://livekit:7880
+      LIVEKIT_API_KEY: ${LIVEKIT_API_KEY:-devkey}
+      LIVEKIT_API_SECRET: ${LIVEKIT_API_SECRET:-secret}
+    extra_hosts:
+      - host.docker.internal:host-gateway
+    depends_on:
+      - livekit
+    restart: unless-stopped
@@ -0,0 +1,10 @@
+# Local development configuration for the Compose stack.
+port: 7880
+bind_addresses:
+  - 0.0.0.0
+rtc:
+  tcp_port: 7881
+  udp_port: 7882
+  use_external_ip: false
+keys:
+  devkey: secret
@@ -1,398 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
-
-"""Realtime client for vLLM-Omni /v1/realtime (audio + text events).
-
-This client:
-1) Reads a local WAV file (must be mono, 16-bit PCM, 16kHz),
-2) Streams PCM16 chunks to /v1/realtime with OpenAI-style events,
-3) Receives response.output_audio.*, response.output_text.*, and transcript events,
-4) Saves synthesized audio to an output WAV file and optional text file.
-
-By default each ``response.output_audio.delta`` is treated as an **incremental PCM**
-chunk and all chunks are concatenated into the final ``--output-wav``.
-
-Optional debugging: pass ``--delta-dump-dir DIR`` to write every
-``response.output_audio.delta`` payload as ``delta_000001.wav``, ``delta_000002.wav``, …
-
-Usage:
-  python openai_realtime_client.py \
-      --url ws://localhost:8091/v1/realtime \
-      --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
-      --input-wav input_16k_mono.wav \
-      --output-wav realtime_output.wav \
-      --server-vad \
-      --delta-dump-dir ./rt_delta_wavs
-
-Dependencies:
-  pip install websockets
-"""
-
-from __future__ import annotations
-
-import argparse
-import asyncio
-import base64
-import json
-import wave
-from pathlib import Path
-from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
-
-try:
-    import websockets
-except ImportError:
-    print("Please install websockets: pip install websockets")
-    raise SystemExit(1)
-
-
-def _read_wav_pcm16(path: Path) -> bytes:
-    with wave.open(str(path), "rb") as wf:
-        nchannels = wf.getnchannels()
-        sampwidth = wf.getsampwidth()
-        framerate = wf.getframerate()
-        comptype = wf.getcomptype()
-        nframes = wf.getnframes()
-
-        if nchannels != 1:
-            raise ValueError(f"Input WAV must be mono (got {nchannels} channels).")
-        if sampwidth != 2:
-            raise ValueError(f"Input WAV must be 16-bit PCM (got sample width={sampwidth}).")
-        if framerate != 16000:
-            raise ValueError(f"Input WAV must be 16kHz (got {framerate} Hz).")
-        if comptype != "NONE":
-            raise ValueError(f"Input WAV must be uncompressed PCM (got comptype={comptype}).")
-        if nframes <= 0:
-            raise ValueError("Input WAV has no audio frames.")
-
-        return wf.readframes(nframes)
-
-
-def _write_wav_pcm16(path: Path, pcm16_bytes: bytes, sample_rate_hz: int) -> None:
-    with wave.open(str(path), "wb") as wf:
-        wf.setnchannels(1)
-        wf.setsampwidth(2)
-        wf.setframerate(sample_rate_hz)
-        wf.writeframes(pcm16_bytes)
-
-
-def _with_realtime_route(url: str, *, server_vad: bool) -> str:
-    parts = urlsplit(url)
-    query = [(key, value) for key, value in parse_qsl(parts.query, keep_blank_values=True) if key != "duplex"]
-    if not server_vad:
-        query.append(("duplex", "0"))
-    return urlunsplit(parts._replace(query=urlencode(query)))
-
-
-async def run_client(
-    url: str,
-    model: str,
-    input_wav: Path,
-    output_wav: Path,
-    output_text: Path | None,
-    chunk_ms: int,
-    send_delay_ms: int,
-    delta_dump_dir: Path | None,
-    server_vad: bool,
-    request_idx: int = 1,
-    total_requests: int = 1,
-) -> None:
-    log_prefix = f"[req {request_idx:02d}/{total_requests:02d}] " if total_requests > 1 else ""
-    pcm16 = _read_wav_pcm16(input_wav)
-    bytes_per_ms = 16000 * 2 // 1000  # mono PCM16 at 16kHz
-    chunk_bytes = max(bytes_per_ms * chunk_ms, 2)
-
-    incremental_pcm_parts: list[bytes] = []
-    output_sample_rate = 24000
-    delta_index = 0
-    text_chunks: list[str] = []
-    final_text: str = ""
-    final_audio_transcript: str = ""
-
-    if delta_dump_dir is not None:
-        delta_dump_dir.mkdir(parents=True, exist_ok=True)
-
-    # Keep the CLI mode authoritative even if the input URL already contains
-    # an explicit duplex query parameter.
-    url = _with_realtime_route(url, server_vad=server_vad)
-    async with websockets.connect(url, max_size=64 * 1024 * 1024) as ws:
-        # 1) Validate model.
-        if server_vad:
-            session_update = {
-                "type": "session.update",
-                "session": {
-                    "model": model,
-                    "audio": {
-                        "input": {
-                            "format": {"type": "audio/pcm", "rate": 16_000},
-                            "turn_detection": {
-                                "type": "server_vad",
-                                "threshold": 0.5,
-                                "prefix_padding_ms": 300,
-                                "silence_duration_ms": 500,
-                                "create_response": True,
-                                "interrupt_response": True,
-                            },
-                        }
-                    },
-                },
-            }
-        else:
-            # Preserve the existing non-duplex wire format.
-            session_update = {"type": "session.update", "model": model}
-        await ws.send(json.dumps(session_update))
-
-        # 2) Preserve the legacy explicit-commit flow unless Server VAD owns endpointing.
-        if not server_vad:
-            await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": False}))
-
-        # 3) Stream audio chunks.
-        for i in range(0, len(pcm16), chunk_bytes):
-            chunk = pcm16[i : i + chunk_bytes]
-            await ws.send(
-                json.dumps(
-                    {
-                        "type": "input_audio_buffer.append",
-                        "audio": base64.b64encode(chunk).decode("utf-8"),
-                    }
-                )
-            )
-            if send_delay_ms > 0:
-                await asyncio.sleep(send_delay_ms / 1000.0)
-
-        # 4) Finalize the turn explicitly, or append silence for Server VAD endpointing.
-        if server_vad:
-            trailing_silence = bytes(16_000 * 2)
-            for i in range(0, len(trailing_silence), chunk_bytes):
-                await ws.send(
-                    json.dumps(
-                        {
-                            "type": "input_audio_buffer.append",
-                            "audio": base64.b64encode(trailing_silence[i : i + chunk_bytes]).decode("utf-8"),
-                        }
-                    )
-                )
-        else:
-            await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
-
-        # 5) Receive server events until audio done.
-        while True:
-            message = await ws.recv()
-            if isinstance(message, bytes):
-                # We only expect JSON text frames.
-                continue
-
-            event = json.loads(message)
-            event_type = event.get("type")
-
-            if event_type == "session.created":
-                continue
-
-            if event_type == "response.output_audio.delta":
-                sr = event.get("sample_rate_hz")
-                if isinstance(sr, int) and sr > 0:
-                    output_sample_rate = sr
-                audio_b64 = event.get("delta" if server_vad else "audio", "")
-                if audio_b64:
-                    pcm_delta = base64.b64decode(audio_b64)
-                    incremental_pcm_parts.append(pcm_delta)
-                    if delta_dump_dir is not None and pcm_delta:
-                        delta_index += 1
-                        dump_path = delta_dump_dir / f"delta_{delta_index:06d}.wav"
-                        _write_wav_pcm16(dump_path, pcm_delta, output_sample_rate)
-                        print(
-                            f"{log_prefix}delta dump #{delta_index}: {dump_path} "
-                            f"(pcm bytes={len(pcm_delta)}, sr={output_sample_rate})"
-                        )
-                continue
-
-            if event_type in {"response.output_text.delta", "transcription.delta"}:
-                delta = event.get("delta", "")
-                if delta:
-                    text_chunks.append(delta)
-                    print(delta, end="", flush=True)
-                continue
-
-            if event_type in {"response.output_text.done", "transcription.done"}:
-                final_text = event.get("text", "") or "".join(text_chunks)
-                usage = event.get("usage")
-                final_text_with_tag = f"Final transcription: {final_text}"
-                if text_chunks:
-                    print()
-                print(f"{log_prefix}{final_text_with_tag}")
-                if usage:
-                    print(f"{log_prefix}text usage: {usage}")
-                continue
-
-            if event_type == "response.output_audio_transcript.done":
-                final_audio_transcript = event.get("transcript", "")
-                continue
-
-            if event_type == "response.output_audio.done":
-                if not server_vad:
-                    break
-                continue
-
-            if event_type == "response.done" and server_vad:
-                await ws.send(json.dumps({"type": "session.close"}))
-                break
-
-            if event_type == "error":
-                raise RuntimeError(f"Server error: {event}")
-
-        all_pcm16 = b"".join(incremental_pcm_parts)
-        if not all_pcm16:
-            raise RuntimeError("No audio received from server.")
-
-        output_wav.parent.mkdir(parents=True, exist_ok=True)
-        _write_wav_pcm16(output_wav, all_pcm16, output_sample_rate)
-        print(f"{log_prefix}Saved realtime audio to: {output_wav} (incremental chunks joined)")
-
-        if output_text is not None:
-            text_to_save = final_text or "".join(text_chunks) or final_audio_transcript
-            output_text.parent.mkdir(parents=True, exist_ok=True)
-            output_text.write_text(text_to_save, encoding="utf-8")
-            print(f"{log_prefix}Saved realtime text to: {output_text}")
-
-
-def _indexed_output_path(path: Path | None, index: int, total: int) -> Path | None:
-    if path is None or total <= 1:
-        return path
-    return path.with_name(f"{path.stem}_{index:02d}{path.suffix}")
-
-
-async def run_clients_concurrent(
-    *,
-    url: str,
-    model: str,
-    input_wav: Path,
-    output_wav: Path,
-    output_text: Path | None,
-    chunk_ms: int,
-    send_delay_ms: int,
-    delta_dump_dir: Path | None,
-    server_vad: bool,
-    num_requests: int,
-    concurrency: int,
-) -> None:
-    sem = asyncio.Semaphore(concurrency)
-
-    async def _run_one(index: int) -> tuple[int, bool, str | None]:
-        per_output_wav = _indexed_output_path(output_wav, index, num_requests)
-        per_output_text = _indexed_output_path(output_text, index, num_requests)
-        per_delta_dir = None
-        if delta_dump_dir is not None:
-            per_delta_dir = delta_dump_dir / f"req_{index:02d}"
-        async with sem:
-            try:
-                await run_client(
-                    url=url,
-                    model=model,
-                    input_wav=input_wav,
-                    output_wav=per_output_wav,
-                    output_text=per_output_text,
-                    chunk_ms=chunk_ms,
-                    send_delay_ms=send_delay_ms,
-                    delta_dump_dir=per_delta_dir,
-                    server_vad=server_vad,
-                    request_idx=index,
-                    total_requests=num_requests,
-                )
-                return index, True, None
-            except Exception as exc:
-                return index, False, str(exc)
-
-    tasks = [asyncio.create_task(_run_one(i), name=f"rt-client-{i}") for i in range(1, num_requests + 1)]
-    results = await asyncio.gather(*tasks)
-
-    failed = [(idx, err) for idx, ok, err in results if not ok]
-    succeeded = num_requests - len(failed)
-    print(f"[summary] succeeded={succeeded}, failed={len(failed)}, total={num_requests}")
-    if failed:
-        for idx, err in failed:
-            print(f"[summary] req {idx:02d} failed: {err}")
-        raise RuntimeError(f"{len(failed)} concurrent request(s) failed")
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description="Realtime audio/text client for vLLM-Omni")
-    parser.add_argument("--url", default="ws://localhost:8091/v1/realtime", help="WebSocket URL")
-    parser.add_argument(
-        "--model",
-        default="Qwen/Qwen3-Omni-30B-A3B-Instruct",
-        help="Model name for session.update",
-    )
-    parser.add_argument("--input-wav", required=True, type=Path, help="Input WAV (mono, PCM16, 16kHz)")
-    parser.add_argument("--output-wav", default=Path("realtime_output.wav"), type=Path, help="Output WAV path")
-    parser.add_argument(
-        "--output-text",
-        default=None,
-        type=Path,
-        help="Optional output text path for final transcription",
-    )
-    parser.add_argument("--chunk-ms", type=int, default=200, help="Input chunk size in milliseconds")
-    parser.add_argument(
-        "--send-delay-ms",
-        type=int,
-        default=0,
-        help="Delay between chunk sends; set >0 to simulate realtime upload",
-    )
-    parser.add_argument(
-        "--delta-dump-dir",
-        type=Path,
-        default=None,
-        help="If set, each response.output_audio.delta is saved as delta_NNNNNN.wav under this directory",
-    )
-    parser.add_argument(
-        "--server-vad",
-        action="store_true",
-        help="Let the server detect speech endpoints instead of sending input_audio_buffer.commit",
-    )
-    parser.add_argument("--num-requests", type=int, default=1, help="Total number of requests to send")
-    parser.add_argument(
-        "--concurrency",
-        type=int,
-        default=1,
-        help="Maximum number of concurrent websocket requests",
-    )
-    args = parser.parse_args()
-
-    if args.num_requests <= 0:
-        raise ValueError("--num-requests must be >= 1")
-    if args.concurrency <= 0:
-        raise ValueError("--concurrency must be >= 1")
-    concurrency = min(args.concurrency, args.num_requests)
-
-    if args.num_requests == 1:
-        asyncio.run(
-            run_client(
-                url=args.url,
-                model=args.model,
-                input_wav=args.input_wav,
-                output_wav=args.output_wav,
-                output_text=args.output_text,
-                chunk_ms=args.chunk_ms,
-                send_delay_ms=args.send_delay_ms,
-                delta_dump_dir=args.delta_dump_dir,
-                server_vad=args.server_vad,
-            )
-        )
-    else:
-        asyncio.run(
-            run_clients_concurrent(
-                url=args.url,
-                model=args.model,
-                input_wav=args.input_wav,
-                output_wav=args.output_wav,
-                output_text=args.output_text,
-                chunk_ms=args.chunk_ms,
-                send_delay_ms=args.send_delay_ms,
-                delta_dump_dir=args.delta_dump_dir,
-                server_vad=args.server_vad,
-                num_requests=args.num_requests,
-                concurrency=concurrency,
-            )
-        )
-
-
-if __name__ == "__main__":
-    main()
@@ -1,7 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
-
-from examples.online_serving.realtime_web.server import main
-
-if __name__ == "__main__":
-    main(default_profile="qwen3-turn")
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+# MODEL=/models/auk-omni bash examples/online_serving/text_to_speech/auk/run_server.sh
+set -euo pipefail
+
+: "${MODEL:?Set MODEL to the assembled AuK or AuK-Flash checkpoint directory}"
+exec vllm serve "$MODEL" --omni --host "${HOST:-127.0.0.1}" --port "${PORT:-8091}" "$@"
@@ -0,0 +1,103 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""AuK speech generation and audio editing through /v1/audio/speech.
+
+See the online TTS user guide for checkpoint preparation and serving. Local
+reference audio is sent as a data URL; it need not exist on the server.
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import mimetypes
+import os
+from pathlib import Path
+
+import httpx
+
+
+def reference_audio(value: str) -> str:
+    if value.startswith(("http://", "https://", "data:")):
+        return value
+    path = Path(value).expanduser()
+    mime = mimetypes.guess_type(path.name)[0] or "audio/wav"
+    return f"data:{mime};base64,{base64.b64encode(path.read_bytes()).decode('ascii')}"
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--api-base", default="http://localhost:8091")
+    parser.add_argument("--api-key", default=os.environ.get("OPENAI_API_KEY", "EMPTY"))
+    parser.add_argument("--model", required=True, help="Model name or assembled bundle path used by the server")
+    parser.add_argument(
+        "--instructions",
+        required=True,
+        help=(
+            "Complete AuK instruction after filling the cookbook template "
+            "(https://github.com/Tencent-Hunyuan/AuK/blob/main/docs/COOKBOOK.md)"
+        ),
+    )
+    parser.add_argument("--ref-audio", help="Reference/source audio: local file, HTTP URL or data URL")
+    parser.add_argument("--duration-seconds", type=float, help="Target duration; defaults to source length with audio")
+    parser.add_argument("--seed", type=int)
+    parser.add_argument("--num-inference-steps", type=int)
+    parser.add_argument("--guidance-scale", type=float)
+    parser.add_argument("--sway", type=float)
+    parser.add_argument("--t-grid", type=float, nargs="+", help="Explicit increasing Euler time grid")
+    parser.add_argument("--vae-sample", action="store_true", help="Sample the VAE posterior instead of taking its mean")
+    parser.add_argument("--stream", action="store_true", help="Receive raw PCM over HTTP after full generation")
+    parser.add_argument("--response-format", choices=["wav", "pcm", "flac", "mp3", "opus"], default="wav")
+    parser.add_argument("--output", type=Path, help="Output file; defaults to output.wav or output.pcm")
+    parser.add_argument("--timeout", type=float, default=300)
+    args = parser.parse_args()
+    if args.duration_seconds is None and args.ref_audio is None:
+        parser.error("--duration-seconds is required without --ref-audio")
+
+    response_format = "pcm" if args.stream else args.response_format
+    payload = {
+        "model": args.model,
+        # OpenAI's shared Speech schema still declares input, while AuK's
+        # canonical request carries all content in the complete instruction.
+        "input": "",
+        "voice": "default",
+        "response_format": response_format,
+    }
+    for name in ("duration_seconds", "seed"):
+        value = getattr(args, name)
+        if value is not None:
+            payload[name] = value
+    payload["instructions"] = args.instructions
+    if args.ref_audio:
+        payload["ref_audio"] = reference_audio(args.ref_audio)
+    extra = {
+        name: getattr(args, name)
+        for name in ("num_inference_steps", "guidance_scale", "sway", "t_grid")
+        if getattr(args, name) is not None
+    }
+    if args.vae_sample:
+        extra["vae_sample"] = True
+    if extra:
+        payload["extra_params"] = extra
+    if args.stream:
+        payload.update(stream=True, stream_format="audio")
+
+    output = args.output or Path(f"output.{response_format}")
+    headers = {"Authorization": f"Bearer {args.api_key}"}
+    with httpx.Client(timeout=args.timeout) as client:
+        with client.stream(
+            "POST", f"{args.api_base.rstrip('/')}/v1/audio/speech", json=payload, headers=headers
+        ) as response:
+            if response.is_error:
+                response.read()
+                raise RuntimeError(f"Speech API returned {response.status_code}: {response.text}")
+            with output.open("wb") as handle:
+                for chunk in response.iter_bytes():
+                    handle.write(chunk)
+    print(f"Saved {output} ({output.stat().st_size} bytes). Native audio: 24000 Hz, mono.")
+    if response_format == "pcm":
+        print("PCM format: signed 16-bit little-endian, 24000 Hz, mono.")
+
+
+if __name__ == "__main__":
+    main()
@@ -56,7 +56,9 @@ dev = [
    # 0.3.12 includes acknowledged TCP writes (Mooncake #2850): completion
    # must mean the receive buffer is ready before zero-copy consumers read it.
    "mooncake-transfer-engine-cuda13==0.3.12.post1",
-    "av==18.0.0", # for ComfyUI tests
+    "av==18.0.0; python_version >= '3.11'", # for ComfyUI tests
+    "av==17.1.0; python_version < '3.11'", # last av release that supports Python 3.10
+    "tomli==2.4.1; python_version < '3.11'", # tomllib backport used by tests/helpers/mark.py
    "openpyxl==3.1.5", # for nightly CI
    "pyttsx3==2.99",
    "opencc==1.4.1",
@@ -159,12 +161,25 @@ vllm-omni = "vllm_omni.entrypoints.cli.main:main"
 vllm_omni_register_models = "vllm_omni.engine.arg_utils:register_omni_models_to_vllm"


+[tool.setuptools]
+include-package-data = false
+
 [tool.setuptools.packages.find]
 where = ["."]
 include = ["vllm_omni*"]

 [tool.setuptools.package-data]
-"vllm_omni" = ["_version.py", "py.typed", "transformers_utils/chat_templates/*.jinja"]
+"vllm_omni" = [
+    "_version.py",
+    "py.typed",
+    "deploy/*.md",
+    "deploy/*.yaml",
+    "diffusion/distributed/csrc/*.cu",
+    "diffusion/distributed/csrc/*.hpp",
+    "model_executor/models/covo_audio/speaker_prompt/*.npy",
+    "model_executor/models/step_audio2/assets/*.wav",
+    "transformers_utils/chat_templates/*.jinja",
+]

 [tool.setuptools_scm]
 # no extra settings needed, presence enables setuptools-scm
@@ -325,6 +340,8 @@ skip_covered = true

 [tool.typos.default]
 extend-ignore-identifiers-re = [
+    # Canonical PyTorch CUDA device property.
+    "^shared_memory_per_block_optin$",
    ".*_thw",
    ".*thw",
    "ein",
@@ -286,7 +286,6 @@ curl -X POST http://localhost:8091/v1/images/generations \
    "size": "1024x1024",
    "seed": 42
  }' | jq -r '.data[0].b64_json' | base64 -d > output-turbo.png
-```

 ### Notes

@@ -418,6 +417,45 @@ curl -s http://localhost:8091/v1/chat/completions \
  supports Boogu-Image-Edit directly
  (`--model Boogu/Boogu-Image-0.1-Edit --guidance-scale 5.0`, optional
  `--guidance-scale-2`).
+- **Request batching (`--max-num-seqs > 1`) is not bit-exact vs. serial.**
+  Compatible TI2I requests (same task type, same `guidance_scale_2`
+  explicitness) can share one forward pass. The instruction encoder batches
+  with `padding="longest"`, so a request's *padded batch shape* depends on
+  whichever partner request(s) happen to co-arrive, even though production
+  RoPE derives each row's own effective position count from that row's own
+  `attention_mask.sum()` rather than from the batch's padded width. In the
+  measurements below, `--max-num-seqs 1` and `--max-num-seqs > 1` do not
+  produce pixel-identical output for the same seed; the exact mechanism
+  (padding-shape-dependent kernel/reduction-order numerics vs. some other
+  batch-shape effect) has not been isolated, so treat this as an observed
+  drift under the specific configs below, not a diagnosed root cause, and
+  do not extrapolate the numbers below as a general upper bound on other
+  configs. Two distinct regimes have been measured so far, and they are
+  **not** interchangeable tolerances:
+    - **Fixed-shape co-batch** (partner instructions encode to the same
+      length): `Boogu-Image-0.1-Edit`, 512x512, 20 steps, guidance 5/2, one
+      fixed reference image and instruction pair, seeds 42-73, B=2 vs. serial
+      -- mean pixel MAE **1.70/255**, max per-image MAE **2.68/255**, min PSNR
+      **34.35 dB** (2/32 image pairs were pixel-exact).
+    - **Heterogeneous partner length** (co-batched instructions encode to
+      different lengths): MAD **2.6-6.0** in the fixed-length partner-mixing
+      check on this PR's test head -- larger than the fixed-shape case above
+      in that check. A targeted current-head A/B run observed a comparable
+      range: `Boogu-Image-0.1-Edit`, 512x512, 20 steps, guidance 5/2, one
+      fixed A request (5-token instruction) co-batched in turn with a
+      2-token and a 49-token partner instruction (both also differing in
+      reference image and content, not length alone; `max_num_seqs=2`,
+      co-batching verified via request tracing, not assumed) --
+      solo vs. `A`+2-token partner MAD **4.06/255** (PSNR 29.9 dB),
+      solo vs. `A`+49-token partner MAD **3.87/255** (PSNR 29.7 dB),
+      2-token vs. 49-token partner MAD **4.05/255** (PSNR 30.2 dB). A
+      controlled ablation (same partner content/reference, varying only
+      instruction length) would be needed to isolate a pure length effect,
+      and has not been run.
+  Pin `--max-num-seqs 1` for any workload that requires strict
+  serial-reproducibility (e.g. golden-image regression tests); use
+  `--max-num-seqs > 1` only when this drift is acceptable for the throughput
+  gain.

 ## Fast image editing (Boogu-Image-0.1-Edit-Turbo)

@@ -0,0 +1,199 @@
+# SeedVR2 video restoration — RTX 5090
+
+## Summary
+
+- Vendor: ByteDance
+- Model: SeedVR2 3B
+- Task: restore or upscale an uploaded video
+- Mode: native offline/HTTP pipeline, one Euler step
+- Hardware: NVIDIA GeForce RTX 5090, 32 GiB per device
+- Maintainer: 0z5a
+
+## When to use this recipe
+
+Run short, constant-frame-rate videos with the native SeedVR2 pipeline. This
+recipe covers the whole-clip path and the separate long-video route. Color
+correction from external applications is a separate execution semantic.
+
+## Supported model contract
+
+| Property | Contract |
+| --- | --- |
+| Model directory | See [checkpoint files and hashes](../../docs/models/seedvr2.md#model-directory) |
+| Input | One uploaded video; blank prompt; positive dimensions divisible by 16 |
+| Output | MP4 at requested geometry, original frame count and source frame rate |
+| Audio | First mono/stereo track, aligned by PTS; re-encoded as AAC |
+| Sampling | One Euler step, CFG=1, per-request seed |
+| Precision | 3B DiT and VAE FP16 |
+| Parallelism | Whole-window SP via `ulysses_degree`; replicated model weights |
+| Frame padding | Internal 4n+1 padding, cropped back; five returns five, six returns six |
+
+## References
+
+- [Canonical SeedVR](https://github.com/ByteDance-Seed/SeedVR)
+- [Model guide](../../docs/models/seedvr2.md)
+- [Integration RFC](https://github.com/vllm-project/vllm-omni/issues/7723)
+
+## Hardware
+
+RTX 5090 with 32 GiB per GPU, PCIe, one GPU for the default command or two/four
+for SP. Each GPU must fit a complete model copy and its activations. Host RAM
+must accommodate checkpoint staging per rank. Admission uses a padded
+five-frame 848×480 output pixel budget by default, allowing longer clips at
+smaller resolutions, plus a 257-frame decoder-work cap. The validated SP4
+configuration with VAE tiling and `vae_patch_parallel_size=4` uses a padded
+five-frame 2560×1472 budget, as does any higher matched degree. On devices
+with more memory the clip budget grows automatically; the other caps are raised
+through `VLLM_OMNI_SEEDVR2_SHARDED_FRAME_PIXELS` and
+`VLLM_OMNI_SEEDVR2_MAX_FRAMES`, and `VLLM_OMNI_SEEDVR2_SHARDED_CLIP_PIXELS`
+overrides the clip budget. The model guide lists
+exact bounds and which profiles have completed GPU validation.
+
+## Software environment
+
+Linux x86-64, Python 3.12, PyTorch 2.13.0+cu130, CUDA 13.0, vLLM 0.29.0, and
+PyAV in the existing environment. Use the vLLM-Omni branch containing this recipe.
+
+## Command
+
+Set `MODEL_DIR` to the authorized checkpoint directory documented in the model
+guide; it must also contain `model_index.json` selecting `SeedVR2Pipeline`.
+
+```bash
+CUDA_VISIBLE_DEVICES=0 vllm serve "$MODEL_DIR" --omni \
+  --model-class-name SeedVR2Pipeline --dtype float16 --enforce-eager \
+  --num-gpus 1 --host 127.0.0.1 --port 8098
+```
+
+For SP=2, expose two GPUs and replace the GPU count with:
+
+```bash
+--num-gpus 2 --distributed-executor-backend mp \
+  --stage-overrides '{"0":{"ulysses_degree":2}}'
+```
+
+Use degree 4 and four visible GPUs for SP=4. Combine any feature-specific
+settings below into the same stage-overrides object; do not repeat that flag.
+The validated 16-frame 720p original-size profile is:
+
+```bash
+CUDA_VISIBLE_DEVICES=0,1,2,3 vllm serve "$MODEL_DIR" --omni \
+  --model-class-name SeedVR2Pipeline --dtype float16 --enforce-eager \
+  --vae-use-tiling --num-gpus 4 --distributed-executor-backend mp \
+  --stage-overrides '{"0":{"ulysses_degree":4,"vae_patch_parallel_size":4,"vae_parallel_mode":"spatial_shard_height"}}' \
+  --host 127.0.0.1 --port 8098
+```
+
+Use `size=1280x720` for a landscape 1280×720 input or `size=720x1280` for a
+portrait 720×1280 input. Both require 16 decoded source frames and retain the
+source frame rate; no upscaling is requested.
+
+## Verification
+
+```bash
+curl --fail-with-body http://127.0.0.1:8098/v1/videos/sync \
+  -F 'prompt= ' -F 'input_references=@input.mp4;type=video/mp4' \
+  -F 'size=224x128' -F 'num_inference_steps=1' \
+  -F 'guidance_scale=1' -F 'seed=7723' --output restored.mp4
+python - <<'PY'
+import av
+with av.open('restored.mp4') as video:
+    stream = video.streams.video[0]
+    print(stream.width, stream.height, stream.average_rate,
+          sum(1 for _ in video.decode(video=0)), len(video.streams.audio))
+PY
+```
+
+Expect HTTP 200 and a completely decodable 224×128 video. Check the frame count,
+FPS, timestamps, and audio against the input. Repeat the same seed and compare
+decoded frames; changing the seed should change the restored video. Inspect
+matching first/middle/last frames at the same display scale. AAC is re-encoded,
+so input/output audio packet hashes need not be identical.
+
+The released 3B checkpoint also passed these four-GPU SP4 HTTP cases with the
+configuration above. Each input had 16 frames at 24 FPS; the output retained
+the original size, 16 decoded frames, timestamps, and decodable audio. All four
+workers and the server exited normally after each run.
+
+| 1× input → output | Earlier six-frame admission | Revised HTTP result | Request time |
+| --- | --- | --- | ---: |
+| 1280×720, 16 frames → 1280×720 | Rejected | HTTP 200; media checks passed | 3.553 s |
+| 720×1280, 16 frames → 720×1280 | Rejected | HTTP 200; media checks passed | 3.388 s |
+
+These are single requests on a shared host, not performance comparisons.
+
+### One long-video request
+
+For a 24 FPS source, the long route can restore up to 7,200 frames at
+768×1344. Use the SP4/VAE height-sharding command above, serve with
+`--api-server-count 1`, and set a persistent
+`VLLM_OMNI_SEEDVR2_LONG_OUTPUT_DIR` before starting. If the source is shorter
+than 300 seconds, `loop_input=true` repeats its frames and audio. The service
+runs the longest model windows the clip budget admits, with four-frame overlap,
+and writes one MP4;
+it does not concatenate separate video files. Download the result within
+`VLLM_OMNI_SEEDVR2_LONG_JOB_TTL_SECONDS`, after which the job directory is
+swept; `DELETE` on the job URL stops a run at the next window boundary.
+
+```bash
+curl --fail-with-body http://127.0.0.1:8098/v1/seedvr2/restore-long \
+  -F 'input_references=@input.mp4;type=video/mp4' \
+  -F 'prompt= ' -F 'size=768x1344' -F 'num_frames=7200' \
+  -F 'loop_input=true' -F 'seed=7723' -F 'extra_params={"color_correction_method":"lab"}'
+# Use the returned id to poll /v1/seedvr2/restore-long/{id};
+# download /v1/seedvr2/restore-long/{id}/content when completed.
+```
+
+The input must have constant 24 FPS timestamps starting at zero. Only one job
+can run per API server, and `--api-server-count` must be one. The existing
+whole-clip `/v1/videos/sync` limits remain in force. Full 7,200-frame GPU
+validation is in progress; use this route only with a memory-checked SP4
+deployment.
+
+Set `VLLM_TEST_SEEDVR2_MODEL_DIR` to the directory containing all model files:
+
+```bash
+python -m pytest -o addopts='' -v tests/diffusion/models/seedvr2/test_seedvr2_e2e.py
+```
+
+The tests run the complete native pipeline for every colour mode and the HTTP
+long-video route at USP 1 and 8 (eight GPUs required for the latter). The HTTP
+cases force a 13-frame window, restore 20 frames across a seam, preserve audio
+and timestamps, reject invalid `extra_params`, and cancel a longer job. They
+use the released model rather than mocking its forward pass.
+
+## Supported features
+
+| Feature | Status |
+| --- | --- |
+| [Window SP](../../docs/models/seedvr2.md) | Model-local regular/shifted window attention |
+| CPU offload, LoRA, compiled execution, CFG/TP/PP | Unsupported |
+| VFR or multichannel audio | Unsupported |
+| VAE temporal/spatial tiling | Available with `--vae-use-tiling` |
+| [Colour correction](../../docs/models/seedvr2.md) | On by default; `color_correction_method=lab\|wavelet\|adain\|none` |
+| Quantization | Separate opt-in feature |
+
+## Measurement scope
+
+Compare identical inputs, seeds, dimensions, GPU counts, and warmups. Separate
+correctness from performance: use repeated interleaved HTTP measurements before
+claiming a stable gain. Keep raw logs, tensors, and generated media outside the
+source diff; link selected visuals from the PR's test results.
+
+## Temporal tiling and VAE patch parallelism
+
+Use `--vae-use-tiling` for causal temporal chunks. On a single GPU this also
+bounds large convolution workspace with exact spatial halos. To shard VAE
+activations across the same two SP ranks, use:
+
+```bash
+--vae-use-tiling --num-gpus 2 --distributed-executor-backend mp \
+  --stage-overrides '{"0":{"ulysses_degree":2,"vae_patch_parallel_size":2,"vae_parallel_mode":"spatial_shard_height"}}'
+```
+
+The VAE patch degree must equal the SP degree. Height sharding is supported;
+width sharding and batch slicing are not. Compare clips that cross a temporal
+chunk boundary as well as five/six-frame clips, and inspect frames adjacent to
+chunk boundaries. Reduced peak memory does not by itself establish lower latency.
+High-resolution clips can still exceed device capacity; validate the intended
+frame count and output size on the target GPUs.
@@ -89,6 +89,7 @@ Install matching vLLM and vLLM-Omni versions, and ensure `ffmpeg` and
 | NVIDIA B200 or H200 | Capacity-based recommendation; not yet verified | All four canonical pipelines |
 | NVIDIA GB200 or GB300 | Capacity-based recommendation; not yet verified | All four canonical pipelines |
 | NVIDIA H100 80 GB | FP8 recipe | Distilled one-stage at 960x544 |
+| Intel Arc Pro B70 32 GB | Verified | Offline Full/SFT one-stage T2V at 768x512 |

 The 1920x1088 two-stage examples require about 114 GB of peak GPU memory.
 80 GB GPUs do not have enough safety margin for the canonical two-stage
@@ -133,6 +134,33 @@ SP attention head counts must be divisible by `ulysses_degree`. Use each
 phase's `H, W` (half/full resolution for two-stage); non-divisible shapes fail.
 Each GPU must still fit the resident weights. Warm up with the production shape.

+### 1x Intel Arc Pro B70 32 GB (XPU)
+
+Offline Full/SFT one-stage T2V at 768x512, 17 frames. Select the legacy
+convolutional VAE with `ltx2_use_conv_vae`; the default DiffVAE decoder loads a
+NATTEN kernel, which has no XPU build. FP8 with layerwise offload streams both
+the transformer and the Gemma-4 text encoder.
+
+Environment: torch 2.14.0+xpu, vLLM 0.29.0, vLLM-Omni `main` at `d8d162d8`.
+
+```bash
+python examples/offline_inference/text_to_video/text_to_video.py \
+  --model Lightricks/LTX-2.5-Diffusers \
+  --model-class-name LTX2Pipeline \
+  --stage-overrides '{"0":{"extras":{"ltx2_use_conv_vae":true}}}' \
+  --prompt "A serene lakeside sunrise with mist over the water." \
+  --num-frames 17 \
+  --quantization fp8 \
+  --enable-layerwise-offload \
+  --vae-use-tiling \
+  --vae-use-slicing \
+  --enforce-eager \
+  --output ltx25_output.mp4
+```
+
+Peaks at 16.3 GiB in 163 s. Two-stage, distilled, and I2V paths were not
+exercised on this hardware.
+
 ## Offline inference

 Choose values from the pipeline table. For example, the distilled two-stage
@@ -165,6 +165,63 @@ curl -X POST http://localhost:8000/v1/videos/sync \
 Use `image_reference` for a URL or JSON-safe image reference. Do not provide it
 together with `input_reference`.

+## Hardware Support
+
+## XPU
+
+### 1x Intel Arc Pro B70 (32 GB)
+
+Offline one-stage T2V at 768x512, 17 frames. FP8 with layerwise offload streams
+both the transformer and the Gemma text encoder, which otherwise stays resident
+beside it.
+
+#### Environment
+
+- OS: Linux
+- Python: 3.10+
+- torch: 2.14.0+xpu
+- vLLM: 0.29.0
+- vLLM-Omni: `main` at `d8d162d8`
+
+#### Command
+
+```bash
+python examples/offline_inference/text_to_video/text_to_video.py \
+  --model diffusers/LTX-2.3-Diffusers \
+  --prompt "A serene lakeside sunrise with mist over the water." \
+  --num-frames 17 \
+  --quantization fp8 \
+  --enable-layerwise-offload \
+  --vae-use-tiling \
+  --vae-use-slicing \
+  --enforce-eager \
+  --output ltx23_output.mp4
+```
+
+The same flags cover every one-stage checkpoint; only `--model` changes. The
+distilled checkpoints additionally need their class and an explicit size,
+because their preset defaults to 1536x1024:
+
+```bash
+  --model diffusers/LTX-2.3-Distilled-Diffusers \
+  --model-class-name LTX2DistilledOneStagePipeline \
+  --height 512 --width 768
+```
+
+| Checkpoint | Peak | Time |
+| --- | ---: | ---: |
+| `Lightricks/LTX-2` | 14.8 GiB | 160 s |
+| `diffusers/LTX-2.3-Diffusers` | 16.4 GiB | 167 s |
+| `dg845/LTX-2.3-Diffusers` | 16.4 GiB | 155 s |
+| `diffusers/LTX-2.3-Distilled-Diffusers` | 14.9 GiB | 128 s |
+
+#### Verification
+
+Each command writes its `--output` MP4. Confirm it decodes and that the sampled
+frames match the prompt.
+
+
+
 ## Guidance

 One-stage and ordinary Stage 1 support independent video/audio CFG,
@@ -289,6 +289,15 @@ The first request took 85.224 seconds. The AR stage generated 4,161 visual token

 The output was a valid 1024 by 1024 RGB PNG.

+### Startup benchmark
+
+The [startup benchmark](../../benchmarks/mammoth_moda2/README.md) provides
+commands, measurement boundaries and a single-A800 eager baseline. On that
+16-CPU-quota machine, parallel stage initialization and 16 CPU threads reduced
+`Omni()` initialization from about 60 s to 37 s. See the benchmark for the
+configuration and limitations; CUDA-graph mode needs separate memory and
+quality validation.
+
 ## MammothModa2-Dev unified inference

 MammothModa2-Dev uses a Qwen3-VL AR backbone, while MammothModa2-Preview uses
@@ -281,6 +281,186 @@ curl -sS -X POST "${API_URL}" \
  -o t2va.mp4
 ```

+This requires the FLASH_ATTN backend and MindIE-SD. H3 automatically applies
+exact power-of-two input pre-scaling (`laser_input_scale=256`) so the
+kernel's fp16 workspace cannot overflow on outlier activations. Measured on
+the FastH3 four-step Dense configuration below, T2VA 15 s at 1344x768, this
+kernel reduced end-to-end latency from ~73 s to ~57 s (about 22%).
+
+
+## FastH3 four-step on A3
+
+[FastH3](https://haoailab.com/blogs/fasth3-preview/) is FastVideo's four-step
+DMD2 student of H3-Base; see
+[MiniMax-H3.md § FastH3 adapter](MiniMax-H3.md#fasth3-adapter) for the adapter
+contract. On A3 the adapter cannot be fused the GPU way. The GPU recipe fuses
+it at load time from `--lora-path`, but that path replicates the full model per
+rank and does not fit in 64 GB HBM. A3 needs distributed layerwise offload, and
+offload is refused with `--lora-path` because it streams weights in without
+going through the fusion (see the GPU recipe's note on why
+`--enable-distributed-layerwise-offload` fails fast with a FastH3 adapter).
+
+The A3 path therefore fuses the adapter **offline**, once, into a native-layout
+checkpoint that distributed layerwise offload can memory-map directly. This
+also lets the offloaded server start without `--lora-path`, so the fusion
+contract check that rejects offload never fires.
+
+### Prepare the fused checkpoint (one-time, offline)
+
+The fusion runs in the vLLM-Omni native namespace
+(`blocks.N.attn.qkv_proj.weight`), matching the base H3 `FL2VA` transformer
+layout. It reads the base transformer and the Dense adapter, adds the adapter's
+low-rank and full-rank deltas per shard, symlinks the unchanged components
+(text encoder, VAEs, tokenizer, processor), and writes the four-step sigma
+ladder into `model_index.json` so the pipeline samples on the release's rungs
+rather than a uniform schedule.
+
+Download the base checkpoint's `FL2VA` partition (native layout) and the Dense
+adapter:
+
+```bash
+export BASE_DIR=/path/to/MiniMax-H3
+hf download MiniMaxAI/MiniMax-H3 --include "FL2VA/*" --local-dir "${BASE_DIR}"
+
+export FASTH3_DIR=/path/to/fasth3
+hf download FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA \
+  dense-datafree/adapter_model.safetensors --local-dir "${FASTH3_DIR}"
+```
+
+> The published full checkpoint
+> `FastVideo/FastVideo-FastH3-4-step-Preview-v1-Dense-DataFree` is stored in the
+> diffusers layout (`transformer_blocks.N.attn.to_q/to_k/to_v`, split QKV,
+> value-first MLP). Distributed layerwise offload memory-maps by exact runtime
+> parameter name and has no diffusers-to-native remap for H3, so that artifact
+> cannot be served directly. Fuse from the base `FL2VA` partition instead — its
+> keys already match the runtime.
+
+Run the fusion:
+
+```python
+# prepare_fasth3_dense.py — fuse base H3 FL2VA + FastH3 Dense adapter (native layout)
+import json, os, shutil, sys, time
+import torch
+from safetensors.torch import load_file, save_file
+from vllm_omni.diffusion.models.minimax_h3.fasth3 import FastH3WeightFusion
+
+BASE = os.environ["BASE_DIR"] + "/FL2VA"          # base H3 FL2VA partition (native layout)
+ADAPTER = os.environ["FASTH3_DIR"] + "/dense-datafree/adapter_model.safetensors"
+OUT = os.environ.get("FUSED_DIR", "/path/to/FastH3-Dense-Fused/FL2VA")
+
+# MiniMax-H3 architecture
+HEAD_DIM, NUM_BLOCKS, NUM_REFINER_BLOCKS = 128, 50, 2
+# FastH3 four-step sigma positions; the pipeline adds per-modality shift on top.
+BASE_SCHEDULE = [0.999, 0.749, 0.5, 0.25, 0.0]
+COMPONENTS = ["audio_vae", "video_vae", "text_encoder", "tokenizer", "processor"]
+
+fusion = FastH3WeightFusion.from_path(
+    ADAPTER, head_dim=HEAD_DIM, num_blocks=NUM_BLOCKS,
+    num_refiner_blocks=NUM_REFINER_BLOCKS)
+assert fusion is not None, "adapter not recognized as FastH3"
+
+os.makedirs(OUT + "/transformer", exist_ok=True)
+src_t = BASE + "/transformer"
+for shard in sorted(f for f in os.listdir(src_t)
+                    if f.endswith(".safetensors") and "index" not in f):
+    data = load_file(os.path.join(src_t, shard), device="cpu")
+    fused = {}
+    for k, v in data.items():
+        fv = fusion.fuse(k, v)
+        fused[k] = fv.to("cpu").to(torch.bfloat16) if fv is not v else v
+    save_file(fused, os.path.join(OUT, "transformer", shard))
+fusion.validate_fully_applied()   # every delta must have met its parameter
+
+for j in os.listdir(src_t):       # copy transformer index/config
+    if j.endswith(".json"):
+        shutil.copy2(os.path.join(src_t, j), os.path.join(OUT, "transformer", j))
+
+for c in COMPONENTS:              # symlink unchanged components
+    s = os.path.join(BASE, c)
+    if os.path.exists(s):
+        os.symlink(os.path.realpath(s), os.path.join(OUT, c))
+
+idx = json.loads(open(BASE + "/model_index.json").read())   # inject sigma ladder
+idx.setdefault("_minimax_h3", {})["base_schedule"] = BASE_SCHEDULE
+open(OUT + "/model_index.json", "w").write(json.dumps(idx, indent=4) + "\n")
+print("fused ->", OUT)
+```
+
+```bash
+export FUSED_DIR=/path/to/FastH3-Dense-Fused/FL2VA
+python prepare_fasth3_dense.py
+```
+
+### Serve the fused checkpoint
+
+Serve the fused directory with the same multi-NPU flags as the base recipe, but
+**without** `--lora-path` — the adapter is already in the weights. Keep
+distributed layerwise offload and, optionally, LaserAttention:
+
+```bash
+export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+export PORT=9098
+export MODEL=/path/to/FastH3-Dense-Fused/FL2VA
+export VLLM_WORKER_MULTIPROC_METHOD=spawn
+export VLLM_OMNI_VIDEO_SYNC_TIMEOUT=1800
+export MINDIE_SD_FA_TYPE="ascend_laser_attention"   # optional
+
+vllm serve "${MODEL}" \
+  --omni \
+  --host 0.0.0.0 \
+  --port "${PORT}" \
+  --trust-remote-code \
+  --task-type t2va \
+  --init-timeout 1800 \
+  --stage-init-timeout 1800 \
+  --num-gpus 8 \
+  --usp 8 \
+  --ring 1 \
+  --text-encoder-tp-size 8 \
+  --enable-distributed-layerwise-offload \
+  --vae-parallel-mode tile \
+  --vae-use-tiling \
+  --vae-patch-parallel-size 8 \
+  --diffusion-attention-backend FLASH_ATTN
+```
+
+Requests must ask for `num_inference_steps=4` and `task=t2va`; FastH3 preview v1
+distills the T2VA path only. The first request includes regional compilation
+(~50 s warmup); exclude it from steady-state numbers.
+
+```bash
+curl -s -X POST "http://localhost:${PORT}/v1/videos/sync" \
+  -F 'prompt=A golden retriever running through a sunflower field at sunset' \
+  -F 'seconds=5' \
+  -F 'aspect_ratio=16:9' \
+  -F 'num_inference_steps=4' \
+  -o out.mp4
+```
+
+### Measured FastH3 four-step evidence
+
+Measured on an Atlas 800I A3 server (8x NPU) with CANN 9.0.1, PyTorch
+2.10.0+cpu, torch_npu 2.10.0.post2, the multi-NPU configuration above, T2VA at
+1344x768, one warmup excluded:
+
+| Duration | LaserAttention | End-to-end |
+| ---: | --- | ---: |
+| 4 s | on | ~12 s (768x768), ~15 s (1344x768) |
+| 8 s | on | ~26 s |
+| 15 s | on | ~57 s |
+| 5 s | off | ~19 s |
+| 10 s | off | ~41 s |
+| 15 s | off | ~73 s |
+
+These describe the validated shapes rather than a general throughput guarantee.
+
+> A native `--lora-path + --enable-distributed-layerwise-offload` path that
+> fuses during the offload memory-map (avoiding the offline step) is possible
+> through the offload backend's per-tensor transform hook, but is left as future
+> work pending upstream design discussion.
+
+
+## HTTP API examples
 Switch tasks via `extra_params.task` (`t2va` / `fl2va` / `ref2va`); FL2VA and
 Ref2VA additionally pass `input_references` (first-frame image, reference
 video/audio). The full parameter table lives in the GPU recipe.
@@ -725,6 +725,9 @@ curl -sS -X POST "${API_URL}" \
 ### 3. Ref2VA: image-only, image/audio, or mixed references

 Run these requests against the combined service or a Ref2VA-only service.
+Reference images keep their original resolution, with dimensions aligned to
+32 pixels. No additional request or server setting is needed.
+
 Image-only Ref2VA omits `audio_reference`; adding one or more audio references
 is optional. The typed fields accept one object or an ordered JSON list.
 `audio_reference` accepts an HTTP(S) URL or a `data:` URL. In one terminal,
@@ -782,6 +785,9 @@ Run this request against the combined service. Repeat the
 `input_references` multipart field once per source video. H3 consumes the
 videos in form order and preserves their original soundtracks during
 conditioning.
+Small reference videos retain their original size; larger videos are scaled
+down to the reference canvas. Dimensions are aligned to 32 pixels. No
+additional request or server setting is needed.

 ```bash
 export SUBJECT_VIDEO=/path/to/green_screen_subject.mp4
@@ -1509,9 +1515,184 @@ vllm serve "${MODEL_ROOT}/FL2VA" \
  --cache-config '{"rel_l1_thresh":0.17}'
 ```

+## Latent super-resolution
+
+H3's video VAE is a ~5B-parameter model, so the usual way to raise output
+resolution -- decode, upscale in pixel space, re-encode -- pays for that VAE
+twice. The community
+[Minimax H3 latent upscaler](https://huggingface.co/LBH-123-AI/Minimax_h3_latent_Upscaler)
+is a learned 3D-convolutional resizer trained on H3's 24-channel latents: it
+runs between the denoise loop and the VAE, so the clip is decoded once, at the
+larger size. Unlike bilinear or bicubic interpolation of a latent, it does not
+introduce ghosting or double-image artifacts.
+
+The network runs one normalization below the pipeline latent: an H3 latent in
+vLLM-Omni is already normalized, and the upscaler was trained on that tensor
+normalized again by the VAE's `latents_mean` / `latents_std`. The stage applies
+and undoes that itself, so nothing about a request changes -- but a port that
+skips it inflates the latent about 5x and decodes as a magenta grid at one tile
+per latent cell.
+
+The checkpoint is optional and is not part of the H3 release. Download one file
+from that repository (`minimax_h3_latent_upscaler_3d_bf16.safetensors`, 691 MB)
+and name it at startup:
+
+```bash
+vllm serve "${MODEL_ROOT}/FL2VA" \
+  --omni \
+  --trust-remote-code \
+  --additional-config '{"latent_upscaler_path":"/models/minimax_h3_latent_upscaler_3d_bf16.safetensors"}'
+```
+
+A request then asks for a size. The three modes are mutually exclusive and all
+resolve to the same 16-pixel latent grid, so the three requests below are the
+same 960x544 -> 1920x1088 upscale:
+
+```bash
+curl -sS -X POST "${API_URL}" \
+  -F 'prompt=...' \
+  -F 'width=960' -F 'height=544' -F 'fps=24' \
+  -F 'extra_params={"task":"t2va","duration":8.7,"latent_upscale":2.0}' \
+  -o t2va_1080p.mp4
+
+# or name the output size in pixels
+#   "latent_upscale":{"width":1920,"height":1088}
+# or name a pixel budget and keep the aspect ratio
+#   "latent_upscale":{"megapixels":2.0}
+```
+
+Over HTTP the field is `extra_params`: the videos request model declares that
+name and nothing else, so an `extra_args` form field is dropped without an
+error and the request silently returns the un-upscaled size. Offline, the same
+value goes in `extra_args`:
+
+```python
+omni = Omni(
+    model=os.path.join(os.environ["MODEL_ROOT"], "FL2VA"),
+    trust_remote_code=True,
+    additional_config={"latent_upscaler_path": "/models/minimax_h3_latent_upscaler_3d_bf16.safetensors"},
+)
+outputs = omni.generate(
+    "A quiet cinematic night scene with matching ambient sound.",
+    OmniDiffusionSamplingParams(
+        height=544,
+        width=960,
+        fps=24,
+        num_inference_steps=50,
+        seed=42,
+        extra_args={"task": "t2va", "duration": 8.7, "latent_upscale": 2.0},
+    ),
+)
+```
+
+### The hi-res route: generate small, refine large
+
+Upscaling alone decodes the enlarged latent directly, which is fast but can
+only interpolate detail the first pass never generated. Adding `latent_refine`
+resumes the denoise schedule at the new size, so the DiT puts real detail back:
+
+1. generate at a cheap size -- far fewer DiT tokens,
+2. upscale the latent,
+3. re-noise it and run the tail of the schedule at the target size.
+
+```bash
+curl -sS -X POST "${API_URL}" \
+  -F 'prompt=...' \
+  -F 'width=960' -F 'height=544' -F 'fps=24' \
+  -F 'num_inference_steps=50' \
+  -F 'extra_params={"task":"t2va","duration":8.7,"latent_upscale":2.0,"latent_refine":0.4}' \
+  -o t2va_1080p.mp4
+```
+
+`latent_refine` is the fraction of the request's steps the second pass re-runs,
+the img2img convention, so it sets both how much the pass may change and what
+it costs: `0.4` of `num_inference_steps=50` is 50 cheap steps followed by 20
+expensive ones. Low values stay close to the first pass and mostly sharpen;
+high values re-generate and may drift from it. `1.0` re-runs the whole schedule
+from noise, which discards the first pass entirely.
+
+Detail does not keep climbing with strength. Measured on a 15s 2688x1536 clip,
+six frames apart, against the same seed and prompt: 0.5 returned 0.98x the
+high-frequency energy of 0.35 and 0.65 returned 0.92x, while the mean per-pixel
+drift from the 0.35 result rose from 6.3 to 12.6 (0-255). What caps detail is
+what the DiT can resolve at the target size, not how noisy a start it is given;
+past that, a higher strength only erases more of the first pass and re-imagines
+it from less. On this checkpoint 0.35 was the best of the three -- treat higher
+values as "re-generate", not as "sharpen".
+
+Why this is faster than generating at the target size: the DiT cost follows the
+video token count, which is quadratic in the latent area for attention. The
+example above is 31,620 video tokens at 960x544 against 126,480 at 1920x1088 --
+4x the tokens, so roughly 4x the linear cost and up to 16x the attention cost
+per step. Paying that on 20 steps instead of 50 is where the saving comes from.
+The gain therefore depends on the resolutions, the strength, and how
+attention-bound the shape is; measure it on your own workload.
+
+Audio is re-noised and refined alongside the video at the same schedule
+position. The two modalities are shifted apart (`flow_shift` 12.0 against
+`audio_flow_shift` 3.0), so each resumes at its own sigma for that position --
+the state the first pass actually passed through, which is what keeps the pass
+in distribution.
+
+`latent_refine` works without `latent_upscale` too, as a plain detail pass at
+the generated size. For FL2VA, the keyframe condition is pinned to the output
+latent size, so a refine after an upscale re-encodes the keyframes from the
+originals at the new size; REF2VA references carry their own shapes and are
+reused as they are.
+
+### Options
+
+| `--additional-config` key | Default | Meaning |
+| --- | --- | --- |
+| `latent_upscaler_path` | unset | Checkpoint file, or a directory holding exactly one. Unset disables the stage and loads no extra weights. |
+| `latent_upscaler_dtype` | the engine dtype | `bf16`, `fp16` or `fp32`. |
+| `latent_upscaler_chunk_frames` | `0` | `0` runs the whole clip in one pass; a positive value bounds activation memory with approximate temporal chunking. |
+| `latent_upscaler_resident` | `false` | Keep the ~700 MB of weights on the GPU between requests instead of in host memory. |
+| `latent_refine_max_tokens_per_rank` | `65536` | Reject larger refine layouts before denoising. Set `0` only after validating a larger deployment. |
+| `latent_upscale` | unset | A default target applied to every request, in the request's own format. A request overrides it, and `false` opts out. |
+| `latent_refine` | unset | A default refine strength applied to every request. A request overrides it, and `false` opts out. |
+
+The `latent_upscale` request value is a multiplier (`2.0`), `false`, or an
+object naming one mode: `{"scale": 2.0}`, `{"width": …, "height": …}` or
+`{"megapixels": …}`, each accepting an `align` (default `32` pixels). The
+checkpoint was trained between 1.0x and 4.0x and only upscales; a smaller
+target is rejected, and `scale` 1.0 is a no-op that skips the stage. The
+`latent_refine` value is a strength in `(0, 1]`, `false`, or
+`{"strength": 0.4}`; it needs no checkpoint of its own.
+
+When `latent_refine` is enabled, both resolved output dimensions must be
+divisible by 32 pixels for the DiT's spatial patches. A custom `align` that
+produces an incompatible target is rejected before denoising; upscale-only
+requests can still use the VAE's 16-pixel grid. Numeric upscale fields must be
+positive and finite, and `width`, `height`, and `align` must be integers.
+
+Temporal chunking can bound activation memory on long clips -- a 15-second clip is
+102 latent frames, more than three 32-frame chunks. It is an approximation rather than a
+partition: the network's GroupNorms pool statistics over whatever clip they are
+handed, so a chunked pass can differ from a single pass across the whole output
+and not only at the chunk seams. How much that costs in practice was not
+measured here; the default single pass buys the exact result for the
+memory a full-length activation volume takes (10.1 GiB against 5.6 GiB at
+2688x1536 for 15 seconds, measured on one B300). Set
+`latent_upscaler_chunk_frames: 32` explicitly if that memory cost is too high.
+
 ## Known limitations

 - TeaCache is calibrated for FL2VA only; Ref2VA requests run uncached.
+- The latent upscaler is a community checkpoint, not part of the H3 release,
+  and is unavailable unless `latent_upscaler_path` names one. It runs
+  replicated on every DiT rank rather than sharded.
+- `latent_refine` is request-mode only. It is a second denoise loop with its
+  own resolution and its own truncated schedule, and the step contract carries
+  one schedule per request, so step execution rejects it. `latent_upscale`
+  without `latent_refine` works in both modes.
+- Large refine layouts can abort workers with `CUDA error: an illegal memory
+  access was encountered`. On B300, observed video token counts of 54,144 per
+  rank complete while 108,288 and 109,360 abort. The preflight limit of 65,536
+  counts the full padded layout, including text, audio, and visual references,
+  per Ulysses rank. It rejects the observed failing sizes before the base pass.
+  This is an empirical B300 guard, not a proven kernel limit on every device;
+  deployments with different hardware can adjust the limit after validation.
 - Combined serving requires sibling `FL2VA` and `Ref2VA` directories, loads
  both task-specific DiTs, and loads shared components once from `FL2VA`.
 - Request mode executes one generation request per diffusion batch. Use
@@ -25,10 +25,10 @@ matching the reference implementation frame for frame on the golden replays used
 as the acceptance gate.

 PersonaPlex is the first Moshi-class (pure-lockstep) model on the vLLM-Omni
-full-duplex serving stack: it plugs into the generic `/v1/duplex` handler
-through the standard plugin seams (`duplex_serving_adapter` /
-`duplex_runtime_extension` in its `pipeline.py`) with a model-specific
-package at `vllm_omni/model_executor/models/personaplex/duplex/`.
+unified full-duplex framework: its `pipeline.py` declares one
+`duplex_plugin` (`PersonaPlexDuplexPlugin`), and the model-specific package at
+`vllm_omni/model_executor/models/personaplex/duplex/` holds the plugin, the
+worker-side lockstep Stage 0 runtime and the 80 ms input framing.

 ## References

@@ -114,16 +114,18 @@ HF_TOKEN=... CUDA_VISIBLE_DEVICES=0 python -m vllm_omni.entrypoints.cli.main ser
  --deploy-config vllm_omni/deploy/personaplex.yaml
 ```

-This exposes `WS /v1/duplex` (native duplex dialect) and
-`WS /v1/realtime?duplex=1` (OpenAI Realtime projection; client API and wire
-protocol in [`docs/serving/realtime_duplex_api.md`](../../docs/serving/realtime_duplex_api.md)).
-Voice and persona are set per session via `extra_body`.
+This exposes `WS /v1/realtime?duplex=1` (alias `WS /v1/duplex`): the OpenAI
+Realtime session protocol, client API and wire vocabulary in
+[`docs/serving/realtime_duplex_api.md`](../../docs/serving/realtime_duplex_api.md).
+Voice (`voice`, a bundled `.pt` basename) and persona (`instructions`) are set
+in `session.update`; the model takes no client commits and serves no
+`/v1/chat/completions` route.

 #### Verification

 ```bash
-# GPU-free contract tests (stage0 runtime + unified serving adapter)
-pytest tests/model_executor/models/personaplex/duplex/ -q
+# GPU-free contract tests (plugin, stage0 runtime, runner scenario)
+pytest tests/model_executor/models/personaplex/duplex/ tests/engine/duplex/test_session_runner_personaplex.py -q

 # GPU e2e: paced 24 kHz PCM over /v1/realtime?duplex=1, two concurrent
 # sessions, overflow admission, slot recycling, non-silent output
@@ -141,6 +141,12 @@ The deploy config at
 is loaded automatically by the model registry (HF `model_type=voxcpm2`).
 Pass `--deploy-config <path>` to override.

+The `engine_extras.hf_overrides.voxcpm2_runtime_config` section controls
+LocDiT CFM generation. `inference_timesteps`, `cfg_value`, and
+`cfg_cutoff_ratio` default to `10`, `2.0`, and `1.0`, respectively.
+`inference_timesteps` is clamped to at least `2`: with CFG-Zero*, a single
+step would skip all estimator calls and return the initial noise.
+
 #### Verification

 **Server cold-start**: ~60 s from `vllm serve` to `Application startup
@@ -35,6 +35,10 @@ the requested `input` speech. Without a nonblank `ref_text`, Local uses
 audio-reference generation. Reference transcripts must match the reference
 audio; they are not style instructions.

+For the optional CUDA MRV2 runner, see the
+[Local 1.5 deployment profile](../../docs/configuration/stage_configs.md#moss-tts-local-15-with-model-runner-v2).
+The default Local deployment continues to use V1.
+
 ## References

 - Offline inference example: [`examples/offline_inference/text_to_speech/moss_tts/`](../../examples/offline_inference/text_to_speech/moss_tts/)
@@ -152,3 +156,296 @@ curl -X POST http://localhost:8091/v1/audio/speech \
 - No `ref_audio` required or accepted for MOSS-SoundEffect.
 - Input field maps to the `ambient_sound` parameter in the upstream processor.
 - Rate: ~12.5 tokens per second; longer descriptions produce longer audio.
+
+## Local 1.5 MRV2 and slot attention
+
+`MOSS-TTS-Local-Transformer-v1.5` supports the native CUDA MRV2 pipeline with
+an explicit deploy profile. The default Local profile continues to use V1.
+Both profiles below preserve 1-frame initial and 15-frame steady codec chunks
+and the model's sampling defaults.
+
+```bash
+vllm serve OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 --omni \
+  --deploy-config vllm_omni/deploy/moss_tts_local_mrv2.yaml
+```
+
+This C64 profile bounds Talker prefill to 512 tokens and retains codec-owned
+CUDA graphs without compiling the codec with Inductor.
+
+For sustained high concurrency on a large-memory CUDA GPU, use:
+
+```bash
+CUDA_VISIBLE_DEVICES=0 OMP_NUM_THREADS=4 \
+vllm serve OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 --omni \
+  --deploy-config vllm_omni/deploy/moss_tts_local_mrv2_high_concurrency.yaml \
+  --stage-init-timeout 1200 --init-timeout 1500
+```
+
+The high-concurrency profile places both stages on one GPU, sets each stage's
+capacity to 128 and fixes the Talker KV budget at 32 GiB. The Talker uses
+mixed FULL CUDA graphs with both its token budget and capture limit set to
+512, so new prompt chunks remain within graph coverage. It also enables
+`hf_overrides.mrv2_gpu_slot_state`: Local hidden states, audio codes and
+continuation control stay in GPU request slots, gathered through MRV2's
+batch mapping. Prefill conditioning and explicit per-request sampling seeds
+retain their original behavior. Published audio codes own their storage, so
+slot reuse cannot overwrite an in-flight output. Set the override to `false`
+to compare with the generic Omni model state. The profile selects
+`codec_attention_backend: triton_slot`, Inductor mode 3 with combo kernels
+disabled, and codec CUDA graph buckets through 128. The configuration was
+validated on one H200. The earlier capacity-256 variant used about 110 GiB of
+sampled peak GPU memory, including loading and warmup. Reduce the stage capacities,
+graph buckets and coalescing target together when adapting it to smaller GPUs. Cold codec
+compilation can take several minutes; subsequent starts can reuse the AOT
+cache. Graph capture is still performed at startup.
+
+The high-concurrency profile also coalesces ready codec streams with
+`connectors.shm.extra.generation_min_batch_size: 32` and
+`generation_max_wait_ms: 12`. The target controls when to stop waiting; it
+neither caps execution at 32 nor guarantees every codec group has 32 rows.
+First and final chunks share the bounded window at high concurrency. When
+fewer than 32 requests remain, dispatch is immediate. Pending outputs are
+retired before waiting, with the original deadline preserved, so waiting does
+not prevent the previous batch from releasing its in-flight state. Cancellation
+and input notifications keep their ordinary scheduler bookkeeping. This option
+requires a stateful native MRV2 generation stage with TP1/PP1. Set the wait to
+`0` to disable coalescing; other profiles keep the immediate-dispatch default.
+
+The high-concurrency profile enables two additional Local model-state
+optimizations under the CUDA stage-0 `hf_overrides`, alongside
+`mrv2_gpu_slot_state: true`:
+
+- `mrv2_batch_prefill: true` reuses the batch's text embeddings and combines
+  CPU reference-code slices and their destination positions into one pinned
+  upload. Host staging is reused only after the upload event completes;
+  GPU-resident references keep the original path.
+- `mrv2_direct_tokens: true` returns the already determined text token after
+  the Local audio/stop decision. It avoids constructing and sampling the
+  full text vocabulary. Audio-code and binary-stop sampling are unchanged.
+  Requests requiring distribution metadata or token constraints, and
+  unsupported execution modes, retain the normal sampler.
+
+Other profiles leave both flags disabled. Set both flags explicitly when
+comparing their combined effect. Deployment
+inheritance replaces a stage's `hf_overrides` mapping, so an overlay must also
+retain `mrv2_gpu_slot_state: true`. Measure first-packet latency as well as
+throughput: faster prompt preparation can change competition between the
+Talker and codec sharing the GPU.
+
+For the single-H200 capacity-128 throughput baseline, keep both switches on.
+In complete Seed-TTS EN1088 runs at client concurrency 128, four combined-switch
+rounds pooled 344.11 audio-s/s versus 335.78 for batch prefill alone; direct
+tokens alone had no reliable gain. The combined path had 685 ms mean first
+audio versus 679 ms across six control rounds, with lower mean completion time.
+A fixed-seed 128-row audio sample showed mean Whisper WER of 3.68% versus
+3.78% for the control. This sample does not establish speaker similarity or
+exclude small quality changes. At concurrency 256, the combined path had
+raised mean first-audio latency by about 0.33 s, so other profiles leave the
+switches off until their latency and quality are checked.
+
+The optional connector setting `generation_coalescing_policy: idle_wait`
+waits for an inbox notification when no stream is runnable, all receivers
+are parked and registered, and no output needs retirement. The first ready
+arrival starts a fresh coalescing window, preserving the batch budget.
+Control messages, including cancellations, can wait up to two windows
+instead of one (24 ms at the 12 ms setting). The default policy remains
+`fixed`; neither policy changes the global orchestration default.
+
+The codec backends differ in state access:
+
+- `sdpa` uses PyTorch attention after gathering and updating the ring cache.
+- `triton` replaces the attention calculation, retaining the ring-cache
+  gather/copy and explicit mask construction.
+- `triton_slot` writes surviving K/V directly into request slots, attends
+  directly to the ring, and advances the active slot offsets in three
+  ordered kernels. Graph-padding rows do not advance persistent state.
+
+The slot path preserves the existing chunk-complete ring semantics, including
+retaining the final cache-capacity tokens when a chunk exceeds the ring.
+It does not introduce another cache owner or change request-slot lifetime.
+Zero-length padding rows skip the attention computation and emit zeros.
+
+The high-concurrency profile includes intermediate codec batch buckets 6, 12
+and 24. To compare with power-of-two buckets, change the stage-1
+`cudagraph_capture_sizes` to `[1, 2, 4, 8, 16, 32, 64, 128]`.
+Keep the maximum bucket equal to the state capacity to retain terminal-tail
+coalescing. Smaller buckets reduce padding work but require more graphs;
+measure complete serving runs before selecting them for a deployment.
+
+For a codec output-path comparison, set
+`connectors.shm.extra.codec_gpu_stream_output: 0` to select the synchronous
+codec output path; `1` selects GPU output snapshots on a private codec
+stream. Both configurations use MRV2 and keep streaming audio responses.
+The option changes input ownership, metadata staging and snapshot handling
+as well as transfer scheduling, so its timing difference is not just D2H
+copy time.
+
+Those chunk-complete semantics truncate the causal window: every ring holds
+exactly `context` entries and a whole chunk is written before attention runs,
+so token `i` of a `T`-token chunk sees `capacity - T + i + 1` keys. With the
+tokenizer-v2 decoder's per-layer contexts (10/10/8/4/2/1 s, i.e. 125/250/400/
+400/400/400 tokens) a 15-frame chunk is 480 tokens at the last transformer, so
+its first 80 tokens attend to nothing and no chunk sees the previous one there.
+Padding a terminal tail to 15 frames therefore decodes it differently from an
+exact-length decode; the difference is deterministic (identical in fp32 and
+fp64), not bf16 noise. The connector option `codec_ring_headroom: 1` sizes each
+ring as `context + max_chunk_frames * tokens_per_frame` (including ramp shapes); chunked streaming
+then reproduces whole-sequence decoding bit-for-bit in fp64 and padded tails
+equal exact tails. It increases state/graph memory and capture time; historical C256 experiments
+observed about a 2% throughput cost. It is off by default and its perceptual
+quality benefit has not been established. Terminal tails are padded
+into the regular 15-frame graph bucket in either mode.
+The slot kernel is specific to the CUDA tokenizer-v2 decoder; other paths
+retain their existing attention implementation. Event-driven orchestration
+remains independently selectable and its default is unchanged.
+
+### Optional progressive chunks
+
+Set `codec_chunk_ramp` under `connectors.shm.extra` to insert smaller chunks
+before steady decoding, for example:
+
+```yaml
+connectors:
+  shm:
+    extra:
+      codec_chunk_ramp: [1, 4, 15]
+```
+
+The ramp's first entry overrides `initial_codec_chunk_frames`. After the last
+entry, chunks use `codec_chunk_frames`. Each request advances independently;
+the final partial chunk is flushed and an empty terminal carries no codec
+tokens. The codec captures every ramp length, sizes the maximum execution
+step for the largest entry, and uses the ramp's first entry for its dedicated
+first-chunk graph. Additional shapes increase startup and graph memory.
+
+Ramps are opt-in. They can reduce early playback underrun while increasing
+codec calls, first-audio latency or total generation time. The measured C128
+throughput presets retain 1→15; their published figures do not establish an
+additional benefit from enabling ramps. The processor and graph-shape tests
+run without GPU or model weights:
+
+```bash
+CUDA_VISIBLE_DEVICES= PYTHONPATH=. python -m pytest -q \
+  tests/model_executor/stage_input_processors/test_moss_tts_async_chunk.py \
+  -m 'core_model and cpu' --run-level core_model
+```
+
+### Low-latency and reference-encoding options
+
+`moss_tts_local_mrv2_low_latency.yaml` keeps both stage capacities at 128 and
+uses prefill 2048, GPU slot state, batch prefill, direct tokens and eager MTP.
+Its codec first-chunk fast path uses dedicated graphs and a stream handoff
+before the regular decoder reuses the request slot. A gate limits contention
+with regular codec work. Slot waits are bounded at 30 seconds. A failed or
+stalled decode raises an error rather than replaying a partially advanced
+slot; a failed output enqueue retains owned PCM for regular delivery.
+Closing the decoder rejects new jobs. Its regular-batch limit of 16 is internal dispatch
+policy; it does not change the client concurrency or stage capacities.
+
+Reference encoding runs in the API layer, independently of MRV2. Enable
+reference graphs explicitly with `VLLM_OMNI_MOSS_REF_GRAPHS=1`; the default
+eager encoder is retained when the option is absent. The graph path uses the
+loaded tokenizer's encoder/quantizer, length buckets and optional compilation
+(`VLLM_OMNI_MOSS_REF_COMPILE=0` disables compilation). It also uses windowed
+attention; `VLLM_OMNI_MOSS_REF_ATTN=sdpa` retains the original attention.
+Compilation and attention changes need not produce bit-identical codes.
+
+For multiple API processes, `VLLM_OMNI_MOSS_REF_CODES_SHARED_DIR` enables shared
+reference-code storage and, by default, a single encoder host with four
+workers. Use a dedicated directory per service, checkpoint and encoding
+configuration. Workers have their own graph resources; all graph captures
+complete before serving begins. `VLLM_OMNI_MOSS_REF_SHARED_ENCODER=0` retains
+separate encoders while sharing codes. More workers/graphs consume memory;
+they do not imply more GPUs. Example after selecting an available GPU and
+configuring any operator-managed MPS daemon:
+
+```bash
+VLLM_WORKER_MULTIPROC_METHOD=spawn \
+VLLM_OMNI_EVENT_DRIVEN_ORCH=1 \
+VLLM_OMNI_CONNECTOR_RECV_POLL_MS=1 \
+VLLM_OMNI_MOSS_REF_GRAPHS=1 \
+VLLM_OMNI_MOSS_REF_ENCODER_WORKERS=4 \
+VLLM_OMNI_MOSS_REF_BATCH_WINDOW_MS=0 \
+VLLM_OMNI_MOSS_REF_INFLIGHT=4 \
+VLLM_OMNI_MOSS_REF_HOST_WINDOW_MS=2 \
+VLLM_OMNI_MOSS_REF_CODES_SHARED_DIR=/dev/shm/moss-local-service \
+vllm serve OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 --omni \
+  --api-server-count 4 \
+  --deploy-config vllm_omni/deploy/moss_tts_local_mrv2_low_latency.yaml \
+  --stage-init-timeout 1200 --init-timeout 1500
+```
+
+This example is a configurable serving profile, not the complete launch
+configuration of a historical benchmark. In particular, the low-latency YAML
+uses synchronous stage-1 scheduling; the retained source46 experiment used
+asynchronous stage-1 scheduling, FP8 backbone, MPS and additional encoder
+environment settings. Do not infer a throughput result from the YAML alone.
+
+### Cold versus hot reference measurements
+
+A cold reference misses the encoded-reference cache: the request pays for
+reference parsing/preparation and GPU encoding before speech generation.
+A hot reference reuses reference codes but still synthesizes new speech;
+generated audio is not cached. Pre-registering a reference moves encoding
+into registration and can make subsequent synthesis hot, but registration
+latency is still part of the first-use cost.
+
+Model startup/compilation is separate from reference coldness. EN1088 contains
+repeated references, so its first pass is not uniformly cold. For an all-unique
+cold pass, use its 666 first-occurrence references, keep order, warm execution
+with disjoint references, then repeat the same 666 requests. A second pass
+through independent API-local caches is not guaranteed hot; verify cache
+coverage before comparing it with a shared-cache result. Report requests/s
+alongside audio-s/s and retain failed, empty, long and near-silent outputs.
+
+### Reproduce the serving benchmark
+
+Use the complete Seed-TTS English test set, including reference audio and text.
+Set `SEED_TTS_DATA` to the directory containing `en/meta.lst` and its 1088
+entries. Run the native benchmark after the server is ready:
+
+```bash
+export VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=48000
+export VLLM_OMNI_BENCH_AUDIO_CHANNELS=2
+export SEED_TTS_WER_EVAL=0
+
+for concurrency in 128; do
+  for phase in warm r1 r2; do
+    vllm bench serve --omni \
+      --model OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 \
+      --backend openai-audio-speech --endpoint /v1/audio/speech \
+      --dataset-name seed-tts --dataset-path "$SEED_TTS_DATA" \
+      --seed-tts-locale en --disable-shuffle --num-prompts 1088 \
+      --num-warmups 0 --ready-check-timeout-sec 0 \
+      --output-len 256 --max-concurrency "$concurrency" \
+      --request-rate inf --seed 42 \
+      --extra-body '{"task_type":"Base","max_new_tokens":256}' \
+      --save-result --save-detailed --result-dir results/moss-local \
+      --result-filename "c${concurrency}-${phase}.json"
+  done
+done
+```
+
+Discard the complete `warm` pass and combine measured runs as total generated
+audio seconds divided by total benchmark duration. Require 1088 successes,
+zero failures and 1088 nonempty-audio metric samples in each run. The output
+cap matches the benchmark protocol; success alone does not establish speech
+quality or that every sentence ended before the cap. Dataset seed 42 does
+not fix an independent sampling seed for every request.
+
+For an attention-only comparison, copy the high-concurrency YAML beside the
+original to preserve relative `base_config` resolution. Change its
+codec `compilation_config.mode` to `0` while retaining `cudagraph_mode: FULL`,
+and compare `codec_attention_backend: triton` against `triton_slot`. Keep all
+other settings, warmup and client concurrency identical. Restart the server
+between configurations and use separate result directories. Comparing the
+compiled slot profile to uncompiled Triton includes both changes.
+
+Kernel and MHA regression tests require CUDA, compatible vLLM/Triton packages,
+and no model weights:
+
+```bash
+python -m pytest -q tests/model_executor/models/moss_tts/test_slot_attention.py \
+  tests/model_executor/models/moss_tts/test_streaming_attention.py \
+  -m 'core_model and cuda' --run-level=core_model
+```
@@ -7,7 +7,8 @@
 - Vendor: Qwen
 - Model: `Qwen/Qwen-Image`
 - Task: Text-to-image generation
- Mode: Online serving with optional step-wise continuous batching
+- Mode: Online serving with optional step-wise continuous batching; offline
+  inference
 - Maintainer: Community

 ## When to use this recipe
@@ -25,11 +26,14 @@ the benchmark assets already bundled in this repository.
  [`examples/online_serving/text_to_image/README.md`](../../examples/online_serving/text_to_image/README.md)
 - Related benchmark:
  [`benchmarks/diffusion/diffusion_benchmark_serving.py`](../../benchmarks/diffusion/diffusion_benchmark_serving.py)
+- Offline inference example used by the XPU section:
+  [`examples/offline_inference/text_to_image/text_to_image.py`](../../examples/offline_inference/text_to_image/text_to_image.py)

 ## Hardware Support

-This recipe currently documents one CUDA GPU serving configuration. Extend it
-with more hardware sections as community validation lands.
+This recipe documents CUDA GPU serving configurations and one Intel XPU
+offline-inference configuration. Extend it with more hardware sections as
+community validation lands.

 ## GPU

@@ -235,3 +239,59 @@ CUTLASS was the fastest validated backend for this checkpoint.
 - The benchmark table intentionally reports Qwen-Image-2512 as
  concurrency-1 request-level data; do not compare it directly with an online
  HTTP concurrency benchmark.
+
+## XPU
+
+### 1x Intel Arc Pro B70 (32 GB)
+
+Offline text-to-image at 1024x1024, with layerwise offload streaming the
+transformer blocks so the 20B model fits one card.
+
+#### Environment
+
+- OS: Linux
+- Python: 3.10+
+- torch: 2.13.0+xpu
+- vLLM: 0.29.0 (`98dff2a8`)
+- vLLM-Omni: `main` at `4c7a98c2`
+
+#### Command
+
+```bash
+python examples/offline_inference/text_to_image/text_to_image.py \
+  --model Qwen/Qwen-Image \
+  --prompt "a cup of coffee on the table" \
+  --num-inference-steps 50 \
+  --enable-layerwise-offload \
+  --vae-use-tiling \
+  --vae-use-slicing \
+  --enforce-eager \
+  --output qwen_image_output.png
+```
+
+Given spare cards, `--enable-cpu-offload` shards the transformer instead of
+streaming it. It shortens the denoise loop but not the run, because the
+per-request model swap costs more than the loop saves.
+
+```bash
+  --tensor-parallel-size 4 \
+  --enable-cpu-offload
+```
+
+| Offload | Cards | Peak VRAM | Time |
+| --- | ---: | ---: | ---: |
+| `--enable-layerwise-offload` | 1 | 16.6 GiB | 50 s |
+| `--enable-cpu-offload` | 2 | 26.4 GiB | 85 s |
+| `--enable-cpu-offload` | 4 | 19.8 GiB | 70 s |
+| `--enable-cpu-offload` | 8 | 16.9 GiB | 87 s |
+
+#### Verification
+
+Confirm `qwen_image_output.png` is written and looks coherent for the prompt.
+
+#### Notes
+
+- Memory usage: peak 16.6 GiB, about 55 s per image.
+- Known limitations: only offline generation was qualified. Step-wise continuous
+  batching, ModelOpt quantization, and online serving are out of scope for this
+  profile.
@@ -53,7 +53,7 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
  --deploy-config /path/to/your_qwen3_omni_overrides.yaml
 ```

-#### Runtime tuning
+### Runtime tuning

 Prefer CLI overrides for day-to-day tuning:

@@ -75,7 +75,7 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
  }'
 ```

-#### Stage-based launch (one stage per process)
+### Stage-based launch (one stage per process)

 Use three terminals (one per stage). Start with the default commands below, then
 add `--max-num-seqs` only if you need explicit per-stage concurrency control.
@@ -135,7 +135,7 @@ CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \

 If you use custom deploy YAML, add `--deploy-config` to each stage command.

-#### Verification
+### Verification

 After server startup, run a multimodal example client:

@@ -185,7 +185,7 @@ python examples/online_serving/qwen3_omni/openai_realtime_client.py \
  --output-wav realtime_output.wav
 ```

-#### Benchmark with `vllm bench`
+### Benchmark with `vllm bench`

 After the server is up, you can run online serving benchmarks with
 `vllm bench serve --omni`.
@@ -240,8 +240,54 @@ vllm bench serve \
  --percentile-metrics ttft,tpot,itl,e2el
 ```

-#### Notes
+### Notes

 - `/v1/realtime` is unsupported while `async_chunk` is enabled.
 - The default deploy uses `SharedMemoryConnector`; this is for single-host
  stage wiring.
+
+## XPU
+
+### 5x Intel Arc Pro B70 (32 GB)
+
+Offline end-to-end text query returning both text and speech. The 62 GiB of
+BF16 thinker weights need four cards, so the `xpu:` block of
+`vllm_omni/deploy/qwen3_omni_moe.yaml` runs the thinker at TP=4 on cards 0-3
+and the talker and code2wav on card 4. It is the registry default here, so
+`--deploy-config` is not needed.
+
+#### Environment
+
+- OS: Linux
+- Python: 3.10+
+- torch: 2.13.0+xpu
+- vLLM: 0.29.0 (`98dff2a8`)
+- vLLM-Omni: `main` at `4c7a98c2`
+
+#### Command
+
+The raised timeouts absorb the first decode step, which JIT-compiles the
+Triton MoE kernels.
+
+```bash
+VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800 \
+python examples/offline_inference/qwen3_omni/end2end.py \
+  --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
+  --query-type text \
+  --stage-init-timeout 1800 \
+  --init-timeout 1800 \
+  --output-wav qwen3omni_output
+```
+
+#### Verification
+
+`qwen3omni_output/` holds one `.txt` answer and the matching `.wav`. Confirm
+the transcript of the audio tracks the text response.
+
+#### Notes
+
+- Runtime: about 260 s for a single text query.
+- The thinker MoE layers have no tuned Triton config for this shape and log
+  `Using default MoE config. Performance might be sub-optimal!`.
+- Known limitations: only `--query-type text` was qualified. The audio, image,
+  and video query types and online serving are out of scope for this profile.
@@ -45,6 +45,7 @@ recipes/
 | [`Baidu/ERNIE-Image.md`](./Baidu/ERNIE-Image.md) | Text-to-image online serving (ERNIE-Image 8B) | 1x or 2x RTX 4090 24GB |
 | [`Bagel/BAGEL-7B-MoT.md`](./Bagel/BAGEL-7B-MoT.md) | Text-to-image with shared online/offline examples | 1x A100 80GB / 2x CUDA GPUs |
 | [`black-forest-labs/FLUX.1-schnell.md`](./black-forest-labs/FLUX.1-schnell.md) | Offline text-to-image generation with CPU offload | 1x NVIDIA GeForce RTX 5090 32GB |
+| [`black-forest-labs/FLUX.2-klein.md`](./black-forest-labs/FLUX.2-klein.md) | Offline image editing (FLUX.2-klein-4B) | 1x Intel Arc Pro B70 32GB |
 | [`Boogu/Boogu-Image.md`](./Boogu/Boogu-Image.md) | Text-to-image online serving (Boogu-Image-0.1-Base) | 1x A100/H100 40GB+ |
 | [`BosonAI/Higgs-Audio-V3-TTS.md`](./BosonAI/Higgs-Audio-V3-TTS.md) | Online + offline multilingual TTS with voice cloning | 1x H100 80GB |
 | [`BreezeBlue/Breeze-TTS-2.md`](./BreezeBlue/Breeze-TTS-2.md) | Online + offline TTS (plain / voice design / voice clone / voice direction, streaming PCM) | 1x NVIDIA L20 48GB |
@@ -62,9 +63,9 @@ recipes/
 | [`MiniMaxAI/MiniMax-H3.md`](./MiniMaxAI/MiniMax-H3.md) | T2VA, FL2VA, image+audio Ref2VA, and multi-video Ref2VA serving | 1x GPU with CPU offload / 4x B300 |
 | [`MiniMaxAI/MiniMax-H3-Spark-GB10.md`](./MiniMaxAI/MiniMax-H3-Spark-GB10.md) | MiniMax H3 T2VA and Ref2VA on DGX Spark | 1x or 2x DGX Spark (GB10) |
 | [`MiniMaxAI/MiniMax-H3-MUSA.md`](./MiniMaxAI/MiniMax-H3-MUSA.md) | MiniMax H3 T2VA, FL2VA, and Ref2VA on MUSA | 4x MTT S5000 (TP4 Ref2VA profile) |
-| [`krea/Krea-2.md`](./krea/Krea-2.md) | Text-to-image (Turbo + Raw), offline + online, with LoRA | 1x H100 80GB |
-| [`LTX/LTX-2.md`](./LTX/LTX-2.md) | LTX-2/LTX-2.3 text-to-video and image-to-video with synchronized audio | H200 141GB / 96GB-class GPU |
-| [`LTX/LTX-2.5.md`](./LTX/LTX-2.5.md) | LTX-2.5-Diffusers: Full/SFT one-stage and distilled two-stage T2V/I2V with synchronized audio | NVIDIA B300; cuDNN-qualified |
+| [`krea/Krea-2.md`](./krea/Krea-2.md) | Text-to-image (Turbo + Raw), offline + online, with LoRA | 1x H100 80GB / 1x Intel Arc Pro B70 32GB |
+| [`LTX/LTX-2.md`](./LTX/LTX-2.md) | LTX-2/LTX-2.3 text-to-video and image-to-video with synchronized audio | H200 141GB / 96GB-class GPU / 1x Intel Arc Pro B70 32GB |
+| [`LTX/LTX-2.5.md`](./LTX/LTX-2.5.md) | LTX-2.5-Diffusers: Full/SFT one-stage and distilled two-stage T2V/I2V with synchronized audio | NVIDIA B300; cuDNN-qualified / 1x Intel Arc Pro B70 32GB |
 | [`MammothModa2/MammothModa2.md`](./MammothModa2/MammothModa2.md) | Preview and Dev text-to-image (AR → DiT); Dev text/image understanding | Preview: 1x L40S 48GB / 1x ≥40GB GPU; Dev: 1x NVIDIA GPU with sufficient cache headroom |
 | [`NVIDIA/SANA-Video-2B.md`](./NVIDIA/SANA-Video-2B.md) | Native and Diffusers-adapter 480p/720p T2V+I2V | 1x RTX 5090 32GB |
 | [`meituan-longcat/LongCat-Video-Avatar-1.5.md`](./meituan-longcat/LongCat-Video-Avatar-1.5.md) | Audio-driven avatar video generation (AT2V / AI2V, single- and multi-speaker, AVC continuation) | 1x H100 80GB |
@@ -82,10 +83,10 @@ recipes/
 | [`NVIDIA/Nemotron-Labs-Audex.md`](./NVIDIA/Nemotron-Labs-Audex.md) | TTS / text-to-audio / audio understanding / cascaded S2S (Audex 2B + 30B-A3B) | 1x H100 80GB |
 | [`NVIDIA/NemotronLabs-VoiceChat.md`](./NVIDIA/NemotronLabs-VoiceChat.md) | Offline speech-to-speech voice chat (11B, frame-locked 12.5 Hz, 3-stage thinker/talker/code2wav) | 1x H100 80GB |
 | [`NVIDIA/SANA-WM.md`](./NVIDIA/SANA-WM.md) | First-frame image-to-video serving with camera control | 1x 24GB+ CUDA GPU; ~40GB disk |
-| [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt mixed FP8/NVFP4 | 1x A100 80GB / 2x B200 |
+| [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt mixed FP8/NVFP4; offline text-to-image on XPU | 1x A100 80GB / 2x B200 / 1x Intel Arc Pro B70 32GB |
 | [`Qwen/Qwen-Image-2512.md`](./Qwen/Qwen-Image-2512.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt FP8 / mixed FP8/NVFP4 | 1x A800 80GB / 2x B200 |
 | [`Qwen/Qwen-Image-Edit.md`](./Qwen/Qwen-Image-Edit.md) | Text-guided single-image editing | 1x or 2x H200 141GB |
-| [`Qwen/Qwen3-Omni.md`](./Qwen/Qwen3-Omni.md) | Online serving for multimodal chat | 1x A100 80GB |
+| [`Qwen/Qwen3-Omni.md`](./Qwen/Qwen3-Omni.md) | Online serving for multimodal chat; offline end-to-end text-to-speech on XPU | 1x A100 80GB / 5x Intel Arc Pro B70 32GB |
 | [`Qwen/Qwen3-TTS.md`](./Qwen/Qwen3-TTS.md) | Text-to-speech serving (CustomVoice / VoiceDesign / Base) | 1x H100/A100 80GB / 1x AMD MI300X |
 | [`SenseNova/SenseNova-U1.md`](./SenseNova/SenseNova-U1.md) | Unified image generation and understanding | 1x or 2x H200 / 1x AMD MI300X |
 | [`SenseNova/SenseNova-U1.5.md`](./SenseNova/SenseNova-U1.5.md) | Unified image generation and understanding, with an 8-step distilled LoRA | 1x A800 80GB |
@@ -93,13 +94,15 @@ recipes/
 | [`Tencent/AuK-H100.md`](./Tencent/AuK-H100.md) | Offline instruction-driven TTS, speech editing, enhancement and separation (AuK base and Flash) | 1x H100 80GB |
 | [`Tencent/Covo-Audio-Chat.md`](./Tencent/Covo-Audio-Chat.md) | Online serving for audio chat | 1x A100 80GB |
 | [`Tencent/HunyuanImage-3.0-Instruct.md`](./Tencent/HunyuanImage-3.0-Instruct.md) | DiT-only text-to-image serving and benchmark, including ModelOpt mixed FP8/NVFP4 | 4x H100/H800 80GB / 2x B200 |
-| [`Wan-AI/Wan2.2-T2V.md`](./Wan-AI/Wan2.2-T2V.md) | Text-to-video serving (Wan2.2 14B) with Skip-Softmax sparse attention | 1x datacenter Blackwell (B200/B300) |
+| [`Tencent/HunyuanVideo-1.5.md`](./Tencent/HunyuanVideo-1.5.md) | Offline text-to-video (HunyuanVideo-1.5 Diffusers) | 1x Intel Arc Pro B70 32GB |
+| [`Tongyi-MAI/Z-Image-Turbo.md`](./Tongyi-MAI/Z-Image-Turbo.md) | Offline few-step text-to-image (Z-Image-Turbo) | 1x Intel Arc Pro B70 32GB |
+| [`Wan-AI/Wan2.2-T2V.md`](./Wan-AI/Wan2.2-T2V.md) | Text-to-video serving (Wan2.2 14B) with Skip-Softmax sparse attention | 8x H20/H100/A100 / 1x datacenter Blackwell (B200/B300) / 1x Intel Arc Pro B70 32GB |
 | [`Wan-AI/Wan2.2-TI2V-5B-A100.md`](./Wan-AI/Wan2.2-TI2V-5B-A100.md) | Unified 720p text-to-video and image-to-video serving (Wan2.2 5B) | 1x A100-SXM4-80GB |
 | [`Wan-AI/Wan2.2-I2V.md`](./Wan-AI/Wan2.2-I2V.md) | Image-to-video serving (Wan2.2 14B) | 8x Ascend NPU (A2/A3) |
 | [`Wan-AI/Wan2.2-S2V.md`](./Wan-AI/Wan2.2-S2V.md) | Speech-to-video serving (Wan2.2 14B) | 2x A100/H100 80GB |
 | [`Wan-AI/Wan2.1-VACE.md`](./Wan-AI/Wan2.1-VACE.md) | Unified T2V, I2V, V2LF, FLF2V, inpaint, and R2V | 1x RTX 5090 (1.3B) / 1x L40S 48GB with layerwise offload (14B) |
 | [`XiaomiMiMo/MiMo-Audio.md`](./XiaomiMiMo/MiMo-Audio.md) | Offline + online omni audio (TTS, ASR, dialogue) | 1x RTX 5090 / 5090D 32GB |
-| [`StabilityAI/Stable-Diffusion-3.5.md`](./StabilityAI/Stable-Diffusion-3.5.md) | Text-to-image serving (SD 3.5-medium and SD 3.5-large) | 1x RTX A6000 48GB |
+| [`StabilityAI/Stable-Diffusion-3.5.md`](./StabilityAI/Stable-Diffusion-3.5.md) | Text-to-image serving (SD 3.5-medium and SD 3.5-large) | 1x RTX A6000 48GB / 1x Intel Arc Pro B70 32GB |
 | [`zai-org/GLM-TTS.md`](./zai-org/GLM-TTS.md) | Online serving for Chinese/English zero-shot voice-cloned TTS | 1x A40 48GB / 2x non-standard RTX 4090 48GB (49,140 MiB each) |
 | [`GLM/GLM-Image.md`](./GLM/GLM-Image.md) | Online serving for image generation | 1x A800 80GB / 2x A800 80GB |
 | [`JD/JoyAI-VL-Interaction.md`](./JD/JoyAI-VL-Interaction.md) | Real-time streaming video-language interaction (proactive speak/silence/delegate) | 1x GPU 24GB+ |
@@ -79,6 +79,9 @@ python examples/offline_inference/text_to_image/text_to_image.py \
 classifier-free guidance; the default `4.0` applies guidance twice and produces a
 blown-out, posterised image.

+Online FP8 does not support the 8-step distilled LoRA.
+Use BF16 without `--quantization fp8` for this LoRA.
+
 ##### Online serving

 ```bash
@@ -90,7 +93,39 @@ python examples/online_serving/sensenova_u1/openai_chat_client.py \

 `-s` takes the base URL; the client appends `/v1` itself.

-#### Measured latency (1x A800 80GB, 25 steps, median of 3 after a warmup)
+##### Online FP8 quantization
+
+`--quantization fp8` quantizes only eligible attention and MLP linears in the
+`language_model` understanding and generation branches at load time. Other layers
+retain their existing precision.
+
+On the tested A800 (SM80) setup with vLLM 0.29.0,
+`VLLM_DISABLED_KERNELS=CutlassFP8ScaledMMLinearKernel` is **required** for online
+FP8 to start successfully. Without it, the selected CUTLASS kernel fails during
+the startup dummy run with `RuntimeError: cutlass_scaled_mm_sm80_epilogue`.
+Disabling this kernel selects Marlin W8A16 (FP8 weights with BF16 activations).
+
+```bash
+VLLM_DISABLED_KERNELS=CutlassFP8ScaledMMLinearKernel \
+vllm serve sensenova/SenseNova-U1.5-8B-MoT --omni \
+    --quantization fp8 --port 8091
+```
+
+#### Measured BF16 vs FP8 performance (1x A800 80GB, 28 steps, P95 over 10 prompts)
+
+vLLM 0.29.0, BF16 base dtype, seed 42, TORCH_SDPA, eager execution, TP=1, batch size 1,
+paged decode off, no LoRA. Each resolution uses the same 10 prompts for BF16 and FP8.
+Both runs set `VLLM_DISABLED_KERNELS=CutlassFP8ScaledMMLinearKernel`.
+
+| Resolution | Peak reserved memory (BF16 → FP8) | Step latency (BF16 → FP8) |
+| --- | --- | --- |
+| 512x512 | 33.166 → 18.697 GiB | 63.674 → 81.136 ms |
+| 1024x1024 | 33.555 → 18.561 GiB | 187.867 → 229.115 ms |
+
+Step latency is the P95 of per-request mean step times, excluding warmup requests; FP8 reduced
+memory usage but increased step latency.
+
+#### Measured BF16 latency (1x A800 80GB, 25 steps, median of 3 after a warmup)

 | Resolution | Step latency | Total |
 | --- | --- | --- |
@@ -34,6 +34,126 @@ text-to-text chat.
 - HuggingFace model page:
  [SenseNova/SenseNova-U1-8B-MoT](https://huggingface.co/SenseNova/SenseNova-U1-8B-MoT)

+## A3B (MoE) variant
+
+[sensenova/SenseNova-U1-A3B-MoT](https://huggingface.co/sensenova/SenseNova-U1-A3B-MoT)
+runs on this same recipe and the same `SenseNovaU1Pipeline` — the checkpoint keeps
+`model_type: neo_chat`, so it resolves without `--model-class-name`. Only the `--model`
+id changes.
+
+The backbone is Qwen3-MoE instead of dense Qwen3, and both MoT branches are sparse:
+`mlp` routes over 128 experts and `mlp_mot_gen` over 32, top-8 each, with per-expert
+width `moe_intermediate_size=768`. `SenseNovaU1Config` picks `SenseNovaU1MoELLMConfig`
+whenever `llm_config` carries `num_experts`, so the dense 8B path is untouched. The
+gen-path knobs (`gen_num_experts`, `gen_num_experts_per_tok`, `gen_moe_intermediate_size`)
+fall back to their understanding-path counterparts when absent.
+
+Experts run through vLLM's `FusedMoE`, whose `moe_forward` op resolves its layer through
+`vllm.forward_context` rather than the diffusion runner's own context, so
+`SenseNovaU1ForCausalLM.forward` enters that context when the model has MoE layers.
+
+### Memory
+
+The weights dominate, and every expert stays resident even though only the top-8 of each
+tower run per token -- the "A3B" in the name is the ~4B activated, not the 38.7B stored.
+Counted from the checkpoint's safetensors headers:
+
+| Component | Params | BF16 |
+| --- | --- | --- |
+| Understanding-path experts (128/layer) | 28.99 B | 54.0 GiB |
+| Generation-path experts (32/layer) | 7.25 B | 13.5 GiB |
+| Attention (dual tower) | 1.81 B | 3.4 GiB |
+| Embedding + LM head | 0.62 B | 1.2 GiB |
+| Routers, norms, vision and flow-matching heads | 0.06 B | 0.1 GiB |
+| Total | 38.74 B | 72.2 GiB |
+
+At 1024x1024 the 72.9 GiB peak is those 72.2 GiB of weights plus about 0.7 GiB of
+activations and workspace. The 2048x2048 edit, the largest activation case below, peaks at
+74.6 GiB, 2.4 GiB above the weights. Text-to-image only routes through the generation tower,
+leaving the 54 GiB of understanding-path experts idle; that tower is reached by the text
+prefix, think mode, `img2text` and `text2text`. One 96 GB card fits TP=1, and TP=2 halves
+the per-GPU footprint.
+
+### Measured (1x H20 96GB, BF16)
+
+Fixed case used for the stage split and the torch.profiler kernel table:
+
+- Prompt: `Close portrait of an elderly woman by a farmhouse window, warm natural light.`
+- 1024x1024, 50 Euler steps, think off, seed 42, `cfg_scale` 4.0, `cfg_norm` none,
+  `timestep_shift` 3.0, `t_eps` 0.02
+- Python 3.12, vLLM 0.30.0, torch 2.13.0+cu130
+- Peak GPU memory is the reserved high-water mark the runner records (`peak_memory_mb`)
+
+| | Time | Memory |
+| --- | --- | --- |
+| Weights after load | ~13 s | 72.2 GiB |
+| Pipeline `forward` | 7.68 s | 72.9 GiB peak |
+
+`--enable-diffusion-pipeline-profiler` on that request:
+
+| Stage | Time | Share |
+| --- | --- | --- |
+| Text prefix (`_t2i_prefix_forward`, understanding tower) | 0.06 s | 0.7% |
+| Denoising loop (`_run_denoising_loop`, generation tower, 50 steps) | 7.56 s | 98.5% |
+| Pipeline `forward` | 7.68 s | 100% |
+
+There is no VAE. The loop ends in unpatchify / denorm / PIL, which is inside the denoise
+row and is a few tens of milliseconds at 1024x1024.
+
+A warm 8-step generate at the same resolution, wrapped in `torch.profiler` after one
+unprofiled warmup request, spent 1.19 s of self-CUDA time as:
+
+| Kernel family | Self CUDA | Share | What it is |
+| --- | --- | --- | --- |
+| `fused_moe_kernel` | 576 ms | 48% | top-8 expert GEMMs on the 32-expert generation tower |
+| FlashAttention `flash_fwd_kernel` | 222 ms | 19% | dual-tower attention |
+| `nvjet` GEMMs | ~233 ms | 20% | `qkv_proj` / `o_proj` (and the small dense leftovers) |
+| `qk_norm_rope_kernel` | 58 ms | 5% | fused RMSNorm + RoPE |
+| MoE routing / `act_and_mul` / elementwise | ~40 ms | 3% | top-k, SwiGLU, residual |
+
+So a denoise step is about half FusedMoE and about two-fifths attention + dense GEMM.
+`--cfg-parallel-size 2` on the same 8-step t2i request (2-branch `cond`+`uncond`)
+is 0.95 s vs 1.48 s on one GPU (~1.56x). Img2img CFG-P=2 also completes for both
+branch counts at 8-step 1024x1024: 2-way (`img_cfg_scale=1`, `cond`+`img_cond`)
+1.09 s; 3-way (`cfg_scale=4`, `img_cfg_scale=2`, `cond`+`img_cond`+`uncond`)
+1.74 s (3 branches on 2 ranks dispatch as `[[0, 2], [1]]`). Cache-DiT and
+layerwise CPU offload do not take effect on A3B; see
+[diffusion features](../../docs/user_guide/diffusion_features.md) note 8.
+
+Other configurations on the same machine (second run, warm kernels):
+
+| Case | Total | Peak GPU memory |
+| --- | --- | --- |
+| text2img 1024x1024, 50 steps, think on | 9.16 s | 73.0 GiB |
+| text2img 1024x1024, 50 steps, think off, TP=2 | 5.64 s | 36.7 GiB per GPU |
+| img2img 2048x2048 output, 25 steps, think off | 20.93 s | 74.6 GiB |
+
+Think-on spends 1.37 s in `_generate_think` (215 tokens). At the same seed TP=2 differs
+from TP=1 only numerically (MAE 2.97/255, identical composition) from the changed
+reduction order.
+
+```bash
+python examples/offline_inference/text_to_image/text_to_image.py \
+    --model sensenova/SenseNova-U1-A3B-MoT \
+    --prompt "Close portrait of an elderly woman by a farmhouse window, warm natural light." \
+    --width 1024 --height 1024 \
+    --seed 42 --num-inference-steps 50 --cfg-scale 4.0 \
+    --extra-body '{"think": false, "cfg_norm": "none", "timestep_shift": 3.0, "t_eps": 0.02}' \
+    --output sensenova_a3b_t2i.png
+```
+
+`img2text` and `text2text` go through the server, as for the dense checkpoint:
+
+```bash
+vllm serve sensenova/SenseNova-U1-A3B-MoT --omni --port 8091
+
+python examples/online_serving/sensenova_u1/openai_chat_client.py \
+    -s http://127.0.0.1:8091 -m img2text -i input.png -p "Describe this image."
+```
+
+All four modalities were verified on A3B: text2img (with and without think), img2img
+editing, img2text and text2text.
+
 ## Hardware Support

 ## GPU
@@ -137,3 +137,39 @@ print('saved output.png')
 - Image quality is noticeably better than medium, with sharper details.
 - `negative_prompt` requires `guidance_scale > 1` to take effect; SD3 defaults to 1.0 (no CFG).
 - 2048×2048 fits within 48 GB but leaves little headroom (~45.5 GiB used out of 49 GiB).
+
+## XPU
+
+### 1x Intel Arc Pro B70 (32 GB) — stable-diffusion-3.5-medium
+
+Offline text-to-image at 1024x1024 with BF16 weights fully resident.
+
+#### Environment
+
+- OS: Linux
+- Python: 3.10+
+- torch: 2.13.0+xpu
+- vLLM: 0.29.0 (`98dff2a8`)
+- vLLM-Omni: `main` at `4c7a98c2`
+
+#### Command
+
+```bash
+python examples/offline_inference/text_to_image/text_to_image.py \
+  --model stabilityai/stable-diffusion-3.5-medium \
+  --prompt "a sunset over mountains, photorealistic" \
+  --num-inference-steps 28 \
+  --vae-use-tiling \
+  --output sd35_medium_output.png
+```
+
+#### Verification
+
+Confirm `sd35_medium_output.png` is written and matches the prompt.
+
+#### Notes
+
+- Memory usage: peak 19.5 GiB against 29.8 GiB free.
+- Known limitations: only `stable-diffusion-3.5-medium` offline generation was
+  qualified. `stable-diffusion-3.5-large` and online serving are out of scope
+  for this profile.
@@ -12,7 +12,7 @@
  both with `Qwen/Qwen2.5-Omni-3B` as the frozen encoder
 - Task: zero-shot TTS, instruct TTS, content and acoustic editing,
  paralinguistic editing, speech enhancement, speaker and music separation
- Mode: offline inference (online chat serving is not yet qualified)
+- Mode: offline inference and online `/v1/audio/speech` (online chat serving is not yet qualified)
 - Hardware: 1x H100 80GB
 - Maintainer: Community

@@ -130,17 +130,35 @@ Measured on one H100 against the upstream implementation with the same seeds
 and mean VAE latents (four cookbook cases, base checkpoint): identical
 transcripts, speaker similarity to the upstream output 0.993 to 0.999, log-mel
 L1 0.11 to 0.47 (the upstream VAE-resampling noise floor is 0.39). Flash:
-0.989 to 0.999 and 0.24 to 0.51. Wall per request on the second call: base
-1.0 to 2.0 s for 3.5 to 10.9 s of audio, Flash 0.17 to 0.30 s; engine start
-about 47 s with a warm page cache.
+0.989 to 0.999 and 0.24 to 0.51.
+
+Latency with the DiT and codec CUDA graphs, measured end to end on one
+H200-class GPU with both stages on it, concurrency 1, median of 10 warm
+requests: base (32 steps, CFG 2) zero-shot 0.25 / 0.31 / 0.40 s for 3 / 6 /
+12 s of audio, instruct TTS 0.17 / 0.23 / 0.32 s; Flash zero-shot 0.08 /
+0.10 / 0.14 s. The first request after start is 1.2 to 1.6 s, because the
+common shapes are compiled at startup. Engine start adds the codec decode
+compile and the DiT warmup to model loading (a few minutes on a cold
+Inductor cache, about 30 s less warm).

 ## Notes

 - Memory usage: about 25 GB peak on the device for a 12 s generation with
  both stages resident (deploy defaults: encoder 0.45, diffusion stage 0.35
-  of device memory).
- Key flags: `enforce_eager` on both stages (the encoder walks the decoder
-  layers itself for the layer fusion). `enable_prefix_caching` must stay off
+  of device memory), plus about 1 GB of CUDA graph pools for the compiled
+  codec decode buckets.
+- Key flags: `enforce_eager` on the encoder stage (it walks the decoder
+  layers itself for the layer fusion); the diffusion stage runs with
+  `enforce_eager: false` so the codec decode is compiled into bucketed CUDA
+  graphs at startup (the deploy config sets 160/320/640 latent frames, i.e.
+  up to 12.8 s; longer clips are decoded in overlapping 640-frame tiles of
+  the same graph;
+  override with `model_config.auk_vae_compile_shapes` and
+  `auk_vae_tile_frames`). Each DiT denoise step replays a per-shape CUDA
+  graph of the regionally compiled double- and single-stream blocks; the
+  per-request conditioning (text projection, reference embedding, padding
+  biases, rotary tables) is prepared once per request outside the graph.
+  `enable_prefix_caching` must stay off
  for the encoder: a cache hit skips prompt positions that the fused
  condition needs. `enable_chunked_prefill` is off by default: forcing it
  (128-token chunks, so two to three chunks per prompt) reproduces the
@@ -161,7 +179,7 @@ about 47 s with a warm page cache.
 | --- | --- | --- |
 | Offline `Omni.generate` | supported (base and Flash) | `docs/getting_started/quickstart.md` |
 | Online `/v1/chat/completions` with audio | not yet qualified | `docs/serving/` |
-| `/v1/audio/speech` | not supported (needs a TTS adapter) | `docs/contributing/model/adding_tts_model.md` |
+| `/v1/audio/speech` | supported (base and Flash, `instructions` or `task_type`) | `docs/user_guide/examples/online_serving/text_to_speech.md` |
 | Streaming / async chunk | not supported | `docs/design/feature/async_chunk.md` |
 | Batching across requests | one request per DiT forward | `docs/user_guide/diffusion/` |
 | Tensor / sequence parallelism | not supported | `docs/configuration/composable_parallel.md` |
@@ -0,0 +1,72 @@
+# HunyuanVideo 1.5
+
+> Text-to-video with the HunyuanVideo-1.5 Diffusers checkpoints
+
+## Summary
+
+- Vendor: Tencent (community Diffusers ports)
+- Model: `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v`
+- Task: Text-to-video
+- Mode: Offline inference
+- Maintainer: Community
+
+## When to use this recipe
+
+Use this recipe to generate a short clip with HunyuanVideo-1.5 on a memory-
+constrained card, trading output size for capacity.
+
+## References
+
+- Model card:
+  <https://huggingface.co/hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v>
+- Shared runnable example:
+  [`examples/offline_inference/text_to_video/text_to_video.py`](../../examples/offline_inference/text_to_video/text_to_video.py)
+- CPU offload guide:
+  [`docs/user_guide/diffusion/cpu_offload.md`](../../docs/user_guide/diffusion/cpu_offload.md)
+
+## Hardware Support
+
+This recipe documents one validated Intel XPU configuration. Extend it with
+more hardware sections as community validation lands.
+
+## XPU
+
+### 1x Intel Arc Pro B70 (32 GB)
+
+BF16 weights with model-level CPU offload at a reduced 320x240, 17-frame
+shape. The 480p and 720p T2V and I2V checkpoints all use the same command; only
+`--model` changes.
+
+#### Environment
+
+- OS: Linux
+- Python: 3.10+
+- torch: 2.13.0+xpu
+- vLLM: 0.29.0 (`98dff2a8`)
+- vLLM-Omni: `main` at `4c7a98c2`
+
+#### Command
+
+```bash
+python examples/offline_inference/text_to_video/text_to_video.py \
+  --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v \
+  --prompt "A serene lakeside sunrise with mist over the water." \
+  --height 240 --width 320 --num-frames 17 \
+  --num-inference-steps 50 \
+  --guidance-scale 6.0 \
+  --enable-cpu-offload \
+  --vae-use-tiling \
+  --vae-use-slicing \
+  --output hunyuan_video_15_output.mp4
+```
+
+#### Verification
+
+Confirm `hunyuan_video_15_output.mp4` decodes and that the sampled frames match
+the prompt.
+
+#### Notes
+
+- Memory usage: 18.3 GiB loaded, 25.7 GiB peak, about 39 s per clip.
+- Known limitations: the native 720p geometry does not fit a 32 GB card with
+  CPU offload. Online serving is out of scope for this profile.
@@ -0,0 +1,68 @@
+# Z-Image Turbo
+
+> Few-step text-to-image with `Tongyi-MAI/Z-Image-Turbo`
+
+## Summary
+
+- Vendor: Tongyi-MAI
+- Model: `Tongyi-MAI/Z-Image-Turbo`
+- Task: Text-to-image
+- Mode: Offline inference
+- Maintainer: Community
+
+## When to use this recipe
+
+Use this recipe to generate 1024x1024 images with the distilled Z-Image Turbo
+checkpoint through the shared offline text-to-image example.
+
+## References
+
+- Model card: <https://huggingface.co/Tongyi-MAI/Z-Image-Turbo>
+- Shared runnable example:
+  [`examples/offline_inference/text_to_image/text_to_image.py`](../../examples/offline_inference/text_to_image/text_to_image.py)
+- CPU offload guide:
+  [`docs/user_guide/diffusion/cpu_offload.md`](../../docs/user_guide/diffusion/cpu_offload.md)
+
+## Hardware Support
+
+This recipe documents one validated Intel XPU configuration. Extend it with
+more hardware sections as community validation lands.
+
+## XPU
+
+### 1x Intel Arc Pro B70 (32 GB)
+
+BF16 weights with model-level CPU offload and VAE tiling/slicing, 25 steps.
+
+#### Environment
+
+- OS: Linux
+- Python: 3.10+
+- torch: 2.13.0+xpu
+- vLLM: 0.29.0 (`98dff2a8`)
+- vLLM-Omni: `main` at `4c7a98c2`
+
+#### Command
+
+```bash
+python examples/offline_inference/text_to_image/text_to_image.py \
+  --model Tongyi-MAI/Z-Image-Turbo \
+  --prompt "a cup of coffee on the table" \
+  --num-inference-steps 25 \
+  --enable-cpu-offload \
+  --vae-use-tiling \
+  --vae-use-slicing \
+  --enforce-eager \
+  --output z_image_turbo_output.png
+```
+
+#### Verification
+
+Confirm `z_image_turbo_output.png` is written as a 1024x1024 PNG matching the
+prompt.
+
+#### Notes
+
+- Memory usage: peak 15.6 GiB, about 58 s per image.
+- Known limitations: only offline generation was qualified. Online serving is
+  out of scope for this profile.
@@ -85,14 +85,78 @@ For online serving client examples and request formats, see
 #### Notes

 - **Key flags:**
-  - `--omni` — enables vLLM-Omni diffusion serving.
-  - `--use-hsdp` — Hybrid Sharded Data Parallelism for the 14B DiT weights.
-  - `--usp <N>` — Unified (Ulysses) Sequence Parallelism degree.
-  - `--cfg-parallel-size <N>` — CFG parallelism; set to 2 for the official model,
+    - `--omni` — enables vLLM-Omni diffusion serving.
+    - `--use-hsdp` — Hybrid Sharded Data Parallelism for the 14B DiT weights.
+    - `--usp <N>` — Unified (Ulysses) Sequence Parallelism degree.
+    - `--cfg-parallel-size <N>` — CFG parallelism; set to 2 for the official model,
    omit for distilled checkpoints.
-  - `--vae-patch-parallel-size 8` / `--vae-use-tiling` — parallel + tiled VAE
+    - `--vae-patch-parallel-size 8` / `--vae-use-tiling` — parallel + tiled VAE
    decoding; disabling patch parallelism can significantly increase VAE latency.

+## XPU
+
+### 1x Intel Arc Pro B70 (32 GB)
+
+Offline text-to-video at a reduced 576x320, 17-frame shape. Layerwise offload
+streams the two 14B expert transformers; VAE slicing bounds the decode
+activations.
+
+#### Environment
+
+- OS: Linux
+- Python: 3.10+
+- torch: 2.13.0+xpu
+- vLLM: 0.29.0 (`98dff2a8`)
+- vLLM-Omni: `main` at `4c7a98c2`
+
+#### Command
+
+```bash
+python examples/offline_inference/text_to_video/text_to_video.py \
+  --model Wan-AI/Wan2.2-T2V-A14B-Diffusers \
+  --prompt "A serene lakeside sunrise with mist over the water." \
+  --negative-prompt "blurry, noisy, artifacts, distorted, low quality, streaks, sparkles, blocky, pixelated" \
+  --height 320 --width 576 --num-frames 17 \
+  --num-inference-steps 60 \
+  --guidance-scale 4.0 \
+  --flow-shift 5.0 \
+  --boundary-ratio 0.875 \
+  --enable-layerwise-offload \
+  --vae-use-slicing \
+  --enforce-eager \
+  --seed 12345 \
+  --output wan22_t2v_output.mp4
+```
+
+Given spare cards, `--enable-cpu-offload` shards the two expert transformers
+instead of streaming them, which nearly halves the denoise loop. Four is the
+fastest count; eight is slower.
+
+```bash
+  --tensor-parallel-size 4 \
+  --enable-cpu-offload
+```
+
+| Offload | Cards | Peak VRAM | Denoise |
+| --- | ---: | ---: | ---: |
+| `--enable-layerwise-offload` | 1 | 9.9 GiB | 160 s |
+| `--enable-cpu-offload` | 2 | 18.9 GiB | 113 s |
+| `--enable-cpu-offload` | 4 | 12.4 GiB | 86 s |
+
+#### Verification
+
+Confirm `wan22_t2v_output.mp4` decodes and that frame 0 is clean.
+
+#### Notes
+
+- Key flags: `--boundary-ratio 0.875` sets the high-noise to low-noise expert
+  switch. `--flow-shift 5.0` with `--guidance-scale 4.0` over 60 steps clears
+  the frame-0 VAE corruption seen at the stock 50-step, `flow_shift=12.0`
+  settings.
+- Known limitations: only offline T2V at 576x320 was qualified. Larger shapes,
+  TI2V, HSDP, CFG parallelism, and online serving are out of scope for this
+  profile.
+
 ## Accelerating with TRTLLM_ATTN + Skip-Softmax (datacenter Blackwell)

 On **datacenter Blackwell** (sm_100 / sm_103), the mask-free Wan pipeline
@@ -153,7 +217,7 @@ instead of `target_sparsity`):
 ### Skip-Softmax controls

 | Key | Valid values | Meaning |
-|---|---|---|
+| --- | --- | --- |
 | `target_sparsity` | finite, `[0, 1]` | Operating point on the checkpoint's calibrated curve (`a·exp(b·s)`). |
 | `threshold` | finite, `≥ 0` | Direct skip threshold, no calibration needed. Mutually exclusive with `target_sparsity`. |
 | `disabled_until_timestep` | finite, `[0, 1]` | Keeps early, high-noise steps dense; skip turns on once the normalized timestep `t ≤ D`. |
@@ -169,7 +233,7 @@ Measured on B300 / SM103, 1280×720 / 81f / 50 steps, torch.compile, official
 ModelOpt calibration (speedup and divergence relative to dense):

 | config | speedup | LPIPS (divergence vs dense) |
-|---|---|---|
+| --- | --- | --- |
 | dense | 1.000x | - |
 | `s=0.65, D=0.86` | 1.062x | 0.041 |
 | `s=0.75, D=1.00` | 1.181x | 0.377 |
@@ -181,6 +245,6 @@ ModelOpt calibration (speedup and divergence relative to dense):
  `flashinfer`. Elsewhere, or without FlashInfer, selecting `TRTLLM_ATTN` raises
  rather than silently degrading.
 - **Known limitations:**
-  - Skip-Softmax is not supported under ring sequence parallelism; use Ulysses SP
+    - Skip-Softmax is not supported under ring sequence parallelism; use Ulysses SP
    (`--usp`), or run Skip-Softmax without ring.
-  - `TRTLLM_ATTN` is BF16-only (no attention-level quantization).
+    - `TRTLLM_ATTN` is BF16-only (no attention-level quantization).
@@ -0,0 +1,69 @@
+# FLUX.2-klein
+
+> Text-guided image editing with `black-forest-labs/FLUX.2-klein-4B`
+
+## Summary
+
+- Vendor: Black Forest Labs
+- Model: `black-forest-labs/FLUX.2-klein-4B`
+- Task: Image editing (image-to-image)
+- Mode: Offline inference
+- Maintainer: Community
+
+## When to use this recipe
+
+Use this recipe to run single-image edits with FLUX.2-klein-4B through the
+shared offline image-edit example.
+
+## References
+
+- Model card: <https://huggingface.co/black-forest-labs/FLUX.2-klein-4B>
+- Shared runnable example:
+  [`examples/offline_inference/image_to_image/image_edit.py`](../../examples/offline_inference/image_to_image/image_edit.py)
+- CPU offload guide:
+  [`docs/user_guide/diffusion/cpu_offload.md`](../../docs/user_guide/diffusion/cpu_offload.md)
+
+## Hardware Support
+
+This recipe documents one validated Intel XPU configuration. Extend it with
+more hardware sections as community validation lands.
+
+## XPU
+
+### 1x Intel Arc Pro B70 (32 GB)
+
+BF16 weights with model-level CPU offload. `--image` takes any RGB image.
+
+#### Environment
+
+- OS: Linux
+- Python: 3.10+
+- torch: 2.13.0+xpu
+- vLLM: 0.29.0 (`98dff2a8`)
+- vLLM-Omni: `main` at `4c7a98c2`
+
+#### Command
+
+```bash
+python examples/offline_inference/image_to_image/image_edit.py \
+  --model black-forest-labs/FLUX.2-klein-4B \
+  --image test_input.png \
+  --prompt "Add a sunset sky with orange and purple clouds" \
+  --num-inference-steps 28 \
+  --guidance-scale 3.5 \
+  --enable-cpu-offload \
+  --enforce-eager \
+  --seed 42 \
+  --output image_edit_output.png
+```
+
+#### Verification
+
+Confirm `image_edit_output.png` follows the prompt while preserving the rest
+of the input image.
+
+#### Notes
+
+- Memory usage: 7.7 GiB loaded in 8.4 s, about 18 s per edit.
+- Known limitations: only offline editing of the 4B checkpoint was qualified.
+  `FLUX.2-klein-9B` and online serving are out of scope for this profile.
@@ -141,3 +141,58 @@ confirm a valid 1024×1024 PNG is written.
  no-CFG, True for Raw CFG).
 - Known limitations: TP / SP / CFG-Parallel are not yet wired for Krea 2. HSDP,
  layerwise CPU offload, and VAE-patch-parallel (decode) are supported.
+
+## XPU
+
+### 1x Intel Arc Pro B70 (32 GB)
+
+Offline text-to-image for the distilled Turbo checkpoint at 1024x1024, with
+layerwise offload streaming the transformer blocks.
+
+#### Environment
+
+- OS: Linux
+- Python: 3.10+
+- torch: 2.13.0+xpu
+- vLLM: 0.29.0 (`98dff2a8`)
+- vLLM-Omni: `main` at `4c7a98c2`
+
+#### Command
+
+```bash
+python examples/offline_inference/text_to_image/text_to_image.py \
+  --model krea/Krea-2-Turbo \
+  --prompt "a photo of a sunset over mountains" \
+  --height 1024 --width 1024 \
+  --num-inference-steps 8 \
+  --enable-layerwise-offload \
+  --vae-use-tiling \
+  --enforce-eager \
+  --output krea2_turbo_output.png
+```
+
+Given spare cards, `--enable-cpu-offload` replaces layerwise streaming. At 8
+steps the swap dominates the run, and cards past the second shard nothing
+further.
+
+```bash
+  --tensor-parallel-size 2 \
+  --enable-cpu-offload
+```
+
+| Offload | Cards | Peak VRAM | Time |
+| --- | ---: | ---: | ---: |
+| `--enable-layerwise-offload` | 1 | 12.3 GiB | 27 s |
+| `--enable-cpu-offload` | 2 | 25.6 GiB | 59 s |
+| `--enable-cpu-offload` | 4 | 25.6 GiB | 59 s |
+
+#### Verification
+
+Confirm `krea2_turbo_output.png` is written as a 1024x1024 PNG matching the
+prompt.
+
+#### Notes
+
+- Memory usage: 7.8 GiB loaded, 12.3 GiB peak, about 26 s per image.
+- Known limitations: only offline Turbo generation was qualified. Krea-2-Raw,
+  LoRA, Cache-DiT, and online serving are out of scope for this profile.
@@ -0,0 +1,122 @@
+# YuE2-3B
+
+> Text-to-music: style + lyrics (optionally an ABC score) in, a 48 kHz stereo song out
+
+## Summary
+
+- Vendor: m-a-p (Multimodal Art Projection)
+- Model: `m-a-p/YuE2-3B` + `m-a-p/YuE2-Vae`
+- Task: Text-to-music generation with an editable symbolic plan (ABC notation)
+- Mode: Offline generation and online `/v1/audio/speech` serving (adapter `tts_adapters/yue2.py`)
+- Integration issue: [#7661](https://github.com/vllm-project/vllm-omni/issues/7661)
+
+## When to use this recipe
+
+Use it to generate songs from a style description and lyrics. Unlike MiniMax
+Music 3, YuE2-3B first writes an ABC score (chords and melody, `cot=full` /
+`cot=melody`) that you can inspect, edit and resubmit (`--abc-file`), then
+renders it with the lyrics. `cot=off` skips the score and generates music
+directly. Weights are **CC BY-NC 4.0** (non-commercial); this recipe loads
+user-downloaded weights and bundles none.
+
+## References
+
+- Model card: [m-a-p/YuE2-3B](https://huggingface.co/m-a-p/YuE2-3B)
+- Upstream: [multimodal-art-projection/YuE](https://github.com/multimodal-art-projection/YuE) (pinned `bd90e4c`)
+- Offline example: `examples/offline_inference/yue2/end2end.py`
+- Deploy config: `vllm_omni/deploy/yue2.yaml` (auto-discovered for `model_type=yue2`)
+
+## Architecture
+
+| Component | Spec |
+| --- | --- |
+| Backbone | Qwen3-1.7B-class decoder, 28 layers, hidden 2048, GQA 16/8, q/k-norm, vocab 184,704, context 24,576 |
+| MoT | NAR path (`nar_self_attn`/`nar_mlp` per layer) shares embed_tokens, final norm and lm_head with the AR path |
+| Frames | Single codebook: one codec token per frame, 25 frames/s, span [151853, 184521) |
+| Sampling | temperature/top-k/top-p + windowed repetition penalty, seeded; model-owned (phase-masked vocab) |
+| Guidance | Off by default (1.0 full/melody, 1.01 off); not exercised |
+| Acoustic | 32-step midpoint flow matching over 64-dim latents on the NAR path, chunked ~12k frames |
+| VAE | `m-a-p/YuE2-Vae` decoder-only, tiled (1024 core + 16 halo frames), FP32 |
+| Output | 48 kHz stereo, whole song delivered when the semantic request finishes |
+
+Generation is one or two requests on a single AR stage: the abc phase writes
+the score, the semantic phase generates codec frames, then queues the ODE and VAE on a
+high-priority CUDA stream. Other requests keep decoding while the finishing
+request emits HOLD tokens (up to 4096 steps, within its token/context budget).
+The scheduler is unchanged. Weights load once; the VAE loads at startup from
+`$YUE2_VAE` or the hub id.
+
+## Running
+
+```bash
+export YUE2_VAE=/models/YuE2-Vae   # or omit and let it resolve from the hub
+python examples/offline_inference/yue2/end2end.py \
+    --model /models/YuE2-3B \
+    --style "A melancholic lo-fi hip-hop track at 85 BPM in F minor" \
+    --lyrics "[Verse]
+Walking down the empty street at midnight
+[Chorus]
+And I keep on walking" \
+    --cot full --seed 831001 --max-frames 200 --output song.wav
+```
+
+`--max-frames 200` caps the song at 8 s (25 frames/s). The end token is
+masked for the preset's first 200 steps, so a 200-frame budget always
+truncates; a longer budget lets the song end on its own end token, and
+`truncated` in the report line says which happened. Serving follows the
+same rule: omit `max_new_tokens` (or set it above 200) for a natural
+ending.
+
+## Notes
+
+The lifecycle e2e regression forces KV-cache preemption with chunked and
+unchunked recompute, mixes ABC and semantic sampling, and cancels a request
+after NAR work has started. It checks history alignment, released synthesis
+buffers, and successful generation after cancellation. Run with one H100/H200
+and cached YuE2-3B and YuE2-Vae weights:
+
+```bash
+export YUE2_MODEL_DIR=/path/to/YuE2-3B
+export YUE2_VAE=/path/to/YuE2-Vae
+CUDA_VISIBLE_DEVICES=0 python -m pytest tests/e2e/online_serving/test_yue2.py \
+    -k preemption_and_synthesis_abort -m 'slow and tts' --run-level full_model -q -s
+```
+
+The test retains the module's weekly TTS routing; it is not a per-PR CI gate.
+
+- **Memory:** `gpu_memory_utilization` budgets the vLLM engine; NAR K/V,
+  acoustic graph buffers and VAE activations also need room during synthesis.
+  The 0.70 / 4-slot defaults require validation on the target card. The earlier
+  RTX 4090 measurement (15.8 GiB after startup, 20.9 GiB peak at 9000 frames)
+  predates the async/compiled implementation; it does not validate this
+  version or long ABC prefixes on a 4090. Reduce the fraction if the target
+  workload needs more synthesis memory, especially for long ABC prefixes and
+  the 9000-frame cap. Releasing completed chunks bounds live NAR buffers,
+  while the shared graph allocator retains reserved storage for reuse.
+- **CUDA graphs:** the default deploy keeps the AR backbone eager. Both ABC
+  and semantic sampling use prewarmed power-of-two graph buckets. Custom
+  sampler captures have a bounded cache and fall back to eager sampling when
+  it fills. The NAR velocity pass uses a chunk graph on the serialized synthesis
+  stream. Chunk graphs share one allocator pool; completed chunks release
+  their engines and buffers after their own events. FA3 runs on Hopper; other
+  CUDA cards use SDPA, which is also warmed at startup.
+- **Serving:** run `vllm serve m-a-p/YuE2-3B --omni` with the default
+  `yue2.yaml`, which uses eager AR with 4 slots. Validate memory capacity
+  for the target card and workload.
+  For the tested single-H200 graph settings, reuse this deploy with
+  `--stage-overrides '{"0":{"max_num_seqs":32,"gpu_memory_utilization":0.5,"enforce_eager":false,"compilation_config":{"cudagraph_mode":"FULL_AND_PIECEWISE"}}}'`.
+  Decode uses FULL and mixed prefill uses PIECEWISE; VAE and the Python
+  synthesis queue remain outside these graphs.
+  Audio is delivered as a whole song; time to first audio equals completion
+  latency. Aborting a running song stops future work units and waits only for
+  the at-most-two submitted units before releasing its buffers.
+- **Preemption:** before drawing for a resumed row, model-owned sampling
+  reconciles the codec history and terminal flags with the engine's retained
+  sequence. Discarded draws cannot advance its penalty window or frame budget.
+- Audio is not expected to match the upstream torch reference bit-for-bit
+  (fused vs eager kernels). Measured against the upstream torch reference
+  (bd90e4c, same style/lyrics/seed 831001, cot=off, 200 frames): prompt token
+  ids 63/63 bit-identical, and the sampled semantic stream agrees
+  bit-for-bit for the first 55 frames before kernel numerics diverge — well
+  past the "first frames agree" acceptance bar of the MiniMax Music 3
+  precedent.
@@ -7,6 +7,11 @@
 # transformers_keys_to_ignore_compat in model_executor/models/utils.py.
 transformers >= 5.13.0, < 5.15
 av>=14.0.0
+# typing_extensions.Concatenate accepts a trailing `...` on 3.10 only from
+# 4.13 (older 4.12.x re-export typing.Concatenate, which rejects it). The
+# Nemotron VoiceChat codec annotates `Concatenate[int, ...]`, so pin the
+# floor instead of relying on the version vLLM/torch pull transitively.
+typing_extensions>=4.13
 omegaconf>=2.3.0
 diffusers==0.40.0
 # Hub kernels, including LTX-2.5 NATTEN. 0.16+ resolves torch-stable-abi*
Author	SHA1	Message	Date
tzhouam	e13468f800	[Bugfix] Restrict OmniRunner cleanup to owned EngineCore workers Signed-off-by: tzhouam <tzhouam@connect.ust.hk>	2026-10-04 03:19:00 +08:00
andyluo7	ee8fdab1dc	[CI/Build] Fix MOSS CPU tests on ROCm workers (#8447 ) Signed-off-by: andyluo7 <andy.luo@amd.com>	2026-10-03 09:13:51 +08:00
Sy03andYueqian Lin	3bc3f1a7d0	[Model] Optimize MOSS Local 1.5 MRV2 serving and progressive chunks (#8213 ) Signed-off-by: Sy03 <1370724210@qq.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com>	2026-10-02 18:03:14 +00:00
Zheng Wengangandamy-why-3459	527982d886	[Bugfix] Skip async-chunk SHM put for stage-0-final requests (#7245 ) Signed-off-by: ZhengWG <zwg0606@gmail.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>	2026-10-02 09:14:39 -07:00
Sy03	2ba5074ca0	[Model] Optimize CosyVoice3 packed streaming with bounded HiFT graphs (#8420 ) Signed-off-by: Sy03 <1370724210@qq.com>	2026-10-02 08:49:50 -07:00
Yueqian Lin	4025bb30f6	[Perf][AuK] Graph the encoder prefill, cache reference latents and fuse the codec activation (#8305 ) Signed-off-by: Yueqian Lin <linyueqian@outlook.com>	2026-10-02 15:43:55 +00:00
Yueqian Lin	13c66da752	[Perf][AuK] Precompute adaLN modulations per request and run the conv position embedding as one GEMM (#8328 ) Signed-off-by: Yueqian Lin <linyueqian@outlook.com>	2026-10-02 08:12:29 -07:00
Nick CaoandCodex	0e6791e9c4	[Bugfix] Initialize eager state in Higgs test fixture (#8426 ) Signed-off-by: Nick Cao <ncao@redhat.com> Co-authored-by: Codex <noreply@openai.com>	2026-10-02 10:22:52 -04:00
Zheng Wengangandamy-why-3459	eb1699bfca	[FEAT][Fronted] Support incremental prefill for realtime video streaming (#5894 ) Signed-off-by: ZhengWG <zwg0606@gmail.com> Signed-off-by: Zheng Wengang <zwg0606@gmail.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>	2026-10-02 22:18:51 +08:00
Sy03	e372aa3a41	[Bugfix] Restore Fish Speech and CosyVoice3 audio quality at high concurrency (#8422 ) Signed-off-by: Sy03 <1370724210@qq.com>	2026-10-02 07:11:04 -07:00
Sy03 princepride Claude Opus 5.5	ae58c2d8b9	[Model] YuE2: async single-stage synthesis and CUDA graph acceleration (#8407 ) Signed-off-by: Sy03 <1370724210@qq.com> Signed-off-by: princepride <wangzhipeng628@gmail.com> Signed-off-by: Wang Zhipeng <wangzhipeng628@gmail.com> Co-authored-by: princepride <wangzhipeng628@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-02 05:17:06 -07:00
linzhenpl07 Claude Opus 5 Zhou Taichang	b10270a01c	[AR-Diffusion] Page in kernel-legal blocks so the default resolution can run (#6481 ) Signed-off-by: linzhenpl07 <linzhenpl07@gmail.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Zhou Taichang <tzhouam@connect.ust.hk>	2026-10-02 20:05:04 +08:00
Sy03	e84014dffb	[Model] Add single-stage Qwen3-TTS with in-Talker streaming codec decode (#8259 ) Signed-off-by: Sy03 <1370724210@qq.com>	2026-10-02 21:34:30 +10:00
Rahul Steiger	596487fe62	[Core] Prototype diffusion attention execution contracts with FA4 (#7379 )	2026-10-02 17:58:47 +08:00
MaciejBalaNVand汪志鹏	97d6c693a6	[feature] Add HSDP pre-sharded option (#7948 ) Signed-off-by: Maciej Bala <mbala@nvidia.com> Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com>	2026-10-02 09:35:09 +00:00
汪志鹏	b742f86136	[CI/Build] Restore streaming executor fixture contract (#8418 ) Signed-off-by: princepride <wangzhipeng628@gmail.com>	2026-10-02 01:45:56 -07:00
andyluo7	53c7340668	test: measure offload memory inside diffusion workers (#8189 ) Signed-off-by: andyluo7 <andy.luo@amd.com>	2026-10-02 14:07:20 +08:00
Sy03	d93e1a8e09	[Model] Optimize Qwen3-Omni MRv2 handoff, code prediction and audio decoding (#8223 ) Signed-off-by: Sy03 <1370724210@qq.com>	2026-10-01 22:22:54 -07:00
wtz2333andYuanheng Zhao	68003cf6ad	[Bugfix] Fix washed-out LingBot Video outputs (#7037 ) Signed-off-by: wtz2333 <2955110911@qq.com> Co-authored-by: Yuanheng Zhao <54058983+yuanheng-zhao@users.noreply.github.com>	2026-10-02 03:57:50 +00:00
andyluo7	bbee488da3	fix: repair ROCm test routing, CPU timeouts, and Breeze autotuning (#8342 ) Signed-off-by: andyluo7 <andy.luo@amd.com>	2026-10-02 07:36:41 +08:00
Joshna-MedisettyandCursor	2560f91859	fix(ming): build ISTFT spectrogram in float32 (#7538 ) Signed-off-by: Joshna Medisetty <joshna.medisetty@intel.com> Co-authored-by: Cursor <cursoragent@cursor.com>	2026-10-01 22:45:09 +00:00
Doug Smith	7b15d22f48	[CI/Build] Package runtime data explicitly (#7956 ) Signed-off-by: Doug Smith <dosmith@redhat.com>	2026-10-01 19:10:21 +00:00
Youyuan Liandprincepride	423f34326e	[Model] Add YuE2-3B text-to-music (model + /v1/audio/speech adapter) (#7886 ) Signed-off-by: Youyuan Li <105433145+Darcy-Lee@users.noreply.github.com> Signed-off-by: princepride <wangzhipeng628@gmail.com> Co-authored-by: princepride <wangzhipeng628@gmail.com>	2026-10-01 18:04:31 +08:00
汪志鹏	a038b38179	[Model] Add MiniMax-H3 latent super-resolution and hi-res refinement (#8322 ) Signed-off-by: princepride <wangzhipeng628@gmail.com>	2026-10-01 14:24:40 +08:00
yzhautouskay	27a6321a63	[Model][Cosmos3] Keep diffusion sampling state in FP32 (#7592 ) Signed-off-by: Yuliya Zhautouskaya <yzhautouskay@nvidia.com>	2026-09-30 11:35:51 -07:00
amy-why-3459	4af28f33bd	[Bugfix] Defer VoxCPM2 NPU patch to avoid platform import cycle (#8330 ) Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>	2026-09-30 05:25:36 -07:00
Sy03	56763cd6c3	[Model] Enable Higgs Audio v3 MRV2 streaming and fix mixed-prefill capture bounds (#8226 ) Signed-off-by: Sy03 <1370724210@qq.com>	2026-09-30 11:32:27 +00:00
psv666andamy-why-3459	b44b2f981c	[Bugfix] Fix Qwen3-Omni audio-input mean_e2el_ms regression (#8306 ) Signed-off-by: psv666 <2693925048@qq.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>	2026-09-30 18:07:55 +08:00
Alicia	c524e1a879	[CI/Build] Refresh targeted perf baselines (#6538 ) Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com>	2026-09-30 09:24:25 +00:00
wangyuandAlicia	d0f073686a	[CI/Build] Split long weekly TTS jobs and route slow full-model cases to nightly (#8239 ) Signed-off-by: wangyu <410167048@qq.com> Co-authored-by: Alicia <115451386+congw729@users.noreply.github.com>	2026-09-30 09:08:24 +00:00
Yan Cao yancaocn Zhou Taichang	004a5711dc	[Bugfix][Helios] Guard cross-attn KV cache against address reuse (#8064 ) Signed-off-by: yancaocn <yancaochn@163.com> Co-authored-by: yancaocn <yancaochn@163.com> Co-authored-by: Zhou Taichang <tzhouam@connect.ust.hk>	2026-09-30 16:44:51 +08:00
WeiQing Chen	e34f8c4f42	[Doc] Update vLLM-Omni WeChat QR code (#8301 ) Signed-off-by: David Chen <530634352@qq.com>	2026-09-30 11:39:05 +08:00
Armaan Amatya Yueqian Lin Hongsheng Liu	aa41a3ac97	[Bugfix] Fix the remaining Python 3.10 breakages (#7985 ) Signed-off-by: Armaan Amatya <armaanamatya2014@gmail.com> Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>	2026-09-30 11:38:23 +08:00
Hongchen Liandredearly123	989161c49d	[Benchmark] Add MammothModa2 startup/loading breakdown (#7312 ) Signed-off-by: redearly123 <65453204+redearly123@users.noreply.github.com> Co-authored-by: redearly123 <65453204+redearly123@users.noreply.github.com>	2026-09-30 11:38:07 +08:00
Mu GuanLinandHongsheng Liu	3a82c85880	[Experimental][Perf] Eliminate redundant host copies from generated-video transport (#7355 ) Signed-off-by: mglyn <1203789601@qq.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>	2026-09-30 11:37:57 +08:00
Egor Chernobrovkin	c27a52a359	[Model] Make VoxCPM2 generation settings configurable (#7010 ) Signed-off-by: Egor Chernobrovkin <33jasooon33@gmail.com>	2026-09-30 11:26:12 +08:00
vOv	25bdb38b11	[Bugfix] Do not block diffusion client shutdown on a dead subprocess (#7812 ) Signed-off-by: cr-gao <gaochenrui@sjtu.edu.cn>	2026-09-30 11:25:37 +08:00
Yueqian Lin	146ccc133a	[Perf][AuK] Compile the DiT step, use cuDNN attention and warm up at startup (#8300 ) Signed-off-by: Yueqian Lin <linyueqian@outlook.com>	2026-09-30 11:21:21 +08:00
yulunandHongsheng Liu	9097fa6547	[Hardware][Ascend] Accelerate VoxCPM2 LocDiT with NPUGraph (#7275 ) Signed-off-by: Big2Wheel <1404207590@qq.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>	2026-09-30 10:11:22 +08:00
boatman Hongsheng Liu Yueqian Lin	35cf6ac355	[Feat][Perf][AuK]Support Online Serving and DiT CUDAGraph (#7469 ) Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com>	2026-09-30 00:18:38 +00:00
BruceLoveDecimalandYueqian Lin	3a28882946	[Perf][Auk] VAE cuda graph and tiling (#7881 ) Signed-off-by: liuqihao <liuqihao970610@gmail.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com>	2026-09-29 23:27:01 +00:00
vraitiandClaude Opus 4.6	8ad6e801f5	[Tests] Fix Qwen3-Omni E2E LiveKit Test (#8294 ) Signed-off-by: vraiti <vraiti@redhat.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>	2026-09-29 16:23:28 -04:00
Sy03	5af3a16526	[Model] Add MiniCPM-o 4.5 turn-mode MRv2 and batched audio inference (#8222 ) Signed-off-by: Sy03 <1370724210@qq.com>	2026-09-29 11:41:39 -07:00
jingchengtianandjingchengtian	43ef1e61fb	[Perf][TTS] Add per-row generator support to MOSS-TTS talker to eliminate serial fallback for seeded batched requests (#7922 ) Signed-off-by: jingchengtian <jingchengtian@users.noreply.github.com> Signed-off-by: jingchengtian <tjc1995@126.com> Co-authored-by: jingchengtian <jingchengtian@users.noreply.github.com>	2026-09-29 13:13:50 -04:00
Alex Brooks	764a109f66	[Config] Disable Async Chunk for Unsupported Pipelines (#5099 ) Signed-off-by: Alex Brooks <albrooks@redhat.com>	2026-09-29 09:52:09 -06:00
Sy03andzack	1ea0ee3b38	[Model] Add opt-in packed Flow and batched HiFT inference for CosyVoice3 (#8224 ) Signed-off-by: Sy03 <1370724210@qq.com> Signed-off-by: zack <huixindaddy@yahoo.com> Co-authored-by: zack <huixindaddy@yahoo.com>	2026-09-29 15:48:36 +00:00
Nick CaoandCodex	c85f1a44fb	[Bugfix] Break duplex OpenAI import cycle (#8287 ) Signed-off-by: Nick Cao <ncao@redhat.com> Co-authored-by: Codex <noreply@openai.com>	2026-09-29 11:07:03 -04:00
andyluo7	f145f66c82	test: define Wan fused BF16 parity budget (#8188 ) Signed-off-by: andyluo7 <andy.luo@amd.com>	2026-09-29 22:43:14 +08:00
Sy03	001f8a80b9	[Core] Extend the shared multi-stage execution framework for mixed V1/MRv2 pipelines (#8184 ) Signed-off-by: Sy03 <1370724210@qq.com>	2026-09-29 07:40:57 -07:00
guangli_Liuandliuguangli	6b4173f50b	[Bugfix] Fix Qwen3-Omni realtime routing with typed stage configs (#8279 ) Signed-off-by: liuguangli <liuguangli35@gmail.com> Co-authored-by: liuguangli <liuguangli35@gmail.com>	2026-09-29 10:23:11 -04:00
0z5a 0z5a princepride	7d6904aa94	SeedVR2: bounded long-video restoration (#8102 ) Signed-off-by: 0z5a <Dezhen.lu@student.uni-tuebingen.de> Signed-off-by: 0z5a <dezhen.lu@uni-tuebingen.de> Signed-off-by: 0z5a <dezhen.lu@student.uni-tuebingen.de> Signed-off-by: princepride <wangzhipeng628@gmail.com> Co-authored-by: 0z5a <dezhen.lu@uni-tuebingen.de> Co-authored-by: princepride <wangzhipeng628@gmail.com>	2026-09-29 11:33:50 +00:00
Canlin Guo	235c430326	[Frontend] Tune reference cache limits and registered MOSS voice requests (#7883 ) Signed-off-by: Canlin Guo <canlinguosdu@gmail.com>	2026-09-29 04:10:17 +00:00
Ding Zuhao	1c1fdf719c	[Bugfix][BAGEL] Keep replicated prefill and cache-update phases off the SP strategy (#8177 ) Signed-off-by: Ding Zuhao <e1583181@u.nus.edu>	2026-09-29 03:45:49 +00:00
Wallbreazzz	5b3abe2f4d	[Feature][TTS] Port static chunk ramp to Moss-TTS streaming codec decode (#8124 ) Signed-off-by: Wallbreazzz <110282866+Wallbreazzz@users.noreply.github.com>	2026-09-29 02:28:15 +00:00
BeatSeat	3e62bba3cb	[Performance][MiniCPM-o] Whole-Euler CUDA graphs for Code2Wav with a shared attention arena (#8007 )	2026-09-29 09:14:12 +10:00
RuQing Xu	0173cb3748	[Kernel] Improve 2 backends importing FlashInfer: FLASHINFER_ATTN and TRTLLM_ATTN (#7431 ) Signed-off-by: Ruqing Xu <7891482+xrq-phys@users.noreply.github.com>	2026-09-28 21:21:58 +00:00
vraiti	d31dbc5b0a	[Frontend] OpenAI-compliant fullduplex /v1/realtime with Qwen3-Omni support (#7285 ) Signed-off-by: vraiti <vraiti@redhat.com>	2026-09-28 20:14:05 +00:00
SYLAR lishunyang12 Roger Wang	7266fc6138	[Bugfix] Avoid enlarging MiniMax H3 reference images and videos (#8253 ) Signed-off-by: lishunyang12 <lishunyang12@users.noreply.github.com> Co-authored-by: lishunyang12 <lishunyang12@users.noreply.github.com> Co-authored-by: Roger Wang <hey@rogerw.io>	2026-09-28 20:12:53 +00:00
Nick CaoandCodex	b656fcf45b	[Tests] Use get_file_store_init_method helper from vllm (#7528 ) Signed-off-by: Nick Cao <ncao@redhat.com> Co-authored-by: Codex <noreply@openai.com>	2026-09-28 14:22:25 -04:00
jingchengtian	12e9280010	[Perf][TTS] Run talker_mtp eagerly when a batch carries explicit seeds (#8009 ) Signed-off-by: jingchengtian <tjc1995@126.com>	2026-09-28 13:31:48 -04:00
vOv	a07384b953	[Bugfix] Raise when a diffusion stage fails sleep or wake_up (#7811 ) Signed-off-by: cr-gao <gaochenrui@sjtu.edu.cn>	2026-09-29 00:12:34 +08:00
Jierui (Jerry) XuandClaude Opus 5	965ad68f1d	[Kernel][MammothModa2] Use shared RMSNorm in the DiT (#7489 ) Signed-off-by: Jerry Xu <xjr2423@gmail.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>	2026-09-29 00:12:16 +08:00
Shaun WalshandClaude Opus 4.6	e823f0d909	[BugFix] Add field constraints to /v1/omni/sleep and /v1/omni/wakeup (#4740 ) Signed-off-by: Shaun Walsh <shaunwalsh24@gmail.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>	2026-09-28 09:30:30 -06:00
R0CKSTAR	29b818a785	feat(magi2): add fused BF16 routed MoE path (#7206 ) Signed-off-by: Xiaodong Ye <xiaodong.ye@mthreads.com>	2026-09-28 23:20:30 +08:00
DMANandHongsheng Liu	79a46a4f54	[Perf][Diffusion] support Sensenova online fp8 (#7955 ) Signed-off-by: Dmaner <2663515256@qq.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>	2026-09-28 23:17:37 +08:00
BeatSeatandHongsheng Liu	2a453c5ad8	feat(cosyvoice3): GPU-resident F0 predictor & pure-PyTorch vocoder (Task C2) (#7518 ) Signed-off-by: BeatSeat <wendavid552@gmail.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>	2026-09-28 23:13:43 +08:00
Clodagh Walsh	9c688aeba0	[TTS] Improve voice delete error message (#7795 ) Signed-off-by: Clodagh Walsh <clodaghwalsh17@gmail.com>	2026-09-28 09:42:34 -04:00
chickeytonandClaude Opus 5.5	5ce6f5dc49	[Bugfix][MiniCPM-o] Forward a speech unit closed by LISTEN after turn_eos to the Talker (#8227 ) Signed-off-by: chickeyton <ngton2014@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>	2026-09-28 11:11:42 +00:00
Aochang Liu	c0331048d0	[Misc] Sort OMNI_PIPELINES registry alphabetically (#8208 ) Signed-off-by: HelloWorldU <asd001liu@gmail.com>	2026-09-27 22:49:40 -07:00
Yueqian Lin	f5bdb20e9d	[Perf][PersonaPlex] Serve 16 concurrent realtime duplex sessions on one GPU (#8192 )	2026-09-28 04:51:38 +00:00
andyluo7andwangyu	36b74f99f6	test: keep Hunyuan LoRA CPU path on native linear (#8166 ) Signed-off-by: andyluo7 <andy.luo@amd.com> Co-authored-by: wangyu <53896905+yenuo26@users.noreply.github.com>	2026-09-28 04:33:40 +00:00
BeatSeatandamy-why-3459	c8a1419e02	[Performance][MiniCPM-o] Zero-copy block-aligned KV sliding window with in-place Re-RoPE for duplex Stage-0 (#7821 ) Signed-off-by: BeatSeat <wendavid552@gmail.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>	2026-09-28 10:11:54 +08:00
kinnara	817f5d0e4d	[Hardware][Ascend] Add FastH3 four-step deployment to MiniMax-H3 NPU … (#7149 ) Signed-off-by: kinnara <kinnara1989@gmail.com>	2026-09-27 17:34:59 +00:00
Ding Zuhao	1b97ff1188	[Model] Support SenseNova-U1-A3B-MoT (MoE) (#8172 ) Signed-off-by: Ding Zuhao <e1583181@u.nus.edu>	2026-09-27 08:48:08 +00:00
andyluo7andAlicia	b63e35ab4f	ci: extend AMD Z-Image cold-start timeout (#8195 ) Signed-off-by: andyluo7 <andy.luo@amd.com> Co-authored-by: Alicia <115451386+congw729@users.noreply.github.com>	2026-09-27 06:11:05 +00:00
chickeyton Claude Fable 5.1 Yueqian Lin	798d52ce55	[Core][Model] Fit PersonaPlex into the Unified Full-duplex Framework (RFC #7181 PR 2) (#7695 ) Signed-off-by: chickeyton <ngton2014@gmail.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com>	2026-09-27 03:02:24 +00:00
Hongsheng Liuandhsliu_ustc	7ab582ae25	[Doc] Update README and docs for v0.30.0 (#8175 ) Signed-off-by: hsliu_ustc <hsliu_ustc@noreply.gitcode.com> Co-authored-by: hsliu_ustc <hsliu_ustc@noreply.gitcode.com>	2026-09-27 10:07:58 +08:00
R0CKSTAR	6715c57b2d	feat: configure Torch Dynamo recompile limit from environment (#7243 ) Signed-off-by: Xiaodong Ye <xiaodong.ye@mthreads.com>	2026-09-26 22:54:47 +08:00
Tianyao Wu	d418004b6b	[Misc] Remove stale offline word_timestamps example resurrected by #5146 (#7233 ) Signed-off-by: Tianyao Wu <rayroy31@gmail.com>	2026-09-26 22:54:06 +08:00
Shenglei FuandClaude	eb67caaaa9	[Diffusion] Enable request-level batching for Boogu-Image (TI2I) (#7227 ) Signed-off-by: Shenglei Fu <sfu@confluent.io> Signed-off-by: Shenglei Fu <117230642+ShengleiFu@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com>	2026-09-26 22:53:35 +08:00
R0CKSTAR	e75278dccc	feat(magi2): add fused SwiGLU7 activation (#7252 ) Signed-off-by: Xiaodong Ye <xiaodong.ye@mthreads.com>	2026-09-26 22:53:04 +08:00
Sy03	e1303e5067	[Model] Optimize Qwen3-TTS streaming throughput and first audio on CUDA (#8163 ) Signed-off-by: Sy03 <1370724210@qq.com>	2026-09-26 22:51:04 +08:00
Anjie HouandHongsheng Liu	8a49c65bf9	[Core][Diffusion] Add byte backpressure to chunk transfers (#7916 ) Signed-off-by: specture724 <specture724@gmail.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>	2026-09-26 17:48:32 +08:00
Anjie Hou Claude Opus 5 Hongsheng Liu	acf6620233	[Refactor][Diffusion] Read the resolved offload policy outside the config layer (#7327 ) Signed-off-by: specture724 <specture724@gmail.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>	2026-09-26 17:45:20 +08:00
andyluo7	ad025defe6	[CI][Diffusion] Restore Krea2 model type marker (#8164 ) Signed-off-by: andyluo7 <andy.luo@amd.com>	2026-09-26 12:55:06 +08:00
Khairul Kabirandkhairulkabir1661	bcb4ae288f	[CI][Diffusion] Add tiny model builder for Krea2Pipeline (#6421 ) Signed-off-by: khairulkabir1661 <khairulkabir1661@users.noreply.github.com> Co-authored-by: khairulkabir1661 <khairulkabir1661@users.noreply.github.com>	2026-09-25 21:54:38 +00:00
Joshna-Medisetty	62ebad2d7c	Add Intel Arc BMG recipes for validated P0 diffusion models. (#7537 ) Signed-off-by: Joshna-Medisetty <joshna.medisetty@intel.com>	2026-09-25 15:42:08 -05:00
Khairul Kabir khairulkabir1661 Claude Opus 4.6	5e36b5a86b	[Bugfix] Apply function module marker per-model instead of hardcoding diffusion (#5875 ) Signed-off-by: khairulkabir1661 <khairulkabir1661@users.noreply.github.com> Co-authored-by: khairulkabir1661 <khairulkabir1661@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>	2026-09-25 14:17:19 -06:00