[Model] Add JoyAI-Image-Edit pipeline support (#4112)

Signed-off-by: nagisa-kun <1434936049@qq.com>
Signed-off-by: nagisa <1434936049@qq.com>
Signed-off-by: hurukawa <61525444+nagisa-kunhah@users.noreply.github.com>

.buildkite/amd/scripts/prewarm_cosmos3_aiter.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Prebuild the BF16 AITER MHA modules used by Cosmos3 on ROCm."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import torch
+
+from vllm_omni.diffusion.attention.backends.utils import fa
+from vllm_omni.platforms import current_omni_platform
+
+ATTENTION_VARIANTS = (
+    (True, "mha_fwd_bf16_nbias_mask_nlse_ndropout_nqscale"),
+    (False, "mha_fwd_bf16_nbias_nmask_nlse_ndropout_nqscale"),
+)
+
+
+def main() -> None:
+    if not current_omni_platform.is_rocm():
+        raise RuntimeError("The Cosmos3 AITER prewarm must run on ROCm")
+    if fa.flash_attn_func is None:
+        raise RuntimeError("ROCm FlashAttention/AITER is unavailable")
+
+    cache_dir_value = os.environ.get("AITER_JIT_DIR")
+    if not cache_dir_value:
+        raise RuntimeError("AITER_JIT_DIR must identify the job-local cache")
+    cache_dir = Path(cache_dir_value)
+
+    device = torch.device(current_omni_platform.device_type)
+    dtype = torch.bfloat16
+    batch_size = 1
+    sequence_length = 64
+    num_heads = 8
+    head_dim = 64
+
+    torch.manual_seed(42)
+    query = torch.randn(
+        batch_size,
+        sequence_length,
+        num_heads,
+        head_dim,
+        device=device,
+        dtype=dtype,
+    )
+    key = torch.randn_like(query)
+    value = torch.randn_like(query)
+
+    for causal, expected_module in ATTENTION_VARIANTS:
+        output = fa.flash_attn_func(
+            query,
+            key,
+            value,
+            dropout_p=0.0,
+            softmax_scale=head_dim**-0.5,
+            causal=causal,
+        )
+        if isinstance(output, tuple):
+            output = output[0]
+        current_omni_platform.synchronize()
+
+        if output.shape != query.shape:
+            raise RuntimeError(f"Unexpected AITER output shape: {output.shape}")
+        if not torch.isfinite(output).all():
+            raise RuntimeError("AITER prewarm produced non-finite output")
+
+        module_path = cache_dir / f"{expected_module}.so"
+        if not module_path.is_file():
+            raise RuntimeError(f"Expected AITER module was not built: {module_path}")
+        print(f"Prebuilt Cosmos3 AITER module: {module_path}", flush=True)
+
+
+if __name__ == "__main__":
+    main()

.buildkite/amd/test-amd-merge.yml

@@ -37,11 +37,35 @@ steps:
       mirror_hardwares: [amdproduction]
       grade: Blocking
       timeout_in_minutes: 45
+      # Buildkite's status-based rule cannot prove every exit 134 is a GPU hang.
+      # It retries 134 once; the Kubernetes job retry starts a fresh pod.
+      # Assertions (1), timeouts (124), and OOMs (137) remain non-retryable.
+      retry:
+        automatic:
+          - exit_status: 134
+            limit: 1
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+        # Keep multi-GPU tests out of this one-GPU pool even when their module
+        # also carries the cpu marker for unit tests.
+        # Keep π0.5 CPU units out of these shards: CUDA READY/MERGE already run
+        # the same suite mandatorily, and AMD covers it non-blocking in nightly.
+        - "pytest -sv tests/diffusion -m 'core_model and cpu and not (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)' --ignore=tests/diffusion/cache/test_teacache_extractors.py --ignore=tests/diffusion/models/pi05/test_pi05_units.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+
+    - label: "Diffusion · AuK VAE GPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 40
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - >-
+          timeout --signal=TERM --kill-after=2m 35m
+          pytest -sv tests/diffusion/models/auk/test_auk_vae_fastpath.py
+          -m 'core_model and rocm and cards_1' --run-level core_model
 
     - label: "Simple · Engine&Entrypoints Test"
       agent_pool: mi300_1
@@ -65,6 +89,25 @@ steps:
 
 - group: ":card_index_dividers: Diffusion Test"
   steps:
+    # AR paged attention is real GPU work, including cold AITER compilation.
+    # Keep its complete numerical matrix out of the CPU-marked Simple shards.
+    - label: "ROCm · AR Diffusion Paged Attention GPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 20
+      commands:
+        - export VLLM_ROCM_USE_AITER=1
+        - export VLLM_OMNI_AR_FA_REQUIRED=1
+        - export AITER_JIT_DIR="/tmp/vllm-omni-ar-aiter-$$BUILDKITE_JOB_ID"
+        - export TORCH_EXTENSIONS_DIR="/tmp/vllm-omni-ar-torch-extensions-$$BUILDKITE_JOB_ID"
+        - mkdir -p "$$AITER_JIT_DIR" "$$TORCH_EXTENSIONS_DIR"
+        - >-
+          timeout --signal=TERM --kill-after=2m 15m
+          pytest -sv tests/diffusion/ar_diffusion/test_paged_attention.py
+          -m 'core_model and rocm and MI325 and cards_1' --run-level core_model
+
     - label: "Diffusion · LoRA Test"
       agent_pool: mi300_1
       depends_on: amd-build
@@ -83,6 +126,7 @@ steps:
       commands:
         - export VLLM_LOGGING_LEVEL=DEBUG
         - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - export DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA
         - pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
         - >-
           pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
@@ -131,6 +175,11 @@ steps:
       grade: Blocking
       commands:
         - pytest -s -v tests/diffusion/distributed/test_tensor_parallel.py
+        - >-
+          timeout 20m pytest -s -v
+          tests/diffusion/attention/test_ulysses_uaa.py::test_ulysses_uaa_2d_mask_layout_matches_baseline
+          -m 'core_model and cards_2'
+          --run-level "core_model"
 
     - label: "Diffusion Model Test"
       agent_pool: mi300_1
@@ -138,7 +187,9 @@ steps:
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
-        - timeout 30m pytest -s -v tests/e2e/offline_inference/test_z_image.py
+        # A cold Z-Image run spends about six minutes loading weights before
+        # the first-request AITER compilation, which can itself exceed 23m.
+        - timeout 55m pytest -s -v tests/e2e/offline_inference/test_z_image.py
 
     - label: "Diffusion GPU Worker Test"
       agent_pool: mi300_2

.buildkite/amd/test-amd-nightly.yml

@@ -82,3 +82,89 @@ steps:
               limit: 1
         commands:
           - pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"
+
+  # π0.5 CPU units are mandatory on CUDA READY/MERGE. On AMD they exercise the
+  # ROCm container/import path with CPU tensors only, so keep them off the
+  # blocking READY/MERGE critical path and retain non-blocking nightly coverage.
+  - group: ":robot_face: Pi0.5 CPU Nightly"
+    steps:
+      - label: "Simple · Pi0.5 CPU Test"
+        agent_pool: mi300_1
+        mirror_hardwares: [amdproduction]
+        grade: NonBlocking
+        timeout_in_minutes: 60
+        commands:
+          - export VLLM_ROCM_USE_AITER=0
+          - "pytest -sv tests/diffusion/models/pi05/test_pi05_units.py -m 'core_model and cpu'"
+
+  - group: ":full_moon: LingBot-Video MI300 Nightly"
+    steps:
+      - label: "LingBot-Video Dense CFG-off Smoke"
+        agent_pool: mi300_1
+        mirror_hardwares: [amdproduction]
+        grade: NonBlocking
+        timeout_in_minutes: 90
+        artifact_paths:
+          - artifacts/rocm-lingbot-nightly/**/*
+        commands:
+          - pytest --collect-only -q tests/e2e/online_serving/test_lingbot_video_expansion.py::test_cfg_off -m "full_model and diffusion and rocm and MI325 and cards_1" --run-level full_model
+          - pytest -s -v tests/e2e/online_serving/test_lingbot_video_expansion.py::test_cfg_off -m "full_model and diffusion and rocm and MI325 and cards_1" --run-level full_model --junitxml=/tmp/lingbot-pytest.xml
+          - |
+            artifact_dir="$${BUILDKITE_BUILD_CHECKOUT_PATH:?}/artifacts/rocm-lingbot-nightly"
+            mkdir -p "$$artifact_dir"
+            cp -v /tmp/lingbot-pytest.xml "$$artifact_dir/"
+
+  - group: ":full_moon: Cosmos3 MI300 Nightly"
+    steps:
+      - label: "Cosmos3 T2I Function"
+        agent_pool: mi300_1
+        mirror_hardwares: [amdproduction]
+        grade: NonBlocking
+        timeout_in_minutes: 90
+        artifact_paths:
+          - artifacts/rocm-cosmos3-nightly/**/*
+        commands:
+          # Cosmos3 uses both masked and unmasked BF16 AITER MHA modules. Build
+          # them before server startup, then reuse the same job-local cache
+          # during the smoke test.
+          - export VLLM_ROCM_USE_AITER=1
+          - export AITER_JIT_DIR="/tmp/vllm-omni-cosmos3-aiter-$$BUILDKITE_JOB_ID"
+          - export TORCH_EXTENSIONS_DIR="/tmp/vllm-omni-cosmos3-torch-extensions-$$BUILDKITE_JOB_ID"
+          - mkdir -p "$$AITER_JIT_DIR" "$$TORCH_EXTENSIONS_DIR"
+          - pytest --collect-only -q tests/e2e/online_serving/test_cosmos3.py::test_text_to_image_001 -m "core_model and rocm and MI325 and cards_1" --run-level full_model
+          - >-
+            timeout --signal=TERM --kill-after=2m 55m
+            python .buildkite/amd/scripts/prewarm_cosmos3_aiter.py
+          - >-
+            timeout --signal=TERM --kill-after=2m 25m
+            pytest -s -v
+            tests/e2e/online_serving/test_cosmos3.py::test_text_to_image_001
+            -m "core_model and rocm and MI325 and cards_1"
+            --run-level full_model
+            --junitxml=/tmp/cosmos3-pytest.xml
+          - |
+            artifact_dir="$${BUILDKITE_BUILD_CHECKOUT_PATH:?}/artifacts/rocm-cosmos3-nightly"
+            mkdir -p "$$artifact_dir"
+            cp -v /tmp/cosmos3-pytest.xml "$$artifact_dir/"
+
+  - group: ":full_moon: HunyuanImage3 MI300 Nightly"
+    steps:
+      - label: "HunyuanImage3 Offline Pixel Accuracy"
+        agent_pool: mi300_4
+        mirror_hardwares: [amdproduction]
+        grade: NonBlocking
+        timeout_in_minutes: 180
+        env:
+          HUNYUAN_IMAGE3_MODEL: "tencent/HunyuanImage-3.0-Instruct"
+          HUNYUAN_IMAGE3_DEVICES: "0,1,2,3"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        artifact_paths:
+          - artifacts/rocm-hunyuanimage3/**/*
+        commands:
+          - pytest --collect-only -q tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py::test_hunyuan_image3_pixel_accuracy_offline -m "full_model and rocm and MI325 and cards_4" --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py::test_hunyuan_image3_pixel_accuracy_offline -m "full_model and rocm and MI325 and cards_4" --run-level full_model --junitxml=/tmp/hunyuanimage3-pytest.xml
+          - |
+            artifact_dir="$${BUILDKITE_BUILD_CHECKOUT_PATH:?}/artifacts/rocm-hunyuanimage3"
+            mkdir -p "$$artifact_dir"
+            cp -v tests/e2e/accuracy/artifacts/HunyuanImage-3_0-Instruct/*.png "$$artifact_dir/"
+            cp -v /tmp/hunyuanimage3-pytest.xml "$$artifact_dir/"

.buildkite/amd/test-amd-ready.yml

@@ -37,11 +37,35 @@ steps:
       mirror_hardwares: [amdproduction]
       grade: Blocking
       timeout_in_minutes: 45
+      # Buildkite's status-based rule cannot prove every exit 134 is a GPU hang.
+      # It retries 134 once; the Kubernetes job retry starts a fresh pod.
+      # Assertions (1), timeouts (124), and OOMs (137) remain non-retryable.
+      retry:
+        automatic:
+          - exit_status: 134
+            limit: 1
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+        # Keep multi-GPU tests out of this one-GPU pool even when their module
+        # also carries the cpu marker for unit tests.
+        # Keep π0.5 CPU units out of these shards: CUDA READY/MERGE already run
+        # the same suite mandatorily, and AMD covers it non-blocking in nightly.
+        - "pytest -sv tests/diffusion -m 'core_model and cpu and not (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)' --ignore=tests/diffusion/cache/test_teacache_extractors.py --ignore=tests/diffusion/models/pi05/test_pi05_units.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+
+    - label: "Diffusion · AuK VAE GPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 40
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - >-
+          timeout --signal=TERM --kill-after=2m 35m
+          pytest -sv tests/diffusion/models/auk/test_auk_vae_fastpath.py
+          -m 'core_model and rocm and cards_1' --run-level core_model
 
     - label: "Simple · Engine&Entrypoints Test"
       agent_pool: mi300_1
@@ -102,6 +126,34 @@ steps:
           tests/diffusion/attention/
           -m 'core_model and rocm and MI325 and cards_1'
           --run-level "core_model"
+        # MammothModa2 exercises the same two AITER MHA modules built by the
+        # preceding suite. Keep its default-backend run in this job so a cold
+        # worker compiles each module once instead of timing out the generic
+        # model job.
+        - >-
+          timeout --signal=TERM --kill-after=2m 10m
+          pytest -s -v -n 1
+          tests/diffusion/models/mammoth_moda2/test_dit_attention_cuda.py
+          --run-level "core_model"
+
+    # AR paged attention is real GPU work, including cold AITER compilation.
+    # Keep its complete numerical matrix out of the CPU-marked Simple shards.
+    - label: "ROCm · AR Diffusion Paged Attention GPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 20
+      commands:
+        - export VLLM_ROCM_USE_AITER=1
+        - export VLLM_OMNI_AR_FA_REQUIRED=1
+        - export AITER_JIT_DIR="/tmp/vllm-omni-ar-aiter-$$BUILDKITE_JOB_ID"
+        - export TORCH_EXTENSIONS_DIR="/tmp/vllm-omni-ar-torch-extensions-$$BUILDKITE_JOB_ID"
+        - mkdir -p "$$AITER_JIT_DIR" "$$TORCH_EXTENSIONS_DIR"
+        - >-
+          timeout --signal=TERM --kill-after=2m 15m
+          pytest -sv tests/diffusion/ar_diffusion/test_paged_attention.py
+          -m 'core_model and rocm and MI325 and cards_1' --run-level core_model
 
     - label: "Diffusion · Batch Test"
       agent_pool: mi300_1
@@ -141,6 +193,11 @@ steps:
         - export VLLM_LOGGING_LEVEL=DEBUG
         - export VLLM_WORKER_MULTIPROC_METHOD=spawn
         - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
+        - >-
+          timeout 20m pytest -s -v
+          tests/diffusion/attention/test_ulysses_uaa.py::test_ulysses_uaa_2d_mask_layout_matches_baseline
+          -m 'core_model and cards_2'
+          --run-level "core_model"
         - >-
           timeout 20m pytest -s -v
           tests/diffusion/models/ltx2/test_ltx2_transformer_ulysses.py
@@ -165,9 +222,18 @@ steps:
         # registered multi-GPU marker from this one-GPU job.
         - >-
           timeout 15m pytest -sv tests/diffusion/models/
+          --ignore=tests/diffusion/models/auk/test_auk_vae_fastpath.py
+          --ignore=tests/diffusion/models/mammoth_moda2/test_dit_attention_cuda.py
           -m 'core_model and cuda and not
           (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
           --run-level "core_model"
+        # Keep the model-specific numerical coverage blocking without paying
+        # the default ROCm backend's cold AITER compilation cost in this lane.
+        - >-
+          timeout 10m env DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA
+          pytest -sv
+          tests/diffusion/models/mammoth_moda2/test_dit_attention_cuda.py
+          --run-level "core_model"
 
     - label: "Diffusion · Wan22 Test"
       agent_pool: mi300_1
@@ -194,7 +260,7 @@ steps:
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
 
-- label: "ROCm · Engine & Model Executor GPU Coverage (R2-01)"
+- label: "ROCm · Model Executor GPU Coverage (R2-01)"
   agent_pool: mi300_1
   depends_on: amd-build
   mirror_hardwares: [amdproduction]
@@ -207,18 +273,17 @@ steps:
     # artifacts relative to its checkout directory.
     - export R2_01_ARTIFACT_DIR="$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/rocm-r2-01"
     - mkdir -p "$$R2_01_ARTIFACT_DIR"
+    # The dedicated Engine Test below owns the only selected tests/engine case;
+    # do not duplicate its model startup in this time-bounded coverage lane.
     # The CUDA ready job also covers tests/worker/test_batched_omni_output.py.
-    # Keep the initial ROCm burn-in scoped to the requested directories.
+    # Keep the initial ROCm burn-in scoped to tests/model_executor/.
     # Verbose pytest output and JUnit retain node IDs without a second
     # collection-only pass over the same GPU-backed test tree.
-    # The CUDA/L4-only CosyVoice3 ragged-flow parity test cold-JITs multiple
-    # AITER kernels on ROCm and is covered by the CUDA ready lane.
     - >-
       pytest -s -v -ra
-      tests/engine/ tests/model_executor/
-      -m 'core_model and cuda and not
+      tests/model_executor/
+      -m 'core_model and cuda and not nvidia_only and not
       (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
-      -k 'not test_code2wav_streaming_batch_matches_ragged_flow_numerics'
       --run-level "core_model"
       --durations=0
       --junitxml="$$R2_01_ARTIFACT_DIR/pytest.xml"

.buildkite/common/ci_source_file_dependencies.yml

@@ -16,10 +16,14 @@ x-qwen3-tts: &qwen3_tts
   - vllm_omni/worker/gpu_model_runner.py
   - vllm_omni/model_executor/models/qwen3_tts/
   - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+  - vllm_omni/model_executor/models/common/fused_sampling.py
+  - vllm_omni/model_executor/models/common/short_kv_attention.py
+  - vllm_omni/model_executor/models/common/talker_first_audio.py
   - vllm_omni/model_executor/models/common/snake_activation.py
   - vllm_omni/model_executor/models/common/alias_free_activation.py
   - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
   - vllm_omni/deploy/qwen3_tts.yaml
+  - vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2_single_gpu.yaml
   - vllm_omni/entrypoints/openai/serving_speech.py
   - vllm_omni/entrypoints/openai/tts_adapters/
   - vllm_omni/engine/stage_pool.py
@@ -30,6 +34,9 @@ x-qwen3-tts: &qwen3_tts
   - vllm_omni/config/omni_config.py
   - vllm_omni/engine/orchestrator.py
   - vllm_omni/engine/stage_engine_core_proc.py
+  - vllm_omni/engine/cuda_mps.py
+  - vllm_omni/engine/stage_runtime.py
+  - vllm_omni/engine/stage_init_utils.py
   - vllm_omni/deploy/qwen3_tts_mrv2.yaml
   - vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2.yaml
   - vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2_b4.yaml
@@ -81,11 +88,16 @@ x-qwen3-omni: &qwen3_omni
   - vllm_omni/entrypoints/openai/serving_chat.py
   - vllm_omni/model_executor/models/qwen3_omni/
   - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+  - vllm_omni/model_executor/models/common/fused_sampling.py
+  - vllm_omni/model_executor/models/common/short_kv_attention.py
+  - vllm_omni/model_executor/models/common/talker_first_audio.py
   - vllm_omni/model_executor/models/common/snake_activation.py
   - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
   - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
   - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
   - vllm_omni/deploy/qwen3_omni_moe.yaml
+  - vllm_omni/deploy/qwen3_omni_moe_mrv2.yaml
+  - vllm_omni/deploy/qwen3_omni_moe_mrv2_h200.yaml
   - vllm_omni/deploy/qwen3_omni_moe_thinking.yaml
 
 x-wan22: &wan22
@@ -103,6 +115,10 @@ x-qwen-image: &qwen_image
   - vllm_omni/diffusion/models/dmd2/
   - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py
 
+x-joy-image: &joy_image
+  - vllm_omni/diffusion/models/joy_image/
+  - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_wan.py
+
 x-hunyuan-image3: &hunyuan_image3
   - vllm_omni/diffusion/models/hunyuan_image3/
   - vllm_omni/model_executor/models/hunyuan_image3/
@@ -137,6 +153,10 @@ x-lingbot-video: &lingbot_video
 x-lingbot-world: &lingbot_world
   - vllm_omni/diffusion/models/lingbot_world/
 
+x-kandinsky6: &kandinsky6
+  - vllm_omni/diffusion/models/kandinsky6/
+  - vllm_omni/model_extras/kandinsky6.py
+
 
 source_file_dependencies:
   # Hardware presets and shared test helpers. Not a job key — do not attach this
@@ -371,6 +391,12 @@ source_file_dependencies:
     - tests/e2e/accuracy/test_qwen_image.py
     - tests/e2e/accuracy/test_diffusers_backend_similarity.py
 
+  diffusion_joy_image_accuracy:
+    - *joy_image
+    - tests/diffusion/models/joy_image/
+    - tests/e2e/accuracy/test_joy_image_edit.py
+    - recipes/JD/JoyAI-Image-Edit.md
+
   diffusion_qwen_image_perf:
     - *qwen_image
     - tests/dfx/perf/scripts/run_diffusion_benchmark.py
@@ -418,6 +444,7 @@ source_file_dependencies:
 
   diffusion_minimax_h3_function:
     - *minimax_h3
+    - tests/helpers/assertions.py
     - tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py
     - tests/e2e/online_serving/minimax_h3/
     - tests/diffusion/models/minimax_h3/
@@ -478,6 +505,11 @@ source_file_dependencies:
     - tests/dfx/perf/scripts/run_benchmark.py
     - tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json
 
+  diffusion_kandinsky6_function:
+    - *kandinsky6
+    - tests/e2e/offline_inference/test_kandinsky6.py
+    - tests/e2e/online_serving/test_kandinsky6.py
+
 
   # Invalid-param jobs: model business code only, plus the reliability scripts.
   reliability_invalid_param_h100:

.buildkite/common/scripts/upload_pipeline.py

@@ -102,6 +102,7 @@ CI_SOURCE_FILE_DEPENDENCIES_PATH = ROOT / ".buildkite/common/ci_source_file_depe
 SOURCE_FILTER_FALLBACK_KEY = "source_filter_fallback"
 CUDA_HF_TOKEN_ENV = "VLLM_CI_HF_TOKEN"
 CUDA_HF_TOKEN_EXPORT = f'if [ -n "$${{{CUDA_HF_TOKEN_ENV}:-}}" ]; then export HF_TOKEN="$${{{CUDA_HF_TOKEN_ENV}}}"; fi'
+CUDA_RUNTIME_CHECK = "python3 .buildkite/cuda/scripts/check_vllm_runtime.py"
 
 # Bootstrap Buildkite ``if`` expressions.
 # ``*_MAIN_IF``: main + env schedule. ``*_LABEL_IF``: PR label (and/or composed with MAIN).
@@ -620,11 +621,11 @@ def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any] | None:
     if any(preset_name == chip or preset_name.startswith(f"{chip}_") for chip in _cuda_mirror_chips()):
         commands = merged.get("commands")
         if isinstance(commands, list):
-            merged["commands"] = [CUDA_HF_TOKEN_EXPORT, *commands]
+            merged["commands"] = [CUDA_HF_TOKEN_EXPORT, CUDA_RUNTIME_CHECK, *commands]
         elif commands is None:
-            merged["commands"] = [CUDA_HF_TOKEN_EXPORT]
+            merged["commands"] = [CUDA_HF_TOKEN_EXPORT, CUDA_RUNTIME_CHECK]
         else:
-            merged["commands"] = [CUDA_HF_TOKEN_EXPORT, commands]
+            merged["commands"] = [CUDA_HF_TOKEN_EXPORT, CUDA_RUNTIME_CHECK, commands]
     # Preset retry (K8S_RETRY on l4_*) must not clobber a step that opted out.
     if "retry" in step:
         merged["retry"] = step["retry"]

.buildkite/cuda/scripts/check_vllm_runtime.py

@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Verify the pinned vLLM and compiled CUDA imports on a GPU CI worker."""
+
+from __future__ import annotations
+
+import importlib
+import re
+from pathlib import Path
+
+
+def main() -> None:
+    dockerfile = Path(__file__).resolve().parents[3] / "docker" / "Dockerfile.ci"
+    match = re.search(
+        r"^ARG VLLM_VERSION=([0-9]+\.[0-9]+\.[0-9]+)$",
+        dockerfile.read_text(),
+        re.MULTILINE,
+    )
+    if match is None:
+        raise RuntimeError(f"Missing vLLM release version in {dockerfile}")
+    expected_version = match.group(1)
+
+    import torch
+    import vllm
+
+    assert vllm.__version__ == expected_version, vllm.__version__
+    assert torch.cuda.is_available(), "vLLM runtime check requires a GPU worker"
+    for module in ("flashinfer", "vllm._C_stable_libtorch"):
+        importlib.import_module(module)
+    print(
+        f"Verified vLLM CUDA runtime: vllm={vllm.__version__} "
+        f"expected={expected_version} torch={torch.__version__} "
+        f"cuda={torch.version.cuda} device={torch.cuda.get_device_name()}"
+    )
+
+
+if __name__ == "__main__":
+    main()

.buildkite/cuda/test-merge.yml

@@ -38,6 +38,17 @@ steps:
   - group: ":card_index_dividers: Diffusion Test"
     depends_on: upload-merge-pipeline
     steps:
+      - label: "Diffusion · AR Paged Attention GPU Test"
+        timeout_in_minutes: 20
+        commands:
+          - export VLLM_OMNI_AR_FA_REQUIRED=1
+          - >-
+            timeout 15m pytest -sv
+            tests/diffusion/ar_diffusion/test_paged_attention.py
+            -m 'core_model and cuda and L4 and cards_1'
+            --run-level "core_model"
+        mirror_hardwares: l4_1
+
       - label: "Diffusion · LoRA Test"
         timeout_in_minutes: 30
         commands:
@@ -124,6 +135,30 @@ steps:
           - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py -m 'advanced_model and cuda' --run-level 'advanced_model'
         mirror_hardwares: h100_2
 
+      - label: "Omni · Qwen3-Omni Async-Chunk SHM Leak"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_omni_shm_leak.py
+          - tests/helpers/runtime.py
+          - tests/helpers/stage_config.py
+          - vllm_omni/core/sched/omni_ar_scheduler.py
+          - vllm_omni/distributed/omni_connectors/connectors/shm_connector.py
+          - vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py
+          - vllm_omni/engine/orchestrator.py
+          - vllm_omni/engine/stage_engine_core_proc.py
+          - vllm_omni/engine/stage_pool.py
+          - vllm_omni/entrypoints/openai/serving_chat.py
+          - vllm_omni/model_executor/models/qwen3_omni/
+          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+          - vllm_omni/model_executor/models/common/snake_activation.py
+          - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
+          - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
+          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
+          - vllm_omni/deploy/qwen3_omni_moe.yaml
+        timeout_in_minutes: 40
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_shm_leak.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+        mirror_hardwares: h100_2
+
       - label: "Omni · MiniCPM-o 4.5 Test"
         source_file_dependencies: omni_minicpmo_4_5_function
         timeout_in_minutes: 50
@@ -154,6 +189,15 @@ steps:
             '
         mirror_hardwares: h100_1
 
+      - label: "Diffusion · Kandinsky6 Test"
+        source_file_dependencies: diffusion_kandinsky6_function
+        commands:
+          - |
+            timeout 40m bash -c "
+              pytest -s -v tests/e2e/offline_inference/test_kandinsky6.py tests/e2e/online_serving/test_kandinsky6.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+            "
+        mirror_hardwares: h100_1
+
       - label: "Diffusion · Wan22 Test"
         source_file_dependencies: diffusion_wan22_function
         commands:
@@ -208,3 +252,14 @@ steps:
             timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_turbo_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
             timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_fasth3_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
         mirror_hardwares: h100_2
+
+      - label: "Diffusion · MiniMax-H3 Latent Upscale and Refine"
+        source_file_dependencies: diffusion_minimax_h3_function
+        timeout_in_minutes: 40
+        commands:
+          - |
+            set -eu
+            VLLM_TEST_MINIMAX_H3_LATENT_UPSCALER=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id="LBH-123-AI/Minimax_h3_latent_Upscaler", filename="minimax_h3_latent_upscaler_3d_conv_v1/minimax_h3_latent_upscaler_3d_conv_v1_bf16.safetensors"))')
+            export VLLM_TEST_MINIMAX_H3_LATENT_UPSCALER
+            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+        mirror_hardwares: h100_2

.buildkite/cuda/test-nightly.yml

@@ -21,7 +21,7 @@ steps:
           - omni_minicpmo_4_5_function
         timeout_in_minutes: 120
         commands:
-          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py --ignore=tests/e2e/online_serving/test_minicpmo_4_5_window.py
 
       - label: ":full_moon: Omni · Function Test with H100 · 2-GPU"
         source_file_dependencies:
@@ -33,9 +33,10 @@ steps:
 
       - label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
         source_file_dependencies: omni_minicpmo_4_5_duplex_function
-        timeout_in_minutes: 50
+        timeout_in_minutes: 120
         commands:
           - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and B200 and omni and cards_1" --run-level "full_model"
+          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_window.py -m "full_model and cuda and H100 and B200 and omni and cards_1" --run-level "full_model"
 
       - label: ":full_moon: Omni · Doc Test with L4 · 4-GPU"
         source_file_dependencies: omni_qwen2_5_omni_doc
@@ -47,6 +48,8 @@ steps:
       - label: ":full_moon: Omni · Doc Test with H100 · 2-GPU"
         source_file_dependencies: omni_qwen3_omni_doc
         timeout_in_minutes: 90
+        artifact_paths:
+          - audio_*.wav
         commands:
           - pytest -s -v tests/examples/ -m "full_model and omni and H100 and B200 and cards_2" --run-level "full_model"
 
@@ -155,6 +158,11 @@ steps:
       - label: ":full_moon: TTS · Function Test with L4"
         source_file_dependencies: tts_qwen3_tts_function
         timeout_in_minutes: 120
+        env:
+          VLLM_OMNI_FAILED_SPEECH_AUDIO_DIR: .
+        artifact_paths:
+          - failed_speech_*.wav
+          - failed_speech_*.json
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
           - export VLLM_USE_DEEP_GEMM="0"
@@ -229,17 +237,6 @@ steps:
             -m "full_model and diffusion and H100 and B200 and cards_1"
             --run-level "full_model"
 
-      - label: ":robot_face: Robot Policy OpenPI · π0.5 · H100 · Single-GPU"
-        timeout_in_minutes: 120
-        commands:
-          - pip install --no-deps "openpi-client @ git+https://github.com/Physical-Intelligence/openpi.git@215abfb217dbac7d5f1273282331b9b1866c0479#subdirectory=packages/openpi-client"
-          - >-
-            pytest -sv
-            tests/e2e/online_serving/test_pi05_expansion.py
-            -m "full_model and diffusion and H100 and cards_1"
-            --run-level "full_model"
-        mirror_hardwares: h100_1
-
       - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU"
         source_file_dependencies: diffusion_qwen_image_function
         timeout_in_minutes: 120
@@ -271,6 +268,12 @@ steps:
           - pytest -s -v tests/e2e/accuracy/test_qwen_image.py -m "H100 and B200 and cards_1" --run-level full_model
           - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m "full_model and H100 and B200 and cards_1" --run-level full_model
 
+      - label: ":full_moon: JoyAI-Image-Edit · Accuracy Test"
+        source_file_dependencies: diffusion_joy_image_accuracy
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/e2e/accuracy/test_joy_image_edit.py -m "full_model and diffusion and H100 and B200 and cards_1" --run-level full_model
+
       - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
         source_file_dependencies: diffusion_hunyuan_image3_accuracy
         timeout_in_minutes: 180

.buildkite/cuda/test-ready.yml

@@ -16,6 +16,25 @@ steps:
           - "pytest -sv tests/model_executor -m 'core_model and cpu'"
         mirror_hardwares: l4_1
 
+      - label: "Simple · CosyVoice3 Packed Flow Test"
+        timeout_in_minutes: 15
+        source_file_dependencies:
+          - tests/model_executor/models/cosyvoice3/test_cosyvoice3_full_batch.py
+          - vllm_omni/model_executor/models/cosyvoice3/
+        commands:
+          - "pytest -sv tests/model_executor/models/cosyvoice3/test_cosyvoice3_full_batch.py -k packed -m 'core_model and cuda' --run-level=core_model"
+        mirror_hardwares: h100_1
+
+      - label: "Simple · MRV2 FA3 Graph Bounds Test"
+        timeout_in_minutes: 15
+        source_file_dependencies:
+          - tests/worker_v2/test_capture_query_bound_gpu.py
+          - tests/worker_v2/test_decode_query_bound_gpu.py
+          - vllm_omni/worker_v2/model_states/omni_model_state.py
+        commands:
+          - "pytest -sv tests/worker_v2/test_capture_query_bound_gpu.py tests/worker_v2/test_decode_query_bound_gpu.py -m 'core_model and cuda and H100' --run-level=core_model"
+        mirror_hardwares: h100_1
+
       - label: "Simple · Diffusion Test"
         timeout_in_minutes: 30
         commands:
@@ -86,7 +105,7 @@ steps:
 
       - label: "Diffusion · Other Test"
         commands:
-          - timeout 15m pytest -sv tests/diffusion/*.py tests/diffusion/ar_diffusion -m 'core_model and cuda' --run-level "core_model"
+          - timeout 15m pytest -sv tests/diffusion/*.py tests/diffusion/ar_diffusion tests/diffusion/layers/test_adalayernorm_*.py -m 'core_model and cuda' --run-level "core_model"
         mirror_hardwares: l4_1
 
   - group: ":card_index_dividers: Tiny Model Tests [Single GPU]"
@@ -103,7 +122,7 @@ steps:
     commands:
       - |
         timeout 15m bash -c "
-                  pytest -sv tests/engine/ tests/model_executor/ tests/worker/test_batched_omni_output.py tests/worker_v2/ -m 'core_model and cuda' --run-level 'core_model'
+                  pytest -sv tests/engine/ tests/model_executor/ tests/worker/test_batched_omni_output.py tests/worker/test_gpu_generation_async_output.py tests/worker_v2/ -m 'core_model and cuda' --run-level 'core_model'
         "
     mirror_hardwares: l4_1
 
@@ -196,6 +215,16 @@ steps:
           - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda'
         mirror_hardwares: h100_1
 
+      - label: "TTS · Higgs v3 Full Response Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_higgs_audio_v3.py
+          - vllm_omni/model_executor/models/higgs_audio_v3/
+          - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py
+          - vllm_omni/deploy/higgs_multimodal_qwen3.yaml
+        commands:
+          - timeout 20m pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py::test_full_response_v1_preserves_audio -m 'core_model and cuda' --run-level 'core_model'
+        mirror_hardwares: h100_1
+
       - label: "TTS · Qwen3-TTS CustomVoice Test"
         source_file_dependencies: tts_qwen3_tts_function
         commands:

.buildkite/cuda/test-weekly.yml

@@ -152,15 +152,122 @@ steps:
             --run-level "full_model"
         mirror_hardwares: h100_4
 
-      - label: "TTS · L4"
-        timeout_in_minutes: 120
+      - label: "TTS · Moss · L4"
+        timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
           - export VLLM_USE_DEEP_GEMM="0"
           - export VLLM_MOE_USE_DEEP_GEMM="0"
           - |
             failed=0
-            for f in $(pytest --collect-only -q tests/e2e/ -m "slow and L4 and tts" --run-level "full_model" | grep '\.py::' | sed -E 's|::.*||' | sort -u); do
+            for f in \
+              tests/e2e/offline_inference/test_moss_tts_expansion.py \
+              tests/e2e/offline_inference/test_moss_tts_nano_expansion.py \
+              tests/e2e/offline_inference/test_moss_tts_realtime.py \
+              tests/e2e/offline_inference/test_moss_tts_realtime_expansion.py \
+              tests/e2e/online_serving/test_moss_tts_expansion.py \
+              tests/e2e/online_serving/test_moss_tts_nano_expansion.py
+            do
+              pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
+            done
+            exit $$failed
+        mirror_hardwares: l4_1
+
+      - label: "TTS · VoxCPM2 · Voxtral · L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - |
+            failed=0
+            for f in \
+              tests/e2e/offline_inference/test_voxcpm2_tts.py \
+              tests/e2e/online_serving/test_voxcpm2_tts.py \
+              tests/e2e/online_serving/test_voxcpm2_tts_expansion.py \
+              tests/e2e/online_serving/test_voxcpm2_tts_lora.py \
+              tests/e2e/offline_inference/test_voxtral_tts_expansion.py \
+              tests/e2e/online_serving/test_voxtral_tts_expansion.py
+            do
+              pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
+            done
+            exit $$failed
+        mirror_hardwares: l4_1
+
+      - label: "TTS · Higgs · L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - |
+            failed=0
+            for f in \
+              tests/e2e/offline_inference/test_higgs_audio_v2_expansion.py \
+              tests/e2e/online_serving/test_higgs_audio_v2_expansion.py \
+              tests/e2e/online_serving/test_higgs_audio_v3.py
+            do
+              pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
+            done
+            exit $$failed
+        mirror_hardwares: l4_1
+
+      - label: "TTS · OmniVoice · Step-Audio · L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - |
+            failed=0
+            for f in \
+              tests/e2e/offline_inference/test_omnivoice_expansion.py \
+              tests/e2e/online_serving/test_omnivoice_expansion.py \
+              tests/e2e/online_serving/test_omnivoice_parity.py \
+              tests/e2e/offline_inference/test_step_audio2_expansion.py
+            do
+              pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
+            done
+            exit $$failed
+        mirror_hardwares: l4_1
+
+      - label: "TTS · Ming · GLM · IndexTTS · Breeze · L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - |
+            failed=0
+            for f in \
+              tests/e2e/offline_inference/test_ming_tts_expansion.py \
+              tests/e2e/online_serving/test_ming_tts_expansion.py \
+              tests/e2e/offline_inference/test_glm_tts_expansion.py \
+              tests/e2e/online_serving/test_glm_tts_expansion.py \
+              tests/e2e/offline_inference/test_indextts2_expansion.py \
+              tests/e2e/online_serving/test_indextts2_expansion.py \
+              tests/e2e/online_serving/test_breeze_tts_2.py
+            do
+              pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
+            done
+            exit $$failed
+        mirror_hardwares: l4_1
+
+      - label: "TTS · Audex · CosyVoice · Qwen3-TTS · L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - |
+            failed=0
+            for f in \
+              tests/e2e/offline_inference/test_audex_tts.py \
+              tests/e2e/offline_inference/test_audex_s2t.py \
+              tests/e2e/offline_inference/test_audex_tta.py \
+              tests/e2e/offline_inference/test_cosyvoice3_expansion.py \
+              tests/e2e/online_serving/test_qwen3_tts_word_timestamps.py
+            do
               pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
             done
             exit $$failed
@@ -224,6 +331,17 @@ steps:
 
       # Split the old shared H100 Single-GPU sweep: one pytest process collected
       # every slow+cards_1 diffusion file and exceeded 150 minutes.
+      - label: ":robot_face: Robot Policy OpenPI · π0.5 · H100 · Single-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - pip install --no-deps "openpi-client @ git+https://github.com/Physical-Intelligence/openpi.git@215abfb217dbac7d5f1273282331b9b1866c0479#subdirectory=packages/openpi-client"
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_pi05_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
       - label: "Diffusion · Qwen-Image · H100 · Single-GPU"
         timeout_in_minutes: 120
         commands:

.buildkite/intel/pipeline-intel.yml

@@ -10,7 +10,7 @@ steps:
           DOCKER_BUILDKIT: "1"
           # Buildkite will automatically replace this with the actual commit hash
           VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.30.0"
+          VLLM_VERSION: "v0.31.0"
         priority: 100
         timeout_in_minutes: 60
         soft_fail: false

.buildkite/npu/test-npu-nightly.yml

@@ -64,6 +64,11 @@ steps:
             tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
             -m "npu and A3 and omni"
             --run-level full_model
+          - >-
+            pytest -s -v
+            tests/e2e/online_serving/test_minicpmo_4_5_window.py
+            -m "full_model and npu and A3 and omni"
+            --run-level full_model
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
         source_file_dependencies: omni_minicpmo_4_5_accuracy
         timeout_in_minutes: 180

.github/workflows/build_wheel.yml

@@ -56,6 +56,35 @@ jobs:
         run: |
           bash scripts/build_wheel.sh --python python
 
+      - name: Validate runtime package data from source archive
+        run: |
+          set -euo pipefail
+          archive_dir="$(mktemp -d)"
+          install_dir="$(mktemp -d)"
+          vcs_wheels=(dist/*.whl)
+          if [[ ${#vcs_wheels[@]} -ne 1 || ! -f "${vcs_wheels[0]}" ]]; then
+            echo "Expected exactly one VCS-built wheel" >&2
+            exit 1
+          fi
+          git archive HEAD | tar -x -C "${archive_dir}"
+          (
+            cd "${archive_dir}"
+            VLLM_OMNI_TARGET_DEVICE=cuda \
+              VLLM_OMNI_VERSION_OVERRIDE=0.0.0 \
+              python -m build --wheel --outdir archive-dist
+          )
+          wheels=("${archive_dir}"/archive-dist/*.whl)
+          if [[ ${#wheels[@]} -ne 1 || ! -f "${wheels[0]}" ]]; then
+            echo "Expected exactly one archive-built wheel" >&2
+            exit 1
+          fi
+          python -m pip install --no-deps --target "${install_dir}" "${wheels[0]}"
+          python tools/check_wheel_package_data.py \
+            --source-root "${archive_dir}" \
+            --wheel "${wheels[0]}" \
+            --vcs-wheel "${vcs_wheels[0]}" \
+            --installed-root "${install_dir}"
+
       - name: Upload Wheel Artifact
         uses: actions/upload-artifact@v7
         with:

.gitignore

@@ -243,6 +243,7 @@ datasets/
 *.csv
 *.json
 !recipes/cosmos3/negative_prompt.json
+!benchmarks/mammoth_moda2/reference_results.json
 !tests/dfx/perf/tests/*.json
 !apps/ComfyUI-vLLM-Omni/example_workflows/*.json
 # Omni-DuplexEval CI config JSON (e.g. accuracy/omni_duplex_eval)
@@ -250,6 +251,7 @@ datasets/
 *.jsonl
 # Shared Hunyuan IT2I input for accuracy tests and vllm bench.
 !tests/assets/hunyuan_image3/it2i.jsonl
+!tests/assets/livekit/client_vad_replay.jsonl
 *.parquet
 
 # Output files

README.md

@@ -16,6 +16,7 @@ Easy, fast, and cheap omni-modality model serving for everyone
 
 *Latest News* 🔥
 
+- [2026/09] We released [0.30.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.30.0), rebased onto vLLM 0.30.0, featuring a unified full-duplex serving framework around engine-owned sessions for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md) and AURA, native cross-stage KV and multimodal payload transfer (Mooncake AR-to-DiT handoff and NIXL connectors), interactive world-model serving with [LingBot World](recipes/Robbyant/LingBot-World-2.0.md), and realtime [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) Turbo inference on NVIDIA Blackwell and Ascend 950.
 - [2026/08] We released [0.28.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.28.0), featuring production-ready [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) serving on GPU and NPU, a unified AR/DiT paged KV cache runtime, and enhanced realtime full-duplex serving for the [MiniCPM-o series](recipes/OpenBMB/MiniCPM-o-4_5.md).
 - [2026/08] [VeRL-Omni](https://github.com/verl-project/verl-omni) `v0.2.0` is released: faster diffusion RL powered by vLLM-Omni (request-level/step-wise batching with FA3), rebuilt Qwen3-Omni multimodal training (DPO & GSPO), plus LTX-2.3, Qwen-Image-Edit support and more. See the [release notes](https://github.com/verl-project/verl-omni/releases/tag/v0.2.0).
 - [2026/08] We released [0.26.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.26.0) - aligned with the vLLM 0.26 release line, featuring [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) joint video/audio generation, an experimental full-duplex realtime runtime for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md), distributed layerwise diffusion offload, and broader model, hardware, streaming, TTS, and quantization support.
@@ -58,9 +59,9 @@ vLLM-Omni is flexible and easy to use with:
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
 - **Omni-modality models** (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
-- **TTS models** (e.g. Qwen3-TTS, IndexTTS 2.5, CosyVoice3)
-- **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
-- **Robot-policy and action models** (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
+- **TTS models** (e.g. Qwen3-TTS, Tencent AuK, Breeze-TTS-2, CosyVoice3)
+- **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, LingBot World, MAGI-2, LTX-2.5, Wan2.2)
+- **Robot-policy and action models** (e.g. π0.5, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
 
 ## Getting Started
 

apps/ComfyUI-vLLM-Omni/README.md

@@ -271,10 +271,10 @@ The [WF-05 template](example_workflows/vLLM-Omni%20MiniMax-H3%20Latent%20Mask%20
 Connect a **Latent Mask Editing** node to **Generate Video → latent_edit** to edit a source clip instead of generating from scratch. It uploads the source media and serializes the video/audio noise masks the MiniMax H3 API accepts:
 
 - `source_video` / `source_audio` — the media to edit.
-- `video_mask` — a ComfyUI mask image; `0` preserves a region, `1` regenerates it, fractional values blend. A 2D mask `[H, W]` is applied to every frame; a 3D mask `[T, H, W]` is treated as a temporal mask (one slice per frame) for continuation or extension. It is resized to the video latent grid.
+- `video_mask` — a ComfyUI mask image; `0` preserves a region, `1` regenerates it, fractional values blend. A 2D mask `[H, W]` is applied to every frame; a 3D mask `[T, H, W]` is treated as a temporal mask (one slice per frame) for continuation or extension. The client area-downsamples it by 16 to keep the upload small, and the server resizes it to the video latent grid.
 - `audio_mask` — a scalar in `[0, 1]`; `0` keeps the source audio, `1` regenerates it, fractional values blend.
 
-A non-trivial mask requires its matching source, and a source without a mask is rejected. This node only forwards inputs to the server; the served model must declare latent-mask editing support (MiniMax-H3).
+The node requires at least one mask; the server then enforces the cross-field rules (a source requires a matching mask, and a non-trivial mask requires its source). This node only forwards inputs to the server; the served model must declare latent-mask editing support (MiniMax-H3).
 
 ### TTS (e.g., Qwen TTS series)
 

apps/ComfyUI-vLLM-Omni/__init__.py

@@ -36,7 +36,6 @@ from .comfyui_vllm_omni.nodes import (
 
 # A dictionary that contains all nodes you want to export with their names
 NODE_CLASS_MAPPINGS = {
-    "VLLMOmniMiniMaxH3TemporalMask": VLLMOmniMiniMaxH3TemporalMask,
     # === Generation ===
     "VLLMOmniGenerateImage": VLLMOmniGenerateImage,
     "VLLMOmniGenerateVideo": VLLMOmniGenerateVideo,
@@ -46,6 +45,7 @@ NODE_CLASS_MAPPINGS = {
     "VLLMOmniVoiceClone": VLLMOmniVoiceClone,
     "VLLMOmniVideoReferences": VLLMOmniVideoReferences,
     "VLLMOmniLatentMaskEditing": VLLMOmniLatentMaskEditing,
+    "VLLMOmniMiniMaxH3TemporalMask": VLLMOmniMiniMaxH3TemporalMask,
     # === Params ===
     "VLLMOmniARSampling": VLLMOmniARSampling,
     "VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling,
@@ -59,7 +59,6 @@ NODE_CLASS_MAPPINGS = {
 
 # A dictionary that contains the friendly/humanly readable titles for the nodes
 NODE_DISPLAY_NAME_MAPPINGS = {
-    "VLLMOmniMiniMaxH3TemporalMask": "MiniMax-H3 Temporal Mask",
     # === Generation ===
     "VLLMOmniGenerateImage": "Generate Image",
     "VLLMOmniGenerateVideo": "Generate Video",
@@ -69,6 +68,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "VLLMOmniVoiceClone": "TTS Voice Cloning",
     "VLLMOmniVideoReferences": "Video References",
     "VLLMOmniLatentMaskEditing": "Latent Mask Editing",
+    "VLLMOmniMiniMaxH3TemporalMask": "MiniMax-H3 Temporal Mask",
     # === Params ===
     "VLLMOmniARSampling": "AR Sampling Params",
     "VLLMOmniDiffusionSampling": "Diffusion Sampling Params",

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py

@@ -8,7 +8,7 @@ import torch
 from comfy_api.input import AudioInput, VideoInput
 
 from .utils.api_client import VLLMOmniClient
-from .utils.latent_mask import _align_frame_count, _video_latent_t
+from .utils.latent_mask import _align_frame_count
 from .utils.logger import get_logger
 from .utils.models import lookup_model_spec
 from .utils.types import (
@@ -1133,13 +1133,13 @@ class VLLMOmniMiniMaxH3TemporalMask:
         else:
             raise ValueError(f"Unknown temporal mask mode: {mode}")
         available = min(frames, math.floor(boundary * 24 + 1e-8))
+        # Snap the preserved prefix to whole VAE clips (5 + 17n frames) so every
+        # latent it maps to is fully preserved.
         prefix = 0 if available < 5 else 5 + 17 * ((available - 5) // 17)
-        preserved = _video_latent_t(prefix) if prefix else 0
-        total = _video_latent_t(frames)
-        mask = torch.ones(total, 1, 1)
-        mask[:preserved] = 0
+        # One slice per output frame; the server pools frames to the latent grid.
+        mask = torch.ones(frames, 1, 1)
+        mask[:prefix] = 0
         indices = torch.arange(frames, device=images.device)
         source_indices = (indices * (source_fps / 24)).floor().long().clamp(max=images.shape[0] - 1)
         preview_images = images.index_select(0, source_indices)
-        preview_mask = mask.index_select(0, torch.arange(frames) * total // frames)
-        return mask, 24.0, preview_images, preview_mask
+        return mask, 24.0, preview_images, mask

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py

@@ -30,7 +30,7 @@ from .format import (
     video_to_base64,
     video_to_bytes,
 )
-from .latent_mask import scalar_mask_to_json, video_mask_to_grid_json
+from .latent_mask import scalar_mask_to_json, video_mask_to_json
 from .logger import get_logger, pretty_printer
 from .models import lookup_model_spec
 from .types import (
@@ -388,6 +388,15 @@ class VLLMOmniClient:
                 content_type="image/png",
             )
 
+        if len(keyframe_images) == 2:
+            for image_filename, image in keyframe_images:
+                form.add_field(
+                    "input_references",
+                    image_tensor_to_png_bytes(image, image_filename),
+                    filename=image_filename,
+                    content_type="image/png",
+                )
+
         # === latent-mask editing (MiniMax H3) ===
         if latent_edit is not None:
             source_video = latent_edit.get("source_video")
@@ -398,21 +407,16 @@ class VLLMOmniClient:
             if video_mask is None and audio_mask is None:
                 raise ValueError("Latent-mask editing requires at least one mask.")
 
-            video_mask_trivial = video_mask is None or bool((video_mask == 1.0).all().item())
-            audio_mask_trivial = audio_mask is None or audio_mask == 1.0
-            if not video_mask_trivial and source_video is None:
-                raise ValueError("A non-trivial video mask requires a source video.")
-            if not audio_mask_trivial and source_audio is None and source_video is None:
-                raise ValueError("A non-trivial audio mask requires a source audio or a source video with audio.")
-
             if source_video is not None:
                 form.add_field(
                     "source_video",
-                    video_to_bytes(source_video, "source.mp4"),
+                    video_to_bytes(source_video),
                     filename="source.mp4",
                     content_type="video/mp4",
                 )
             if source_audio is not None:
+                # The filename extension selects the audio codec (mp3), so it is
+                # meaningful even though aiohttp uses the explicit filename below.
                 form.add_field(
                     "source_audio",
                     audio_to_bytes(source_audio, "source_audio.mp3"),
@@ -420,7 +424,7 @@ class VLLMOmniClient:
                     content_type="audio/mpeg",
                 )
             if video_mask is not None:
-                mask_json = video_mask_to_grid_json(video_mask, width=width, height=height, num_frames=num_frames)
+                mask_json = video_mask_to_json(video_mask)
                 form.add_field(
                     "video_noise_mask",
                     mask_json.encode("utf-8"),
@@ -435,15 +439,6 @@ class VLLMOmniClient:
                     content_type="application/json",
                 )
 
-        if len(keyframe_images) == 2:
-            for image_filename, image in keyframe_images:
-                form.add_field(
-                    "input_references",
-                    image_tensor_to_png_bytes(image, image_filename),
-                    filename=image_filename,
-                    content_type="image/png",
-                )
-
         # === model specific params. Either use a specialized builder, or add flattened fields as-is ===
         if model_params is not None:
             model_params = dict(model_params)

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py

@@ -1,11 +1,25 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
+"""Client-side serialization of MiniMax-H3 latent-edit masks.
+
+A video mask is sent in frame space — a 2D spatial ``[H, W]`` (applied to every
+frame) or a 3D ``[T, H, W]`` (one slice per frame) — and the server resolves it
+to the latent grid. An audio mask is a scalar applied to all time steps.
+"""
+
 import json
+import math
 
 import torch
 import torch.nn.functional as F
 
+# The server rejects mask JSON above 8 MiB, so the spatial axes are
+# area-downsampled by the VAE spatial stride before upload. This depends only on
+# the stride, not on the H3 shape lattice, which the server still owns.
+_VAE_SPATIAL_STRIDE = 16
+_MASK_DECIMALS = 4
+
 
 def scalar_mask_to_json(value: float) -> str:
     if not 0.0 <= value <= 1.0:
@@ -13,6 +27,27 @@ def scalar_mask_to_json(value: float) -> str:
     return str(value)
 
 
+def video_mask_to_json(mask: torch.Tensor) -> str:
+    """Serialize a frame-space video mask to JSON for the ``video_noise_mask`` field.
+
+    ``mask`` is a 2D spatial mask or a 3D frame-space mask. Its spatial axes are
+    area-downsampled by the VAE stride; the server resizes the result to the
+    latent grid, so the client neither floors the canvas nor maps frames to
+    latents.
+    """
+    if mask.ndim not in (2, 3):
+        raise ValueError(f"expected a 2D or 3D mask tensor, got {mask.ndim}D")
+    frames = mask.unsqueeze(0) if mask.ndim == 2 else mask
+    size = tuple(max(1, math.ceil(dim / _VAE_SPATIAL_STRIDE)) for dim in frames.shape[1:])
+    frames = F.interpolate(frames.unsqueeze(1).float(), size=size, mode="area").squeeze(1)
+    if mask.ndim == 2:
+        frames = frames.squeeze(0)
+    # Round in float64 so tolist() emits short decimals rather than float32 noise.
+    return json.dumps(frames.double().round(decimals=_MASK_DECIMALS).tolist(), separators=(",", ":"))
+
+
+# The temporal-mask node snaps its preserve boundary to the H3 frame lattice;
+# this mirrors the server shape planner.
 def _align_frame_count(frame_count: int) -> int:
     if frame_count <= 0:
         return 1
@@ -20,32 +55,3 @@ def _align_frame_count(frame_count: int) -> int:
     while current % 17 != 5:
         current += 1
     return current
-
-
-def _video_latent_t(frame_count: int) -> int:
-    if frame_count <= 5:
-        return 2
-    return ((int(frame_count) - 5) // 17) * 5 + 2
-
-
-def video_mask_to_grid(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> torch.Tensor:
-    tv = _video_latent_t(_align_frame_count(num_frames))
-    height = int(height) // 32 * 32
-    width = int(width) // 32 * 32
-    gh, gw = height // 16, width // 16
-
-    if mask.ndim == 2:
-        mask = mask.unsqueeze(0)
-    elif mask.ndim != 3:
-        raise ValueError(f"expected a 2D or 3D mask tensor, got {mask.ndim}D")
-
-    grid = F.interpolate(mask.unsqueeze(1).float(), size=(gh, gw), mode="area").squeeze(1)
-    if grid.shape[0] == 1:
-        grid = grid.expand(tv, gh, gw)
-    elif grid.shape[0] != tv:
-        grid = F.interpolate(grid.unsqueeze(0).unsqueeze(0), size=(tv, gh, gw), mode="nearest").squeeze(0).squeeze(0)
-    return grid
-
-
-def video_mask_to_grid_json(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> str:
-    return json.dumps(video_mask_to_grid(mask, width=width, height=height, num_frames=num_frames).tolist())

apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md

@@ -12,6 +12,6 @@ Edit a source video with MiniMax-H3 using spatial or temporal masks. The workflo
 Open **vLLM-Omni MiniMax-H3 Latent Mask Editing.json** from the example workflows and set the Generate Video URL/model for the case you want to run.
 
 1. Upload your source video in Load Video.
-2. Adjust the case's mask. For removal and inpainting, set Mask Canvas to the video dimensions, Mask Rectangle to the region size, and Combine Masks x/y to its position. For continuation and extension, set MiniMax-H3 Temporal Mask's mode and duration; continuation also uses `preserve_fraction`. Match the duration in Generate Video, and use a duration longer than the source for extension. Mask values are `0` to preserve and `1` to regenerate.
+2. Adjust the case's mask. For removal and inpainting, set Mask Canvas to the video dimensions, Mask Rectangle to the region size, and Combine Masks x/y to its position, or replace that chain with Load Image (as Mask) for an arbitrary-shape mask. For continuation and extension, set MiniMax-H3 Temporal Mask's mode and duration; continuation also uses `preserve_fraction`. Match the duration in Generate Video, and use a duration longer than the source for extension. Mask values are `0` to preserve and `1` to regenerate.
 3. Adjust the prompt to describe the desired content. Set the output dimensions and duration in Generate Video. If editing audio, update the sound description and audio mask (`0` preserves, `1` regenerates the whole clip).
 4. Select the case's Mask Preview Save Video node and choose **Execute to selected output nodes** to preview the mask as a red overlay without inference, or select its Result Save Video node to generate the edited video. Running the entire workflow generates all four cases.

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json

@@ -1,499 +0,0 @@
-{
-  "id": "latent-mask-editing-7380",
-  "revision": 0,
-  "last_node_id": 7,
-  "last_link_id": 5,
-  "nodes": [
-    {
-      "id": 1,
-      "type": "MarkdownNote",
-      "pos": [
-        40,
-        20
-      ],
-      "size": [
-        420,
-        210
-      ],
-      "flags": {},
-      "order": 0,
-      "mode": 0,
-      "inputs": [],
-      "outputs": [],
-      "title": "Latent-mask editing server prerequisite",
-      "properties": {},
-      "widgets_values": [
-        "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends."
-      ],
-      "color": "#432",
-      "bgcolor": "#000"
-    },
-    {
-      "id": 2,
-      "type": "LoadVideo",
-      "pos": [
-        40,
-        280
-      ],
-      "size": [
-        420,
-        120
-      ],
-      "flags": {},
-      "order": 1,
-      "mode": 0,
-      "inputs": [
-        {
-          "localized_name": "file",
-          "name": "file",
-          "type": "COMBO",
-          "widget": {
-            "name": "file"
-          },
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "localized_name": "video",
-          "name": "video",
-          "type": "VIDEO",
-          "links": [
-            1
-          ]
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "LoadVideo"
-      },
-      "widgets_values": [
-        ""
-      ]
-    },
-    {
-      "id": 3,
-      "type": "LoadImageMask",
-      "pos": [
-        40,
-        460
-      ],
-      "size": [
-        420,
-        160
-      ],
-      "flags": {},
-      "order": 2,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "image",
-          "type": "STRING",
-          "widget": {
-            "name": "image"
-          },
-          "link": null
-        },
-        {
-          "name": "channel",
-          "type": "STRING",
-          "widget": {
-            "name": "channel"
-          },
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "localized_name": "mask",
-          "name": "mask",
-          "type": "MASK",
-          "links": [
-            2
-          ]
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "LoadImageMask"
-      },
-      "widgets_values": [
-        "",
-        "red"
-      ]
-    },
-    {
-      "id": 4,
-      "type": "VLLMOmniLatentMaskEditing",
-      "pos": [
-        560,
-        260
-      ],
-      "size": [
-        420,
-        300
-      ],
-      "flags": {},
-      "order": 3,
-      "mode": 0,
-      "inputs": [
-        {
-          "localized_name": "source_video",
-          "name": "source_video",
-          "type": "VIDEO",
-          "link": 1,
-          "shape": 7
-        },
-        {
-          "localized_name": "source_audio",
-          "name": "source_audio",
-          "type": "AUDIO",
-          "link": null
-        },
-        {
-          "localized_name": "video_mask",
-          "name": "video_mask",
-          "type": "MASK",
-          "link": 2,
-          "shape": 7
-        },
-        {
-          "localized_name": "audio_mask",
-          "name": "audio_mask",
-          "type": "FLOAT",
-          "widget": {
-            "name": "audio_mask"
-          },
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "localized_name": "latent_edit",
-          "name": "latent_edit",
-          "type": "LATENT_MASK_EDITING",
-          "links": [
-            3
-          ]
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "VLLMOmniLatentMaskEditing"
-      },
-      "widgets_values": [
-        0.5
-      ]
-    },
-    {
-      "id": 5,
-      "type": "VLLMOmniMiniMaxH3Params",
-      "pos": [
-        40,
-        680
-      ],
-      "size": [
-        420,
-        140
-      ],
-      "flags": {},
-      "order": 4,
-      "mode": 0,
-      "inputs": [
-        {
-          "localized_name": "audio_flow_shift",
-          "name": "audio_flow_shift",
-          "type": "FLOAT",
-          "widget": {
-            "name": "audio_flow_shift"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "flow_shift",
-          "name": "flow_shift",
-          "type": "FLOAT",
-          "widget": {
-            "name": "flow_shift"
-          },
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "localized_name": "video params",
-          "name": "video params",
-          "type": "VIDEO_PARAMS",
-          "links": [
-            4
-          ]
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
-      },
-      "widgets_values": [
-        3.0,
-        12.0
-      ]
-    },
-    {
-      "id": 6,
-      "type": "VLLMOmniGenerateVideo",
-      "pos": [
-        1080,
-        200
-      ],
-      "size": [
-        540,
-        680
-      ],
-      "flags": {},
-      "order": 5,
-      "mode": 0,
-      "inputs": [
-        {
-          "localized_name": "frame",
-          "name": "frame",
-          "type": "IMAGE",
-          "link": null,
-          "shape": 7
-        },
-        {
-          "localized_name": "references",
-          "name": "references",
-          "type": "VIDEO_REFERENCES",
-          "link": null,
-          "shape": 7
-        },
-        {
-          "localized_name": "sampling_params",
-          "name": "sampling_params",
-          "type": "SAMPLING_PARAMS",
-          "link": null
-        },
-        {
-          "localized_name": "lora",
-          "name": "lora",
-          "type": "REMOTE_LORA",
-          "link": null
-        },
-        {
-          "localized_name": "model_params",
-          "name": "model_params",
-          "type": "VIDEO_PARAMS",
-          "link": 4,
-          "shape": 7
-        },
-        {
-          "localized_name": "fast_h3",
-          "name": "fast_h3",
-          "type": "FASTH3_DEPLOYMENT",
-          "link": null
-        },
-        {
-          "localized_name": "latent_edit",
-          "name": "latent_edit",
-          "type": "LATENT_MASK_EDITING",
-          "link": 3,
-          "shape": 7
-        },
-        {
-          "localized_name": "url",
-          "name": "url",
-          "type": "STRING",
-          "widget": {
-            "name": "url"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "model",
-          "name": "model",
-          "type": "STRING",
-          "widget": {
-            "name": "model"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "prompt",
-          "name": "prompt",
-          "type": "STRING",
-          "widget": {
-            "name": "prompt"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "negative_prompt",
-          "name": "negative_prompt",
-          "type": "STRING",
-          "widget": {
-            "name": "negative_prompt"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "width",
-          "name": "width",
-          "type": "INT",
-          "widget": {
-            "name": "width"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "height",
-          "name": "height",
-          "type": "INT",
-          "widget": {
-            "name": "height"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "fps",
-          "name": "fps",
-          "type": "INT",
-          "widget": {
-            "name": "fps"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "duration",
-          "name": "duration",
-          "type": "FLOAT",
-          "widget": {
-            "name": "duration"
-          },
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "localized_name": "video",
-          "name": "video",
-          "type": "VIDEO",
-          "links": [
-            5
-          ]
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "VLLMOmniGenerateVideo"
-      },
-      "widgets_values": [
-        "http://127.0.0.1:8000/v1",
-        "MiniMaxAI/MiniMax-H3",
-        "Partially restyle the clip and soundtrack",
-        "",
-        160,
-        120,
-        24,
-        4.458
-      ]
-    },
-    {
-      "id": 7,
-      "type": "SaveVideo",
-      "pos": [
-        1700,
-        300
-      ],
-      "size": [
-        420,
-        160
-      ],
-      "flags": {},
-      "order": 6,
-      "mode": 0,
-      "inputs": [
-        {
-          "localized_name": "video",
-          "name": "video",
-          "type": "VIDEO",
-          "link": 5,
-          "shape": 7
-        },
-        {
-          "localized_name": "filename_prefix",
-          "name": "filename_prefix",
-          "type": "STRING",
-          "widget": {
-            "name": "filename_prefix"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "format",
-          "name": "format",
-          "type": "COMFY_DYNAMICCOMBO_V3",
-          "widget": {
-            "name": "format"
-          },
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "localized_name": "video",
-          "name": "video",
-          "type": "VIDEO",
-          "links": []
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "SaveVideo"
-      },
-      "widgets_values": [
-        "vllm-omni/latent-edit",
-        "mp4"
-      ]
-    }
-  ],
-  "links": [
-    [
-      1,
-      2,
-      0,
-      4,
-      0,
-      "VIDEO"
-    ],
-    [
-      2,
-      3,
-      0,
-      4,
-      2,
-      "MASK"
-    ],
-    [
-      3,
-      4,
-      0,
-      6,
-      6,
-      "LATENT_MASK_EDITING"
-    ],
-    [
-      4,
-      5,
-      0,
-      6,
-      4,
-      "VIDEO_PARAMS"
-    ],
-    [
-      5,
-      6,
-      0,
-      7,
-      0,
-      "VIDEO"
-    ]
-  ],
-  "groups": [],
-  "config": {},
-  "extra": {},
-  "version": 0.4
-}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json

@@ -1,560 +0,0 @@
-{
-  "last_node_id": 9,
-  "last_link_id": 7,
-  "nodes": [
-    {
-      "id": 2,
-      "type": "LoadVideo",
-      "pos": [
-        40,
-        100
-      ],
-      "size": [
-        420,
-        120
-      ],
-      "flags": {},
-      "order": 1,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "file",
-          "type": "STRING",
-          "widget": {
-            "name": "file"
-          },
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "name": "video",
-          "type": "VIDEO",
-          "links": [
-            1
-          ]
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "LoadVideo"
-      },
-      "widgets_values": [
-        ""
-      ]
-    },
-    {
-      "id": 3,
-      "type": "SolidMask",
-      "pos": [
-        40,
-        300
-      ],
-      "size": [
-        420,
-        160
-      ],
-      "flags": {},
-      "order": 2,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "value",
-          "type": "FLOAT",
-          "widget": {
-            "name": "value"
-          },
-          "link": null
-        },
-        {
-          "name": "width",
-          "type": "INT",
-          "widget": {
-            "name": "width"
-          },
-          "link": null
-        },
-        {
-          "name": "height",
-          "type": "INT",
-          "widget": {
-            "name": "height"
-          },
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "name": "mask",
-          "type": "MASK",
-          "links": [
-            2
-          ]
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "SolidMask"
-      },
-      "widgets_values": [
-        0.0,
-        160,
-        120
-      ]
-    },
-    {
-      "id": 4,
-      "type": "SolidMask",
-      "pos": [
-        40,
-        500
-      ],
-      "size": [
-        420,
-        160
-      ],
-      "flags": {},
-      "order": 3,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "value",
-          "type": "FLOAT",
-          "widget": {
-            "name": "value"
-          },
-          "link": null
-        },
-        {
-          "name": "width",
-          "type": "INT",
-          "widget": {
-            "name": "width"
-          },
-          "link": null
-        },
-        {
-          "name": "height",
-          "type": "INT",
-          "widget": {
-            "name": "height"
-          },
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "name": "mask",
-          "type": "MASK",
-          "links": [
-            3
-          ]
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "SolidMask"
-      },
-      "widgets_values": [
-        1.0,
-        160,
-        120
-      ]
-    },
-    {
-      "id": 5,
-      "type": "BatchMasksNode",
-      "pos": [
-        250,
-        380
-      ],
-      "size": [
-        200,
-        100
-      ],
-      "flags": {},
-      "order": 4,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "masks.mask0",
-          "type": "MASK",
-          "link": 2,
-          "slot_index": 0
-        },
-        {
-          "name": "masks.mask1",
-          "type": "MASK",
-          "link": 3,
-          "slot_index": 1
-        }
-      ],
-      "outputs": [
-        {
-          "name": "MASK",
-          "type": "MASK",
-          "links": [
-            4
-          ]
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "BatchMasksNode"
-      },
-      "widgets_values": []
-    },
-    {
-      "id": 6,
-      "type": "VLLMOmniLatentMaskEditing",
-      "pos": [
-        450,
-        180
-      ],
-      "size": [
-        400,
-        120
-      ],
-      "flags": {},
-      "order": 5,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "source_video",
-          "type": "VIDEO",
-          "link": 1
-        },
-        {
-          "name": "source_audio",
-          "type": "AUDIO",
-          "link": null
-        },
-        {
-          "name": "video_mask",
-          "type": "MASK",
-          "link": 4
-        },
-        {
-          "name": "audio_mask",
-          "type": "FLOAT",
-          "widget": {
-            "name": "audio_mask"
-          },
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "name": "latent_edit",
-          "type": "LATENT_MASK_EDITING",
-          "links": [
-            5
-          ]
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "VLLMOmniLatentMaskEditing"
-      },
-      "widgets_values": [
-        -1.0
-      ]
-    },
-    {
-      "id": 7,
-      "type": "VLLMOmniMiniMaxH3Params",
-      "pos": [
-        40,
-        700
-      ],
-      "size": [
-        420,
-        160
-      ],
-      "flags": {},
-      "order": 6,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "audio_flow_shift",
-          "type": "FLOAT",
-          "widget": {
-            "name": "audio_flow_shift"
-          },
-          "link": null
-        },
-        {
-          "name": "flow_shift",
-          "type": "FLOAT",
-          "widget": {
-            "name": "flow_shift"
-          },
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "name": "video params",
-          "type": "VIDEO_PARAMS",
-          "links": [
-            6
-          ]
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
-      },
-      "widgets_values": [
-        3.0,
-        12.0
-      ]
-    },
-    {
-      "id": 8,
-      "type": "VLLMOmniGenerateVideo",
-      "pos": [
-        700,
-        180
-      ],
-      "size": [
-        450,
-        380
-      ],
-      "flags": {},
-      "order": 7,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "frame",
-          "type": "IMAGE",
-          "link": null
-        },
-        {
-          "name": "references",
-          "type": "VIDEO_REFERENCES",
-          "link": null
-        },
-        {
-          "name": "sampling_params",
-          "type": "SAMPLING_PARAMS",
-          "link": null
-        },
-        {
-          "name": "lora",
-          "type": "LORA",
-          "link": null
-        },
-        {
-          "name": "model_params",
-          "type": "VIDEO_PARAMS",
-          "link": 6
-        },
-        {
-          "name": "fast_h3",
-          "type": "FAST_H3",
-          "link": null
-        },
-        {
-          "name": "latent_edit",
-          "type": "LATENT_MASK_EDITING",
-          "link": 5
-        },
-        {
-          "name": "url",
-          "type": "STRING",
-          "widget": {
-            "name": "url"
-          },
-          "link": null
-        },
-        {
-          "name": "model",
-          "type": "STRING",
-          "widget": {
-            "name": "model"
-          },
-          "link": null
-        },
-        {
-          "name": "prompt",
-          "type": "STRING",
-          "widget": {
-            "name": "prompt"
-          },
-          "link": null
-        },
-        {
-          "name": "negative_prompt",
-          "type": "STRING",
-          "widget": {
-            "name": "negative_prompt"
-          },
-          "link": null
-        },
-        {
-          "name": "width",
-          "type": "INT",
-          "widget": {
-            "name": "width"
-          },
-          "link": null
-        },
-        {
-          "name": "height",
-          "type": "INT",
-          "widget": {
-            "name": "height"
-          },
-          "link": null
-        },
-        {
-          "name": "fps",
-          "type": "INT",
-          "widget": {
-            "name": "fps"
-          },
-          "link": null
-        },
-        {
-          "name": "duration",
-          "type": "FLOAT",
-          "widget": {
-            "name": "duration"
-          },
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "name": "video",
-          "type": "VIDEO",
-          "links": [
-            7
-          ]
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "VLLMOmniGenerateVideo"
-      },
-      "widgets_values": [
-        "http://127.0.0.1:8000/v1",
-        "MiniMaxAI/MiniMax-H3",
-        "temporal mask: regenerate the ending",
-        "",
-        160,
-        120,
-        24,
-        4.458
-      ]
-    },
-    {
-      "id": 9,
-      "type": "SaveVideo",
-      "pos": [
-        900,
-        180
-      ],
-      "size": [
-        420,
-        200
-      ],
-      "flags": {},
-      "order": 8,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "video",
-          "type": "VIDEO",
-          "link": 7
-        },
-        {
-          "name": "filename_prefix",
-          "type": "STRING",
-          "widget": {
-            "name": "filename_prefix"
-          },
-          "link": null
-        },
-        {
-          "name": "format",
-          "type": "STRING",
-          "widget": {
-            "name": "format"
-          },
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "name": "video",
-          "type": "VIDEO",
-          "links": []
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "SaveVideo"
-      },
-      "widgets_values": [
-        "temporal_mask",
-        "mp4"
-      ]
-    }
-  ],
-  "links": [
-    [
-      1,
-      2,
-      0,
-      6,
-      0,
-      "VIDEO"
-    ],
-    [
-      2,
-      3,
-      0,
-      5,
-      0,
-      "MASK"
-    ],
-    [
-      3,
-      4,
-      0,
-      5,
-      1,
-      "MASK"
-    ],
-    [
-      4,
-      5,
-      0,
-      6,
-      2,
-      "MASK"
-    ],
-    [
-      5,
-      6,
-      0,
-      8,
-      6,
-      "LATENT_MASK_EDITING"
-    ],
-    [
-      6,
-      7,
-      0,
-      8,
-      4,
-      "VIDEO_PARAMS"
-    ],
-    [
-      7,
-      8,
-      0,
-      9,
-      0,
-      "VIDEO"
-    ]
-  ],
-  "groups": [],
-  "config": {},
-  "extra": {},
-  "version": 0.4
-}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json

@@ -1,511 +0,0 @@
-{
-  "id": "latent-mask-editing-7380",
-  "revision": 0,
-  "last_node_id": 7,
-  "last_link_id": 5,
-  "nodes": [
-    {
-      "id": 1,
-      "type": "MarkdownNote",
-      "pos": [
-        40,
-        20
-      ],
-      "size": [
-        420,
-        210
-      ],
-      "flags": {},
-      "order": 0,
-      "mode": 0,
-      "inputs": [],
-      "outputs": [],
-      "title": "Latent-mask editing server prerequisite",
-      "properties": {},
-      "widgets_values": [
-        "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends."
-      ],
-      "color": "#432",
-      "bgcolor": "#000"
-    },
-    {
-      "id": 2,
-      "type": "LoadVideo",
-      "pos": [
-        40,
-        280
-      ],
-      "size": [
-        420,
-        120
-      ],
-      "flags": {},
-      "order": 1,
-      "mode": 0,
-      "inputs": [
-        {
-          "localized_name": "file",
-          "name": "file",
-          "type": "COMBO",
-          "widget": {
-            "name": "file"
-          },
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "localized_name": "video",
-          "name": "video",
-          "type": "VIDEO",
-          "links": [
-            1
-          ]
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "LoadVideo"
-      },
-      "widgets_values": [
-        ""
-      ]
-    },
-    {
-      "id": 3,
-      "type": "SolidMask",
-      "pos": [
-        40,
-        460
-      ],
-      "size": [
-        420,
-        160
-      ],
-      "flags": {},
-      "order": 2,
-      "mode": 0,
-      "inputs": [
-        {
-          "localized_name": "value",
-          "name": "value",
-          "type": "FLOAT",
-          "widget": {
-            "name": "value"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "width",
-          "name": "width",
-          "type": "INT",
-          "widget": {
-            "name": "width"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "height",
-          "name": "height",
-          "type": "INT",
-          "widget": {
-            "name": "height"
-          },
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "localized_name": "mask",
-          "name": "mask",
-          "type": "MASK",
-          "links": [
-            2
-          ]
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "SolidMask"
-      },
-      "widgets_values": [
-        1.0,
-        160,
-        120
-      ]
-    },
-    {
-      "id": 4,
-      "type": "VLLMOmniLatentMaskEditing",
-      "pos": [
-        560,
-        260
-      ],
-      "size": [
-        420,
-        300
-      ],
-      "flags": {},
-      "order": 3,
-      "mode": 0,
-      "inputs": [
-        {
-          "localized_name": "source_video",
-          "name": "source_video",
-          "type": "VIDEO",
-          "link": 1,
-          "shape": 7
-        },
-        {
-          "localized_name": "source_audio",
-          "name": "source_audio",
-          "type": "AUDIO",
-          "link": null
-        },
-        {
-          "localized_name": "video_mask",
-          "name": "video_mask",
-          "type": "MASK",
-          "link": 2,
-          "shape": 7
-        },
-        {
-          "localized_name": "audio_mask",
-          "name": "audio_mask",
-          "type": "FLOAT",
-          "widget": {
-            "name": "audio_mask"
-          },
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "localized_name": "latent_edit",
-          "name": "latent_edit",
-          "type": "LATENT_MASK_EDITING",
-          "links": [
-            3
-          ]
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "VLLMOmniLatentMaskEditing"
-      },
-      "widgets_values": [
-        0.5
-      ]
-    },
-    {
-      "id": 5,
-      "type": "VLLMOmniMiniMaxH3Params",
-      "pos": [
-        40,
-        680
-      ],
-      "size": [
-        420,
-        140
-      ],
-      "flags": {},
-      "order": 4,
-      "mode": 0,
-      "inputs": [
-        {
-          "localized_name": "audio_flow_shift",
-          "name": "audio_flow_shift",
-          "type": "FLOAT",
-          "widget": {
-            "name": "audio_flow_shift"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "flow_shift",
-          "name": "flow_shift",
-          "type": "FLOAT",
-          "widget": {
-            "name": "flow_shift"
-          },
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "localized_name": "video params",
-          "name": "video params",
-          "type": "VIDEO_PARAMS",
-          "links": [
-            4
-          ]
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
-      },
-      "widgets_values": [
-        3.0,
-        12.0
-      ]
-    },
-    {
-      "id": 6,
-      "type": "VLLMOmniGenerateVideo",
-      "pos": [
-        1080,
-        200
-      ],
-      "size": [
-        540,
-        680
-      ],
-      "flags": {},
-      "order": 5,
-      "mode": 0,
-      "inputs": [
-        {
-          "localized_name": "frame",
-          "name": "frame",
-          "type": "IMAGE",
-          "link": null,
-          "shape": 7
-        },
-        {
-          "localized_name": "references",
-          "name": "references",
-          "type": "VIDEO_REFERENCES",
-          "link": null,
-          "shape": 7
-        },
-        {
-          "localized_name": "sampling_params",
-          "name": "sampling_params",
-          "type": "SAMPLING_PARAMS",
-          "link": null
-        },
-        {
-          "localized_name": "lora",
-          "name": "lora",
-          "type": "REMOTE_LORA",
-          "link": null
-        },
-        {
-          "localized_name": "model_params",
-          "name": "model_params",
-          "type": "VIDEO_PARAMS",
-          "link": 4,
-          "shape": 7
-        },
-        {
-          "localized_name": "fast_h3",
-          "name": "fast_h3",
-          "type": "FASTH3_DEPLOYMENT",
-          "link": null
-        },
-        {
-          "localized_name": "latent_edit",
-          "name": "latent_edit",
-          "type": "LATENT_MASK_EDITING",
-          "link": 3,
-          "shape": 7
-        },
-        {
-          "localized_name": "url",
-          "name": "url",
-          "type": "STRING",
-          "widget": {
-            "name": "url"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "model",
-          "name": "model",
-          "type": "STRING",
-          "widget": {
-            "name": "model"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "prompt",
-          "name": "prompt",
-          "type": "STRING",
-          "widget": {
-            "name": "prompt"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "negative_prompt",
-          "name": "negative_prompt",
-          "type": "STRING",
-          "widget": {
-            "name": "negative_prompt"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "width",
-          "name": "width",
-          "type": "INT",
-          "widget": {
-            "name": "width"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "height",
-          "name": "height",
-          "type": "INT",
-          "widget": {
-            "name": "height"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "fps",
-          "name": "fps",
-          "type": "INT",
-          "widget": {
-            "name": "fps"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "duration",
-          "name": "duration",
-          "type": "FLOAT",
-          "widget": {
-            "name": "duration"
-          },
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "localized_name": "video",
-          "name": "video",
-          "type": "VIDEO",
-          "links": [
-            5
-          ]
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "VLLMOmniGenerateVideo"
-      },
-      "widgets_values": [
-        "http://127.0.0.1:8000/v1",
-        "MiniMaxAI/MiniMax-H3",
-        "Partially restyle the clip and soundtrack",
-        "",
-        160,
-        120,
-        24,
-        4.458
-      ]
-    },
-    {
-      "id": 7,
-      "type": "SaveVideo",
-      "pos": [
-        1700,
-        300
-      ],
-      "size": [
-        420,
-        160
-      ],
-      "flags": {},
-      "order": 6,
-      "mode": 0,
-      "inputs": [
-        {
-          "localized_name": "video",
-          "name": "video",
-          "type": "VIDEO",
-          "link": 5,
-          "shape": 7
-        },
-        {
-          "localized_name": "filename_prefix",
-          "name": "filename_prefix",
-          "type": "STRING",
-          "widget": {
-            "name": "filename_prefix"
-          },
-          "link": null
-        },
-        {
-          "localized_name": "format",
-          "name": "format",
-          "type": "COMFY_DYNAMICCOMBO_V3",
-          "widget": {
-            "name": "format"
-          },
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "localized_name": "video",
-          "name": "video",
-          "type": "VIDEO",
-          "links": []
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "SaveVideo"
-      },
-      "widgets_values": [
-        "vllm-omni/latent-edit",
-        "mp4"
-      ]
-    }
-  ],
-  "links": [
-    [
-      1,
-      2,
-      0,
-      4,
-      0,
-      "VIDEO"
-    ],
-    [
-      2,
-      3,
-      0,
-      4,
-      2,
-      "MASK"
-    ],
-    [
-      3,
-      4,
-      0,
-      6,
-      6,
-      "LATENT_MASK_EDITING"
-    ],
-    [
-      4,
-      5,
-      0,
-      6,
-      4,
-      "VIDEO_PARAMS"
-    ],
-    [
-      5,
-      6,
-      0,
-      7,
-      0,
-      "VIDEO"
-    ]
-  ],
-  "groups": [],
-  "config": {},
-  "extra": {},
-  "version": 0.4
-}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json

@@ -2293,10 +2293,10 @@
       "title": "Usage Guide",
       "properties": {},
       "widgets_values": [
-        "Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference."
+        "Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. For an arbitrary-shape spatial mask, replace a case's Mask Rectangle chain with Load Image (as Mask). Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference."
       ],
       "widgets_values_named": {
-        "text": "Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference."
+        "text": "Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. For an arbitrary-shape spatial mask, replace a case's Mask Rectangle chain with Load Image (as Mask). Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference."
       },
       "color": "#432",
       "bgcolor": "#000"
@@ -2823,7 +2823,7 @@
         "Node name for S&R": "VLLMOmniGenerateVideo"
       },
       "widgets_values": [
-        "http://127.0.0.1:8001/v1",
+        "http://127.0.0.1:8000/v1",
         "MiniMaxAI/MiniMax-H3",
         "integrated_multimodal_description: The masked area is a flat, seamless continuation of the surrounding wall, matching its color, texture, and lighting. The wall surface remains consistent throughout the video.\n\noverall_soundscape: \n\nnon_diegetic_music: ",
         "",
@@ -2833,7 +2833,7 @@
         5
       ],
       "widgets_values_named": {
-        "url": "http://127.0.0.1:8001/v1",
+        "url": "http://127.0.0.1:8000/v1",
         "model": "MiniMaxAI/MiniMax-H3",
         "prompt": "integrated_multimodal_description: The masked area is a flat, seamless continuation of the surrounding wall, matching its color, texture, and lighting. The wall surface remains consistent throughout the video.\n\noverall_soundscape: \n\nnon_diegetic_music: ",
         "negative_prompt": "",
@@ -3677,7 +3677,7 @@
         "Node name for S&R": "VLLMOmniGenerateVideo"
       },
       "widgets_values": [
-        "http://127.0.0.1:8001/v1",
+        "http://127.0.0.1:8000/v1",
         "MiniMaxAI/MiniMax-H3",
         "integrated_multimodal_description: A small orange cat rests curled up in the masked area, gently swaying its tail from side to side. Its appearance matches the surrounding anime style, lighting, and shadows.\n\noverall_soundscape:\n\nnon_diegetic_music: ",
         "",
@@ -3687,7 +3687,7 @@
         5
       ],
       "widgets_values_named": {
-        "url": "http://127.0.0.1:8001/v1",
+        "url": "http://127.0.0.1:8000/v1",
         "model": "MiniMaxAI/MiniMax-H3",
         "prompt": "integrated_multimodal_description: A small orange cat rests curled up in the masked area, gently swaying its tail from side to side. Its appearance matches the surrounding anime style, lighting, and shadows.\n\noverall_soundscape:\n\nnon_diegetic_music: ",
         "negative_prompt": "",
@@ -3916,7 +3916,7 @@
         "Node name for S&R": "VLLMOmniGenerateVideo"
       },
       "widgets_values": [
-        "http://127.0.0.1:8001/v1",
+        "http://127.0.0.1:8000/v1",
         "MiniMaxAI/MiniMax-H3",
         "integrated_multimodal_description: The sky turns completely overcast, with the sun hidden and no direct sunlight, as torrential rain pours over the school entrance in dense visible streaks and splashes on the wet pavement. Distant lightning flashes in the sky behind the school buildings.\n\noverall_soundscape: Loud, steady rainfall, splashing water, and gusts of wind.\n\nnon_diegetic_music: None.",
         "",
@@ -3926,7 +3926,7 @@
         5
       ],
       "widgets_values_named": {
-        "url": "http://127.0.0.1:8001/v1",
+        "url": "http://127.0.0.1:8000/v1",
         "model": "MiniMaxAI/MiniMax-H3",
         "prompt": "integrated_multimodal_description: The sky turns completely overcast, with the sun hidden and no direct sunlight, as torrential rain pours over the school entrance in dense visible streaks and splashes on the wet pavement. Distant lightning flashes in the sky behind the school buildings.\n\noverall_soundscape: Loud, steady rainfall, splashing water, and gusts of wind.\n\nnon_diegetic_music: None.",
         "negative_prompt": "",
@@ -4363,7 +4363,7 @@
         "Node name for S&R": "VLLMOmniGenerateVideo"
       },
       "widgets_values": [
-        "http://127.0.0.1:8001/v1",
+        "http://127.0.0.1:8000/v1",
         "MiniMaxAI/MiniMax-H3",
         "integrated_multimodal_description: Cut to the same school on a summer night beneath a deep blue sky dotted with stars. A firework rises from behind the school buildings, leaving a glowing trail; the camera slowly tilts upward to follow it into the sky, ending as it bursts into a large bloom of colorful sparks.\n\noverall_soundscape: Soft summer insect sounds, followed by the firework rising with a whistle and bursting with a distant boom and crackling sparks.\n\nnon_diegetic_music: N/A",
         "",
@@ -4373,7 +4373,7 @@
         10
       ],
       "widgets_values_named": {
-        "url": "http://127.0.0.1:8001/v1",
+        "url": "http://127.0.0.1:8000/v1",
         "model": "MiniMaxAI/MiniMax-H3",
         "prompt": "integrated_multimodal_description: Cut to the same school on a summer night beneath a deep blue sky dotted with stars. A firework rises from behind the school buildings, leaving a glowing trail; the camera slowly tilts upward to follow it into the sky, ending as it bursts into a large bloom of colorful sparks.\n\noverall_soundscape: Soft summer insect sounds, followed by the firework rising with a whistle and bursting with a distant boom and crackling sparks.\n\nnon_diegetic_music: N/A",
         "negative_prompt": "",

benchmarks/README.md

@@ -66,6 +66,13 @@ Accuracy benchmarks for image generation/editing models, adapting external suite
 - **Layout**: `accuracy/text_to_image/` (GEBench), `accuracy/image_to_image/` (GEdit-Bench)
 - **Method**: generation and judge scoring both run through local `vllm-omni serve` endpoints
 
+### [MammothModa2](mammoth_moda2/README.md) — Startup and Weight Loading
+
+Startup / model-loading benchmark for the two-stage MammothModa2 (AR → DiT) deployment, plus a raw safetensors loading micro-benchmark.
+
+- **Layout**: `mammoth_moda2/bench_startup.py` (single-process startup and first/steady request timing), `mammoth_moda2/parse_startup_log.py` (per-stage breakdown from engine logs), `mammoth_moda2/bench_storage_scenarios.sh` (cold/warm page cache on local disk vs. network storage), `mammoth_moda2/raw_load_bench.py` (safetensors → GPU without vLLM)
+- **Key metrics**: time to engine ready, per-stage spawn/init/weight-load/profile time, first vs. steady-state request latency
+
 ### Common serving metrics framework
 
 `vllm_omni/benchmarks/` extends `vllm bench serve --omni` with Omni-specific datasets, backends, and multimodal metrics. Key metrics include:

benchmarks/diffusion/backends.py

@@ -107,6 +107,8 @@ async def async_request_image_edits(
 
     if extra_body.get("guidance_scale") is not None:
         form.add_field("guidance_scale", str(extra_body["guidance_scale"]))
+    if extra_body.get("guidance_scale_2") is not None:
+        form.add_field("guidance_scale_2", str(extra_body["guidance_scale_2"]))
     if extra_body.get("negative_prompt") is not None:
         form.add_field("negative_prompt", str(extra_body["negative_prompt"]))
     if extra_body.get("true_cfg_scale") is not None:

benchmarks/diffusion/bench_wan_vae_decode.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """Benchmark and exactness check for the Wan VAE decoder fast path.
 
 Loads a diffusers Wan VAE (by default the Cosmos3 one), decodes seeded latents
@@ -30,10 +30,13 @@ return the full video on rank 0 only, so equality and PSNR are computed there.
 from __future__ import annotations
 
 import argparse
+import json
 import math
 import os
+import statistics
 import time
 from collections.abc import Iterable
+from unittest.mock import patch
 
 import torch
 import torch.distributed as dist
@@ -82,6 +85,10 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--profile", action="store_true", help="Print a torch.profiler CUDA kernel table")
     parser.add_argument("--profile-rows", type=int, default=25)
     parser.add_argument("--save-output", default=None, help="Save the `off` decode output to this .pt path")
+    parser.add_argument(
+        "--first-frame-ablation", action="store_true", help="Compare channels_last with/without first-frame Conv2d"
+    )
+    parser.add_argument("--latents", help="For ablation: saved VAE-input tensor (.pt), after latent mean/std scaling")
     parser.add_argument(
         "--vae-patch-parallel-size",
         type=int,
@@ -254,8 +261,99 @@ def run_level(
     return output.detach(), stats
 
 
+@torch.inference_mode()
+def run_first_frame_ablation(args: argparse.Namespace, dtype: torch.dtype, device: torch.device) -> None:
+    """Same-model ABBA comparison; disable only the first-frame helper in the reference arm."""
+    from vllm_omni.diffusion.distributed.autoencoders.wan_vae_fastpath import forwards
+
+    if args.iters < 2 or args.warmup < 0:
+        raise ValueError("Ablation requires --iters >= 2 and --warmup >= 0")
+    vae = load_vae(args, dtype, device, parallel=False)
+    report = install_wan_vae_fastpath(vae, level="channels_last")
+    if not report.installed:
+        raise RuntimeError(f"channels_last installation failed: {report.reason}")
+    latents = make_latents(vae, args, dtype, device)
+    if args.latents:
+        saved = torch.load(args.latents, map_location="cpu", weights_only=True)
+        saved = saved[:, :, : latents.shape[2]]
+        if saved.shape != latents.shape:
+            raise ValueError(f"Saved latent shape {saved.shape} does not match {latents.shape}")
+        latents = saved.to(device=device, dtype=dtype).contiguous()
+    helpers = {"baseline": lambda conv, x, bias: None, "candidate": forwards._first_frame_conv2d}
+    optimized_calls = 0
+
+    def counted(conv, x, bias):
+        nonlocal optimized_calls
+        result = helpers["candidate"](conv, x, bias)
+        optimized_calls += result is not None
+        return result
+
+    with patch.object(forwards, "_first_frame_conv2d", counted):
+        decode(vae, latents)
+        sync()
+    if optimized_calls == 0:
+        raise RuntimeError("The selected workload did not exercise first-frame Conv2d")
+    measurements = []
+    outputs: dict[str, list[torch.Tensor]] = {arm: [] for arm in helpers}
+    for _ in range(args.warmup):
+        for helper in helpers.values():
+            with patch.object(forwards, "_first_frame_conv2d", helper):
+                decode(vae, latents)
+                sync()
+    for iteration in range(args.iters):
+        # Alternating AB and BA pairs give ABBA order without dropping odd runs.
+        arms = ("baseline", "candidate") if iteration % 2 == 0 else ("candidate", "baseline")
+        for arm in arms:
+            with patch.object(forwards, "_first_frame_conv2d", helpers[arm]):
+                sync()
+                torch.accelerator.reset_peak_memory_stats()
+                start = time.perf_counter()
+                output = decode(vae, latents)
+                sync()
+                elapsed = (time.perf_counter() - start) * 1000
+                peak = torch.accelerator.max_memory_allocated()
+            measurements.append({"arm": arm, "wall_ms": elapsed, "peak_allocated_bytes": peak})
+            if len(outputs[arm]) < 2:
+                outputs[arm].append(output.cpu())
+            del output
+    baseline, candidate = outputs["baseline"][0], outputs["candidate"][0]
+    difference = candidate.float() - baseline.float()
+    stats = {}
+    for arm in helpers:
+        times = [row["wall_ms"] for row in measurements if row["arm"] == arm]
+        stats[arm] = {
+            "median_ms": statistics.median(times),
+            "mean_ms": statistics.mean(times),
+            "std_ms": statistics.pstdev(times),
+            "min_ms": min(times),
+            "max_ms": max(times),
+        }
+    result = {
+        "scope": "VAE decode; same-head channels_last first-frame Conv2d ablation",
+        "args": vars(args),
+        "torch": torch.__version__,
+        "cuda": torch.version.cuda,
+        "baseline": stats["baseline"],
+        "candidate": stats["candidate"],
+        "latency_reduction": 1 - stats["candidate"]["median_ms"] / stats["baseline"]["median_ms"],
+        "optimized_calls_per_decode": optimized_calls,
+        "outputs_finite": bool(torch.isfinite(baseline).all() and torch.isfinite(candidate).all()),
+        "self_repeat_equal": {arm: torch.equal(*values) for arm, values in outputs.items()},
+        "candidate_vs_baseline": {
+            "torch_equal": torch.equal(candidate, baseline),
+            "max_abs": difference.abs().max().item(),
+            "relative_l2": (difference.norm() / baseline.float().norm().clamp_min(1e-30)).item(),
+            "psnr_db": psnr(candidate, baseline),
+        },
+        "measurements": measurements,
+    }
+    print("FIRST_FRAME_ABLATION " + json.dumps(result))
+
+
 def main() -> None:
     args = parse_args()
+    if args.latents and not args.first_frame_ablation:
+        raise SystemExit("--latents requires --first-frame-ablation")
     if not torch.cuda.is_available():
         raise SystemExit("CUDA device required")
     rank, world_size = init_distributed()
@@ -267,6 +365,12 @@ def main() -> None:
         )
     device = torch.device("cuda", torch.accelerator.current_device_index()) if world_size > 1 else torch.device("cuda")
     dtype = DTYPES[args.dtype]
+    if args.first_frame_ablation:
+        if world_size != 1 or parallel:
+            raise SystemExit("--first-frame-ablation requires one GPU")
+        with torch.backends.cudnn.flags(benchmark=False, allow_tf32=False):
+            run_first_frame_ablation(args, dtype, device)
+        return
     levels: Iterable[str] = [level.strip() for level in args.fast_path.split(",") if level.strip()]
     for level in levels:
         if level not in VAE_FAST_PATH_LEVELS:

benchmarks/diffusion/benchmark_mammoth_moda2_qk_norm_rope.py

@@ -0,0 +1,262 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Benchmark MammothModa2's Q/K RMSNorm + adjacent-pair real RoPE.
+
+Example:
+
+    CUDA_VISIBLE_DEVICES=0 python \
+      benchmarks/diffusion/benchmark_mammoth_moda2_qk_norm_rope.py \
+      --tokens 512 4173 8346 --warmup 50 --iters 200
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.metadata
+import json
+import os
+import statistics
+from collections.abc import Callable
+
+import torch
+
+from vllm_omni.diffusion.layers.fused_qk_norm_rope import (
+    _fused_cuda_supported,
+    fused_qk_norm_rope,
+)
+from vllm_omni.diffusion.layers.norm import RMSNorm
+from vllm_omni.diffusion.models.mammoth_moda2.rope_real import (
+    apply_real_rotary_emb,
+)
+
+_HEAD_DIM = 120
+_Q_HEADS = 21
+_KV_HEADS = 7
+_EPS = 1e-5
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--tokens", nargs="+", type=int, default=[512, 4173, 8346])
+    parser.add_argument("--warmup", type=int, default=50)
+    parser.add_argument("--iters", type=int, default=200)
+    parser.add_argument("--rounds", type=int, default=3)
+    parser.add_argument("--attention-sequences", nargs="*", type=int, default=[])
+    return parser.parse_args()
+
+
+def _version(distribution: str) -> str | None:
+    try:
+        return importlib.metadata.version(distribution)
+    except importlib.metadata.PackageNotFoundError:
+        return None
+
+
+def _measure(
+    fn: Callable[[], object],
+    warmup: int,
+    iters: int,
+) -> dict[str, float]:
+    for _ in range(warmup):
+        fn()
+    torch.accelerator.synchronize()
+
+    torch.accelerator.reset_peak_memory_stats()
+    baseline_bytes = torch.accelerator.memory_allocated()
+    samples = []
+    for _ in range(iters):
+        start = torch.cuda.Event(enable_timing=True)
+        end = torch.cuda.Event(enable_timing=True)
+        start.record()
+        fn()
+        end.record()
+        end.synchronize()
+        samples.append(float(start.elapsed_time(end)))
+
+    ordered = sorted(samples)
+    return {
+        "median_ms": statistics.median(ordered),
+        "mean_ms": statistics.mean(ordered),
+        "stddev_ms": statistics.pstdev(ordered),
+        "min_ms": ordered[0],
+        "max_ms": ordered[-1],
+        "p90_ms": ordered[min(len(ordered) - 1, int(len(ordered) * 0.9))],
+        "extra_peak_mib": (torch.accelerator.max_memory_allocated() - baseline_bytes) / 2**20,
+    }
+
+
+@torch.inference_mode()
+def _run_shape(tokens: int, warmup: int, iters: int, *, fused_first: bool = False) -> dict[str, object]:
+    dtype = torch.bfloat16
+    device = torch.device("cuda")
+    torch.manual_seed(42)
+    query = torch.randn(tokens, _Q_HEADS, _HEAD_DIM, device=device, dtype=dtype)
+    key = torch.randn(tokens, _KV_HEADS, _HEAD_DIM, device=device, dtype=dtype)
+    norm_q = RMSNorm(_HEAD_DIM, eps=_EPS).to(device=device, dtype=dtype)
+    norm_k = RMSNorm(_HEAD_DIM, eps=_EPS).to(device=device, dtype=dtype)
+    norm_q.weight.normal_(mean=1.0, std=0.2)
+    norm_k.weight.normal_(mean=1.0, std=0.2)
+    angles = torch.rand(tokens, _HEAD_DIM // 2, device=device) * (2 * torch.pi)
+    cos = angles.cos().repeat_interleave(2, dim=-1).to(dtype)
+    sin = angles.sin().repeat_interleave(2, dim=-1).to(dtype)
+
+    def native() -> tuple[torch.Tensor, torch.Tensor]:
+        q_out = apply_real_rotary_emb(norm_q(query).unsqueeze(0), cos.unsqueeze(0), sin.unsqueeze(0)).squeeze(0)
+        k_out = apply_real_rotary_emb(norm_k(key).unsqueeze(0), cos.unsqueeze(0), sin.unsqueeze(0)).squeeze(0)
+        return q_out, k_out
+
+    def fused() -> tuple[torch.Tensor, torch.Tensor]:
+        rope_table = torch.cat((cos[..., 0::2], sin[..., 0::2]), dim=-1)
+        return fused_qk_norm_rope(
+            query,
+            key,
+            norm_q.weight,
+            norm_k.weight,
+            rope_table,
+            _EPS,
+            head_dim=_HEAD_DIM,
+            rotary_dim=_HEAD_DIM,
+            interleaved=True,
+        )
+
+    expected_q, expected_k = native()
+    actual_q, actual_k = fused()
+    errors = {
+        "q_max_abs": (actual_q.float() - expected_q.float()).abs().max().item(),
+        "q_mean_abs": (actual_q.float() - expected_q.float()).abs().mean().item(),
+        "k_max_abs": (actual_k.float() - expected_k.float()).abs().max().item(),
+        "k_mean_abs": (actual_k.float() - expected_k.float()).abs().mean().item(),
+    }
+    torch.testing.assert_close(actual_q, expected_q, atol=0.0625, rtol=0.02)
+    torch.testing.assert_close(actual_k, expected_k, atol=0.0625, rtol=0.02)
+
+    functions = {"native": native, "fused": fused}
+    order = ("fused", "native") if fused_first else ("native", "fused")
+    measured = {name: _measure(functions[name], warmup, iters) for name in order}
+    native_stats, fused_stats = measured["native"], measured["fused"]
+    return {
+        "tokens": tokens,
+        "native": native_stats,
+        "fused_including_rope_pack": fused_stats,
+        "median_speedup": native_stats["median_ms"] / fused_stats["median_ms"],
+        "errors": errors,
+    }
+
+
+@torch.inference_mode()
+def _run_attention(seq: int, warmup: int, iters: int, *, fused_first: bool = False) -> dict[str, object]:
+    from vllm_omni.diffusion.models.mammoth_moda2.mammothmoda2_dit_model import TransformerBlock
+
+    torch.manual_seed(42)
+    block = (
+        TransformerBlock(
+            2520,
+            _Q_HEADS,
+            _KV_HEADS,
+            multiple_of=256,
+            ffn_dim_multiplier=1.0,
+            norm_eps=_EPS,
+            rope_repeats_pairs=True,
+        )
+        .to(device="cuda", dtype=torch.bfloat16)
+        .eval()
+    )
+    block.attn.norm_q.weight.normal_(1.0, 0.2)
+    block.attn.norm_k.weight.normal_(1.0, 0.2)
+    hidden = torch.randn(2, seq, 2520, device="cuda", dtype=torch.bfloat16)
+    mask = torch.ones(2, seq, device="cuda", dtype=torch.bool)
+    mask[0, seq - min(300, seq // 2) :] = False
+    mask[1, seq - min(17, seq // 2) :] = False
+    angles = torch.rand(2, seq, _HEAD_DIM // 2, device="cuda") * (2 * torch.pi)
+    # The production producer supplies FP32 tables; each example has its own positions.
+    rotary = angles.cos().repeat_interleave(2, dim=-1), angles.sin().repeat_interleave(2, dim=-1)
+    env = "VLLM_OMNI_FUSED_QK_NORM_ROPE_MIN_TOKENS"
+    previous = os.environ.get(env)
+
+    def run():
+        return block.attn(hidden, hidden, attention_mask=mask, image_rotary_emb=rotary)
+
+    try:
+        outputs = {}
+        timings = {}
+        order = ("fused", "native") if fused_first else ("native", "fused")
+        for name in order:
+            os.environ[env] = "0" if name == "fused" else str(2 * seq + 1)
+            outputs[name] = run()
+            timings[name] = _measure(run, warmup, iters)
+        diff = (outputs["native"].float() - outputs["fused"].float()).abs()[mask]
+        max_diff, mean_diff = diff.max().item(), diff.mean().item()
+        assert max_diff < 2e-2 and mean_diff < 1e-3, (max_diff, mean_diff)
+        assert torch.count_nonzero(outputs["fused"][~mask]) == 0
+        return {
+            "sequence": seq,
+            "batch": 2,
+            "backend": block.attn.omni_attn.attn_backend.__name__,
+            "native": timings["native"],
+            "fused": timings["fused"],
+            "median_speedup": timings["native"]["median_ms"] / timings["fused"]["median_ms"],
+            "max_abs_diff": max_diff,
+            "mean_abs_diff": mean_diff,
+        }
+    finally:
+        if previous is None:
+            os.environ.pop(env, None)
+        else:
+            os.environ[env] = previous
+
+
+def main() -> None:
+    args = _parse_args()
+    if (
+        args.rounds < 1
+        or args.iters < 1
+        or args.warmup < 0
+        or any(value < 1 for value in [*args.tokens, *args.attention_sequences])
+    ):
+        raise ValueError("rounds, iters and sequence lengths must be positive; warmup must be nonnegative")
+    if not torch.cuda.is_available():
+        raise RuntimeError("This benchmark requires CUDA")
+
+    probe_q = torch.empty(1, _Q_HEADS, _HEAD_DIM, device="cuda", dtype=torch.bfloat16)
+    probe_k = torch.empty(1, _KV_HEADS, _HEAD_DIM, device="cuda", dtype=torch.bfloat16)
+    if not _fused_cuda_supported(probe_q, probe_k, _HEAD_DIM, _HEAD_DIM, interleaved=True):
+        raise RuntimeError("The fused CUDA QK norm/RoPE path is unavailable")
+
+    result = {
+        "environment": {
+            "gpu": torch.cuda.get_device_name(),
+            "torch": torch.__version__,
+            "cuda": torch.version.cuda,
+            "triton": _version("triton"),
+            "vllm": _version("vllm"),
+            "vllm_omni": _version("vllm-omni"),
+        },
+        "config": {
+            "dtype": "bfloat16",
+            "q_heads": _Q_HEADS,
+            "kv_heads": _KV_HEADS,
+            "head_dim": _HEAD_DIM,
+            "warmup": args.warmup,
+            "iters": args.iters,
+            "rounds": args.rounds,
+            "inference_mode": True,
+            "alternating_order": True,
+            "norm_weights": "normal(mean=1.0, std=0.2)",
+        },
+        "results": [
+            {"round": round_id, **_run_shape(tokens, args.warmup, args.iters, fused_first=bool(round_id % 2))}
+            for round_id in range(args.rounds)
+            for tokens in args.tokens
+        ],
+        "attention_results": [
+            {"round": round_id, **_run_attention(seq, args.warmup, args.iters, fused_first=bool(round_id % 2))}
+            for round_id in range(args.rounds)
+            for seq in args.attention_sequences
+        ],
+    }
+    print(json.dumps(result, indent=2))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/diffusion/diffusion_benchmark_serving.py

@@ -1107,6 +1107,11 @@ def calculate_metrics(
     error_outputs = [o for o in outputs if not o.success]
 
     num_success = len(success_outputs)
+    error_log_path = os.environ.get("BENCH_ERROR_LOG")
+    if error_outputs and error_log_path:
+        with open(error_log_path, "a") as f:
+            for output in error_outputs:
+                f.write(f"ERROR latency={output.latency:.3f} detail={output.error!r}\n")
     latencies = [o.latency for o in success_outputs]
     peak_memories = [o.peak_memory_mb for o in success_outputs if o.peak_memory_mb > 0]
 

benchmarks/diffusion/performance_dashboard/mammoth_moda2_serving_performance.md

@@ -0,0 +1,209 @@
+# MammothModa2-Preview Serving Performance Dashboard
+
+This document describes how to deploy and benchmark **bytedance-research/MammothModa2-Preview** (T2I) using vLLM-Omni. It includes service startup configuration, benchmark methodology, dataset settings, and performance results.
+
+MammothModa2 is a two-stage AR→DiT text-to-image pipeline: stage 0 is an autoregressive LLM that encodes the prompt into a visual-token grid, and stage 1 is a generation-LLM DiT that renders the final image (`final_output_type: image`). Unlike single-stage diffusion pipelines, `/v1/images/generations` requests traverse both stages, and the DiT stage ran with `max_num_seqs: 1` in the then-default deploy config, so concurrent requests serialized on the DiT stage while the AR stage batched (request-level DiT batching landed later in #7476).
+
+> **Legacy-topology baseline**: all numbers in this dashboard were measured on the legacy topology, where the MammothModa2 DiT stage runs as an LLM-generation stage (`stage_type: llm`) built from `VllmConfig`, served through the LLM-typed image-stage compatibility path. #7134 migrates this pipeline to the shared diffusion runtime (DiT registered as a standard `DIFFUSION` stage built from `OmniDiffusionConfig`). These results should be treated as the pre-migration baseline and are expected to be rerun after #7134 lands; they are not directly comparable with post-migration numbers.
+
+---
+
+# 1. Overview
+
+This document covers:
+
+* Service launch configuration
+* Benchmark scripts and usage
+* Dataset and workload settings
+* Performance measurement results
+* Reproducibility guidelines
+
+---
+
+# 2. Test Environment
+
+| Component | Specification |
+|------------|----------------|
+| GPU | NVIDIA A800-SXM4-80GB (1x) |
+| CPU | 18 cores |
+| Model weights | `bytedance-research/MammothModa2-Preview` |
+| Deploy config | `vllm_omni/deploy/mammoth_moda2.yaml` (bundled) |
+| Stage 0 (AR) | `gpu_memory_utilization: 0.5`, `enforce_eager: true`, `max_num_seqs: 100` |
+| Stage 1 (DiT) | `gpu_memory_utilization: 0.3`, `enforce_eager: true`, `max_num_seqs: 1` |
+| Sampling params (per request) | `text_guidance_scale: 9.0`, `cfg_range: [0.0, 1.0]`, `num_inference_steps: 50` sent via `--extra-body` / `--num-inference-steps` |
+
+---
+
+# 3. Service Launch Configuration
+
+## 3.1 Basic Serving Command
+
+```bash
+vllm-omni serve bytedance-research/MammothModa2-Preview --omni \
+    --port 8091 \
+    --trust-remote-code
+```
+
+When serving from a local weights directory (no `model_index.json`), pass the bundled deploy config explicitly so the `mammoth_moda2` pipeline is selected:
+
+```bash
+vllm-omni serve /path/to/MammothModa2-Preview --omni \
+    --port 8091 \
+    --trust-remote-code \
+    --deploy-config vllm_omni/deploy/mammoth_moda2.yaml
+```
+
+## 3.2 Key Parameters
+
+| Parameter | Description |
+|-----------|-------------|
+| `--deploy-config` | Path to the deploy YAML; defaults to the bundled `mammoth_moda2.yaml` for the HF checkpoint |
+| `size` (request) | `WxH`, e.g. `1024x1024`; routed to the AR grid and the DiT `target_w/target_h` |
+| `num_inference_steps` (request) | DiT sampling steps (default 50) |
+| `guidance_scale` (request) | Text guidance scale (default 9.0) |
+
+Note: the benchmark script's `--warmup-num-inference-steps` shortcut is not usable for this pipeline — a reduced step count truncates the AR stage's visual-token output and crashes the DiT stage. Use `--warmup-requests 0` and warm up with a real request instead.
+
+---
+
+# 4. Benchmark Script
+
+## 4.1 Benchmark Entry
+
+```bash
+python benchmarks/diffusion/diffusion_benchmark_serving.py \
+    --base-url http://localhost:8091 \
+    --model bytedance-research/MammothModa2-Preview \
+    --endpoint /v1/images/generations \
+    --task t2i \
+    --dataset random \
+    --num-prompts 8 \
+    --width <W> --height <H> \
+    --num-inference-steps <S> \
+    --extra-body '{"text_guidance_scale": 9.0, "cfg_range": [0.0, 1.0]}' \
+    --max-concurrency <C> \
+    --warmup-requests 0 \
+    --seed 142 \
+    --output-file <OUT>.json
+```
+
+## 4.2 Dataset
+
+`random` synthetic T2I prompts, fixed seed 142, 8 prompts per configuration (4 for 2048x2048 due to per-request latency).
+
+---
+
+# 5. Performance Results
+
+All requests succeeded (0 failures across the sweep). Latency is end-to-end per request (AR + DiT + VAE decode + base64 response).
+
+> Note: these are legacy-topology baseline results (see the callout in the overview); rerun after the #7134 diffusion-runtime migration before comparing across topologies.
+
+## 5.1 Latency
+
+| Resolution | Steps | Concurrency | Mean (s) | P50 (s) | P95 (s) | P99 (s) |
+|------------|-------|-------------|----------|---------|---------|---------|
+| 512x512    | 20    | 1           | 21.8     | 21.7    | 22.1    | 22.2    |
+| 512x512    | 20    | 2           | 24.8     | 25.2    | 25.6    | 25.8    |
+| 512x512    | 20    | 4           | 28.5     | 29.2    | 31.1    | 31.8    |
+| 1024x1024  | 50    | 1           | 96.3     | 95.8    | 99.8    | 100.8   |
+| 1024x1024  | 50    | 2           | 121.5    | 125.4   | 127.9   | 128.5   |
+| 1024x1024  | 50    | 4           | 150.9    | 157.6   | 178.6   | 186.3   |
+| 2048x2048  | 50    | 1           | 271.9    | 271.6   | 273.6   | 273.8   |
+
+## 5.2 Throughput
+
+| Resolution | Steps | Concurrency | Throughput (img/s) |
+|------------|-------|-------------|--------------------|
+| 512x512    | 20    | 1           | 0.0459             |
+| 512x512    | 20    | 2           | 0.0798             |
+| 512x512    | 20    | 4           | 0.1329             |
+| 1024x1024  | 50    | 1           | 0.0104             |
+| 1024x1024  | 50    | 2           | 0.0162             |
+| 1024x1024  | 50    | 4           | 0.0238             |
+| 2048x2048  | 50    | 1           | 0.0037             |
+
+## 5.3 Stage Breakdown (concurrency 1, from server logs)
+
+Per-stage wall time from the server's `[OmniTiming]` records across the §5.1/§5.4/§5.5 sweeps (n = 18/9/5 per row; at concurrency 1 stage wall time equals `stage_gen_time_ms` in the engine stats table):
+
+| Resolution | Steps | AR stage (s) | DiT stage (s) | E2E (s) | AR share | AR output tokens | AR ms/token |
+|------------|-------|--------------|---------------|---------|----------|------------------|-------------|
+| 512x512    | 20    | 20.3         | 1.7           | 22.0    | 92%      | 1,057            | 19.2        |
+| 1024x1024  | 50    | 78.9         | 17.2          | 96.1    | 82%      | 4,161            | 19.0        |
+| 2048x2048  | 50    | 156.3        | 114.8         | 271.2   | 58%      | 8,154            | 19.2        |
+
+Stage 0 is the AR prompt encoder (TTFT 82 ms at 1024x1024); stage 1 is the DiT renderer. Three observations:
+
+1. **AR per-token cost is constant (~19 ms/token) across resolutions**, so AR time is fully predictable: the AR stage decodes a fixed visual grid whose length scales with resolution (1,057 → 4,161 → 8,154 tokens), giving `AR ≈ tokens × 19 ms`. This is also why §5.4's step sweep is sub-linear — the AR token count is step-invariant.
+2. **DiT time scales super-linearly with pixels**: a 16x pixel increase (512 → 2048) multiplies DiT time by 67 (1.7 → 114.8 s).
+3. **The dominant stage flips with resolution**: AR is 92% of E2E at 512x512 (DiT optimization is nearly pointless there) but 58% at 2048x2048, where both stages matter.
+
+At concurrency > 1, per-stage wall times additionally include intra-stage queueing (each stage serializes at `max_num_seqs: 1`), so the c2/c4 rows in §5.1 cannot be decomposed additively from this table.
+
+## 5.4 Inference-Step Sensitivity (1024x1024 / 512x512, concurrency 1)
+
+Steps 25 at 1024x1024 (half the default 50) and steps 25 at 512x512, same protocol as §5.1:
+
+| Resolution | Steps | Concurrency | Mean (s) | P50 (s) | P95 (s) | P99 (s) |
+|------------|-------|-------------|----------|---------|---------|---------|
+| 1024x1024  | 25    | 1           | 85.2     | 85.3    | 87.8    | 88.7    |
+| 1024x1024  | 25    | 4           | 113.7    | 115.8   | 126.1   | 129.4   |
+| 512x512    | 25    | 1           | 21.9     | 21.8    | 22.2    | 22.3    |
+
+Halving DiT steps does not halve end-to-end latency (96.3 → 85.2 s at 1024): the AR stage emits a fixed count of visual tokens (4,161 at 1024x1024) regardless of step count, so its ~79 s contribution (§5.3) is step-invariant; only the DiT sampling time scales with steps. At 512x512, steps 20 → 25 adds ~0.1 s per step (21.8 → 21.9 s).
+
+## 5.5 Prompt-Length Sensitivity (long vs short prompts)
+
+A ~430-word long prompt (repeated via `--random-request-config '[{"weight":1,"prompt":"..."}]'`) against the default short synthetic prompt, same seed and settings:
+
+| Resolution | Steps | Concurrency | Prompt | Mean (s) | P99 (s) | Throughput (img/s) |
+|------------|-------|-------------|--------|----------|---------|---------------------|
+| 1024x1024  | 50    | 1           | short  | 96.3     | 100.8   | 0.0104              |
+| 1024x1024  | 50    | 1           | long   | 93.6     | 94.4    | 0.0107              |
+| 1024x1024  | 50    | 4           | short  | 150.9    | 186.3   | 0.0238              |
+| 1024x1024  | 50    | 4           | long   | 141.7    | 174.2   | 0.0255              |
+| 512x512    | 20    | 1           | short  | 21.8     | 22.2    | 0.0459              |
+| 512x512    | 20    | 1           | long   | 22.2     | 22.5    | 0.0450              |
+
+Prompt length has no material effect on latency (the long-prompt rows are within run-to-run variance and even slightly faster with tighter P99 tails). This confirms the AR stage's cost is dominated by decoding the fixed 4,161-token visual grid, not by prompt encoding: the KV prefill of a few hundred extra text tokens is negligible against 4,161 decode steps.
+
+## 5.6 Peak GPU Memory (per-stage process, nvidia-smi polling at 1 Hz)
+
+Sampled during the §5.4/§5.5 sweeps (stage processes identified by PID; both stages share the single A800):
+
+| Configuration (worst case) | stage0_ar (MB) | stage1_dit (MB) |
+|----------------------------|----------------|-----------------|
+| 1024x1024, 50 steps, c4    | 39,290         | 11,104          |
+| all other configs          | 39,194         | 11,102–11,104   |
+
+Memory is essentially flat across the sweep: the AR stage grows by only ~96 MB at concurrency 4 and the DiT stage by 2 MB, because both stages pre-allocate from `gpu_memory_utilization` budgets (0.5 / 0.3) at startup. Combined steady-state usage ≈ 50.4 GiB of the 80 GiB device.
+
+## 5.7 DiT-Stage Component Attribution (pipeline profiler)
+
+With `enable_diffusion_pipeline_profiler: true` in the stage-1 deploy config (`MammothModa2DiTPipeline` targets: `gen_transformer.forward`, `gen_vae.decode`, `gen_image_condition_refiner.forward`), three warm 1024x1024 / 50-step requests (profiler adds a `synchronize` per call, so timings are indicative, not throughput-grade):
+
+| Component | Per request (s) | Share of DiT pipeline |
+|-----------|-----------------|-----------------------|
+| `gen_transformer.forward` (100 calls: 50 steps x 2 passes) | 14.01–14.07 | 98.6% |
+| `gen_vae.decode` (1 call) | 0.148 | 1.0% |
+| `gen_image_condition_refiner.forward` (1 call) | 0.005 | 0.03% |
+| whole `MammothModa2DiTPipeline.forward` | 14.18–14.27 | — |
+
+The DiT latency is ~99% transformer sampling; VAE decode costs only ~0.15 s per image at 1024x1024 and is not a bottleneck. Per-step transformer time is ~140 ms (two forward passes per step: ~2.8 s nominal x CFG), consistent with §5.3's 17.2 s image-TTFT figure once base64 encoding and stage handoff are included.
+
+## 5.8 Endpoint Fix Verification
+
+Before the serving fix, `/v1/images/generations` returned HTTP 503 ("No diffusion stage found in multi-stage pipeline") because the pipeline's image-output stage is a generation-LLM DiT rather than a `diffusion`-typed stage. After the fix all requests return HTTP 200 with a valid base64 image (verified across every sweep configuration above).
+
+---
+
+# 6. Reproducibility
+
+1. Start the server as in §3.1 and wait for `Application startup complete`.
+2. Warm up with one real request (any size) — do not use reduced-step warmups.
+3. Run the sweep commands from §4.1 for each row in §5.
+4. Server-side per-stage timings are printed in the engine stats table (`stage_gen_time_ms`, `output_unit_count`) at `--log-stats` level info.
+5. New rows use the same seed (`--seed 142`), `--num-prompts 8`, and `--warmup-requests 0` protocol as §4.1; when saving results with `--output-file`, name them `<res>_s<steps>_c<conc>[_long].json` to keep runs comparable (steps-25, long-prompt, peak-memory and profiler runs).
+6. To reproduce §5.6 peak-memory numbers, poll `nvidia-smi --query-gpu=memory.used --format=csv -l 1` per stage PID while a sweep is in flight (e.g. `nvidia-smi pmon` / `--query-compute-apps`).
+7. To reproduce §5.7, copy the bundled deploy config, set `enable_diffusion_pipeline_profiler: true` on the stage-1 entry, restart, and grep `[DiffusionPipelineProfiler]` lines from the server log; timings include a `torch.cuda.synchronize` per call.

benchmarks/kernels/_magi2_musa_bench.py

@@ -0,0 +1,194 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Cold-cache torch.cuda timing and correctness gates for MAGI-2 microbenchmarks."""
+
+import argparse
+import importlib
+import importlib.metadata
+import json
+import logging
+import math
+import os
+import subprocess
+import sys
+from collections.abc import Callable
+from pathlib import Path
+from types import ModuleType
+from typing import Any
+
+LOGGER = logging.getLogger(__name__)
+
+
+def emit(**fields: Any) -> None:
+    LOGGER.info(json.dumps(fields, sort_keys=True))
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--tokens", type=int, nargs="+", default=[1, 4, 16, 64, 129, 4096])
+    parser.add_argument("--dtype", choices=["bfloat16", "float16", "float32"], default="bfloat16")
+    parser.add_argument("--num-tests", type=int, default=30, help="Measured samples per implementation.")
+    parser.add_argument("--warmup", type=int, default=10, help="Interleaved warmup calls before sampling.")
+    parser.add_argument("--seed", type=int, default=42)
+
+
+def load_musa(args: argparse.Namespace, parser: argparse.ArgumentParser) -> Any:
+    if min(*args.tokens, args.num_tests) <= 0 or args.warmup < 0:
+        parser.error("tokens and num-tests must be positive; warmup must be nonnegative")
+    logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout)
+    torch = importlib.import_module("torch")
+    cuda = getattr(torch, "cuda")
+    # torchada is needed only on MUSA; after it is loaded, use torch.cuda for
+    # both CUDA and MUSA as documented by torchada.
+    if hasattr(torch.version, "musa") and torch.version.musa is not None:
+        try:
+            importlib.import_module("torchada")
+        except ModuleNotFoundError as error:
+            if error.name != "torchada":
+                raise
+    has_gpu = cuda.is_available() or (
+        hasattr(torch.version, "musa") and torch.version.musa is not None and torch.musa.is_available()
+    )
+    if not has_gpu:
+        emit(status="skip", reason="CUDA/MUSA device is unavailable", torch=torch.__version__)
+        return None
+    torch.manual_seed(args.seed)
+    cuda.manual_seed_all(args.seed)
+    return torch
+
+
+def report_environment(torch: Any, args: argparse.Namespace, module: ModuleType) -> None:
+    root = Path(__file__).resolve().parents[2]
+    packages = {}
+    for name in ("torch", "torch_musa", "torchada", "triton", "vllm", "vllm-omni"):
+        try:
+            packages[name] = importlib.metadata.version(name)
+        except importlib.metadata.PackageNotFoundError:
+            packages[name] = None
+    cuda = getattr(torch, "cuda")
+    properties = cuda.get_device_properties(cuda.current_device())
+    try:
+        source_sha = subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True).strip()
+        source_dirty = bool(subprocess.check_output(["git", "-C", str(root), "status", "--porcelain"], text=True))
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        source_sha = os.environ.get("MAGI2_BENCH_SOURCE_SHA", "unknown")
+        source_dirty = None
+    emit(
+        status="environment",
+        command=sys.argv,
+        source_root=str(root),
+        source_sha=source_sha,
+        source_dirty=source_dirty,
+        kernel_source=module.__file__,
+        device=cuda.get_device_name(cuda.current_device()),
+        sm_count=getattr(properties, "multi_processor_count", None),
+        packages=packages,
+        env={name: os.environ.get(name) for name in ("MUSA_VISIBLE_DEVICES", "TORCHDYNAMO_DISABLE")},
+        config=vars(args),
+        mode="eager operator diagnostic; no compile or graph capture",
+        comparison="forward_native versus the default CustomOp dispatch",
+        timing="torch.cuda.Event; alternating native/fused calls; vanilla 512 MiB flush outside timed regions",
+        timing_scope="whole callable, including device launch gaps; not isolated kernel duration",
+        l2_flush=True,
+        l2_flush_bytes=512 * 1024 * 1024,
+    )
+
+
+class _LaunchProbe:
+    def __init__(self, kernel: Any) -> None:
+        self.kernel = kernel
+        self.count = 0
+
+    def __getitem__(self, grid: Any) -> Any:
+        self.count += 1
+        return self.kernel[grid]
+
+
+def check_pair(
+    torch: Any,
+    reference: Callable,
+    fused: Callable,
+    module: ModuleType,
+    kernels: tuple[str, ...],
+    *,
+    rtol: float,
+    atol: float,
+) -> dict:
+    cuda = getattr(torch, "cuda")
+    expected = reference()
+    probes = {name: _LaunchProbe(getattr(module, name)) for name in kernels}
+    try:
+        for name, probe in probes.items():
+            setattr(module, name, probe)
+        actual = fused()
+        cuda.synchronize()
+    finally:
+        for name, probe in probes.items():
+            setattr(module, name, probe.kernel)
+    if any(probe.count != 1 for probe in probes.values()):
+        raise RuntimeError(f"Expected one launch per fused kernel; got {[(k, p.count) for k, p in probes.items()]}")
+    expected = expected if isinstance(expected, tuple) else (expected,)
+    actual = actual if isinstance(actual, tuple) else (actual,)
+    errors = []
+    for result, baseline in zip(actual, expected, strict=True):
+        for value in (baseline, result):
+            if not torch.isfinite(value).all().item():
+                raise AssertionError("Nonfinite benchmark output")
+            if value.numel() > 1 and (value == value.flatten()[0]).all().item():
+                raise AssertionError("Constant benchmark output")
+        torch.testing.assert_close(result, baseline, rtol=rtol, atol=atol)
+        difference = (result.float() - baseline.float()).abs()
+        errors.append(
+            {
+                "max_abs": difference.max().item(),
+                "max_rel": (difference / baseline.float().abs().clamp_min(1e-6)).max().item(),
+            }
+        )
+    return {"parity": "pass", "errors": errors, "rtol": rtol, "atol": atol, "kernels": list(kernels)}
+
+
+def _percentile(samples: list[float], quantile: float) -> float:
+    ordered = sorted(samples)
+    position = (len(ordered) - 1) * quantile
+    lower, upper = math.floor(position), math.ceil(position)
+    return ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower)
+
+
+def measure_pair(torch: Any, reference: Callable, fused: Callable, args: argparse.Namespace) -> dict:
+    cuda = getattr(torch, "cuda")
+    functions = (reference, fused)
+    order: list[int] = []
+
+    def alternate() -> Any:
+        mode = len(order) % 2
+        order.append(mode)
+        return functions[mode]()
+
+    times_ms = []
+    for _ in range(args.warmup):
+        alternate()
+    cuda.synchronize()
+    order.clear()
+    flush = torch.empty(512 * 1024 * 1024, dtype=torch.uint8, device="cuda")
+    for _ in range(2 * args.num_tests):
+        flush.zero_()
+        cuda.synchronize()
+        start = cuda.Event(enable_timing=True)
+        end = cuda.Event(enable_timing=True)
+        start.record()
+        alternate()
+        end.record()
+        end.synchronize()
+        times_ms.append(start.elapsed_time(end))
+    samples: tuple[list[float], list[float]] = ([], [])
+    for mode, milliseconds in zip(order[-len(times_ms) :], times_ms, strict=True):
+        if not math.isfinite(milliseconds) or milliseconds <= 0:
+            raise RuntimeError(f"Invalid GPU event sample: {milliseconds}")
+        samples[mode].append(milliseconds * 1000)
+    stats = {}
+    for label, values in zip(("native", "fused"), samples, strict=True):
+        if len(values) != args.num_tests:
+            raise RuntimeError("GPU timing harness returned an unexpected sample count")
+        p50 = _percentile(values, 0.5)
+        stats[label] = {"median_us": p50, "p50_us": p50, "p90_us": _percentile(values, 0.9), "samples": len(values)}
+    stats["speedup_p50"] = stats["native"]["p50_us"] / stats["fused"]["p50_us"]
+    return stats

benchmarks/kernels/benchmark_magi2_bf16_moe.py

@@ -0,0 +1,310 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Compare MAGI-2 routed MoE with equal preprocessing boundaries.
+
+python -m benchmarks.kernels.benchmark_magi2_bf16_moe --tokens 2 4096 10000
+
+Default (`--mode dispatch`): the same MAGI-2 layer is timed with its native
+reference and default BF16 fast path, including routing and all preprocessing.
+With `--mode routed`, routing probabilities/IDs are inputs to BOTH timed paths. Sorting,
+padding, packing, allocations and reduction remain inside their respective
+calls. The reference uses deterministic routed storage in diagnostic mode.
+Use --mode dispatch to measure the
+actual layer with the native and default fast paths, including the router. Unsupported
+dispatch fails explicitly; it is never silently replaced by another baseline.
+
+Stage timings are standalone diagnostics, not additive or env speedup claims.
+These synthetic local shapes are not full-model performance measurements.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib
+import json
+import math
+import os
+import statistics
+import subprocess
+import time
+from contextlib import contextmanager
+from pathlib import Path
+from unittest.mock import patch
+
+
+def parse_args(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--tokens", type=int, nargs="+", default=[2, 4096])
+    parser.add_argument("--heads", type=int, default=3, help="EP4 local head count for MAGI-2 Preview.")
+    parser.add_argument("--experts", type=int, default=256, help="Released experts per head.")
+    parser.add_argument("--top-k", type=int, default=6)
+    parser.add_argument("--warmup", type=int, default=10)
+    parser.add_argument("--iters", type=int, default=50)
+    parser.add_argument("--seed", type=int, default=419)
+    parser.add_argument("--mode", choices=["routed", "dispatch"], default="dispatch")
+    parser.add_argument(
+        "--native-deterministic",
+        action="store_true",
+        help="Use the portable deterministic native reference (needed by Triton-MUSA 3.2).",
+    )
+    parser.add_argument("--stages", action="store_true")
+    parser.add_argument("--output", type=Path)
+    args = parser.parse_args(argv)
+    if min(*args.tokens, args.heads, args.experts, args.top_k, args.iters) < 1 or args.warmup < 0:
+        parser.error("Shapes/iters must be positive; warmup must be nonnegative")
+    if args.top_k > args.experts:
+        parser.error("top-k must not exceed experts per head")
+    return args
+
+
+@contextmanager
+def env_switch(enabled):
+    # Keep the legacy benchmark context API; dispatch is no longer opt-in.
+    with patch.dict(os.environ, MAGI2_DETERMINISTIC="0"):
+        yield
+
+
+def stats(values):
+    if not values or any(not math.isfinite(x) or x <= 0 for x in values):
+        raise ValueError("Missing/non-positive/non-finite timing sample")
+    ordered = sorted(values)
+    return {
+        "mean_us": statistics.mean(values),
+        "p50_us": statistics.median(values),
+        "p90_us": ordered[math.ceil(0.9 * len(ordered)) - 1],
+        "samples_us": values,
+    }
+
+
+def measure(torch, calls, args):
+    """Alternating AB/BA; cold-cache flush is outside each timed region."""
+    cuda = getattr(torch, "cuda")
+    names = list(calls)
+    sequence = names + names[::-1]
+    wall_samples = []
+    labels = []
+    counter = 0
+
+    def invoke():
+        nonlocal counter
+        name = sequence[counter % len(sequence)]
+        counter += 1
+        cuda.synchronize()
+        start = time.perf_counter_ns()
+        output = calls[name]()
+        cuda.synchronize()
+        elapsed = (time.perf_counter_ns() - start) / 1000
+        wall_samples.append(elapsed)
+        labels.append(name)
+        return output
+
+    count = len(names) * args.iters
+    cache = torch.empty(512 * 1024 * 1024, device="cuda", dtype=torch.uint8)
+    gpu_ms = []
+    for index in range(len(names) * args.warmup + count):
+        cache.zero_()
+        cuda.synchronize()
+        begin, end = cuda.Event(enable_timing=True), cuda.Event(enable_timing=True)
+        begin.record()
+        invoke()
+        end.record()
+        end.synchronize()
+        if index >= len(names) * args.warmup:
+            gpu_ms.append(begin.elapsed_time(end))
+    labels, wall_samples = labels[-count:], wall_samples[-count:]
+    result = {}
+    for name in names:
+        wall = [x for label, x in zip(labels, wall_samples) if label == name]
+        event = [float(x) * 1000 for label, x in zip(labels, gpu_ms) if label == name]
+        if len(wall) != args.iters or len(event) != args.iters:
+            raise RuntimeError("Timing harness did not return the requested sample counts")
+        result[name] = {"synchronized_wall": stats(wall), "gpu_event": stats(event)}
+    return result
+
+
+def make_inputs(torch, tokens, args):
+    # Fan-in-scaled synthetic weights. Unlike the old script, W_up and W_gate
+    # have the same distribution and top-k IDs are unique within each token.
+    torch.manual_seed(args.seed)
+    x = torch.randn(tokens, args.heads, 256, device="cuda").to(torch.bfloat16)
+    shape = (args.heads * args.experts, 256, 1280)
+    gate, up = [(torch.randn(shape, device="cuda") / 16).to(torch.bfloat16) for _ in range(2)]
+    down = (torch.randn(shape[0], 1280, 256, device="cuda") / math.sqrt(1280)).to(torch.bfloat16)
+    logits = torch.randn(args.heads, tokens, args.experts, device="cuda")
+    selected = logits.topk(args.top_k, dim=-1)
+    probabilities = selected.values.softmax(-1)
+    return x, probabilities, selected.indices, gate, up, down
+
+
+def check_and_capture(torch, moe, kernels, data, args):
+    x, probabilities, indices, gate, up, down = data
+    gather, weights, offsets = moe.global_sort_routes(probabilities, indices, args.experts)
+    sorted_routes, launches = [], []
+    scatter, invoke = kernels._deterministic_scatter, moe.invoke_fused_moe_bf16
+
+    def capture_scatter(routes, *params):
+        sorted_routes.append(routes.detach().clone())
+        return scatter(routes, *params)
+
+    def capture_launch(*params, **kwargs):
+        launches.append((params, kwargs))
+        return invoke(*params, **kwargs)
+
+    with patch.object(kernels, "_deterministic_scatter", capture_scatter):
+        moe.triton_mh_moe_forward(x, gather, weights, offsets, gate, up, down, deterministic=True)
+    packed_w13 = moe._pack_bf16_w13(gate, up)
+    route_buffers = moe._allocate_bf16_route_buffers(
+        x.shape[0] * args.heads * args.top_k,
+        args.heads * args.experts,
+        128,
+        x.device,
+    )
+    with patch.object(moe, "invoke_fused_moe_bf16", capture_launch):
+        actual = moe._bf16_fused_moe_forward(x, probabilities, indices, packed_w13, down, route_buffers)
+    if len(sorted_routes) != 1 or len(launches) != 2:
+        raise RuntimeError("Unexpected kernel dispatch: benchmark contract needs updating")
+    flat_experts = (indices + torch.arange(args.heads, device=x.device)[:, None, None] * args.experts).flatten()
+    order = flat_experts.argsort(stable=True)
+    new_routes = launches[1][0][2].reshape(-1, 256).index_select(0, order)
+    scatter_ids = gather.long() * args.heads + flat_experts[order] // args.experts
+    expected = torch.zeros(x.shape[0] * args.heads, 256, device=x.device, dtype=torch.float32)
+    expected.index_add_(0, scatter_ids, sorted_routes[0].float())
+    expected = expected.to(x.dtype).view_as(x)
+    errors = []
+    # Same expert/FP32 reduction contract as test_bf16_moe_wiring.py, rather
+    # than relaxing tolerances on the inherited BF16 scatter reduction.
+    for result, reference in ((new_routes, sorted_routes[0]), (actual, expected)):
+        if not (torch.isfinite(result).all() and torch.isfinite(reference).all()):
+            raise RuntimeError("Non-finite MoE output")
+        torch.testing.assert_close(result, reference, rtol=2e-2, atol=0.5)
+        relative_l2 = (result.float() - reference.float()).norm() / reference.float().norm().clamp_min(1e-6)
+        if relative_l2.item() >= 1e-3:
+            raise AssertionError(f"MoE relative L2 {relative_l2.item()} >= 1e-3")
+        errors.append(
+            {"max_abs": (result.float() - reference.float()).abs().max().item(), "relative_l2": relative_l2.item()}
+        )
+    return launches, errors
+
+
+def benchmark(torch, moe, kernels, tokens, args):
+    data = make_inputs(torch, tokens, args)
+    x, probabilities, indices, gate, up, down = data
+    launches, errors = check_and_capture(torch, moe, kernels, data, args)
+    packed_w13 = moe._pack_bf16_w13(gate, up)
+    route_count = tokens * args.heads * args.top_k
+    route_buffers = moe._allocate_bf16_route_buffers(route_count, args.heads * args.experts, 128, x.device)
+
+    def reference():
+        gather, weights, offsets = moe.global_sort_routes(probabilities, indices, args.experts)
+        return moe.triton_mh_moe_forward(x, gather, weights, offsets, gate, up, down, deterministic=True)
+
+    calls = {
+        "reference_deterministic_routed": reference,
+        "candidate_routed": lambda: moe._bf16_fused_moe_forward(
+            x, probabilities, indices, packed_w13, down, route_buffers
+        ),
+    }
+    if args.mode == "dispatch":
+        config = moe.Magi2MultiHeadMoEConfig(
+            args.heads * 256, args.heads, args.experts, args.top_k, 1280, torch.bfloat16
+        )
+        layer = moe.Magi2MultiHeadMoE(config).to(x.device)
+        for parameter, value in ((layer.W_gate, gate), (layer.W_up, up), (layer.W_down, down)):
+            parameter.copy_(value)
+        layer.gate.normal_(std=1 / 16)
+
+        flat_x = x.reshape(tokens, -1)
+
+        def native_dispatch():
+            x_heads = flat_x.view(tokens, args.heads, 256)
+            probs, idx = layer._route(x_heads)
+            gather, weights, offsets = moe.global_sort_routes(probs, idx, args.experts)
+            return moe.triton_mh_moe_forward(
+                x_heads,
+                gather,
+                weights,
+                offsets,
+                layer.W_gate,
+                layer.W_up,
+                layer.W_down,
+                deterministic=args.native_deterministic,
+            ).reshape(tokens, -1)
+
+        calls = {"native_dispatch": native_dispatch, "fused_dispatch": lambda: layer(flat_x)}
+        torch.testing.assert_close(calls["fused_dispatch"](), calls["native_dispatch"](), rtol=2e-2, atol=0.5)
+    timings = measure(torch, calls, args)
+    names = list(calls)
+    result = {"status": "pass", "tokens": tokens, "mode": args.mode, "correctness": errors, "timings": timings}
+    result["speedup_p50_wall"] = (
+        timings[names[0]]["synchronized_wall"]["p50_us"] / timings[names[1]]["synchronized_wall"]["p50_us"]
+    )
+    if args.stages:
+        route_ids = (indices.int() + torch.arange(args.heads, device=x.device)[:, None, None] * args.experts).reshape(
+            -1, args.top_k
+        )
+
+        def gems():
+            for params, kwargs in launches:
+                moe.invoke_fused_moe_bf16(*params, **kwargs)
+
+        result["standalone_stages_not_additive"] = measure(
+            torch,
+            {
+                "reference_sort": lambda: moe.global_sort_routes(probabilities, indices, args.experts),
+                "candidate_align": lambda: moe._align_bf16_routes(route_ids, args.heads * args.experts, 128),
+                "candidate_pack_w13": lambda: torch.stack((gate.transpose(1, 2), up.transpose(1, 2)), dim=2).reshape(
+                    args.heads * args.experts, 2560, 256
+                ),
+                "candidate_two_gemms_prepared": gems,
+            },
+            args,
+        )
+    return result
+
+
+def main():
+    args = parse_args()
+    import torch
+
+    if hasattr(torch.version, "musa") and torch.version.musa is not None:
+        importlib.import_module("torchada")  # before importing vLLM / capturing CUDA APIs
+    cuda = getattr(torch, "cuda")
+    has_gpu = cuda.is_available() or (
+        hasattr(torch.version, "musa") and torch.version.musa is not None and torch.musa.is_available()
+    )
+    if not has_gpu:
+        print(json.dumps({"status": "skip", "reason": "No visible CUDA/MUSA device"}))
+        return
+    import vllm_omni.diffusion.models.magi2.fused_moe_kernels as kernels
+    import vllm_omni.diffusion.models.magi2.mh_moe as moe
+
+    root = Path(__file__).resolve().parents[2]
+    try:
+        sha = subprocess.check_output(
+            ["git", "-C", str(root), "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL
+        ).strip()
+    except (OSError, subprocess.CalledProcessError):
+        sha = os.environ.get("MAGI2_BENCH_SOURCE_SHA", "unknown")
+    report = {
+        "device": cuda.get_device_name(),
+        "sm_count": cuda.get_device_properties(0).multi_processor_count,
+        "source_sha": sha,
+        "module_path": moe.__file__,
+        "torch": torch.__version__,
+        "args": {**vars(args), "output": str(args.output)},
+        "cache": "512 MiB flush outside each region",
+        "capture": "eager; JIT warmed; no graph/compile",
+        "results": [],
+    }
+    with torch.inference_mode():
+        for tokens in args.tokens:
+            result = benchmark(torch, moe, kernels, tokens, args)
+            report["results"].append(result)
+            print(json.dumps(result), flush=True)
+    if args.output:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(json.dumps(report, indent=2) + "\n")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/kernels/benchmark_magi2_swiglu7.py

@@ -0,0 +1,87 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Compare native and fused MAGI-2 SwiGLU7 operators on CUDA or MUSA.
+
+Run from the checkout root in an environment with PyTorch and the selected
+backend dependencies.  torchada is loaded automatically only on MUSA::
+
+    python -m benchmarks.kernels.benchmark_magi2_swiglu7 \
+        --tokens 1 4 16 64 129 4096 --intermediate-size 1280 --num-tests 30
+
+The released MAGI-2 config supplies the 1280 intermediate width. Token counts
+are a synthetic sweep; pass per-rank sizes from the target trace for production
+comparisons. Input is contiguous interleaved gate/up [tokens, 2 * intermediate].
+Both paths evaluate FP32 arithmetic with one cast to the requested output dtype.
+This measures eager operator latency, not compiled serving performance.
+No CUDA/MUSA device (or no PyTorch) produces a JSON skip record and exit status zero.
+"""
+
+import argparse
+from functools import partial
+
+from benchmarks.kernels._magi2_musa_bench import (
+    add_arguments,
+    check_pair,
+    emit,
+    load_musa,
+    measure_pair,
+    report_environment,
+)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    add_arguments(parser)
+    parser.add_argument("--intermediate-size", type=int, default=1280)
+    parser.add_argument("--out-dtype", choices=["bfloat16", "float16", "float32"], default=None)
+    args = parser.parse_args()
+    if args.intermediate_size <= 0:
+        parser.error("intermediate-size must be positive")
+    torch = load_musa(args, parser)
+    if torch is None:
+        return
+    cuda = getattr(torch, "cuda")
+    from vllm_omni.diffusion.layers import swiglu7
+
+    report_environment(torch, args, swiglu7)
+    dtype = getattr(torch, args.dtype)
+    out_dtype = getattr(torch, args.out_dtype or args.dtype)
+    device = torch.device("cuda")
+    rtol = {torch.bfloat16: 1.6e-2, torch.float16: 2e-3, torch.float32: 1e-5}[out_dtype]
+    atol = 1e-5 if out_dtype == torch.float32 else 2e-3
+    op = swiglu7.SwiGLU7()
+    with torch.inference_mode():
+        for tokens in args.tokens:
+            # Values outside both clamp boundaries exercise the released formula.
+            x = (torch.randn(tokens, 2 * args.intermediate_size, device=device) * 10).to(dtype)
+            reference = partial(op.forward_native, x, out_dtype=out_dtype)
+            fused = partial(op, x, out_dtype=out_dtype)
+            parity = check_pair(torch, reference, fused, swiglu7, ("_swiglu7_kernel",), rtol=rtol, atol=atol)
+            timing = measure_pair(torch, reference, fused, args)
+            output_elements = tokens * args.intermediate_size
+            minimum_io_bytes = (
+                x.numel() * x.element_size() + output_elements * torch.empty((), dtype=out_dtype).element_size()
+            )
+            emit(
+                status="pass",
+                op="swiglu7",
+                input_shape=list(x.shape),
+                input_stride=list(x.stride()),
+                dtype=args.dtype,
+                out_dtype=args.out_dtype or args.dtype,
+                alpha=1.702,
+                limit=7.0,
+                minimum_io_bytes=minimum_io_bytes,
+                # Six scalar arithmetic operations and one exp per output;
+                # clamps and transcendental costs are not modeled as FLOPs.
+                scalar_flops_excluding_exp_and_clamps=6 * output_elements,
+                effective_fused_gbps=minimum_io_bytes / timing["fused"]["p50_us"] / 1000,
+                **parity,
+                **timing,
+            )
+            del x, reference, fused
+            cuda.empty_cache()
+
+
+if __name__ == "__main__":
+    main()

benchmarks/mammoth_moda2/README.md

@@ -0,0 +1,124 @@
+# MammothModa2 startup benchmark
+
+Measures `Omni()` initialization and first/subsequent image requests for
+MammothModa2-Preview. This is a measurement baseline for
+[#7075](https://github.com/vllm-project/vllm-omni/issues/7075); it changes no runtime code.
+
+## Run
+
+From the repository root, with the model downloaded and vLLM-Omni installed:
+
+```bash
+set -o pipefail
+python benchmarks/mammoth_moda2/bench_startup.py \
+    --model /path/MammothModa2-Preview \
+    --deploy-config vllm_omni/deploy/mammoth_moda2.yaml \
+    --height 512 --width 512 --seed 42 \
+    --extra-body '{"text_guidance_scale": 4.0, "cfg_range": [0.0, 1.0], "num_inference_steps": 8}' \
+    --repeat 1 --label baseline --output-json baseline.json --save-image baseline.png \
+    2>&1 | tee baseline.log
+python benchmarks/mammoth_moda2/parse_startup_log.py baseline.log --require-complete --markdown
+```
+
+Use `OMP_NUM_THREADS=16` to set CPU threads and `--parallel-stage-init` to
+initialize stages concurrently. For first/subsequent requests, use
+`--height 1024 --width 1024 --repeat 3` and 50 inference steps. Memory polling
+is opt-in (`--sample-memory`); collect it in separate runs to avoid perturbing
+the timing comparison. `--save-image` saves only the first request's image.
+Compare that image across runs; a fixed seed alone does not guarantee equality
+across execution modes or software versions.
+
+## Measurement boundaries
+
+| Metric | Boundary / limitation |
+| --- | --- |
+| Imports | Before torch import → benchmark imports complete; excludes interpreter startup |
+| Time to ready | `Omni()` call → return; excludes client imports and requests |
+| Time to first image | Benchmark clock start → first `generate()` return, including request setup |
+| First / subsequent request | Each `generate()` call → return; same prompt and settings |
+| Stage spawn / device init | Log timestamps: launch → `world_size=` → `Starting to load model`; 1 s resolution |
+| Weight load | Engine-reported `Loading weights took`; read/cast/copy are merged |
+| Model setup estimate | Reported model-load time minus weight-load time; not independently instrumented |
+| Profile + KV + warmup | One merged interval after loading; compilation/capture may be nested within it |
+| Compile / graph capture | Explicit engine log durations when present; N/A for eager execution |
+| Memory | Optional 1 s samples: sum of process-tree RSS and maximum device memory used across GPUs reported by `nvidia-smi`; includes other GPU users and can miss transient peaks |
+
+Stage timelines overlap with parallel initialization; do not sum them to
+calculate time to ready. Missing log fields stay unmeasured. The parser marks
+logs without `BENCH_JSON` as incomplete; `--require-complete` makes this an error.
+
+## Reference results
+
+A800-SXM4-80GB; Ubuntu 22.04; 16-CPU cgroup quota on a 128-core host;
+vLLM 0.28.0+cu129, torch 2.13.0+cu129, vLLM-Omni `34aa1d2a`.
+Checkpoint: 34.5 GiB, 8 shards, mostly fp32, on local NVMe.
+Committed eager deploy config, GPU memory fractions 0.5 / 0.3.
+
+The 2026-09-10 rerun used the corrected benchmark: one warm-up process,
+then three rounds of four configurations with order rotated between rounds.
+The checkpoint page cache was warmed before the matrix; no cache eviction
+was requested during it. Each fresh process sent one 512×512, 8-step request
+(guidance 4.0, seed 42). Memory polling was disabled. All 12 measured runs
+are retained in [the results](reference_results.json).
+Values below are `Omni()` seconds, mean ± sample standard deviation, n=3;
+client imports took approximately another 10 s.
+
+| CPU threads | Serial init | Parallel init |
+| --- | --- | --- |
+| Default (64) | 60.1 ± 0.2 | 41.1 ± 1.0 |
+| 16 | 56.5 ± 0.9 | 36.7 ± 0.2 |
+
+On this configuration, parallel initialization reduced time to ready by about
+32%; combining it with 16 threads reduced it by about 39%. This supports
+trying these settings on similar deployments, not changing defaults for all
+models or hardware. Fixed 512×512 samples matched across all 12 measured runs;
+this is a smoke check, not a broad quality evaluation.
+
+For context, in the earlier serial baseline AR / DiT spent approximately 11 / 11 s in
+spawn and imports, 6 / 6 s in device setup, 6 / 4 s loading weights, and
+5 / 2 s in profile + KV + warmup. About 6 s followed the last stage becoming
+ready. These coarse log intervals locate work; they are not independent
+operator timings.
+
+Separate observations from the earlier experiment (not rerun in this matrix):
+
+- Requested page-cache eviction: 80.8 s; FlashInfer JIT cache removed:
+  129.1 s (means, n=3 each). These are separate cache conditions, not guaranteed
+  cold-machine startup or isolated disk/JIT costs.
+- Three 1024×1024, 50-step eager requests in one process took
+  95.2 / 94.9 / 94.6 s. No large first-request penalty was observed in that run.
+- With AR `enforce_eager: false` and serial init, time to ready was
+  62.5 ± 0.1 s (warm caches, n=3); logged graph capture was 4 s / 0.20 GiB.
+  The model reported torch.compile unsupported. KV capacity fell from about
+  15.7 to 2.0 GiB, and one parallel-init trial failed. The reported non-KV
+  profiling difference is not a measurement of graph allocation. Graph-mode
+  samples differed from eager. This needs separate memory and quality analysis
+  before recommending the configuration.
+
+## Loading diagnostic and storage scenarios
+
+```bash
+OMP_NUM_THREADS=16 python benchmarks/mammoth_moda2/raw_load_bench.py \
+    --model /path/MammothModa2-Preview --shards 6,7,8 --mode cpu_cast
+# Compare --mode gpu_cast in a fresh process with the same cache conditions.
+
+MODEL_LOCAL=/local/MammothModa2-Preview OUT_DIR=./startup-bench \
+    bash benchmarks/mammoth_moda2/bench_storage_scenarios.sh
+# Optional MODEL_NFS=/nfs/MammothModa2-Preview; SCENARIOS="local-cold local-warm".
+```
+
+The diagnostic clones each mmap-backed tensor into owned CPU memory before
+casting. `materialize_s` includes page faults, CPU allocation and copying;
+`cast_s` measures dtype conversion; `h2d_s` includes GPU allocation and copy.
+CUDA context/operation warmup is outside the timer. Total also includes file
+opening and bookkeeping. This extra CPU copy differs from the engine loader,
+so these numbers do not decompose its loading time or measure disk bandwidth.
+The revised diagnostic was run three times each for CPU cast with 64 / 16
+threads and GPU cast with 16 threads; per-run values are in the results file.
+Both CPU materialization and conversion were faster with 16 threads here; this
+does not isolate how much of the engine-level improvement comes from either.
+
+Storage scenarios explicitly read shards before warm runs and request
+`posix_fadvise(DONTNEED)` before cold runs. Eviction is not guaranteed, and
+server-side NFS caches are uncontrolled. The script stops on failure and
+summarizes only logs from the current invocation.

benchmarks/mammoth_moda2/__init__.py

@@ -1,4 +1,2 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
-
-"""Qwen3-Omni turn-based browser call UI."""

benchmarks/mammoth_moda2/bench_startup.py

@@ -0,0 +1,338 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Startup / loading benchmark for multi-stage omni models (MammothModa2 first).
+
+Measures, from a single fresh process:
+  * python import time (torch + vllm_omni)
+  * Omni() construction wall time (all stages: weight load, profiling, KV cache, warmup)
+  * first request latency (includes JIT / lazy init that only happens on first run)
+  * steady-state request latency (repeat N-1 more requests)
+  * optional host RSS / device memory samples (--sample-memory, once a second)
+
+Per-stage breakdown (weight loading, model-load memory, engine init) is emitted by the
+stage engine-core subprocesses into the log; use parse_startup_log.py on the captured
+stdout/stderr to merge them with the JSON written here.
+
+Usage (keep the log: the per-stage breakdown is parsed from it afterwards):
+  python benchmarks/mammoth_moda2/bench_startup.py --model /path/MammothModa2-Preview \
+      --deploy-config vllm_omni/deploy/mammoth_moda2.yaml \
+      --height 1024 --width 1024 --seed 42 \
+      --extra-body '{"text_guidance_scale": 4.0, "cfg_range": [0.0, 1.0], "num_inference_steps": 50}' \
+      --repeat 2 --label nfs-warm --output-json out.json 2>&1 | tee run.log
+  python benchmarks/mammoth_moda2/parse_startup_log.py run.log --markdown
+"""
+
+import argparse
+import json
+import os
+import platform
+import subprocess
+import threading
+import time
+
+T_PROCESS_START = time.perf_counter()
+
+import torch  # noqa: E402
+
+from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args  # noqa: E402
+from vllm_omni.entrypoints.omni import Omni  # noqa: E402
+from vllm_omni.entrypoints.openai.stage_params import clone_sampling_params  # noqa: E402
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams  # noqa: E402
+from vllm_omni.model_extras import (  # noqa: E402
+    build_text_to_image_prompt as build_model_text_to_image_prompt,
+)
+from vllm_omni.model_extras import (  # noqa: E402
+    get_extra_body_params,
+    get_model_class_name,
+    should_init_extra_args_for_non_diffusion_stages,
+)
+from vllm_omni.platforms import current_omni_platform  # noqa: E402
+
+T_IMPORTS_DONE = time.perf_counter()
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--model", required=True)
+    p.add_argument("--deploy-config", default=None)
+    p.add_argument("--prompt", default="A stylish woman riding a motorcycle in NYC, movie poster style")
+    p.add_argument("--negative-prompt", default=None)
+    p.add_argument("--height", type=int, default=1024)
+    p.add_argument("--width", type=int, default=1024)
+    p.add_argument("--seed", type=int, default=42)
+    p.add_argument("--extra-body", type=json.loads, default=None, help="JSON object forwarded as model extra_body")
+    p.add_argument(
+        "--repeat", type=int, default=2, help="number of generate() calls; 1st = first-request, rest = steady"
+    )
+    p.add_argument("--label", default="run", help="free-form label, e.g. nfs-cold / nfs-warm / local-nvme")
+    p.add_argument("--output-json", default=None)
+    p.add_argument("--save-image", default=None, help="optional path to save the first request's image")
+    p.add_argument("--sample-memory", action="store_true", help="sample memory once a second (adds overhead)")
+    p.add_argument(
+        "--parallel-stage-init",
+        action="store_true",
+        help="initialize stages that share a GPU concurrently (VllmOmniOrchestratorConfig.parallel_stage_init)",
+    )
+    args = p.parse_args()
+    if args.repeat < 1:
+        p.error("--repeat must be at least 1")
+    return args
+
+
+def gpu_info() -> dict:
+    info = {"torch": torch.__version__, "cuda": torch.version.cuda, "python": platform.python_version()}
+    try:
+        import vllm
+
+        info["vllm"] = vllm.__version__
+    except Exception:  # pragma: no cover
+        pass
+    try:
+        import vllm_omni
+
+        info["vllm_omni"] = getattr(vllm_omni, "__version__", "src")
+    except Exception:  # pragma: no cover
+        pass
+    try:
+        out = subprocess.run(
+            ["nvidia-smi", "--query-gpu=name,driver_version,memory.total", "--format=csv,noheader"],
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+        info["gpu"] = out.stdout.strip()
+    except Exception:
+        info["gpu"] = "n/a"
+    return info
+
+
+class PeakSampler:
+    """Poll host RSS of this process tree and GPU memory in use once a second (benchmark-side only)."""
+
+    def __init__(self, interval_s: float = 1.0):
+        self.interval_s = interval_s
+        self.host_rss_peak_gib = 0.0
+        self.gpu_used_peak_gib = 0.0
+        self._stop = threading.Event()
+        self._thread = threading.Thread(target=self._run, daemon=True)
+
+    def _sample(self) -> None:
+        try:
+            import psutil
+
+            root = psutil.Process()
+            rss = root.memory_info().rss + sum(
+                c.memory_info().rss for c in root.children(recursive=True) if c.is_running()
+            )
+            self.host_rss_peak_gib = max(self.host_rss_peak_gib, rss / 2**30)
+        except Exception:
+            pass
+        try:
+            out = subprocess.run(
+                ["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"],
+                capture_output=True,
+                text=True,
+                timeout=5,
+            )
+            used = max(float(x) for x in out.stdout.split() if x.strip())
+            self.gpu_used_peak_gib = max(self.gpu_used_peak_gib, used / 1024)
+        except Exception:
+            pass
+
+    def _run(self) -> None:
+        while not self._stop.is_set():
+            self._sample()
+            self._stop.wait(self.interval_s)
+
+    def start(self) -> "PeakSampler":
+        self._thread.start()
+        return self
+
+    def stop(self) -> dict:
+        self._stop.set()
+        self._thread.join(timeout=10)
+        self._sample()
+        return {
+            "host_rss_peak_gib": round(self.host_rss_peak_gib, 2),
+            "gpu_used_peak_gib": round(self.gpu_used_peak_gib, 2),
+        }
+
+
+def fs_type(path: str) -> str:
+    try:
+        out = subprocess.run(["stat", "-f", "-c", "%T", path], capture_output=True, text=True, timeout=10)
+        return out.stdout.strip()
+    except Exception:
+        return "unknown"
+
+
+def build_request(omni: Omni, args: argparse.Namespace, generator: torch.Generator):
+    model_class_name = get_model_class_name(omni)
+    declared = get_extra_body_params(model_class_name)
+
+    prompt_dict = {"prompt": args.prompt}
+    if args.negative_prompt:
+        prompt_dict["negative_prompt"] = args.negative_prompt
+    prompt_dict = build_model_text_to_image_prompt(
+        model_class_name=model_class_name, prompt=prompt_dict, height=args.height, width=args.width
+    )
+
+    diffusion_params = OmniDiffusionSamplingParams(
+        height=args.height,
+        width=args.width,
+        seed=args.seed,
+        generator=generator,
+    )
+    user_extra = dict(args.extra_body or {})
+    if declared:
+        apply_declared_extra_args(diffusion_params, declared, user_extra)
+    else:
+        diffusion_params.extra_args.update({k: v for k, v in user_extra.items() if v is not None})
+
+    init_non_diffusion = should_init_extra_args_for_non_diffusion_stages(model_class_name)
+    sampling_params_list = [clone_sampling_params(p) for p in (omni.default_sampling_params_list or [])]
+    if not sampling_params_list:
+        sampling_params_list = [diffusion_params]
+
+    replaced = False
+    for idx, params in enumerate(sampling_params_list):
+        if isinstance(params, OmniDiffusionSamplingParams):
+            sampling_params_list[idx] = diffusion_params
+            replaced = True
+        elif init_non_diffusion and hasattr(params, "extra_args"):
+            params.extra_args = {**(params.extra_args or {}), **(diffusion_params.extra_args or {})}
+            if hasattr(params, "seed"):
+                params.seed = args.seed
+            # MammothModa2 AR stage: one visual token per grid cell + one EOL per row + 1 look-ahead.
+            info = prompt_dict.get("additional_information", {})
+            if idx == 0 and info.get("omni_task") == ["t2i"]:
+                ar_w = int(info.get("ar_width", [0])[0])
+                ar_h = int(info.get("ar_height", [0])[0])
+                if ar_w > 0 and ar_h > 0:
+                    params.max_tokens = ar_h * (ar_w + 1) + 1
+    if not replaced and len(sampling_params_list) == 1:
+        sampling_params_list = [diffusion_params]
+    return prompt_dict, sampling_params_list
+
+
+def stage_metrics(outputs) -> dict:
+    """Pull vllm-omni's per-stage timing out of the request output when present."""
+    res = {}
+    for out in outputs or []:
+        metrics = getattr(out, "metrics", None)
+        if isinstance(metrics, dict) and "stage_metrics" in metrics:
+            for sid, m in metrics["stage_metrics"].items():
+                if not isinstance(m, dict):
+                    continue
+                res[str(sid)] = {
+                    k: m.get(k)
+                    for k in ("stage_gen_time_ms", "num_tokens_in", "num_tokens_out", "vllm_ttft_ms", "vllm_tpot_ms")
+                    if k in m
+                }
+    return res
+
+
+def main() -> None:
+    args = parse_args()
+    result = {
+        "label": args.label,
+        "model": args.model,
+        "prompt": args.prompt,
+        "negative_prompt": args.negative_prompt,
+        "sample_memory": args.sample_memory,
+        "model_fs": fs_type(args.model),
+        "deploy_config": args.deploy_config,
+        "height": args.height,
+        "width": args.width,
+        "seed": args.seed,
+        "extra_body": args.extra_body,
+        "parallel_stage_init": args.parallel_stage_init,
+        "omp_num_threads": os.environ.get("OMP_NUM_THREADS"),
+        "env": gpu_info(),
+        "phases_s": {"imports": round(T_IMPORTS_DONE - T_PROCESS_START, 3)},
+        "requests": [],
+    }
+
+    omni_kwargs = {"model": args.model, "mode": "text-to-image", "log_stats": True}
+    if args.deploy_config:
+        omni_kwargs["deploy_config"] = args.deploy_config
+    if args.parallel_stage_init:
+        omni_kwargs["parallel_stage_init"] = True
+
+    sampler = PeakSampler().start() if args.sample_memory else None
+    t0 = time.perf_counter()
+    omni = None
+    try:
+        omni = Omni(**omni_kwargs)
+        t1 = time.perf_counter()
+        result["peak_mem_startup"] = sampler.stop() if sampler else {}
+        sampler = None
+        result["phases_s"]["engine_init"] = round(t1 - t0, 3)
+        result["phases_s"]["process_to_engine_ready"] = round(t1 - T_PROCESS_START, 3)
+        print(
+            f"BENCH engine_init={t1 - t0:.2f}s process_to_engine_ready={t1 - T_PROCESS_START:.2f}s "
+            f"peak_mem_startup={result['peak_mem_startup']}",
+            flush=True,
+        )
+        sampler = PeakSampler().start() if args.sample_memory else None
+
+        generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
+        outputs = None
+        first_outputs = None
+        first_image_elapsed = 0.0
+        for i in range(args.repeat):
+            prompt_dict, sampling_params_list = build_request(omni, args, generator)
+            tr0 = time.perf_counter()
+            outputs = omni.generate(prompt_dict, sampling_params_list=sampling_params_list)
+            tr1 = time.perf_counter()
+            if first_outputs is None:
+                first_outputs = outputs
+                first_image_elapsed = tr1 - T_PROCESS_START
+            rec = {
+                "index": i,
+                "kind": "first" if i == 0 else "steady",
+                "latency_s": round(tr1 - tr0, 3),
+                "stages": stage_metrics(outputs),
+            }
+            result["requests"].append(rec)
+            print(f"BENCH request[{i}] {rec['kind']} latency={tr1 - tr0:.2f}s stages={rec['stages']}", flush=True)
+
+        result["peak_mem_requests"] = sampler.stop() if sampler else {}
+        sampler = None
+        steady = [r["latency_s"] for r in result["requests"] if r["kind"] == "steady"]
+        result["summary_s"] = {
+            "imports": result["phases_s"]["imports"],
+            "engine_init": result["phases_s"]["engine_init"],
+            "first_request": result["requests"][0]["latency_s"],
+            "steady_request_avg": round(sum(steady) / len(steady), 3) if steady else None,
+            "time_to_first_image_from_process_start": round(first_image_elapsed, 3),
+        }
+
+        for key in ("host_rss_peak_gib", "gpu_used_peak_gib"):
+            result["summary_s"][key] = (
+                max(result["peak_mem_startup"][key], result["peak_mem_requests"][key]) if args.sample_memory else None
+            )
+
+        from vllm_omni.diffusion.utils.image_output import extract_images_from_outputs
+
+        images = extract_images_from_outputs(first_outputs)
+        if not images:
+            raise RuntimeError("First request returned no image")
+        if args.save_image:
+            os.makedirs(os.path.dirname(os.path.abspath(args.save_image)), exist_ok=True)
+            images[0].save(args.save_image)
+
+        print("BENCH_JSON " + json.dumps(result), flush=True)
+        if args.output_json:
+            with open(args.output_json, "w") as f:
+                json.dump(result, f, indent=2)
+            print(f"BENCH wrote {args.output_json}", flush=True)
+    finally:
+        if sampler:
+            sampler.stop()
+        if omni is not None:
+            omni.close()
+
+
+if __name__ == "__main__":
+    main()

benchmarks/mammoth_moda2/bench_storage_scenarios.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+# MODEL_LOCAL=/local/model MODEL_NFS=/nfs/model OUT_DIR=./startup-bench bash "$0"
+# Optional SCENARIOS: local-cold local-warm nfs-cold nfs-warm.
+set -euo pipefail
+HERE=$(cd "$(dirname "$0")" && pwd)
+OUT_DIR=${OUT_DIR:-./startup-bench}
+DEPLOY_CONFIG=${DEPLOY_CONFIG:-vllm_omni/deploy/mammoth_moda2.yaml}
+STEPS=${STEPS:-50}
+SIZE=${SIZE:-1024}
+REPEAT=${REPEAT:-2}
+SEED=${SEED:-42}
+EXTRA=${EXTRA:-'{"text_guidance_scale": 4.0, "cfg_range": [0.0, 1.0], "num_inference_steps": '$STEPS'}'}
+mkdir -p "$OUT_DIR/logs" "$OUT_DIR/json" "$OUT_DIR/images"
+rm -f "$OUT_DIR/summary.json" "$OUT_DIR/summary.md"
+
+prepare_cache() {
+  python - "$1" "$2" <<'PY'
+import os
+import sys
+from pathlib import Path
+
+files = sorted(Path(sys.argv[1]).glob("*.safetensors"))
+if not files:
+    raise SystemExit("no checkpoint shards found")
+for path in files:
+    with path.open("rb") as f:
+        if sys.argv[2] == "cold":
+            os.posix_fadvise(f.fileno(), 0, 0, os.POSIX_FADV_DONTNEED)
+        else:
+            while f.read(8 * 1024 * 1024):
+                pass
+print("[bench]", "requested eviction of" if sys.argv[2] == "cold" else "read", len(files), "shards")
+PY
+}
+
+default_scenarios=""
+if [ -n "${MODEL_LOCAL:-}" ]; then default_scenarios="local-cold local-warm"; fi
+if [ -n "${MODEL_NFS:-}" ]; then default_scenarios="$default_scenarios nfs-cold nfs-warm"; fi
+read -r -a scenarios <<< "${SCENARIOS:-$default_scenarios}"
+if [ "${#scenarios[@]}" -eq 0 ]; then echo "Set MODEL_LOCAL or MODEL_NFS" >&2; exit 1; fi
+logs=()
+for label in "${scenarios[@]}"; do
+  case "$label" in
+    local-cold|local-warm) model=${MODEL_LOCAL:?Set MODEL_LOCAL} ;;
+    nfs-cold|nfs-warm) model=${MODEL_NFS:?Set MODEL_NFS} ;;
+    *) echo "Unknown scenario: $label" >&2; exit 1 ;;
+  esac
+  prepare_cache "$model" "${label##*-}"
+  log="$OUT_DIR/logs/$label.log"
+  # Remove previous outputs for this label so a failed rerun cannot leave stale results.
+  rm -f "$OUT_DIR/json/$label.json" "$OUT_DIR/images/$label.png"
+  if python "$HERE/bench_startup.py" --model "$model" --deploy-config "$DEPLOY_CONFIG" \
+    --height "$SIZE" --width "$SIZE" --seed "$SEED" --extra-body "$EXTRA" --repeat "$REPEAT" \
+    --label "$label" --output-json "$OUT_DIR/json/$label.json" --save-image "$OUT_DIR/images/$label.png" \
+    > "$log" 2>&1; then
+    logs+=("$log")
+  else
+    rc=$?
+    echo "[bench] $label failed (exit $rc); see $log" >&2
+    exit "$rc"
+  fi
+done
+python "$HERE/parse_startup_log.py" "${logs[@]}" --require-complete --markdown \
+  --json-out "$OUT_DIR/summary.json" | tee "$OUT_DIR/summary.md"
+echo BENCH_ALL_DONE

benchmarks/mammoth_moda2/parse_startup_log.py

@@ -0,0 +1,245 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Merge per-stage startup numbers from a bench_startup.py log with its BENCH_JSON line.
+
+The stage engine cores run in subprocesses and report weight-loading / init timings
+only through log lines such as:
+
+  (StageEngineCoreProc_stage0_replica0 pid=1) INFO ... Loading weights took 39.66 seconds
+  (StageEngineCoreProc_stage0_replica0 pid=1) INFO ... Model loading took 21.45 GiB memory and 40.59 seconds
+  (StageEngineCoreProc_stage0_replica0 pid=1) INFO ... init engine (profile, create kv cache, warmup model) took 7.00 s
+  (StageEngineCoreProc_stage0_replica0 pid=1) INFO ... Filesystem type for checkpoints: NFS4.
+      Checkpoint size: 34.52 GiB. Available RAM: 22.76 GiB.
+  INFO ... [Omni] AsyncOmniEngine initialized in 159.74 seconds
+
+Usage:
+  python benchmarks/mammoth_moda2/parse_startup_log.py run.log [more.log ...] [--markdown] [--json-out summary.json]
+"""
+
+import argparse
+import json
+import re
+import sys
+from datetime import datetime
+from typing import Any
+
+STAGE = r"\(StageEngineCoreProc_stage(?P<stage>\d+)_replica\d+ pid=\d+\)"
+PATTERNS = {
+    "weights_load_s": re.compile(STAGE + r".*Loading weights took (?P<v>[\d.]+) seconds"),
+    "model_load_s": re.compile(STAGE + r".*Model loading took (?P<mem>[\d.]+) GiB memory and (?P<v>[\d.]+) seconds"),
+    "engine_init_s": re.compile(
+        STAGE + r".*init engine \(profile, create kv cache, warmup model\) took (?P<v>[\d.]+) s"
+    ),
+    # Only present when the stage does not run with enforce_eager.
+    "graph_capture_s": re.compile(
+        STAGE + r".*Graph capturing finished in (?P<v>[\d.]+) secs, took (?P<mem>[\d.]+) GiB"
+    ),
+    "compile_s": re.compile(STAGE + r".*torch\.compile takes (?P<v>[\d.]+) s in total"),
+    "ckpt_fs": re.compile(
+        STAGE
+        + r".*Filesystem type for checkpoints: (?P<fs>\w+)\. Checkpoint size: (?P<size>[\d.]+) GiB\."
+        + r" Available RAM: (?P<ram>[\d.]+) GiB"
+    ),
+}
+ENGINE_READY = re.compile(r"AsyncOmniEngine initialized in (?P<v>[\d.]+) seconds")
+PREFETCH_SKIP = re.compile(STAGE + r".*exceeds 90% of available RAM")
+COMPILE_UNSUPPORTED = re.compile(STAGE + r".*`torch\.compile` is turned on, but the model .* does not support it")
+
+# Startup timeline: (milestone key, regex). Timestamps in vllm logs are "MM-DD HH:MM:SS" (1 s resolution).
+TS = re.compile(r"(?:INFO|WARNING|ERROR) (?P<ts>\d\d-\d\d \d\d:\d\d:\d\d)")
+MILESTONES = [
+    ("omni_init_start", re.compile(r"\[Omni\] Initializing with model")),
+    ("stage{stage}_launch", re.compile(r"Stage-(?P<stage>\d+) set runtime devices")),
+    ("stage{stage}_proc_up", re.compile(STAGE + r".*world_size=\d+ rank=\d+")),
+    ("stage{stage}_load_start", re.compile(STAGE + r".*Starting to load model")),
+    ("stage{stage}_load_done", re.compile(STAGE + r".*Loading weights took")),
+    ("stage{stage}_init_done", re.compile(STAGE + r".*init engine \(profile, create kv cache, warmup model\) took")),
+    ("stage{stage}_ready", re.compile(r"\[StageRuntime\] Stage (?P<stage>\d+) initialized")),
+    ("engine_ready", re.compile(r"AsyncOmniEngine initialized in")),
+]
+
+
+def _ts_seconds(ts: str) -> int:
+    # Use a leap year because engine timestamps do not include the year.
+    value = datetime.strptime("2000-" + ts, "%Y-%m-%d %H:%M:%S")
+    return int((value - datetime(2000, 1, 1)).total_seconds())
+
+
+def timeline(lines: list[str]) -> dict:
+    """Return milestone timestamps (s, relative to omni_init_start) and derived per-stage phase durations."""
+    marks: dict[str, int] = {}
+    for line in lines:
+        tsm = TS.search(line)
+        if not tsm:
+            continue
+        for key, pat in MILESTONES:
+            m = pat.search(line)
+            if m:
+                k = key.format(stage=m.group("stage")) if "{stage}" in key else key
+                marks.setdefault(k, _ts_seconds(tsm.group("ts")))
+                break
+    if "omni_init_start" not in marks:
+        return {}
+    t0 = marks["omni_init_start"]
+    rel = {k: (v - t0) % (366 * 86400) for k, v in marks.items()}
+    phases: dict[str, Any] = {}
+    names = ("launch", "proc_up", "load_start", "load_done", "init_done", "ready")
+    for s in sorted({k.split("_")[0] for k in rel if k.startswith("stage")}):
+        found = {name: rel.get(f"{s}_{name}") for name in names}
+        if any(v is None for v in found.values()):
+            continue
+        ts = {name: int(v) for name, v in found.items() if v is not None}
+        phases[s] = {
+            "spawn_import_config_s": ts["proc_up"] - ts["launch"],
+            "device_dist_init_s": ts["load_start"] - ts["proc_up"],
+            "model_and_weights_load_s": ts["load_done"] - ts["load_start"],
+            "profile_kv_warmup_s": ts["init_done"] - ts["load_done"],
+            "ready_handoff_s": ts["ready"] - ts["init_done"],
+            "total_s": ts["ready"] - ts["launch"],
+        }
+    if "engine_ready" in rel:
+        last_ready = max((v for k, v in rel.items() if k.endswith("_ready") and k != "engine_ready"), default=0)
+        phases["post_stages_wiring_s"] = rel["engine_ready"] - last_ready
+        phases["engine_ready_s"] = rel["engine_ready"]
+    return {"marks_rel_s": rel, "phases": phases}
+
+
+def parse(path: str) -> dict:
+    stages: dict[str, dict] = {}
+    rec: dict[str, Any] = {"log": path, "stages": stages}
+    with open(path, errors="replace") as f:
+        lines = f.readlines()
+    rec["timeline"] = timeline(lines)
+    for line in lines:
+        if line.startswith("BENCH_JSON "):
+            rec["bench"] = json.loads(line[len("BENCH_JSON ") :])
+            continue
+        m = ENGINE_READY.search(line)
+        if m:
+            rec["engine_ready_s_reported"] = float(m.group("v"))
+            continue
+        m = PREFETCH_SKIP.search(line)
+        if m:
+            stages.setdefault(m.group("stage"), {})["page_cache_prefetch_skipped"] = True
+            continue
+        m = COMPILE_UNSUPPORTED.search(line)
+        if m:
+            stages.setdefault(m.group("stage"), {})["torch_compile_unsupported"] = True
+            continue
+        for key, pat in PATTERNS.items():
+            m = pat.search(line)
+            if not m:
+                continue
+            st = stages.setdefault(m.group("stage"), {})
+            if key == "ckpt_fs":
+                st["checkpoint_fs"] = m.group("fs")
+                st["checkpoint_size_gib"] = float(m.group("size"))
+                st["available_ram_gib"] = float(m.group("ram"))
+            elif key == "graph_capture_s":
+                st["graph_capture_s"] = float(m.group("v"))
+                st["graph_capture_mem_gib"] = float(m.group("mem"))
+            elif key == "model_load_s":
+                st["model_load_s"] = float(m.group("v"))
+                st["model_load_mem_gib"] = float(m.group("mem"))
+                # "Model loading took" covers model construction + weight loading; "Loading weights took" only
+                # the weights. The difference also includes loader setup/teardown overhead.
+                if "weights_load_s" in st:
+                    st["model_setup_estimate_s"] = round(st["model_load_s"] - st["weights_load_s"], 2)
+            else:
+                st[key] = float(m.group("v"))
+            break
+    rec["status"] = "complete" if "bench" in rec else "incomplete"
+    return rec
+
+
+def _table(columns: dict[str, str], records: list[dict]) -> str:
+    def row(values):
+        return "| " + " | ".join("" if v is None else str(v) for v in values) + " |"
+
+    lines = [row(columns.values()), row(["---"] * len(columns))]
+    lines.extend(row(record.get(key) for key in columns) for record in records)
+    return "\n".join(lines)
+
+
+def to_markdown(recs: list[dict]) -> str:
+    summary, stages, phases, warnings = [], [], [], []
+    for r in recs:
+        b = r.get("bench", {})
+        label = b.get("label", r["log"])
+        if r.get("status") == "incomplete":
+            warnings.append(f"Incomplete log: {r['log']} (no BENCH_JSON).")
+        summary.append({"label": label, **b.get("summary_s", {})})
+        for sid, st in sorted(r["stages"].items()):
+            stages.append({"label": label, "stage": sid, **st})
+            if st.get("torch_compile_unsupported"):
+                stages[-1]["compile_s"] = "unsupported"
+        ph = r.get("timeline", {}).get("phases", {})
+        for sid, values in sorted(ph.items()):
+            if sid.startswith("stage"):
+                phases.append({"label": label, "stage": sid, **values})
+        if "post_stages_wiring_s" in ph:
+            phases.append({"label": label, "stage": "after last stage", "total_s": ph["post_stages_wiring_s"]})
+    tables = [
+        (
+            {
+                "label": "label",
+                "imports": "imports (s)",
+                "engine_init": "engine ready (s)",
+                "first_request": "first request (s)",
+                "steady_request_avg": "subsequent avg (s)",
+                "time_to_first_image_from_process_start": "process→first image (s)",
+                "host_rss_peak_gib": "sampled host RSS (GiB)",
+                "gpu_used_peak_gib": "sampled GPU used (GiB)",
+            },
+            summary,
+        ),
+        (
+            {
+                "label": "label",
+                "stage": "stage",
+                "model_setup_estimate_s": "model setup estimate (s)",
+                "weights_load_s": "weights load (s)",
+                "compile_s": "compile (s)",
+                "graph_capture_s": "graph capture (s)",
+                "engine_init_s": "profile/KV/warmup reported (s)",
+            },
+            stages,
+        ),
+        (
+            {
+                "label": "label",
+                "stage": "stage",
+                "spawn_import_config_s": "spawn/import (s)",
+                "device_dist_init_s": "device init (s)",
+                "model_and_weights_load_s": "model+weights load (s)",
+                "profile_kv_warmup_s": "profile/KV/warmup interval (s)",
+                "total_s": "total (s)",
+            },
+            phases,
+        ),
+    ]
+    return "\n\n".join(warnings + [_table(columns, rows) for columns, rows in tables])
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("logs", nargs="+")
+    ap.add_argument("--markdown", action="store_true")
+    ap.add_argument("--json-out", default=None)
+    ap.add_argument("--require-complete", action="store_true", help="fail if any log has no BENCH_JSON")
+    args = ap.parse_args()
+    recs = [parse(p) for p in args.logs]
+    if args.require_complete and any(r["status"] != "complete" for r in recs):
+        ap.error("incomplete benchmark log: missing BENCH_JSON")
+    if args.json_out:
+        with open(args.json_out, "w") as f:
+            json.dump(recs, f, indent=2)
+    if args.markdown:
+        print(to_markdown(recs))
+    else:
+        json.dump(recs, sys.stdout, indent=2)
+        print()
+
+
+if __name__ == "__main__":
+    main()

benchmarks/mammoth_moda2/raw_load_bench.py

@@ -0,0 +1,127 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Raw safetensors -> GPU micro-benchmark, without vLLM's loader.
+
+Compares CPU and GPU dtype conversion after materializing each mmap-backed tensor.
+Materialization includes page faults, CPU allocation and copying; it is not a disk
+bandwidth measurement. CUDA setup is excluded; transfer includes GPU allocation.
+
+  --mode cpu_cast         materialize -> cast to bf16 on the CPU -> copy to GPU
+  --mode gpu_cast         materialize -> copy source dtype to GPU -> cast on the GPU
+
+  python benchmarks/mammoth_moda2/raw_load_bench.py --model /path/MammothModa2-Preview --shards 6,7,8
+  OMP_NUM_THREADS=4 python benchmarks/mammoth_moda2/raw_load_bench.py --model /path --shards 6,7,8 --mode cpu_cast
+"""
+
+import argparse
+import json
+import os
+import re
+import time
+
+import torch
+from safetensors import safe_open
+
+SHARD_NO = re.compile(r"-(\d+)-of-\d+\.safetensors$")
+
+
+def shard_files(model: str, shards: str) -> list[str]:
+    """Resolve ``--shards`` to files, refusing empty or partial matches so a typo cannot pass as a fast load."""
+    files = sorted(f for f in os.listdir(model) if f.endswith(".safetensors"))
+    if not files:
+        raise SystemExit(f"no .safetensors files under {model}")
+    if shards != "all":
+        want = {int(x) for x in shards.split(",")}
+        found = {int(m.group(1)): f for f in files if (m := SHARD_NO.search(f))}
+        missing = sorted(want - found.keys())
+        if missing:
+            raise SystemExit(f"shard(s) {missing} not found under {model}; available: {sorted(found)}")
+        files = [found[n] for n in sorted(want)]
+    return [os.path.join(model, f) for f in files]
+
+
+def sync() -> None:
+    if torch.cuda.is_available():
+        torch.accelerator.synchronize()
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--model", required=True)
+    ap.add_argument("--shards", default="all", help="comma list of 1-based shard numbers, or 'all'")
+    ap.add_argument("--mode", choices=("cpu_cast", "gpu_cast"), default="cpu_cast")
+    ap.add_argument("--label", default="raw")
+    ap.add_argument("--no-cuda", action="store_true", help="read + CPU cast only (mode must be cpu_cast)")
+    a = ap.parse_args()
+    if a.no_cuda and a.mode != "cpu_cast":
+        ap.error("--no-cuda only makes sense with --mode cpu_cast")
+
+    paths = shard_files(a.model, a.shards)
+    total_bytes = sum(os.path.getsize(p) for p in paths)
+
+    if not a.no_cuda:
+        # Initialize the context and exercise both operations outside the timer.
+        warmup = torch.zeros(1).to("cuda")
+        warmup = warmup.to(torch.bfloat16)
+        sync()
+        del warmup
+
+    t0 = time.perf_counter()
+    n_tensors, cpu_bytes = 0, 0
+    dtypes: dict[str, int] = {}
+    t_materialize = t_cast = t_h2d = 0.0
+    for p in paths:
+        with safe_open(p, framework="pt", device="cpu") as f:
+            for k in f.keys():
+                r0 = time.perf_counter()
+                t = f.get_tensor(k).clone()  # touch every page and own the CPU storage
+                t_materialize += time.perf_counter() - r0
+                dtypes[str(t.dtype)] = dtypes.get(str(t.dtype), 0) + 1
+                cpu_bytes += t.numel() * t.element_size()
+                needs_cast = t.is_floating_point() and t.dtype != torch.bfloat16
+                if a.mode == "cpu_cast":
+                    c0 = time.perf_counter()
+                    if needs_cast:
+                        t = t.to(torch.bfloat16)
+                    t_cast += time.perf_counter() - c0
+                    if not a.no_cuda:
+                        h0 = time.perf_counter()
+                        g = t.to("cuda")
+                        sync()
+                        t_h2d += time.perf_counter() - h0
+                        del g
+                else:
+                    h0 = time.perf_counter()
+                    g = t.to("cuda")
+                    sync()
+                    t_h2d += time.perf_counter() - h0
+                    c0 = time.perf_counter()
+                    if needs_cast:
+                        g = g.to(torch.bfloat16)
+                    sync()
+                    t_cast += time.perf_counter() - c0
+                    del g
+                n_tensors += 1
+                del t
+    total = time.perf_counter() - t0
+    res = {
+        "label": a.label,
+        "mode": a.mode,
+        "files": [os.path.basename(p) for p in paths],
+        "file_bytes_gib": round(total_bytes / 2**30, 2),
+        "tensor_bytes_gib": round(cpu_bytes / 2**30, 2),
+        "n_tensors": n_tensors,
+        "dtypes": dtypes,
+        "torch_threads": torch.get_num_threads(),
+        "cpu_affinity": len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else None,
+        "materialize_s": round(t_materialize, 3),
+        "cast_s": round(t_cast, 3),
+        "h2d_s": round(t_h2d, 3),
+        "total_s": round(total, 3),
+        "cuda_setup_included": False,
+    }
+    print("RAWLOAD_JSON " + json.dumps(res))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/mammoth_moda2/reference_results.json

@@ -0,0 +1,39 @@
+{
+  "date": "2026-09-10",
+  "runtime_commit": "34aa1d2afa59547945fc59f88ab6f4f25f52a777",
+  "environment": {"torch": "2.13.0+cu129", "cuda": "12.9", "python": "3.12.14", "vllm": "0.28.0", "vllm_omni": "0.28.1.dev81+g34aa1d2af.d20260908", "gpu": "NVIDIA A800-SXM4-80GB, 580.126.09, 81920 MiB"},
+  "method": "One excluded warm-up/canary; warm page cache; three rounds of the 2x2 eager configurations with rotated order; fresh process for every run; memory polling disabled.",
+  "request": {"height": 512, "width": 512, "seed": 42, "steps": 8, "guidance": 4.0, "prompt": "A stylish woman riding a motorcycle in NYC, movie poster style"},
+  "gpu_memory_utilization": [0.5, 0.3],
+  "cpu_quota": 16,
+  "startup_columns": ["configuration", "round", "engine_init_s", "imports_s", "first_request_s", "time_to_first_image_s", "image_md5"],
+  "startup_runs": [
+    ["default-serial", 1, 60.022, 10.158, 20.827, 91.089, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["default-serial", 2, 60.015, 10.082, 20.766, 90.909, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["default-serial", 3, 60.34, 9.911, 21.089, 91.381, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["omp-serial", 1, 55.45, 9.965, 20.711, 86.169, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["omp-serial", 2, 57.01, 9.667, 20.697, 87.418, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["omp-serial", 3, 57.044, 9.707, 20.415, 87.211, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["default-parallel", 1, 40.258, 10.112, 20.392, 70.829, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["default-parallel", 2, 42.177, 10.135, 20.692, 73.093, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["default-parallel", 3, 40.848, 9.997, 20.995, 71.887, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["omp-parallel", 1, 36.88, 9.666, 20.319, 66.932, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["omp-parallel", 2, 36.534, 9.6, 20.314, 66.517, "60c1fe4d464fba8c6ffd1b752fbbd40c"],
+    ["omp-parallel", 3, 36.689, 9.582, 20.467, 66.802, "60c1fe4d464fba8c6ffd1b752fbbd40c"]
+  ],
+  "raw_loading_columns": ["configuration", "round", "torch_threads", "n_tensors", "materialize_s", "cast_s", "h2d_s", "total_s"],
+  "raw_loading_runs": [
+    ["cpu64", 1, 64, 620, 3.174, 2.309, 1.217, 7.796],
+    ["cpu64", 2, 64, 620, 2.957, 2.443, 1.433, 7.846],
+    ["cpu64", 3, 64, 620, 2.916, 2.613, 1.067, 7.942],
+    ["cpu16", 1, 16, 620, 0.857, 1.137, 0.713, 3.643],
+    ["cpu16", 2, 16, 620, 0.871, 1.174, 0.746, 3.608],
+    ["cpu16", 3, 16, 620, 0.814, 1.142, 0.724, 3.487],
+    ["gpu16", 1, 16, 620, 0.938, 0.028, 1.41, 3.74],
+    ["gpu16", 2, 16, 620, 0.926, 0.029, 1.467, 3.83],
+    ["gpu16", 3, 16, 620, 0.87, 0.029, 1.395, 3.661]
+  ],
+  "raw_loading_workload": {"shards": [6, 7, 8], "source_dtype": "float32", "tensor_gib": 13.3, "warmup": "CUDA context, copy, cast and synchronize before timer", "order_per_round": ["cpu64", "cpu16", "gpu16"]},
+  "executed_script_sha256": {"parse_startup_log.py": "aa260de2cfcaec2fc4a678b245bb7c16fa464543aaec72b947fd4f769d547011", "raw_load_bench.py": "1f70f6b3c205402b2c2c7c52b224e1be3c04a12d12d11044a86f1b9b6c7a1d0f", "bench_startup.py": "04b1e8b9398b4d9885fbb6fe341e8a9ee98792d0be200c15c57e1fc5183c6ff0", "__init__.py": "b1d617f45f34dbe6f5f58d792d90c84af8d8d0a7e7def77c48170ca93495ca0d"},
+  "notes": ["Standard deviations use ddof=1.", "Only first-request ONGs are compared.", "Raw materialization includes CPU allocation/copy; H2D includes GPU allocation; not engine-loader timings."]
+}

benchmarks/tts/bench_tts.py

@@ -1,4 +1,7 @@
 #!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 """Universal TTS benchmark CLI for vllm-omni.
 
 Runs ``vllm bench serve --omni`` with model-aware defaults loaded from
@@ -87,6 +90,7 @@ def build_bench_args(
     served_model_name: str | None = None,
     num_warmups: int = 2,
     request_seed: int | None = None,
+    duration_seconds: float | None = None,
 ) -> list[str]:
     """Build the ``vllm bench serve --omni`` command for one (task, concurrency) run."""
     dataset_name = _TASK_TO_DATASET[task]
@@ -95,7 +99,10 @@ def build_bench_args(
     task_extra_body: dict[str, Any] = dict((model_cfg.get("task_extra_body") or {}).get(task) or {})
     if request_seed is not None:
         task_extra_body["seed"] = request_seed
-
+    if duration_seconds is not None:
+        if not math.isfinite(duration_seconds) or duration_seconds <= 0:
+            raise ValueError("duration_seconds must be finite and positive")
+        task_extra_body["duration_seconds"] = duration_seconds
     # Resolve dataset path
     if dataset_path:
         resolved_dataset_path = dataset_path
@@ -194,7 +201,7 @@ def print_summary_table(results: list[dict[str, Any]]) -> None:
     if not results:
         return
     header = (
-        f"{'Task':<16} {'Concurrency':>11} {'RTF mean':>10} "
+        f"{'Task':<16} {'Task Type':<12} {'Concurrency':>11} {'RTF mean':>10} "
         f"{'TTFP (ms)':>10} {'Throughput':>12} {'WER':>7} {'SIM':>7} {'UTMOS':>7}"
     )
     print(f"\n{'=' * len(header)}")
@@ -204,6 +211,7 @@ def print_summary_table(results: list[dict[str, Any]]) -> None:
     print("-" * len(header))
     for r in results:
         task = r.get("_task", "?")
+        task_type = r.get("_task_type", "-")
         conc = r.get("_concurrency", "?")
         rtf = r.get("mean_audio_rtf", float("nan"))
         ttfp = r.get("mean_audio_ttfp_ms", float("nan"))
@@ -216,7 +224,7 @@ def print_summary_table(results: list[dict[str, Any]]) -> None:
             return f"{v:.{digits}f}" if not math.isnan(v) else "  n/a"
 
         print(
-            f"{task:<16} {str(conc):>11} {fmt(rtf):>10} {fmt(ttfp, 0):>10} "
+            f"{task:<16} {task_type:<12} {str(conc):>11} {fmt(rtf):>10} {fmt(ttfp, 0):>10} "
             f"{fmt(throughput):>12} {fmt(wer):>7} {fmt(sim):>7} {fmt(utmos):>7}"
         )
     print("=" * len(header))
@@ -249,6 +257,12 @@ def main() -> None:
         help="Number of prompts per run. If one value, applied to all concurrency levels.",
     )
     parser.add_argument("--num-warmups", type=int, default=2, help="Warmup requests before each measured run")
+    parser.add_argument(
+        "--duration-seconds",
+        type=float,
+        default=None,
+        help="Target audio duration per request (AuK defaults to 5 seconds). Independent of --output-len.",
+    )
     parser.add_argument(
         "--request-seed",
         type=int,
@@ -277,6 +291,8 @@ def main() -> None:
 
     if args.num_warmups < 0:
         parser.error("--num-warmups cannot be negative")
+    if args.duration_seconds is not None and (not math.isfinite(args.duration_seconds) or args.duration_seconds <= 0):
+        parser.error("--duration-seconds must be finite and positive")
 
     model_configs = load_model_configs(Path(args.model_configs))
     if args.model not in model_configs:
@@ -333,11 +349,16 @@ def main() -> None:
                 served_model_name=args.served_model_name,
                 num_warmups=args.num_warmups,
                 request_seed=args.request_seed,
+                duration_seconds=args.duration_seconds,
             )
             result = run_one_benchmark(cmd)
             if result is not None:
                 result["_task"] = task
                 result["_concurrency"] = concurrency
+                task_body = model_cfg.get("task_extra_body", {}).get(task, {})
+                task_type = task_body.get("task_type")
+                if task_type is not None:
+                    result["_task_type"] = task_type
                 all_results.append(result)
                 # Persist the metadata so plot_results.py can pick it up.
                 if args.output_dir and result_filename:

benchmarks/tts/model_configs.yaml

@@ -7,6 +7,22 @@
 # is tracked here.
 
 models:
+  tencent/AuK: &auk
+    supported_tasks: [default_voice, voice_clone]
+    backend: openai-audio-speech
+    endpoint: /v1/audio/speech
+    trust_remote_code: true
+    task_extra_body:
+      default_voice:
+        duration_seconds: 5.0
+        voice: default
+        task_type: CustomVoice
+      voice_clone:
+        duration_seconds: 5.0
+        voice: default
+        task_type: Base
+  tencent/AuK-Flash: *auk
+
   BreezeBlue/Breeze-TTS-2:
     supported_tasks: [default_voice, voice_design, voice_clone]
     backend: openai-audio-speech

docker/Dockerfile.ci

@@ -1,10 +1,7 @@
-# Target vLLM version: v0.30.0 (tag @ ced6857afa0e).
-# Use the matching release image and its coherent CUDA/PyTorch/FlashInfer
-# runtime. Do not reinstall a commit wheel or repair dependencies on top of
-# an older base image at a release boundary. For a future unreleased target,
-# restore an explicit wheel pin and validate its complete runtime stack.
+# Use the official release image with its matching CUDA/PyTorch stack.
+ARG VLLM_VERSION=0.31.0
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai
-ARG VLLM_BASE_TAG=v0.30.0
+ARG VLLM_BASE_TAG=v${VLLM_VERSION}
 
 # We use registry-based layer caching for the dependencies,
 # where the tag is determined based on the hash of the contents
@@ -12,6 +9,7 @@ ARG VLLM_BASE_TAG=v0.30.0
 # everything except the actual project dir, since that should not
 # change much across branches.
 FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG} AS deps
+ARG VLLM_VERSION
 # Keep APP_DIR outside /workspace so k8s Buildkite workspace mounts cannot
 # shadow the editable install target (#5364). Docker WORKDIR creates /opt if
 # needed.
@@ -36,12 +34,17 @@ RUN apt-get update && \
 # for CI
 COPY pyproject.toml setup.py ./
 COPY requirements/ requirements/
-ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0
+# Scope the override to Omni; dependency source builds need their own versions.
+ENV VLLM_OMNI_VERSION_OVERRIDE=0.0.0
 # Editable install: deps + project metadata are cached in this layer, while
 # the package itself is linked to APP_DIR (populated later by COPY --link).
 # Do NOT use a non-editable install here — that copies an empty stub into
 # site-packages and breaks ``import vllm_omni.entrypoints``.
 RUN uv pip install --system -e ".[dev]"
+# Fail the image build if dependency resolution replaced the selected vLLM.
+# The CPU builder has no NVIDIA driver; CUDA imports are checked at the start
+# of each GPU test job by .buildkite/cuda/scripts/check_vllm_runtime.py.
+RUN python3 -c "import vllm; assert vllm.__version__ == '${VLLM_VERSION}', vllm.__version__"
 
 # The final stage just copies the working dir on top of the
 # dependencies image. This uses --link so that we can skip

docker/Dockerfile.ci.dockerignore

@@ -1,6 +1,6 @@
 # NOTE: this is largely taken from vLLM
 
-# Exclude .git for CI builds — version is set via SETUPTOOLS_SCM_PRETEND_VERSION
+# Exclude .git for CI builds — version is set via VLLM_OMNI_VERSION_OVERRIDE
 .git
 .venv
 /build

docker/Dockerfile.cuda

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai:v0.30.0
+ARG BASE_IMAGE=vllm/vllm-openai:v0.31.0
 FROM ${BASE_IMAGE}
 
 ARG COMMON_WORKDIR=/app

docker/Dockerfile.npu

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_VERSION=v0.30.0
+ARG VLLM_ASCEND_VERSION=v0.31.0
 ARG VLLM_ASCEND_DEVICE_SUFFIX=
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_VERSION}${VLLM_ASCEND_DEVICE_SUFFIX}
 

docker/Dockerfile.npu.ci

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_VERSION=v0.30.0
+ARG VLLM_ASCEND_VERSION=v0.31.0
 ARG VLLM_ASCEND_DEVICE_SUFFIX=
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_VERSION}${VLLM_ASCEND_DEVICE_SUFFIX}
 

docker/Dockerfile.rocm

@@ -1,7 +1,7 @@
 # Keep the ROCm runtime aligned with the vLLM release targeted by
 # docker/Dockerfile.ci. A stale base image can build successfully while failing
 # at runtime when vLLM-Omni imports APIs added by the newer vLLM release.
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.30.0
+ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.31.0
 
 # Keep the dependency-heavy portion independent from the source tree. The
 # registry cache is keyed on these inputs, while the final source copy can be
@@ -12,11 +12,11 @@ FROM ${BASE_IMAGE} AS deps
 # BASE_IMAGE pins the default prebuilt runtime when USE_NIGHTLY_BUILD=0.
 # VLLM_VERSION_OR_COMMIT_HASH is consumed only by the opt-in source rebuild when
 # USE_NIGHTLY_BUILD=1. Keep both aligned with docker/Dockerfile.ci's
-# VLLM_BASE_TAG. This is enforced by tests/buildkite/test_rocm_dockerfile.py. An
+# release version. This is enforced by tests/buildkite/test_rocm_dockerfile.py. An
 # intentional custom source pin must update that regression in the same change,
 # and should return to the stable release as soon as one is available.
 ARG USE_NIGHTLY_BUILD=0
-ARG VLLM_VERSION_OR_COMMIT_HASH=v0.30.0
+ARG VLLM_VERSION_OR_COMMIT_HASH=v0.31.0
 ARG ARG_PYTORCH_ROCM_ARCH
 ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}}
 
@@ -75,7 +75,8 @@ WORKDIR ${COMMON_WORKDIR}/vllm-omni
 RUN mkdir -p vllm_omni && touch vllm_omni/__init__.py
 COPY pyproject.toml setup.py ./
 COPY requirements/ requirements/
-ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0
+# Scope the override to Omni; dependency source builds need their own versions.
+ENV VLLM_OMNI_VERSION_OVERRIDE=0.0.0
 
 RUN uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" \
         --no-cache-dir -e ".[dev]" --no-build-isolation && \

docker/Dockerfile.xpu

@@ -1,4 +1,4 @@
-# Target vLLM version: v0.30.0. Keep VLLM_VERSION aligned with the target vLLM
+# Target vLLM version: v0.31.0. Keep VLLM_VERSION aligned with the target vLLM
 # release on every rebase.
 #
 # This file layers vLLM-Omni on top of the published upstream XPU image, which
@@ -18,7 +18,7 @@
 #
 # .buildkite/intel/scripts/run-xpu-test.sh automates that pull-then-build fallback.
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai-xpu
-ARG VLLM_VERSION=v0.30.0
+ARG VLLM_VERSION=v0.31.0
 ARG VLLM_BASE=${VLLM_BASE_IMAGE}:${VLLM_VERSION}
 
 FROM ${VLLM_BASE} AS vllm-omni

docs/.nav.yml

@@ -15,7 +15,7 @@ nav:
         - Image Edit: serving/image_edit_api.md
         - Videos: serving/videos_api.md
       - Streaming and Realtime:
-        - Realtime Audio: serving/realtime_api.md
+        - Full Duplex OpenAI: serving/realtime_openai_duplex_api.md
         - Full Duplex: serving/full_duplex_api.md
         - Realtime Duplex Reference: serving/realtime_duplex_api.md
         - Streaming Video Input: serving/video_stream_api.md
@@ -47,6 +47,7 @@ nav:
     - configuration/*
   - Models:
     - models/supported_models.md
+    - SeedVR2: models/seedvr2.md
   - Features:
     - Overview: features/README.md
     - Runtime and Stage Execution:
@@ -151,6 +152,7 @@ nav:
           - Hybrid Sharded Data Parallel (HSDP): design/feature/hsdp.md
           - Pipeline Parallel: design/feature/pipeline_parallel.md
           - Sequence Parallel: design/feature/sequence_parallel.md
+          - Window Sequence Parallel: design/feature/window_sequence_parallel.md
           - Tensor Parallel: design/feature/tensor_parallel.md
           - VAE Patch Parallelism: design/feature/vae_parallel.md
         - Attention Optimization:

docs/README.md

@@ -56,9 +56,9 @@ vLLM-Omni is flexible and easy to use with:
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
 - Omni-modality models (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
-- TTS models (e.g. Qwen3-TTS, IndexTTS 2.5, CosyVoice3)
-- Diffusion models — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
-- Robot-policy and action models (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
+- TTS models (e.g. Qwen3-TTS, Tencent AuK, Breeze-TTS-2, CosyVoice3)
+- Diffusion models — image, video, and audio generation (e.g. MiniMax H3, LingBot World, MAGI-2, LTX-2.5, Wan2.2)
+- Robot-policy and action models (e.g. π0.5, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
 
 For more information, checkout the following:
 

docs/configuration/README.md

@@ -2,7 +2,7 @@
 
 This section lists the most common options for running vLLM-Omni.
 
-For options within a vLLM Engine, please refer to the [vLLM 0.29 configuration guide](https://docs.vllm.ai/en/v0.29.0/configuration/index.html).
+For options within a vLLM Engine, please refer to the [vLLM 0.31 configuration guide](https://docs.vllm.ai/en/v0.31.0/configuration/index.html).
 
 Each model defines fixed topology in a registered `PipelineConfig` and runtime
 overrides in a deploy YAML.

docs/configuration/environment_variables.md

@@ -64,6 +64,7 @@ The lifecycle labels used below are:
 | Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
 | --- | --- | --- | --- | --- |
 | `DIFFUSION_ATTENTION_BACKEND` | Backend name or `auto`; default `auto` (platform selection) | Diffusion stages; read when `OmniDiffusionConfig` is constructed | `diffusion_attention_config.default` wins. An unknown backend fails during backend resolution. | Stable fallback |
+| `DIFFUSION_ATTENTION_QUANT` | `<dtype_qk>:<dtype_vo>[:<q_block_size>:<k_block_size>]`; default unset | Diffusion stages; read when `OmniDiffusionConfig` is constructed | Used only when `DIFFUSION_ATTENTION_BACKEND` sets a non-`auto` backend and no explicit attention default exists. Quantization support varies by backend. Malformed fields raise `ValueError`. | Stable fallback |
 | `DIFFUSION_CACHE_BACKEND` | `none`, `cache_dit`, `tea_cache`, `mag_cache`, `step_cache`, `stepcache`, or `step_cache_dit`; default `none` | Diffusion runner startup | Explicit `cache_backend` in config wins. Otherwise this name wins over the deprecated alias. Unsupported values raise `ValueError` during runner setup. | Stable fallback |
 | `DIFFUSION_CACHE_ADAPTER` | Same values as `DIFFUSION_CACHE_BACKEND`; default `none` | Diffusion runner startup | Used only when neither explicit `cache_backend` nor `DIFFUSION_CACHE_BACKEND` is set. Unsupported values raise `ValueError`. | Deprecated; use `DIFFUSION_CACHE_BACKEND` |
 | `OMNI_DIFFUSION_PROMPT_EMBED_CACHE` | Boolean: `1`, `true`, `yes`, `on`, `0`, `false`, `no`, or `off`; default disabled | Each diffusion runner; resolved during model setup | A recognized environment value overrides the explicit enable setting. An unrecognized value is ignored. | Experimental |
@@ -77,6 +78,13 @@ Backend names for `DIFFUSION_ATTENTION_BACKEND` are the members of
 `SAGE_ATTN`, `FLASHINFER_ATTN`, and `TRTLLM_ATTN`. Platform support still
 depends on the installed kernels and model path.
 
+For `DIFFUSION_ATTENTION_QUANT`, the dtype fields accept `float16`,
+`bfloat16`, `int8`, or `fp8_e4m3`; the selected backend may support only a
+subset. For example, `DIFFUSION_ATTENTION_BACKEND=TRTLLM_ATTN` with
+`DIFFUSION_ATTENTION_QUANT=int8:bfloat16:1:16` selects INT8 Q/K SageAttention.
+Explicit `diffusion_attention_config.default` takes precedence over both
+environment variables.
+
 ### Serving and runtime
 
 | Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
@@ -122,7 +130,7 @@ settings.
 
 | Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
 | --- | --- | --- | --- | --- |
-| `VLLM_OMNI_FUSED_QK_NORM_ROPE_MIN_TOKENS` | Non-negative integer; unset or empty means each consumer's own default (Boogu-Image: `2048`) | Token gate of the shared `fused_qk_norm_rope` op: a rotary table is packed for the fused kernel only when it spans at least this many positions (`B*S`); read at forward time | Environment-only. `0` = always fuse; a very large value disables the fused path. Any other value raises `ValueError` on the first forward. The crossover is host-dependent; re-benchmark before overriding. | Experimental performance control |
+| `VLLM_OMNI_FUSED_QK_NORM_ROPE_MIN_TOKENS` | Non-negative integer; unset or empty means each consumer's own default (Boogu-Image: `2048`; MammothModa2: `0`) | Token gate of the shared `fused_qk_norm_rope` op: a rotary table is packed for the fused kernel only when it spans at least this many positions (`B*S`); read at forward time | Environment-only. `0` = always fuse; a very large value disables the fused path. Any other value raises `ValueError` on the first forward. The crossover is host-dependent; re-benchmark before overriding. | Experimental performance control |
 | `VLLM_OMNI_SKIP_NVFP4_NAN_CLAMP` | Boolean truthy spellings: `1`, `true`, `yes`, `on`; default false | ModelOpt NVFP4 compatibility patch; read when `vllm_omni.patch` imports | Environment-only escape hatch. Any other value means false. Set only to diagnose the upstream NaN-scale issue. | Diagnostic and temporary |
 | `VLLM_OMNI_USE_QUACK_FP8` | Boolean truthy spellings: `1`, `true`, `yes`, `on`; unset means hardware auto-detection | FP8 scaled matrix multiplication; evaluated when quack capability is selected | A set value overrides auto-detection. Any non-truthy value forces quack off. If quack cannot load, vLLM-Omni warns and falls back to FlashInfer. | Experimental performance control |
 
@@ -130,6 +138,22 @@ settings.
 Omni-owned variable, even though vLLM-Omni supplies a persistent default when
 it is unset.
 
+### Torch compilation
+
+| Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
+| --- | --- | --- | --- | --- |
+| `VLLM_OMNI_TORCH_DYNAMO_RECOMPILE_LIMIT` | Positive integer; unset preserves the current Torch setting | Process default for Torch Dynamo; read when `vllm_omni` is imported | Sets `torch._dynamo.config.recompile_limit` to the requested value. A non-positive or non-integer value raises `ValueError`. Later backend-specific overrides still take precedence. | Diagnostic |
+
+Set this variable before launching Omni, for example:
+
+```bash
+export VLLM_OMNI_TORCH_DYNAMO_RECOMPILE_LIMIT=64
+```
+
+The setting applies across model families and platforms. vLLM compilation
+contexts and backends can subsequently apply their own limits; this variable
+does not replace those backend-specific policies.
+
 ## Per-stage environment
 
 Deploy configurations can set arbitrary environment keys for one stage:
@@ -162,7 +186,7 @@ their keys only.
 ## Inherited vLLM variables
 
 vLLM-Omni also reads variables through its aligned vLLM dependency. Refer to
-the [vLLM 0.29 environment-variable reference](https://docs.vllm.ai/en/v0.29.0/configuration/env_vars.html)
+the [vLLM 0.31 environment-variable reference](https://docs.vllm.ai/en/v0.31.0/configuration/env_vars.html)
 for their definitions. This includes vLLM launch, cache, logging, plugin, ROCm,
 XPU, ModelScope, and FlashInfer workspace settings.
 
@@ -190,7 +214,7 @@ collection, examples, bug reports, or logs.
 
 ## Model-specific variables
 
-The audit found 58 variables read by a single model or pipeline family. They are
+The inventory includes 93 variables read by a single model or pipeline family. They are
 not listed as public usage options here because doing so would turn implementation
 escape hatches into an accidental compatibility contract.
 
@@ -199,11 +223,11 @@ Every audited model-specific name has a migration disposition in the
 
 | Disposition | Count | Required outcome |
 | --- | ---: | --- |
-| Promote | 35 | Move a stable setting into typed stage or model configuration. |
-| Request scope | 6 | Move request-varying behavior into a declared request-option schema. |
+| Promote | 64 | Move a stable setting into typed stage or model configuration. |
+| Request scope | 5 | Move request-varying behavior into a declared request-option schema. |
 | External | 0 | Retain only when a supported third-party library owns the contract. |
-| Internalize | 11 | Keep a debug or diagnostic switch out of public documentation and configuration. |
-| Deprecate/remove | 6 | Remove a compatibility escape hatch that has no continuing contract. |
+| Internalize | 17 | Keep a debug or diagnostic switch out of public documentation and configuration. |
+| Deprecate/remove | 7 | Remove a compatibility escape hatch that has no continuing contract. |
 
 The disposition is a migration target, not a statement that the existing
 environment switch is stable. Promote or request-scope work should land in

docs/configuration/hsdp_pre_sharded_loading.md

@@ -0,0 +1,22 @@
+# Pre-sharded HSDP loading
+
+Use `--use-hsdp --hsdp-shard-size 8 --hsdp-weight-load-strategy pre_sharded`
+to shard transformer parameters before loading checkpoint values. Set the shard
+size to the available GPU count (or configure replication explicitly).
+The default strategy, `full`, retains the existing load-then-shard behavior.
+The same `hsdp_weight_load_strategy` option is available through Python and stage
+engine configuration.
+
+The pre-sharded strategy reads rank-local slices from ordinary Hugging Face
+safetensors checkpoints into FSDP-owned storage. No converted checkpoint is
+required. Cosmos 3 and Cosmos 3 Edge transformer blocks release their initialized
+parameter storage to meta as they are constructed; nonpersistent buffers retain
+their values.
+Encoders and VAEs follow their existing loading paths.
+
+This strategy requires the default diffusion pipeline loader, complete dedicated
+transformer checkpoint sources, and supported checkpoint key mappings. Tensor
+layouts must match the runtime layout; tensor transforms are unsupported.
+Checkpoint tensor dtypes must match runtime tensor dtypes at preflight; dtype conversion is unsupported.
+Quantization and LoRA are unsupported. Every worker needs checkpoint access.
+`num_weight_load_threads` controls the safetensors reader's threads per rank.

docs/configuration/stage_configs.md

@@ -35,7 +35,7 @@ Common `StagePipelineConfig` fields include:
 | `final_output` / `final_output_type` | Whether the stage produces a user-visible output and its modality. |
 | `owns_tokenizer` | Whether this stage owns the pipeline tokenizer. |
 | `model_arch`, `hf_config_name` | Stage-specific model architecture and nested HF config selector. |
-| `engine_output_type` | Runtime output representation such as `text`, `latent`, or `audio`. |
+| `engine_output_type` | Canonical runtime output type: `text`, `image`, `audio`, `latent`, or `token_ids`. |
 | `custom_process_input_func` | Processor applied to this stage's incoming payload. |
 | `custom_process_next_stage_input_func` | Processor used for full-payload handoff to the next stage. |
 | `async_chunk_process_next_stage_input_func` | Processor used for async chunk handoff. |
@@ -45,6 +45,50 @@ To add or change topology, define and register a new pipeline variant. Use
 deploy YAML only for runtime placement, resource sizing, connectors, and other
 deployment overrides.
 
+### Engine output types
+
+`engine_output_type` is parsed when the LLM stage's output processor is built
+during service initialization. Invalid values fail startup before requests
+are accepted; engine replicas may already have been initialized at that point
+and are cleaned up on failure.
+
+Use the exact lowercase names listed above. Combinations such as `text+audio`
+or `text,audio` remain supported, with every component canonical. Omitted or
+`None` values retain the default behavior (`TEXT` in the output processor).
+Empty strings, whitespace, case variants, and aliases are rejected.
+
+For existing custom pipelines or direct callers, replace old aliases explicitly:
+
+| Old value | Canonical value |
+| ------- | ------- |
+| `speech`, `wav`, `waveform` | `audio` |
+| `images`, `pixel_values`, `pixels` | `image` |
+| `latents` | `latent` |
+| `tokens` | `text` for text output, or `token_ids` for discrete IDs |
+
+`token_ids` is a separate type, not a `text` alias. It describes discrete
+vocabulary or codebook indices, distinct from continuous latent representations.
+Existing models using `latent` retain their historical payload contracts.
+GLM-Image's AR stage keeps `engine_output_type="token_ids"`; generated IDs still
+travel through `cumulative_token_ids`, and source-image IDs through
+`ids.prior_image`.
+
+Bare primary tensors and generic producer keys (`model_outputs` or `hidden`)
+use the declared type as their default output key, including `token_ids`.
+The normalizer does not inspect or convert their dtype. Distinct auxiliary
+outputs must use explicit field names: `model_outputs` and `hidden` both map
+to the same default key and must not represent two independent outputs.
+GLM-Image disables auxiliary hidden output for its default GPU execution.
+Some runner paths still attach it; that field is then named `token_ids` even
+though it contains hidden states. The GLM-Image bridge ignores this auxiliary
+field and reads generated IDs from `cumulative_token_ids` instead.
+
+Token-ID tensors use the existing dimension-0 accumulation default, subject to
+registered per-key overrides, and remain available after DELTA emissions.
+This is separate from the existing accumulation of `cumulative_token_ids`.
+`final_output_type` and Diffusion's `output_type` are separate settings and are
+unchanged by these rules.
+
 ## Deploy configuration schema
 
 The new deploy schema lives under `vllm_omni/deploy/` and is paired with a frozen `PipelineConfig` registered by the model's `pipeline.py`. Each deploy YAML has these top-level fields:
@@ -78,7 +122,57 @@ explicit CLI or stage override, or the downstream vLLM engine default.
 Note: for the diffusion path, an omitted `distributed_executor_backend` selects
 `uni` on a single GPU (in-process worker, no MessageQueue / `/dev/shm` output
 segments) and `mp` when `num_gpus > 1`. Set `mp` explicitly to keep a worker
-subprocess on one GPU. `ray` / `external_launcher` are not fully supported yet.
+subprocess on one GPU. Set `ray` for Ray-managed single- or multi-node
+diffusion workers. `external_launcher` is not supported yet.
+
+The diffusion `ray` backend runs one worker actor per GPU and supports workers
+spanning Ray nodes. Enable it in the deploy YAML:
+
+```yaml
+distributed_executor_backend: ray
+```
+
+The executor reuses Ray when already initialized; otherwise it calls
+`ray.init()`, which handles cluster discovery or local startup and respects
+`RAY_ADDRESS`. There is no diffusion-specific address setting. Ray must be
+installed on every node.
+
+Ray diffusion workers receive the driver's `VLLM_*`, `OMNI_*`, `DIFFUSION_*`,
+`NCCL_*`, `TORCH_NCCL_*`, `UCX_*`, and Hugging Face environment settings,
+plus `PYTHONPATH`, `CUDA_LAUNCH_BLOCKING`, and `OMP_NUM_THREADS`.
+Explicit stage `runtime.env` entries are also forwarded and take precedence.
+Device visibility, rank/rendezvous settings, worker host addresses, and `RAY_*`
+variables are excluded so each worker retains its own assignment and identity.
+
+For multiple nodes, start the cluster before launching Omni. For example,
+with head IP `10.0.0.1`:
+
+```bash
+# On the head node:
+ray start --head --node-ip-address=10.0.0.1 --port=6379
+
+# On each additional worker node:
+ray start --address=10.0.0.1:6379
+
+# On the host launching Omni (MODEL and my_deploy.yaml are placeholders):
+RAY_ADDRESS=10.0.0.1:6379 vllm serve MODEL --omni --deploy-config my_deploy.yaml
+```
+
+Omni creates actors on the resources of that cluster; it does not start Ray
+on additional machines.
+
+### Stage-level runner selection
+
+`model_runner: v1` or `v2` at the deploy level sets the default runner.
+A `model_runner` on an individual stage overrides that default, allowing
+one stage to migrate or roll back independently. This does not change
+`PipelineConfig` topology, input adapters, or the connector payload contract.
+An omitted stage override preserves the deploy-level selection.
+
+MRv2 native downstream receivers currently support turn-based requests only;
+streaming sessions and resumable input require the V1 prompt-replacement path.
+Selecting a runner does not add the session capabilities it lacks. Platform
+fallbacks and stage overrides are validated after configuration resolution.
 
 ### Stage fields
 
@@ -306,7 +400,9 @@ ROCm and MUSA. Select one of these deployment profiles:
 | `qwen3_tts_mrv2.yaml` | V2 | B1 | Explicit MRV2 profile (same runner selection as the default) |
 | `qwen3_tts_high_concurrency_mrv2.yaml` | V2 | B1, B2 | Opt-in throughput tuning |
 | `qwen3_tts_high_concurrency_mrv2_b4.yaml` | V2 | B1, B2, B3, B4 | Experimental throughput / buffered playback |
+| `qwen3_tts_high_concurrency_mrv2_single_gpu.yaml` | V2 | B1–B8 | Experimental two-stage deployment on one GPU, with MPS |
 | `qwen3_tts_high_concurrency.yaml` | V1 | Existing defaults | V1 high-concurrency control |
+| `qwen3_tts_fused_single_gpu.yaml` | V2 | N/A (in-Talker decoder) | Opt-in single-stage CUDA pipeline on one GPU |
 
 ```bash
 # MRV2 is the default; pass a copy with `model_runner: v1` to force V1.
@@ -337,6 +433,102 @@ supply `mtp_sampling_params` and `get_mtp_seed(sampling_params)` for model-local
 (with `mtp_sample_steps` and `mtp_sample_vocab_size`) as needed. Qwen3-TTS retains
 its existing `talker_mtp` entry point for V1.
 
+### Selecting one or two Qwen3-TTS stages
+
+Choose the deployment profile explicitly with `--deploy-config`. The default
+`qwen3_tts` pipeline retains two stages: the Talker publishes codec chunks to
+Code2Wav, which produces audio. Its stage devices can be placed on the same
+GPU or on separate GPUs. `qwen3_tts_fused` is a separate, opt-in pipeline for
+one CUDA GPU: the Talker owns the stateful codec decoder and emits PCM directly,
+without a Code2Wav engine or stage connector.
+
+```bash
+# Two stages on one GPU; the two-stage pipeline remains independently selectable.
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni \
+  --deploy-config vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2_single_gpu.yaml
+
+# One stage on one GPU; no inter-stage transport or Code2Wav scheduling.
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni \
+  --deploy-config vllm_omni/deploy/qwen3_tts_fused_single_gpu.yaml
+```
+
+The single-stage profile requires CUDA, V2, asynchronous chunks, TP=PP=1,
+an in-process worker and disabled prefix caching. Select the full profile;
+adding `talker_stream_decode: true` to a two-stage Talker is rejected because
+its latent output belongs to the Code2Wav input contract. Single-stage options
+live in `stages[].additional_config`; two-stage transport and first-frame
+options live in `connectors.*.extra`.
+
+The single-stage profile defaults `talker_stream_first_audio` to `false` for
+throughput. To opt into a separate first-frame delivery queue, save this
+overlay beside the bundled deployment files and pass its path to
+`--deploy-config`:
+
+```yaml
+base_config: qwen3_tts_fused_single_gpu.yaml
+stages:
+  - stage_id: 0
+    additional_config:
+      talker_stream_decode: true
+      talker_stream_first_audio: true
+      ref_code_context_frames: 72
+      code_predictor_kv_cache: true
+      code_predictor_fused_sampling: true
+      code_predictor_fused: true
+```
+
+This queue uses PCM from the single-stage decoder. The two-stage
+`talker_first_audio` option below uses its own first-frame decoder and does
+not require single-stage execution. The two options cannot be enabled
+together. Single-stage prefix-cache reset with
+`reset_running_requests=True` returns `False` while requests are active;
+wait for completion or abort them before retrying.
+
+Reference-cloning context is selected independently by each profile. The
+single-stage profile explicitly uses 72 reference frames, matching the
+`qwen3_tts.yaml` context boundary. A shorter
+`additional_config.ref_code_context_frames` (for example 25) can change
+timbre continuity even when WER is similar;
+the throughput measurements for CustomVoice do not qualify Base voice
+cloning quality. Reference layout, grouping and priming are model hooks;
+the shared worker does not select Qwen's context length or codebook layout.
+
+### Qwen3-TTS first-frame delivery and rollback
+
+The standard `qwen3_tts.yaml`, `qwen3_tts_high_concurrency.yaml`,
+`qwen3_tts_mrv2.yaml` and `qwen3_tts_high_concurrency_mrv2.yaml` profiles
+enable the `talker_first_audio` connector option, as does the experimental
+two-stage `qwen3_tts_high_concurrency_mrv2_single_gpu.yaml` profile.
+This changes the default CUDA MRV2 streaming path:
+residual prediction runs eagerly after the Talker sample, the Talker loads
+an additional first-frame decoder with its weights and CUDA graphs, and the
+orchestrator orders its audio before subsequent Code2Wav chunks.
+
+Code2Wav retains full-audio prefix graphs alongside the optional state-only
+graphs. Each request's delivery marker selects the graph and audio trimming;
+enabling the option alone never suppresses audio or disables prefix batching
+for requests that retain regular codec delivery.
+
+The path requires asynchronous chunks, TP/PP 1, an in-process executor and
+disabled prefix caching. Requests with reference codes and unsupported
+runners or platforms retain regular Code2Wav delivery.
+
+To restore regular codec delivery and avoid loading the Talker's additional
+decoder, use a deploy overlay with the relevant base profile:
+
+```yaml
+base_config: qwen3_tts.yaml
+connectors:
+  connector_of_shared_memory:
+    extra:
+      talker_first_audio: false
+```
+
+First-packet latency measures when PCM starts arriving. Time to first audible
+audio (TTFA) also includes any leading silence in the generated audio. An
+earlier first packet therefore does not necessarily improve TTFA; measure
+both for the intended voice and workload.
+
 ### Included performance work
 
 - Request snapshots have a fast path for immutable scalar leaves, including
@@ -372,14 +564,86 @@ speaker similarity before adopting either batching preset for a production
 workload. Floating-point decoder outputs can differ across batch sizes; this PR
 does not claim bitwise or quality equivalence.
 
-### Optional MPS deployment
+### Experimental MPS deployment
 
-NVIDIA MPS is an optional operator setting for colocated CUDA processes, not a
-YAML option or a library default. This PR does not establish a throughput or
-first-packet latency benefit from MPS. Measure the exact deployment with and
-without MPS before enabling it.
+NVIDIA MPS lets colocated CUDA stage processes share GPU execution resources.
+It is disabled by default. The experimental
+`qwen3_tts_high_concurrency_mrv2_single_gpu.yaml` profile sets `cuda_mps: true`,
+alongside cached residual prediction, fused sampling, time-major codec
+convolutions, first-frame delivery, and larger graph batches.
 
-Use only assigned GPUs and an independent MPS pipe directory. A private MPS
-server does not provide exclusive GPU ownership or MIG isolation. For a
-single-GPU deployment, explicitly place both stages on that GPU; the supplied
-high-concurrency profile places its two stages on different GPUs by default.
+The runtime requires `nvidia-cuda-mps-control` on `PATH` and one explicit CUDA
+GPU per local EngineCore stage, with `parallel_stage_init: false`.
+Use numeric GPU ordinals for stage placement and visibility so initialization
+locks identify the physical GPU before MPS remaps it. It starts a private MPS daemon for each selected
+GPU and stops its own daemon after the stages exit. If
+`CUDA_MPS_PIPE_DIRECTORY` already names an operator-managed daemon, the runtime
+reuses it without stopping it. Diffusion and remote stages are unsupported.
+Set `cuda_mps: false` in a deploy overlay to disable automatic MPS management.
+
+Stage `env.CUDA_MPS_PIPE_DIRECTORY` overrides the parent setting when selecting
+the daemon; an explicit empty string selects a private daemon. Stages sharing
+a GPU reuse the first stage's daemon. Later stages may omit the setting or
+explicitly match that policy; a conflicting explicit setting fails before
+that stage starts. The parent environment is unchanged.
+
+MPS can improve throughput under concurrent load while increasing first-packet
+or first-audible-audio latency, particularly at low request rates. For a
+latency-sensitive workload, compare the same profile with `cuda_mps: false`.
+An inherited operator-managed MPS daemon must also be disabled by its owner
+for that comparison; this switch only controls automatic MPS management.
+
+MPS does not reserve a GPU. Use only assigned GPUs, explicitly place stages on
+the intended GPU, and warm the complete pipeline before measuring performance.
+
+## MOSS-TTS Local 1.5 with Model Runner V2
+
+`OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5` defaults to CUDA MRV2 using
+the shared runtime introduced for Qwen3-TTS. To select the C64 profile without
+MPS explicitly:
+
+```bash
+vllm serve OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 --omni \
+  --deploy-config vllm_omni/deploy/moss_tts_local_mrv2.yaml
+```
+
+This profile inherits the batching, codec graph buckets and 1-frame/15-frame
+chunk geometry from `moss_tts_local_v1.yaml`, and selects V2 for both the Local
+Talker and codec stages. The Local depth predictor exposes the MRV2 `mtp`
+capabilities while retaining its V1 `talker_mtp` implementation, sampling
+defaults and explicit request-seed handling. Codec chunks use the native data
+plane; the internal Talker retains its final-only orchestrator output policy.
+
+On CUDA, the profile bounds stage-0 prefill work to 512 tokens per iteration.
+The codec retains its own CUDA Graph capture/replay, including batch buckets
+through 64, but disables Inductor compilation (`compilation_config.mode: 0`)
+of the stateful decoder to avoid lengthy compilation at startup. This is
+independent of `enforce_eager`; the profile keeps `enforce_eager: false`.
+It explicitly selects `cudagraph_mode: FULL`, which works without Inductor;
+the default `FULL_AND_PIECEWISE` would otherwise normalize to `NONE` and clear
+the codec's capture buckets when compilation is disabled.
+
+For sustained C128 serving on a large-memory CUDA GPU, select the
+separate `moss_tts_local_mrv2_high_concurrency.yaml` profile. It uses 128
+stream slots per stage, a 32 GiB Talker KV budget, and the codec's
+`triton_slot` backend with Inductor compilation and codec-owned CUDA graphs.
+The bounded KV budget leaves room for codec state and graphs; it does not
+guarantee that 128 maximum-length prompts fit simultaneously. See the
+[MOSS recipe](gh-file:recipes/OpenMOSS/MOSS-TTS.md#local-15-mrv2-and-slot-attention)
+for activation, backend comparisons, memory requirements and benchmark commands.
+
+Omitting `--deploy-config` selects the C128 system profile on CUDA GPUs with
+at least 140 GiB memory and MPS available. Smaller GPUs or a failed memory
+query use C64 with utilization-based memory budgets; missing MPS selects
+C64 without MPS. Selecting `moss_tts_local.yaml` explicitly uses C64 with MPS.
+Use `moss_tts_local_v1.yaml` for the previous V1 deployment.
+NPU, XPU, ROCm and MUSA overrides retain V1. These profiles do not enable
+MRV2 for MOSS Delay, Realtime or Nano. Local 1.5 outputs 48 kHz stereo audio;
+set `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=48000` and
+`VLLM_OMNI_BENCH_AUDIO_CHANNELS=2` when benchmarking raw PCM.
+
+Event-driven orchestration remains independently selectable with
+`VLLM_OMNI_EVENT_DRIVEN_ORCH=0` or `1`. Keep the runner and deployment identical
+when comparing these modes. Model-runner selection does not change the
+orchestration default. MPS, codec dispatch and graph settings belong to the
+deployment profile; the automatic CUDA default selects MPS when available.

docs/design/feature/attention_backend_selection.md

@@ -4,6 +4,9 @@ This document defines the selection and extension contract for diffusion
 attention backends. User-facing backend choices, installation, and tuning are
 in the [attention backend guides](../../user_guide/diffusion/attention_backends.md).
 
+The [execution contract PoC](attention_execution_contract_poc.md) describes the
+experimental post-selection capability and lifecycle contract from RFC #7226.
+
 ## Scope
 
 The contract applies to DiT and other diffusion attention layers. It is

docs/design/feature/attention_execution_contract_poc.md

@@ -0,0 +1,154 @@
+# Diffusion attention execution contract PoC
+
+This prototype accompanies [RFC #7226](https://github.com/vllm-project/vllm-omni/issues/7226)
+and extends the [attention selection design](attention_backend_selection.md).
+It demonstrates path-specific capabilities with dense BF16 FA4, NPU/ROCm
+routing examples, and tensor-state lifetime with a test-only attention module.
+
+## Execution contract
+
+`ExecutionContext` describes the requested execution path. `ExecutionPathResult`
+reports its identity, support status and reason, and compilation mode.
+`AttentionBackend.resolve_capabilities()` provides conservative pre-construction
+results. After initialization, `Attention.resolve_execution_path()` supplies the
+active parallel, paged-KV, and HSDP context; the backend combines it with the
+selected kernel and normalized metadata.
+
+Resolution runs outside compiled execution. FA4 reports `SUPPORTED` and
+`CUSTOM_OP` for dense, noncausal BF16 without parallel or HSDP boundaries when
+its kernel accepts the head dimensions. Dimension validation delegates to FA4's
+architecture-specific rules through `backends/utils/fa.py`. Kernel rejections
+report `UNSUPPORTED` with an actionable reason. Missing private validators and
+other unmigrated paths report `UNMIGRATED` with advisory `EAGER_ONLY` defaults.
+`requested_support()` checks a fullgraph request; it does not enforce selection
+or change existing execution.
+
+FA4 execution uses an opaque custom op. Its fake output preserves Q's batch,
+sequence, and head count and uses V's head dimension. Output is contiguous even
+when Q is noncontiguous. Metadata normalization is
+shared with dispatch. Producers must update published mask semantics when masks
+change; unpublished masks remain runtime-dependent to avoid synchronization.
+Callers resolve again when execution metadata changes.
+
+## NPU and ROCm worked examples
+
+The same `FLASH_ATTN` backend resolves MindIE paths on NPU and AITER paths on
+ROCm. Both examples report `UNMIGRATED`: their route is inspected, while hardware
+correctness and compilation remain unvalidated. `EAGER_ONLY` is an advisory
+default, not a measured compiler limitation. Existing execution is unchanged.
+
+| Input | NPU / MindIE | ROCm / AITER |
+| --- | --- | --- |
+| Dense | `npu_dense` | `rocm_dense` |
+| Padding mask | `npu_masked`: expand the key mask to `[B, 1, Q, K]` | `rocm_masked_varlen`: unpad, call varlen attention, then restore rows |
+| Packed inputs | Opt-in `[real, pad]` contract; choose varlen or prefix-KV slicing from the environment | Forward complete packed-document boundaries to varlen attention |
+| Incomplete packing | Rebuild a mask when possible; otherwise reject before the kernel | Shared CUDA-like normalization rejects incomplete metadata |
+| Unpublished mask semantics | Route to the masked wrapper | Keep resolution runtime-dependent without reading mask values |
+
+NPU resolution reuses `_resolve_packed_seq_npu`; ROCm reuses the metadata
+normalization used by dispatch. AITER identity comes from initialization rather
+than caller-provided context. Quantized, paged, piecewise, and parallel paths
+remain outside these worked examples.
+
+The new L1 tests use CPU tensors and replace only vendor kernel calls. They
+check path identity, forwarded masks/boundaries, fallback rejection, and
+conservative fullgraph-request handling. They do not test vendor numerics or
+compiler support. Existing L1 CI already collects this test file.
+
+```bash
+python -m pytest tests/diffusion/attention/test_flash_attn.py \
+  -k 'npu_contract or rocm_contract' -m 'core_model and cpu' -q
+```
+
+### Hardware validation required before migration
+
+On an Ascend/MindIE or ROCm/AITER runner, record device, driver, PyTorch, vendor
+library, and compiler versions, then validate the corresponding rows above:
+
+1. Compare eager outputs with FP32 SDPA for dense and padding-mask inputs,
+   including unequal Q/K lengths. Compare valid query rows; do not assume both
+   platforms define padded-query outputs identically.
+2. Compare packed outputs with separate per-document references. On NPU, cover
+   both varlen and laser prefix-slicing modes, plus mask reconstruction and the
+   missing-mask error. A padding fallback does not establish multi-document
+   isolation. On ROCm, cover multiple real documents.
+3. Check output shape, dtype, device, finite values, and unchanged inputs.
+   Start BF16 comparisons at `atol=rtol=1e-2` and have the platform maintainer
+   confirm the tolerance against its reference tests.
+4. Run `torch.compile(fullgraph=True, dynamic=True)` with the platform's supported
+   compiler over multiple sequence lengths. Record graph breaks and recompiles;
+   choose `TRACEABLE`, `CUSTOM_OP`, or verified `EAGER_ONLY` from that evidence.
+   A custom-op path additionally needs schema/fake checks and repeated replay.
+
+Promote only the paths validated on that platform. These hardware checks have
+not been run locally; ROCm/NPU fullgraph support is not claimed.
+
+## State lifetime example
+
+`test_attention_state_lifetime.py` prepares an owned query-scale tensor eagerly
+and passes it explicitly to an opaque attention op. The compiled module retains
+its state after the caller drops its reference and releases ownership when the
+compiled callable is deleted. Equivalent instances reuse a graph with their own
+state values.
+
+This example tests ownership and release, dynamic replay across instances,
+custom-op schema/fake behavior, and Inductor execution. It adds no production
+backend or state registry. Planning keys, caching, fallback-policy declarations,
+opaque handles, CUDA graph capture, and TRTLLM integration are deferred.
+
+## Validation
+
+```bash
+python -m pytest tests/diffusion/attention/test_flash_attn_compile.py \
+  tests/diffusion/attention/test_flash_attn.py \
+  tests/diffusion/attention/test_attention_capabilities.py \
+  tests/diffusion/attention/test_attention_state_lifetime.py -q -rs
+```
+
+The focused suite covers capability decisions and metadata changes, existing
+padding/mask regressions, tensor-state lifetime, and real FA4 fullgraph execution
+against FP32 SDPA and eager FA4. Representative Q/K and V dimensions are
+(32, 32), (64, 64), (80, 48), (192, 128), and (256, 256), with batches 1 and 2
+and sequence lengths up to 1,024. Invalid dimensions are checked against actual
+kernel errors. These samples exercise the contract rather than define support.
+BF16 comparisons use `atol=rtol=1e-2`; `torch.library.opcheck` checks schema and
+fake-output correctness.
+
+Single-graph reuse is asserted in the tensor-state test. The FA4 numerical tests
+retain compiler caches across shapes within each case but do not require one
+graph across different batch sizes, Q/K length equality, or head dimensions.
+
+Tested environment: GB300, PyTorch `2.13.0+cu130`, FA4 `4.0.0b18`, CUTLASS DSL
+`4.6.2` with CUDA 13 libraries, Quack `0.6.4`, and TVM FFI `0.1.11`.
+Real-kernel tests require CUDA and CuTe FA4; their availability is checked at runtime.
+
+Before the rebase, the focused suite passed all 97 tests without skips, including
+11 CPU NPU/ROCm routing tests. After rebasing onto upstream `e3be42e05`, the
+expanded suite reports 99 passed and 11 failed in the local vLLM 0.28 environment.
+All 11 failures are import errors: upstream now requires `compute_layout_strides`
+from vLLM, which this environment lacks. Both noncontiguous FA4 regression cases
+and real-kernel schema/fake checks pass. The full suite must be rerun with the
+vLLM version required by upstream. NPU/ROCm device numerics and compilation remain
+unvalidated. Pre-commit passes with the CI hook skips. Upstream PyTorch/CUTLASS
+warnings remain.
+
+## TRTLLM dense execution
+
+`TRTLLM_ATTN` declares `SUPPORTED` / `CUSTOM_OP` for noncausal dense BF16 on
+B200/GB200 (SM100) and B300/GB300 (SM103), head dimension 128, and equal Q/K/V
+head counts, without parallel, paged-KV, piecewise, or HSDP boundaries. Pre-construction, SAGE, skip-softmax, packed,
+and other unverified paths remain `UNMIGRATED`. Resolution and dispatch share
+metadata validation; workspace mutation is explicit in the custom-op schema.
+
+CPU contract tests cover fullgraph replay and schema/fake consistency using a
+substituted dispatcher. Real-kernel validation requires Blackwell with FlashInfer:
+
+```bash
+python -m pytest tests/diffusion/attention/test_trtllm_attn.py \
+  -k dense_contract_fullgraph_matches_sdpa -q -rs
+```
+
+These hardware tests compare eager/compiled output with FP32 SDPA and check
+schema/fake agreement. The TRTLLM and contract suite passed on Blackwell before the final architecture
+restriction was added: 61 passed, no skips (19 dependency deprecation warnings).
+The SM100/SM103 restriction is additionally covered by CPU contract tests.

docs/design/feature/sequence_parallel.md

@@ -281,6 +281,10 @@ NOTE: be careful to test adequately when refactoring classes that take this styl
 | `expected_dims` | int \| None | Expected tensor rank for validation (optional) |
 | `split_output` | bool | `False`: shard **input** params; `True`: shard **output** tensors |
 | `auto_pad` | bool | Auto-pad if sequence not divisible by world_size (default: `False`) |
+| `clone_shard` | bool | Give multi-rank shards their own contiguous storage (default: `False`); single-rank inputs are unchanged |
+
+With `clone_shard=True`, memory savings require callers to drop their references to the full tensor.
+Other references to the same storage also keep the full allocation alive.
 
 **SequenceParallelOutput Parameters:**
 

docs/design/feature/window_sequence_parallel.md

@@ -0,0 +1,186 @@
+# Window-Aligned Sequence Parallelism (SeedVR2)
+
+## Motivation
+
+SeedVR2's diffusion transformer (`NaDiT`) has no global attention: every layer
+partitions the video tokens into 3D windows and attends only inside a window.
+The window size is normalised against a fixed 720p reference area, so raising the
+output resolution grows the *number* of windows, never their size.
+
+Ulysses redistributes the sequence and head dimensions around attention.
+For SeedVR2, window-aligned SP offers a different communication tradeoff: one
+hidden-state exchange per layout boundary versus Q/K/V and output exchanges
+for Ulysses. The actual performance crossover requires measurement.
+Window-aligned SP instead assigns **whole windows** to ranks:
+
+* every rank keeps **all** attention heads, so no head-count divisibility is
+  required,
+* attention inside a layer needs **zero** communication,
+* regular and shifted layouts alternate at every layer, so the released
+  32-layer model has 31 inter-layer transitions, plus one text reduction per
+  layer and a final video gather. A transition can be a local permutation
+  when ownership does not change.
+
+## Window layouts
+
+`nadit.py` ports the reference windowing verbatim (Apache-2.0,
+ByteDance SeedVR2). For a post-patch token grid `(T, H, W)`:
+
+```text
+scale   = sqrt(45 * 80 / (H * W))          # 720p reference area
+wh, ww  = ceil(round(H * scale) / nh), ceil(round(W * scale) / nw)
+wt      = ceil(min(T, 30) / nt)            # BYTEDANCE_MAX_TEMPORAL_WINDOW
+```
+
+The regular method tiles with `slice(it * wt, min((it + 1) * wt, T))`; the
+shifted method moves every cut by half a window and **clips at the boundary**
+(`max(..., 0)` / `min(..., size)`) -- it is not a cyclic `torch.roll`. Both
+methods produce an exact partition of the grid; the shifted layout is genuinely
+different, so an `A -> B -> A -> B` layer schedule needs both directions of
+re-sharding.
+
+Windows are enumerated `for iw ... for ih ... for it ...` (w outermost, t
+innermost) and tokens inside a window are row-major in `(t, h, w)`. The
+canonical token id is `(t * H + h) * W + w`, which is exactly what the reference
+obtains by windowing `torch.arange`.
+
+Consequences that matter (all verified against the reference):
+
+* Window token counts are **ragged** (edge windows are clipped), so a shard must
+  be balanced by token count, not by window count.
+* The number of layout transitions follows the actual layer schedule. For the
+  released 3B checkpoint (`num_layers=32`, alternating methods) there are 31
+  inter-layer transitions, not 16; `ensure_layout(current, required)` derives
+  them instead of assuming `layer % 2`.
+
+## Planner
+
+`nadit.py` implements a deterministic LPT assignment:
+
+```text
+windows sorted by (-video_token_count, original_window_id)
+each window is placed on the rank with the smallest (assigned_tokens, rank_id)
+each rank then lists its windows in ascending original order
+```
+
+The cache key covers the post-patch token grid, the full window method and
+parameters (via a SHA-1 geometry fingerprint), the geometry version, the **SP
+world size** and the planner version, so a different degree can never reuse a
+stale assignment. Caches are LRU-bounded.
+
+## Plan A redistribution
+
+At a layout boundary the activations must move from the source assignment to the
+destination assignment. With `Z_src[r]` / `Z_dst[d]` the canonical token ids held
+by each rank and `C[r, d]` the number of rows rank `d` needs from rank `r`:
+
+* `input_split_sizes` of rank `r` is row `r` of `C`, `output_split_sizes` is
+  column `r`,
+* each peer segment is packed in **destination-local order**, so the receiver can
+  reorder with a single `argsort` of the concatenated destination positions,
+* `network_exchange_required` is computed from the global count matrix, so every
+  rank takes the same branch; a rank that happens to exchange nothing never
+  skips the collective alone.
+
+The runtime is a variable-split `torch.distributed.all_to_all_single` over an
+regular SP process group (`get_sp_group().device_group`), plus a local-permutation fast
+path when the whole group stays in place (which is the SP=1 case). The first
+version is deliberately synchronous: no extra streams, no per-layer
+`cuda.synchronize()` or barrier "fixups", and no hidden all-gather.
+
+## Text stream
+
+Text is replicated on every rank and stays identical: each layer's attention
+produces one text output per window, ranks sum their local window outputs,
+one SUM all-reduce runs on the window-SP group, and every rank divides by the
+**global** window count of the current layout.
+
+```text
+T_next = (sum_r sum_{w in Omega_r} T_w) / W_global
+```
+
+This is neither a token-weighted mean nor a mean of rank means; a rank with no
+window contributes a zero tensor (with the *same dtype as its peers* -- a dtype
+mismatch changes the collective and deadlocks NCCL, which is why the empty-rank
+path is covered by a GPU test).
+
+## Existing SP configuration
+
+Configure `ulysses_degree=N` to create the existing N-rank SP group. Pass
+`DiffusionParallelConfig` to `SeedVR2NaDiT.build_runtime(parallel_config=...)`;
+the model validates the configuration and obtains `get_sp_group().device_group`.
+This adds no framework parallel option or group-construction branch.
+
+For SeedVR2 this selects **window-aligned Plan A**, not head-sharded Ulysses.
+Every rank retains all 20 attention heads, so SP=8 does not require the head
+count to divide by eight. The model has no `_sp_plan`, and its attention uses
+`skip_sequence_parallel=True`; generic sequence hooks and Ulysses attention
+exchanges are not installed. Reports name the actual execution path explicitly.
+
+SeedVR2 rejects ring, allgather, `advanced_uaa`, and `ulysses_a2a_permute`.
+Other diffusion models keep their existing Ulysses/Ring/AllGather behavior.
+
+## SeedVR2 integration
+
+`SeedVR2WindowRuntime` owns, per request: the layer schedule, the entry
+distribution, `ensure_layout` transitions, the text reduction and the final
+reconstruction back to canonical token order. The DiT forward keeps only this
+rank's video rows from the patch embedding onwards; the patch projection,
+norms, ADA modulation, residual and MLP are all tokenwise, so they run locally.
+
+The block never calls a parallel attention strategy: the shared `Attention`
+layer is created with `skip_sequence_parallel=True`, and window attention is a
+packed-varlen call over `[video window, replicated text]` segments (with a
+grouped-SDPA fallback for backends without a packed-varlen entry point).
+
+## Reference quirks reproduced on purpose
+
+* `vid_out_ada` is declared with `layers=["out"]`, whose embedding regrouping is
+  arithmetically incompatible with its 1-D parameters. The reference never uses
+  that value: it asks the forward-wide cache for `emb_repeat_0_vid`, a key the
+  first block already wrote, and therefore applies the *block* attention
+  modulation together with the learned `out_scale` / `out_shift`. The port
+  applies that effective modulation directly; changing it would break parity.
+* `mm_layers=10` controls weight sharing only. Text is consumed and produced by
+  all 32 blocks; only the last block freezes its text MLP path.
+* RoPE positions are window-local: a video token at window-local `(dt, dh, dw)`
+  uses axial indices `(text_len + dt, dh, dw)`, and every window restarts at 0.
+* The rotary span is `3 * (rope_dim // 3)` (126 of 128 channels for the 3B), so
+  the last two channels pass through unrotated.
+
+## Checkpoint compatibility
+
+The parameter layout of the port mirrors the reference, with one explicit
+exception: the released checkpoint stores the per-block RoPE table as
+`blocks.<i>.attn.rope.rope.freqs`, while this port registers it as
+`blocks.<i>.attn.rope.freqs`. The validation loader normalizes exactly that
+suffix (and only that suffix); every other key must already match.
+
+> The validation loader explicitly normalizes the reference RoPE buffer suffix
+> from `.rope.rope.freqs` to `.rope.freqs`. Checkpoint validation fails on any
+> remaining missing or unexpected key after normalization.
+
+Normalization is restricted and collision-checked: a key that is already
+normalized is left untouched, and two source keys mapping onto the same target
+key raise instead of silently overwriting a tensor. Loading then fails on any
+remaining missing, unexpected or shape-mismatched key, so a checkpoint that does
+not match the port cannot reach inference. Full 32-layer loading is the
+acceptance path; truncating the block list is a development fixture and requires
+an explicit flag.
+
+The native serving loader applies the same suffix normalization and strict
+checkpoint check.
+
+## Integration limits
+
+The native serving pipeline passes its parallel configuration to the DiT runtime
+and returns restored video through the standard media path. The causal VAE can
+run replicated, tiled, or height-sharded across the window-SP group.
+
+## Limitations
+
+* Plan A re-shards the full activation at every layout boundary. Plan B halo
+  exchange is a follow-up with a separate crossover evaluation.
+* The packed-varlen attention kernel needs a backend that accepts
+  `cu_seqlens`; the grouped-SDPA path is the portable fallback and is what the
+  pinned L20 environment exercises.

docs/design/fullduplex-personaplex.md

@@ -1,142 +1,137 @@
-# PersonaPlex Unified Full-Duplex Design
+# PersonaPlex on the Unified Full-Duplex Framework
 
 ## Status and target
 
-This design adapts PersonaPlex PR #4771 to the unified full-duplex runtime on
-top of:
-
-- vLLM-Omni `origin/main`: `67c54777bb22e9e7e08fdf7c47a64f06b566fc47`
-- PersonaPlex PR head: `477fb7c225f0c06991bc8aa55eadbd908ba282e4`
-
-The target is the engine-native path:
+PersonaPlex (`nvidia/personaplex-7b-v1`, a Moshi finetune) is served by the
+[Unified Full-Duplex Framework](fullduplex.md) through one model plugin,
+`PersonaPlexDuplexPlugin` (`vllm_omni/model_executor/models/personaplex/duplex/plugin.py`),
+selected by `PipelineConfig.duplex_plugin`. The serving path is:
 
 ```text
 /v1/realtime?duplex=1
-  -> OpenAI Realtime session actor
-  -> DuplexRequestClient
-  -> AsyncOmni correlated RPC
-  -> DuplexControlPlane
-  -> resumable Stage 0 request
-  -> PersonaPlex Talker
-  -> streaming PersonaPlex Code2Wav
-  -> PersonaPlex data-plane projector
+  -> OmniDuplexSessionHandler (thin: websocket I/O, Realtime <-> DuplexCommand/DuplexEvent)
+  -> DuplexOmni / DuplexOmniEngine
+  -> DuplexSessionManager -> DuplexSessionRunner (one per session)
+       plugin.plan_append: one 80 ms frame -> one resumable Stage 0 append
+  -> PersonaPlex Talker (Stage 0, lockstep temporal transformer + depformer)
+  -> streaming PersonaPlex Code2Wav (Stage 1, Mimi)
+  -> plugin.data_plane: cumulative audio/text -> deltas
   -> response.output_audio.delta + response.output_audio_transcript.delta
 ```
 
-The standalone `/api/chat` and `/v1/audio/duplex` server that accompanied the
-original PR was demo-only and has been removed from the tree; the unified
-engine path above is the only serving surface. It was never evidence that the
-unified engine path works.
-
-## Why configuration-only enablement was invalid
-
-The staged pipeline as shipped by the original PR was explicitly turn based.
-Its Talker read `pplex_user_codes`, `pplex_prefill_text`, and
-`pplex_silence_codes`, but no production staged input path wrote those fields.
-The voice prompt, persona prefill, and streaming Mimi state lived only in the
-standalone `PersonaPlexEngine` (demo-only, since removed).
-
-Setting only the following fields would therefore advertise an endpoint whose
-model never receives the live microphone stream:
-
-```python
-duplex_control_enabled = True
-duplex_runtime_extension = "..."
-duplex_serving_adapter = "..."
-```
-
-The adapter must supply a real scheduler data plane, not just endpoint
-capabilities.
+The pre-framework pair (engine runtime extension plus serving adapter) and
+the standalone `/api/chat` / `/v1/audio/duplex` server are gone; the plugin is
+the only integration surface, and this document describes it.
 
 ## Supported scope
 
-The unified implementation supports:
+The integration supports:
 
-- up to two engine-owned sessions on one replica;
-- 24 kHz mono float PCM input;
-- one 1920-sample, 80 ms model frame per physical append unit;
-- continuous user input while assistant audio is generated or played;
-- bundled `.pt` voice prompts and a session persona;
-- greedy text and depformer sampling, matching the current PersonaPlex port;
-- `/v1/realtime?duplex=1` (the wire vocabulary is catalogued in the
-  [Realtime Duplex API](../serving/realtime_duplex_api.md) serving guide);
+- up to `duplex_session.max_sessions` engine-owned sessions on one replica
+  (the shipped deploy sets two);
+- 24 kHz mono float PCM input, one 1920-sample (80 ms) model frame per
+  physical append unit; clients may send any chunking, the session buffers
+  whole frames;
+- continuous user input while assistant audio is generated or played
+  (pure lockstep: the model listens while it speaks);
+- bundled `.pt` voice prompts (`voice`) and a session persona (`instructions`);
+- greedy text and depformer sampling (one temporal token per frame);
 - the public client preset
   `vllm_omni.clients.personaplex.create_duplex_session_config()` (24 kHz
-  `pcm_f32le` input format, voice prompt, persona) as the canonical
-  session-config source for `DuplexClient` consumers;
-- engine lease close, disconnect cleanup, reconnect after cleanup, and explicit
-  response cancellation without cross-session state reuse.
+  `pcm_f32le` input, voice, persona) as the canonical session-config source
+  for `DuplexClient` / `InlineDuplexClient` consumers;
+- engine lease close, disconnect cleanup and explicit response cancellation
+  without cross-session state reuse.
 
-The implementation does not claim:
+It does not claim:
 
-- more than two simultaneous PersonaPlex sessions on one replica;
-- arbitrary WAV voice cloning;
-- turn-based `response.create` semantics for an otherwise continuous model;
-- scheduler migration of live codec state between replicas;
-- exact output equality with the standalone engine after different scheduling
-  boundaries.
+- arbitrary WAV voice cloning (voices are bundled basenames, resolved by the
+  worker under the checkpoint);
+- turn-based `response.create` semantics, client commits or external turn
+  signals (`supports_client_commit=false`, `supports_external_turn_signal=false`);
+- text seeding, hence no `/v1/chat/completions` route
+  (`supports_chat_completions=false`);
+- session resume across a transport drop (`supports_session_resume=false`);
+- destructive output interruption or model-state rewind at a playback cursor
+  (`supports_barge_in=false`, `supports_audio_truncate=false`): overlapping
+  speech is native model behaviour, not a barge-in contract.
 
-The capability payload derives multi-session support from the configured
-session limit. The shipped two-session deployment reports
-`supports_multi_session=true` and `supports_multi_session_same_replica=true`.
 `duplex_session.max_sessions` is the only capacity source: config resolution
-propagates it to every stage as `duplex_max_sessions`, and both Mimi pools read
-that model-config value. Connector extras do not carry a second model-specific
-capacity knob that could drift from engine admission.
-It still reports `supports_barge_in=false`: the generic epoch fence can suppress
-stale transport output, but neither PersonaPlex nor the current MiniCPM-o 4.5
-adapter proves that model-owned streaming state can be destructively rewound or
-restarted at a playback cursor. Continuous overlapping speech is model-native
-duplex behavior, not by itself a barge-in contract.
+propagates it to every stage as `duplex_max_sessions`, and both Mimi pools
+(Stage 0 encoders, Stage 1 decoders) read that model-config value. The
+capability payload derives `supports_multi_session` /
+`supports_multi_session_same_replica` from it.
 
 ## Components and ownership
 
-### Serving adapter
+### The plugin (engine side)
 
-`PersonaPlexServingRuntimeAdapter` owns only serving-side state:
+`PersonaPlexDuplexPlugin` owns both halves of the contract.
 
-- a transactional PCM append buffer;
-- validation of `voice_prompt` and `instructions`;
-- public capabilities;
-- PersonaPlex data-plane output projection.
+Engine policy:
 
-It accepts 24 kHz `pcm_f32le`, groups client packets into whole 1920-sample
-frames, zero-pads only the final residual, and rolls a reservation back when an
-engine append fails. It never loads CUDA weights and never encodes user audio.
+- `configure_sampling_params`: Stage 0 greedy (`temperature=0`, `top_k=1`,
+  `max_tokens=1`), other stages untouched.
+- `plan_append`: validates exactly one 1920-sample 24 kHz `pcm_f32le` frame
+  (through `model_executor/common/duplex/payload.py`) and reserves one
+  scheduler slot per frame plus, on the first append of an epoch (`seq == 1`),
+  the `personaplex_prefill_slots` of the voice/persona prefill. The prompt
+  carries the fence, `seq`, the payload and the runtime config under
+  `model_intermediate_buffer["duplex"]`.
+- `decide_output`: never decides. PersonaPlex is always-clocked; what the
+  client hears comes from the final-stage data plane.
+- `silence_unit_payload`: one frame of zeros at 24 kHz
+  (`silence_continuation_samples=1920`,
+  `silence_continuation_sample_rate_hz=24000`). The runner uses it to keep a
+  model turn clocked when the client pauses; the startup warmup sends it.
 
-Client input cannot provide local filesystem paths. A voice is a bundled
-basename such as `NATF2.pt`; the worker resolves it under the local model
-checkpoint. `instructions` is the persona string.
+Session policy:
 
-### Runtime extension
+- `capabilities`: `personaplex_capabilities(max_sessions)` (80 ms units,
+  append-only, no commits, no barge-in, no resume, no chat route).
+- `prepare_runtime_config`: `voice` must be a bundled `.pt` basename,
+  `instructions` defaults to the shipped persona; the prefill slot count is
+  computed once per `(model, voice, persona)` off the orchestrator loop
+  (`personaplex_prefill_slots`, itself caching the tokenizer and the voice
+  bundle row count). The result is server-owned runtime config
+  (`personaplex_model_path`, `personaplex_voice_prompt`,
+  `personaplex_persona`, `personaplex_prefill_slots`); the same keys are
+  refused in a client's `extra_body`.
+- `runtime_config_for_update`: persona and voice are immutable for the
+  session (`persona_update_unsupported`, `voice_update_unsupported`).
+- session state, extra-body validation and data-plane context are the
+  framework defaults (`DefaultDuplexModelSessionState` with the 80 ms
+  `PersonaPlexPcmAppendBuffer`, `DuplexDataPlaneContext`).
 
-`PersonaPlexDuplexRuntimeExtension` is pure model policy. It:
+Because `supports_client_commit` is off, the session auto-responds without
+`extra_body.auto_response`: a stock Realtime client streams audio and hears
+the model without any vendor flag.
 
-- configures greedy Stage 0 sampling and bounded segment lengths;
-- maps each accepted PCM append to a scheduler prompt;
-- places immutable session identity, append sequence, PCM payload, voice, and
-  persona under `model_intermediate_buffer["duplex"]`;
-- reserves exactly one scheduler prompt slot per encoded Mimi frame, plus the
-  first-append voice/persona prefill length;
-- never performs model inference or owns session state.
+### Input framing
 
-The extension returns no turn/listen decision. PersonaPlex is an always-clocked
-model, so visible audio/text comes from the final stage data plane.
+`PersonaPlexPcmAppendBuffer` is the `FixedFramePcmAppendBuffer` of
+`model_executor/common/duplex/pcm_buffer.py` at 24 kHz / 1920 samples /
+80 ms. It accepts 24 kHz `pcm_f32le` only, groups client packets into whole
+frames, takes one frame out per append as a reservation (committed when the
+stage accepted the append, rolled back to the front of the buffer when it did
+not), zero-pads only a final residual on commit, and never encodes audio.
 
-### Stage 0 streaming runtime
+### Stage 0 streaming runtime (worker side)
 
-The Talker owns a `PersonaPlexStage0DuplexRuntime` helper, analogous to
-MiniCPM-o's Stage 0 helper but with PersonaPlex lockstep semantics.
+The Talker owns a `PersonaPlexStage0DuplexRuntime`
+(`model_executor/models/personaplex/duplex/stage0.py`), created lazily in
+`_duplex_stage0_runtime()` and released through `on_requests_finished`.
 
-For each admitted session it owns:
+For each live `(session_id, epoch)` it owns:
 
-- streaming Mimi encoder convolution and transformer state;
-- the selected voice embedding bundle;
-- persona tokenization and prefill embeddings;
-- the prior user code frame needed by the one-frame acoustic delay;
-- append identity used to make a retried scheduler update idempotent.
+- the streaming Mimi encoder convolution and transformer state;
+- the selected voice embedding bundle and the persona prefill embeddings;
+- the prior user code frames needed by the one- and two-frame acoustic
+  delays;
+- the append identity `(epoch, seq)` that makes a retried scheduler update
+  idempotent.
 
-The first append builds this ordered prefill:
+The first append of an epoch builds this ordered prefill:
 
 ```text
 voice embeddings
@@ -146,45 +141,47 @@ voice embeddings
   -> first live user frame
 ```
 
-Later appends encode only new 1920-sample frames. The helper returns the
+Later appends encode only new 1920-sample frames. The runtime returns the
 per-frame user codes and prompt embeddings through the request's
-`model_intermediate_buffer`. The normal vLLM runner remains authoritative for
-attention metadata, block tables, KV allocation, scheduling, and sampling.
+`model_intermediate_buffer`; the normal vLLM runner remains authoritative for
+attention metadata, block tables, KV allocation, scheduling and sampling.
 
-The Talker must distinguish resumable prompt prefill from decode positions.
-Prompt rows consume the exact prepared embeddings; sampled decode rows continue
-to use the existing delayed agent/user frame construction. No code path may
-fall back to an all-initial user stream for a duplex request.
+**Epochs.** The framework identifies a Stage 0 request by
+`(session_id, epoch)`; `response.cancel` and `output_audio_buffer.clear`
+advance the epoch, abort the current request and start the next append at
+`seq == 1` on a fresh request with fresh KV. The Stage 0 runtime therefore
+keys its state by `(session_id, epoch)`: when an append of a newer epoch
+arrives, any older-epoch state of the same session is closed first (its Mimi
+encoder returns to the pool, so the codec budget never counts a superseded
+epoch), and the voice/persona prefill is replayed because the plan reserved
+the slots again. The user-visible consequence is that a cancel restarts the
+model's conversation context. A late `on_requests_finished` for the aborted
+request finds nothing to close.
 
-Cleanup is keyed by the full `(session_id, incarnation)` identity. Every live
-session has an independent Mimi encoder instance; encoder convolution/KV state
-is never shared between asynchronously scheduled sessions. A finished or
+Every live session has an independent Mimi encoder instance; encoder state is
+never shared between asynchronously scheduled sessions. A finished or
 aborted scheduler request resets and returns only that session's encoder.
 
 ### Stage 1 streaming decoder
 
-The current `PersonaPlexCode2Wav` calls one-shot `MimiModel.decode` and is not
-CUDA-graph safe. Unified duplex uses eager Stage 1 and maintains an independent
-streaming Mimi decoder for every active request. Decoder ownership is keyed by
-the stable Stage 1 request id and released by `on_requests_finished`; a mixed
-batch must never advance another request's convolution or transformer state.
-
-Each Stage 0 segment emits de-delayed agent codebooks. Stage 1 decodes only the
-new code frames, emits only the new PCM suffix, and resets state when the
-request is closed. Connector chunk boundaries retain the final raw code frame
-needed to de-delay the next chunk.
-
-The deploy default sets Stage 1 `enforce_eager: true`; a default configuration
-that fails during CUDA graph capture is not an acceptable deployment profile.
+`PersonaPlexCode2Wav` runs eager (`enforce_eager: true` in the deploy) and
+maintains an independent streaming Mimi decoder for every active request,
+keyed by the Stage 1 request id and released by `on_requests_finished`. Each
+Stage 0 segment emits de-delayed agent codebooks; Stage 1 decodes only the new
+code frames and emits only the new PCM suffix. Connector chunk boundaries
+retain the final raw code frame needed to de-delay the next chunk.
 
 ### Data-plane projector
 
-`PersonaPlexDataPlaneSession` converts cumulative or delta Stage 1 output into
-model-neutral native results:
+`PersonaPlexDataPlaneSession` is the `CumulativeAudioTextDataPlane` of
+`model_executor/common/duplex/data_plane.py` with a 24 kHz default rate. It
+keeps one audio/text cursor per request so cumulative Stage 1 output cannot
+replay old audio, and yields one internal result per new suffix:
 
 ```python
 {
     "stage_role": "tts",
+    "is_listen": False,
     "data_plane_request_id": request_id,
     "text": text_delta,
     "audio_data": encoded_audio_delta,
@@ -192,82 +189,74 @@ model-neutral native results:
     "sample_rate_hz": 24000,
     "audio_duration_ms": delta_duration,
     "end_of_turn": False,
+    "runtime_impl": "scheduler_data_plane",
+    ...
 }
 ```
 
-It owns per-request audio and text cursors so a cumulative output cannot replay
-old audio. The generic projector turns each native result into the Realtime
-`response.output_audio.delta` / `response.output_audio_transcript.delta` pair (and
-`response.output_text.delta` for text) under one `response_id`; the full
-mapping is the name map in the
-[Realtime Duplex API](../serving/realtime_duplex_api.md) serving guide.
-PersonaPlex keeps one visible response open while continuous output arrives.
-Session close or cancellation terminates that response through the generic
-Realtime lifecycle; a codec segment finishing is not a conversational turn
-boundary. PersonaPlex advertises `supports_barge_in=false` and
-`supports_session_resume=false`, so `barge_in`, `turn_detection.server_vad`,
-and `session.resume` are rejected on this model.
+The runner turns each result into the Realtime
+`response.output_audio.delta` / `response.output_audio_transcript.delta` pair
+under one `response_id`. PersonaPlex keeps one visible response open while
+continuous output arrives; session close or cancellation terminates it through
+the generic Realtime lifecycle. A codec segment finishing is not a
+conversational turn boundary.
 
 ## Error and lifecycle contracts
 
-- Unsupported sample rate, malformed base64, non-finite PCM, invalid voice
-  basename, or changed format fails before scheduler submission.
+- Unsupported sample rate, malformed base64, non-finite PCM, a partial frame
+  at the plan, an invalid voice basename or a changed format fails before
+  scheduler submission, as a typed `error` event.
 - Append is prepare/submit/commit. Failure rolls back the exact reserved PCM
   bytes and does not advance the model frame cursor.
-- A repeated `operation_id` must not encode or submit the same frame twice.
-- Input iterator exceptions execute the same cleanup as explicit close.
-- Cancellation does not release the engine lease until the stage request and
-  any in-flight codec operation are actually finished.
-- A bounded cleanup timeout returns a cleanup error and keeps the session in
-  the closing admission set; it must not make the slot available while work
-  still mutates shared state.
-- New sessions cannot observe the previous voice, persona, PCM tail, Mimi
-  convolution state, or Talker delayed code frame.
-
-(The legacy `PersonaPlexDuplexRuntime.run()` and standalone server drain path,
-which followed the same exception-safe rule, have been removed from the tree.)
+- A repeated `(epoch, seq)` identity does not encode or submit the same frame
+  twice.
+- Cancellation advances the epoch; the aborted request's Stage 0 state is
+  released by the next-epoch append or by `on_requests_finished`, whichever
+  comes first.
+- Close releases the admission slot only once stage cleanup succeeded; the
+  reaper retries a failed cleanup while the slot stays held.
+- New sessions cannot observe a previous voice, persona, PCM tail, Mimi
+  convolution state or Talker delayed code frame.
 
 ## Testing and acceptance
 
-### Contract tests
+### CPU contract tests
 
-Tests first cover:
+- `tests/model_executor/models/personaplex/duplex/test_plugin.py`: pipeline
+  binding and plugin load, honest capabilities, private keys, voice/persona
+  resolution and caching, immutable updates, greedy sampling, one-slot-plus-
+  prefill planning, frame validation, the 24 kHz silence unit, and the
+  CPU-checked helpers of the E2E driver.
+- `tests/model_executor/models/personaplex/duplex/test_stage0_runtime.py`:
+  first-append prefill, depformer teacher forcing, causal user-frame delays,
+  idempotent retries, independent encoders per session, capacity, epoch
+  restart and late finish of a superseded request.
+- `tests/engine/duplex/test_session_runner_personaplex.py`: the session
+  runner with the real plugin -- one submission per frame, prefill on the
+  first append only, half frames buffered, wrong rate refused, cumulative
+  Code2Wav output projected as 24 kHz deltas, cancel restarting the epoch,
+  close aborting the request.
+- `tests/model_executor/executor_common/`: the shared toolbox (PCM helpers, payload
+  validation, fixed-frame buffer, cumulative data plane, request-output
+  readers).
 
-- pipeline registration enables the control plane and selects both PersonaPlex
-  adapters;
-- the capability payload is two-session, 80 ms, append-only, and honest;
-- PCM reservation commit/rollback, partial-frame flush, invalid input, and
-  operation idempotency;
-- runtime prompt fields and exact token budgeting;
-- first-append voice/persona prefill followed by live user codes;
-- later appends retain per-session Mimi state and do not replay prefill;
-- interleaved Stage 0 and Stage 1 work preserves independent codec histories;
-- output projection emits only audio/text deltas;
-- close, exception, timeout, and reconnect cleanup;
-- ordinary non-duplex imports do not load PersonaPlex modules.
+### GPU validation
 
-### Remote H20 validation
+`tests/e2e/online_serving/personaplex_realtime_duplex.py` (wrapped by
+`tests/e2e/online_serving/test_personaplex_duplex.py` when
+`PERSONAPLEX_MODEL_PATH` is set) drives the default `personaplex.yaml`
+deployment and requires:
 
-Validation runs in an isolated remote worktree using the ModelScope
-`nv-community/personaplex-7b-v1` checkpoint and its Mimi dependency.
-
-The required evidence is:
-
-1. default `personaplex.yaml` reaches ready without a local eager override;
-2. `/health` returns 200;
-3. `/v1/realtime?duplex=1` reports `model_native_duplex`, `chunk_period_ms=80`,
-   and two-session admission;
-4. paced 24 kHz PCM appends produce finite, non-silent 24 kHz audio deltas and
-   text deltas;
-5. microphone input continues during assistant output without cancelling the
-   scheduler request;
-6. two paced sessions simultaneously produce independent non-empty audio, and
-   a third session is rejected with `resource_exhausted`;
-7. closing either session frees only its scheduler request and GPU codec state,
-   after which a replacement session can use a different persona without state
-   leakage;
-8. malformed input returns one typed error and does not poison the next append;
-9. all focused unit tests and `git diff --check` pass.
+1. the server reaches ready without a local eager override; `/health` is 200;
+2. `/v1/realtime?duplex=1` reports `model_native_duplex`, `chunk_period_ms=80`
+   and two-session admission, with server-allocated session ids;
+3. paced 24 kHz PCM appends produce finite, non-silent, whole-frame 24 kHz
+   audio deltas and text deltas;
+4. two paced sessions simultaneously produce independent non-empty audio, and
+   a third session is refused with `resource_exhausted`;
+5. closing one session frees only its scheduler request and codec state, after
+   which a replacement session with a different persona is admitted with a
+   fresh id and no state leakage.
 
 Audio that is empty, all zero, non-finite, or only a protocol `listen` event is
 not a successful end-to-end result.

docs/design/fullduplex.md

@@ -220,6 +220,7 @@ vllm_omni/
 │       │                            DUPLEX_REALTIME_CAPABILITIES, duplex_response_format
 │       ├── events.py                re-export shim (DuplexEvent = RealtimeEvent) + the runner's
 │       │                            epoch-filter sets DOMAIN_TERMINAL_EVENTS / MODEL_OUTPUT_EVENTS
+│       ├── delivery.py              DuplexOutputBuffer (bounded output shared with the session handle)
 │       ├── realtime_events.py       RealtimeProjectionState: internal event -> typed events
 │       ├── messages.py              queue envelopes (Open/Close/Resume/Touch/Command/Result/Event), DuplexSessionError
 │       ├── config.py                DuplexSessionConfig, DuplexCapabilities, ResponseCreateOptions
@@ -284,7 +285,8 @@ command  handle.submit(DuplexCommand)
          -> engine.submit_command_async -> DuplexSessionCommandMessage on the request queue [one-way]
          -> DuplexSessionManager.dispatch: unknown_session / input_backpressure checks, then runner mailbox
 output   DuplexOrchestrator._intercept_stage_output -> runner.on_stage_output -> mailbox -> typed events
-         -> output_sink (DuplexSessionEventMessage) -> DuplexOmni._route_engine_message -> handle.events()
+         -> session's bounded output buffer -> handle.events(); closure and errors without a live session
+            use output_sink (DuplexSessionEventMessage) -> DuplexOmni._route_engine_message
 detach   DuplexOmni.detach_session(expected_lease_generation) -> touch(DETACH): engine-owned disconnect
          grace, refused for a lease newer than the one the caller held; expiry -> SessionExpired
 resume   DuplexOmni.resume_session(expected_lease_generation) -> lease CAS keyed by the control id (a replay
@@ -303,7 +305,9 @@ reap     DuplexSessionManager.reaper_loop: idle TTL / disconnect grace expiry, c
 
 ### Concurrency and ordering model of the runner
 
-Everything below runs on the orchestrator asyncio loop; there is no lock.
+The runner's session state is owned by the orchestrator asyncio loop and
+needs no lock. The output buffer shared with the caller thread uses a lock
+for queue updates and the final validity check before delivery.
 
 - **Single writer per session.** `DuplexEngineSession` is mutated only by its
   runner. Inputs reach the runner through one mailbox in this order: client
@@ -332,13 +336,15 @@ Everything below runs on the orchestrator asyncio loop; there is no lock.
   on one loop in program order, nothing can be emitted for an old epoch after
   its terminal event; `runner.emit()` still drops a terminal carrying a stale
   epoch and stamps `epoch` on every event for clients that filter
-  defensively. This is a statement about order, not about latency: the engine
-  output queue and each `DuplexSessionHandle` outbox are unbounded and FIFO,
-  so a cancellation is delivered behind whatever audio was already emitted for
-  the response it cancels. A client that must stop quickly cancels its own
-  playback on `response.done` / `audio.cancelled` rather than waiting for the
-  stream to drain. Bounding those buffers and letting an accepted invalidation
-  skip undelivered media is left to the follow-up RFC.
+  defensively. In this experimental follow-up, public output goes directly to
+  one bounded buffer shared by the engine and `DuplexSessionHandle`. Accepted
+  cancellation removes queued `AudioDelta` events for that response through
+  the cancelled epoch. Text increments, audio/transcript completion markers,
+  other responses and normal completion retain FIFO order. The consumer also
+  checks an event already taken from the buffer: WebSocket delivery repeats
+  that check under the connection lock, before sequencing and journaling.
+  Already sequenced replay entries are unchanged; already delivered audio
+  still requires the client to stop its own playback.
 - **Backpressure before the mailbox.** `DuplexSessionManager.dispatch`
   checks `max_pending_input_bytes_per_session` and reserves a pending turn for
   `Commit` before the put; a rejected command is answered with
@@ -349,6 +355,46 @@ Everything below runs on the orchestrator asyncio loop; there is no lock.
   `ResponseDone.stage_metrics` and `metadata.vllm_omni.stage_metrics` on the
   wire keep their meaning.
 
+### Experimental output limits
+
+`max_pending_output_bytes_per_session` defaults to 2 MiB and
+`max_pending_output_events_per_session` to 512. The byte limit counts compact
+Realtime JSON, including base64 audio, not decoded PCM or Python object
+overhead. Audio is already base64-encoded by the producer; counting its length
+avoids serializing or decoding that large field again. Neither default has
+been tuned from a performance study.
+
+Errors and response endings have an additional 64 KiB / eight-event reserve
+and retain FIFO order. One final session-closure notification has an independent
+slot, so an exhausted reserve cannot prevent closure or break delivery to
+other sessions. If that reserve also fills, an error or response-ending
+notification may be omitted; the final session-closure notification still
+has its independent slot. Oversized close details (over 4 KiB with generated identity)
+are replaced by a compact reason, preserving terminal type and identity.
+Late writes after closure are ignored; a locally ended iterator is not reopened
+by a late terminal, and a failed open closes the abandoned handle. Events other
+than errors are not forwarded once their session's buffer has been removed.
+One event held by the consumer is outside the queued budget.
+
+The overflow policy fails the active response without committing its partial
+text to history and closes only the affected session. If overflow occurs while
+a completion batch is being delivered, its not-yet-queued `ResponseDone` is
+converted to failure with its identity and output preserved; any history entry
+for that response is removed. An ending already accepted by the buffer is not
+replaced or followed by a second ending. After overflow, newly
+emitted ordinary events are suppressed, including text/audio completion,
+content-part and output-item completion markers. Previously queued non-audio
+events retain FIFO order; error and failed response-ending notifications may
+use the reserve before the independent closure notification. This differs from
+accepted cancellation, which removes only matching audio. See the
+[public error and closure fields](../serving/realtime_duplex_api.md#output-limits-and-slow-consumers).
+Response-only recovery is not implemented. These limits do not bound the runner's raw stage
+output mailbox, replay journal, or client playback queues. The shared buffer
+requires the current same-process, separate-thread engine arrangement; it is
+not a cross-process delivery protocol. Direct handle iteration and WebSocket
+delivery share buffer validity, but end-to-end client behavior still needs
+independent real-model verification.
+
 ## Orchestrator seams
 
 `OrchestratorBase` keeps the generic machinery (request handler skeleton,

docs/design/index.md

@@ -68,6 +68,7 @@ guides list selectable backends, platform defaults, installation, and tuning.
 The design contracts separate selection mechanics from backend algorithms:
 
 - [Attention Backend Selection](feature/attention_backend_selection.md)
+- [Attention Execution Contract PoC](feature/attention_execution_contract_poc.md)
 - [Skip-Softmax](feature/skip_softmax.md)
 
 #### CPU offloading
@@ -115,3 +116,6 @@ The design contracts separate selection mechanics from backend algorithms:
 The pre-#5137 pages are preserved in the
 [legacy module archive](module/archive/README.md) for historical reference and
 are not active design contracts.
+
+- [Qwen3-Omni MRv2 performance](qwen3_omni_mrv2_performance.md)
+- [MiniCPM-o 4.5 turn-mode MRv2 performance](minicpm_o45_mrv2_performance.md)

docs/design/minicpm_o45_mrv2_performance.md

@@ -0,0 +1,36 @@
+# MiniCPM-o 4.5 MRv2 profiles
+
+Turn mode uses the stage-level MRv2 contracts from #8184. Thinker emits live
+latent metadata outside graph replay; Talker keeps codec history and EOS
+control on device. Code2Wav reuses mainline Whole-Euler Flow graphs and shared
+prompt state. The previous experimental block compilation, tiled attention,
+channels-last and merged-CFM implementations are removed.
+
+`minicpmo_4_5_turn_mrv2_h200.yaml` selects Talker capacity 16 and 4 GiB KV;
+the generic MRv2 profile selects capacity 8 and 2 GiB KV. These configurations
+require new end-to-end performance measurements after the mainline integration.
+Previously reported numbers do not describe this revised codec backend.
+
+The generic MRv2 profile and the opt-in native V1 duplex H200 profile enable
+ordinary TF32 for CFM DiT GEMMs within Code2Wav forward/capture only
+(`torch.backends.cuda.matmul.allow_tf32` on dense QKV/MLP, Triton
+`input_precision="tf32"` on tiled attention). This is not compensated TF32x3.
+The previous process matmul policy is restored afterwards. cuDNN's TF32 policy
+is independent. HiFT stays IEEE FP32. TF32 changes rounding.
+Duplex keeps the mainline V1 session path. Turn results do not establish duplex
+performance or interruption correctness.
+
+The shared asynchronous output snapshot and batched Talker preprocessing also
+apply to V1 when async chunking/scheduling is enabled. Default V1 behavior must
+therefore be included in end-to-end regression validation.
+
+CFM graph execution defaults to the fused DiT body on CUDA when TF32 is
+allowed. NVIDIA SM80+ uses tiled TF32 attention; older CUDA devices use the
+fused body's SDPA fallback. Unsupported DiT layouts or non-FP32 attention
+caches retain the original body. Other platforms retain their existing default.
+Set `cfm_fused_body: false` in connector `extra` to restore the original body.
+Disabling TF32 with `token2wav_allow_tf32: false` or
+`MINICPMO_CODE2WAV_TF32=off` also disables the default fusion choice; an explicit
+`cfm_fused_body: true` overrides that choice and may use TF32 attention.
+Slot pooling and row-offset merging remain opt-in. Fusion changes floating-point
+rounding and does not promise identical waveforms.

docs/design/minicpmo_encoder_cuda_graph.md

@@ -0,0 +1,61 @@
+# MiniCPM-o 4.5 input encoder CUDA graphs
+
+The vision transformer and stateless audio encoder reuse vLLM's encoder graph
+manager for exact input shapes. Each encoder retains at most four graphs by
+default, and captures on the second call with the same shape and CUDA stream.
+Startup profiling can consume slots. Graphs are never evicted: once the cache
+is full, other shapes run eagerly for the lifetime of the model.
+
+Use `--hf-overrides` to configure both encoders:
+
+```json
+{
+  "encoder_cuda_graph": true,
+  "encoder_cuda_graph_max_graphs": 4,
+  "encoder_cuda_graph_min_capture_calls": 2,
+  "encoder_cuda_graph_min_free_bytes": 1073741824,
+  "encoder_cuda_graph_share_pools": true
+}
+```
+
+`max_graphs` is a nonnegative integer per encoder; zero disables capture.
+`min_capture_calls` is an integer of at least two. Increasing it reduces capture
+of short-lived shapes. Admission history is bounded to four times `max_graphs`,
+so an evicted history entry must accumulate its calls again. Increasing the
+graph cap can improve coverage but retains more GPU memory. Existing graphs
+are not replaced, avoiding repeated capture costs when traffic changes.
+
+By default, graphs in the same encoder on the same device and replay stream
+share a memory pool and capture stream. Different encoders and replay streams
+remain isolated. Graph inputs and output buffers are allocated outside capture;
+only transient capture allocations share the pool. Returned outputs remain
+cloned. A late capture waits for earlier replay work and output clones before
+reusing the pool. Set `encoder_cuda_graph_share_pools` to `false` for an A/B
+comparison with a separate pool and capture stream per graph.
+
+The graph count is **not a byte limit**. Sharing pools reuses intermediate
+storage but does not eliminate per-shape static input and output buffers.
+Before a new capture, the adapter checks device-free memory against
+`min_free_bytes`, a nonnegative integer (default: 1 GiB; zero disables the check).
+Below this floor the call runs eagerly, and a later call may try admission
+again. Existing graphs can still replay. This check is conservative about
+reusable allocator memory and does not predict capture allocations, reserve
+memory, or guarantee that capture fits. Size the floor for the deployment;
+disable encoder graphs when their additional memory cannot be accommodated.
+
+Each adapter exposes `get_cumulative_stats()`: graph hits, total misses, hit
+rate, graph count, and separate ineligible, capacity, warmup and memory misses.
+These include calls rejected before the upstream manager, so a full cache's
+lost coverage is visible. Vision uses `vpm._encoder_graph`; audio uses
+`_audio_encoder_graph` on the model. The counters are per adapter, not global.
+
+Capture failures propagate the original exception and make that adapter
+unusable. Subsequent calls raise without retrying capture or running CUDA eager
+work. Restart the worker, optionally with encoder graphs disabled. This is a
+fatal contract, not automatic recovery: a failed CUDA capture can leave the
+device context unusable. `capture_failures` records this terminal state.
+
+`--enforce-eager` always disables these graphs. CPU, grad/autocast, stateful
+streaming audio and the other model-specific eager fallbacks remain unchanged.
+The adapter clones graph outputs through upstream postprocessing so retained
+embeddings survive later replays.

docs/design/module/ar_runtime.md

@@ -47,6 +47,29 @@ adapter, not injected through unrelated scheduler state.
 
 **Rule:** Workers and model runners MUST NOT implement cross-stage routing.
 
+## Token-only output payloads
+
+An AR stage whose downstream decoder consumes sampled tokens rather than
+hidden states can set `omni_pooler_payload_include_hidden = False`. Hidden
+states remain available for logits and sampling; prefill conditioning and
+other required multimodal outputs still cross the stage boundary.
+
+A chunk processor that needs IDs on steps without a tensor payload declares
+`requires_token_updates = True`. The scheduler then enqueues nonempty sampled
+IDs while preserving final-stage exclusions and the processor's EOF handling.
+Processors without this declaration retain their tensor-payload trigger.
+
+When hidden and multimodal payloads are absent, the shared runner skips empty
+per-request payload construction. Prefix-cache merges and pending model
+postprocess retain the normal path. This policy is independent of whether
+output materialization runs inline or in the background.
+
+Inline construction consumes query offsets, scheduled-token counts and
+scheduler metadata before the next execution can reuse them. Background
+construction snapshots those inputs. Both paths reuse the request IDs already
+copied by upstream bookkeeping; connector collection stays on the main thread
+because it can run a tensor-parallel collective.
+
 ## Safe-change guide
 
 Test request lifecycle, abort, cache state, and every affected worker execution

docs/design/module/diffusion/diffusion_runtime.md

@@ -36,6 +36,7 @@ validation_paths:
   - tests/diffusion/test_diffusion_worker.py
   - tests/diffusion/test_multiproc_engine_concurrency.py
   - tests/diffusion/test_uniproc_executor.py
+  - tests/diffusion/test_ray_executor.py
   - tests/diffusion/test_result_pump.py
   - tests/diffusion/test_async_output_worker.py
   - tests/diffusion/test_diffusion_ipc.py
@@ -96,8 +97,10 @@ The executor then chooses how the worker runs:
   RPC.
 - **`mp`** (default when `num_gpus > 1`, or when set explicitly):
   `MultiprocDiffusionExecutor` starts one `WorkerProc` per device.
+- **`ray`** (explicit): `RayDiffusionExecutor` starts one GPU actor per worker
+  across one or more Ray nodes.
 
-Either way, each worker owns its device, distributed state,
+In all backends, each worker owns its device, distributed state,
 `DiffusionWorker`, `DiffusionModelRunner`, and pipeline instance.
 
 ```mermaid
@@ -108,7 +111,7 @@ flowchart TB
         stageProc["StageDiffusionProc<br/>stage transport and lifecycle"]
         engine["DiffusionEngine<br/>admission and control loop"]
         scheduler["RequestScheduler or StepScheduler<br/>request state and policy"]
-        executor["DiffusionExecutor<br/>uni or mp backend"]
+        executor["DiffusionExecutor<br/>uni, mp, or ray backend"]
         streams["Per-request output streams"]
 
         stageProc --> engine
@@ -132,6 +135,8 @@ flowchart TB
     client -.->|"inline: direct call"| engine
     executor -->|"mp: broadcast / result queues"| workerProc
     executor -.->|"uni: in-process call"| worker
+    executor -->|"ray: actor RPC"| rayActor["RayDiffusionWorkerWrapper"]
+    rayActor --> worker
     streams -->|"process-backed"| stageProc
     stageProc -->|"ZMQ results"| client
     streams -.->|"inline"| client
@@ -154,6 +159,7 @@ flowchart TB
 | `DiffusionExecutor` | Execution backend contract, worker RPC, health, shutdown | Admission and request-state transitions |
 | `UniProcDiffusionExecutor` | Single in-process worker for `num_gpus == 1`; no IPC or async output pump | Multi-GPU execution |
 | `MultiprocDiffusionExecutor` | Worker processes, shared-memory message queues, result dispatch, worker monitoring | In-process single-GPU path |
+| `RayDiffusionExecutor` | GPU actors, placement groups, remote RPC, actor monitoring | Local worker subprocesses and shared-memory IPC |
 | `WorkerProc` | One `mp` worker process's IPC loop and reply rules | Scheduling policy; unused by `uni` |
 | `DiffusionWorker` | Device/distributed setup, LoRA activation, profiling, sleep/wake, runner delegation | Request admission |
 | `DiffusionModelRunner` | Pipeline loading, request-local model state, cache/compile setup, request or step execution | Queueing and cross-stage routing |
@@ -219,12 +225,12 @@ uses fused `execute_model_batch()` and requires the pipeline to declare
 request-batch support.
 
 **Distributed layerwise offload with AllGather (DLO DP concurrency)** is a
-separate multiproc dispatch path. It activates when
+separate dispatch path supported by `mp` and `ray`. It activates when
 `data_parallel_size > 1`, `enable_distributed_layerwise_offload` is set, and
 `dlo_use_allgather` is true. The engine then sets `dp_concurrent = True` and
 raises `scheduler.max_num_running_reqs` to `dp_size`, overriding the
 `max_num_seqs` value from `initialize()`. The scheduler still emits a
-multi-request `DiffusionSchedulerOutput`; the multiproc executor routes that
+multi-request `DiffusionSchedulerOutput`; these executors route that
 wave through `execute_request()` instead of fused `execute_model_batch()`.
 
 Optional admission coalescing can fill a `dp_size` wave: when
@@ -233,7 +239,7 @@ the first schedule of a wave (under `dp_concurrent`, the stable window is
 `min(0.3s, wait/2)`). With the default `request_batch_max_wait_ms == 0`,
 there is no wait.
 
-Before dispatch, `MultiprocDiffusionExecutor.execute_request()` rejects waves
+Before dispatch, both backends reject waves
 whose requests differ in sampling-parameter compatibility or `extra_args`
 (AllGather requires every DP rank to follow the same forward schedule). Each
 worker rank then picks one envelope from the wave:
@@ -297,8 +303,9 @@ head-of-line blocking.
 | --- | --- | --- |
 | `uni` | Default for `num_gpus == 1`, or set explicitly | One in-process worker |
 | `mp` | Default for `num_gpus > 1`, or set explicitly | One worker process per device |
+| `ray` | Set explicitly | One GPU actor per worker across Ray nodes |
 
-Ray and external-launcher diffusion backends are not implemented. A custom
+The external-launcher diffusion backend is not implemented. A custom
 `DiffusionExecutor` subclass or import path is also accepted.
 
 ### Uniproc backend
@@ -337,11 +344,27 @@ result path and does not start those pumps.
 See [Async diffusion output](../../feature/async_diffusion_output.md) for the
 multiproc request-mode timeline.
 
+### Ray backend
+
+`RayDiffusionExecutor` reuses initialized Ray or calls `ray.init()` (respecting
+`RAY_ADDRESS`). It reuses the current placement group or reserves one GPU
+bundle per worker with `PACK` placement. Ray owns GPU placement; PyTorch
+owns distributed collectives through a shared TCP rendezvous. Global ranks
+are grouped by host, while each actor uses local device 0.
+
+Request batches, step execution, and DLO DP waves use actor RPCs with
+rank-aware replies. Returned tensors move to CPU before Ray serialization;
+this backend does not use the multiproc async output pumps. Driver inference
+settings and explicit stage `runtime.env` are forwarded, excluding worker
+identity and device assignment. See [Stage configuration](../../../configuration/stage_configs.md)
+for cluster setup and environment details.
+
 ## Worker and runner boundary
 
 On the `mp` path, `WorkerProc` receives messages and calls methods through
 `WorkerWrapperBase`. On the `uni` path, the executor owns that wrapper
-directly. In both cases, `DiffusionWorker` handles the parts tied to a device:
+directly. Ray actors also construct `WorkerWrapperBase`. In all cases,
+`DiffusionWorker` handles the parts tied to a device:
 distributed initialization, model-runner construction, LoRA activation,
 profiling, and memory sleep/wake. It delegates actual model work to
 `DiffusionModelRunner`.
@@ -399,6 +422,10 @@ streams with an error, closes scheduler state, and shuts down the executor.
   async-output futures as failed, then clears them. Completed async outputs
   cached for later `wait_output_ready()` calls are dropped with the executor
   object.
+- **`ray`**: the executor gives actors up to five seconds to shut down, then
+  kills them and removes placement groups it owns. Actor death is monitored
+  even while idle; actor errors and RPC timeouts fail the group and notify the
+  stage. The stage client allows subprocess cleanup before signal fallback.
 - **`uni`**: the executor shuts down the in-process worker, drops the final
   model reference, and empties the accelerator cache so a later engine can reuse
   the device.
@@ -407,7 +434,7 @@ streams with an error, closes scheduler state, and shuts down the executor.
 
     Do not continue using a worker group after a fatal collective timeout or
     unexpected worker exit. Distributed state may be incomplete. The
-    multiprocess executor deliberately fails closed so the stage can restart.
+    `mp` and `ray` executors deliberately fail closed so the stage can restart.
     The uniproc executor fails closed when the accelerator context itself is
     poisoned.
 
@@ -417,10 +444,11 @@ streams with an error, closes scheduler state, and shuts down the executor.
   `diffusion_model_runner_cls` configuration still wins.
 - Tests and custom engine integrations may inject a `BaseScheduler` subclass.
   `SchedulerInterface` remains only as a deprecated compatibility name.
-- `distributed_executor_backend` may be `"uni"`, `"mp"`, a custom `DiffusionExecutor`
+- `distributed_executor_backend` may be `"uni"`, `"mp"`, `"ray"`, a custom `DiffusionExecutor`
   subclass, or an import path. A backend must preserve scheduler output,
   request identity, health, and cleanup contracts. Use `"mp"` when you need
-  process isolation or multi-GPU; `"uni"` is the single-GPU default.
+  local process isolation or multi-GPU, or `"ray"` for workers across nodes;
+  `"uni"` is the single-GPU default.
 - Worker extensions go through `WorkerWrapperBase`. They add worker methods;
   they do not become a second scheduler or request lifecycle.
 
@@ -466,7 +494,7 @@ Test the smallest affected slice, then cover its neighboring boundary:
 | --- | --- |
 | Admission or state transitions | `test_diffusion_scheduler.py`, including duplicate IDs, compatibility, abort, and missing output |
 | Engine loop or output delivery | `test_diffusion_engine.py`, `test_diffusion_engine_cleanup.py`, `test_diffusion_engine_rpc_routing.py` |
-| Executor or IPC | `test_multiproc_engine_concurrency.py`, `test_uniproc_executor.py`, `test_result_pump.py`, `test_diffusion_ipc.py`, `test_async_output_worker.py` |
+| Executor or IPC | `test_multiproc_engine_concurrency.py`, `test_uniproc_executor.py`, `test_ray_executor.py`, `test_result_pump.py`, `test_diffusion_ipc.py`, `test_async_output_worker.py` |
 | Worker or runner | `test_diffusion_worker.py`, `test_diffusion_model_runner.py` |
 | Stage boundary | `test_stage_diffusion_proc.py`, `test_inline_stage_diffusion_client.py` |
 | Warmup or streaming | `test_diffusion_engine_dummy_run.py`, `test_diffusion_streaming_output.py` |

docs/design/qwen3_omni_mrv2_performance.md

@@ -0,0 +1,22 @@
+# Qwen3-Omni MRv2 performance
+
+This model adaptation builds on #8184's existing stage runtime and output
+contracts. It adds Talker CPU metadata parsing, sampled-token embedding
+handoff, an optional first-frame decoder, length-grouped Code2Wav graph
+buckets, and opt-in incremental short-KV code prediction. The H200 profile
+also enables conservative fused sampling and bounded cuDNN algorithm search.
+The shared predictor keeps the upstream re-prefill helper and its default
+sampling behavior; short KV is explicitly selected by the deployment profile.
+
+Use `vllm_omni/deploy/qwen3_omni_moe_mrv2_h200.yaml` for the experimental H200
+profile. Preserve a control with `code_predictor_fused_sampling=false` and
+`codec_cudnn_benchmark=false` when C64 P99 is the primary requirement.
+cuDNN search restores process settings on exit and has startup cost. Kernel
+arithmetic is not bitwise-equivalent; the first-frame decoder uses additional
+model memory. Full-duplex performance is not established by turn-mode tests.
+
+First-frame delivery requires `talker_first_audio: true` in connector extra,
+a one-frame initial codec chunk, CUDA, a local single-rank Talker and prefix caching disabled.
+Other deployments retain the regular Code2Wav path.
+`codec_fused_snake` explicitly opts CUDA profiles into fused decoder blocks;
+the base V1 profile retains its original blocks and indexed text cache.

docs/design/qwen3_omni_tts_performance_optimization.md

@@ -93,13 +93,13 @@ Each optimization stacks on the previous one. The summary plots below show the c
 
 **Benchmark environment:**
 
-| | Qwen3-Omni                  | Qwen3-TTS |
-| --- |-----------------------------| --- |
-| **GPU** | A100                        | H200 |
+| | Qwen3-Omni | Qwen3-TTS |
+| --- | ----------------------------- | --- |
+| **GPU** | A100 | H200 |
 | **Model** | Qwen3-Omni-30B-A3B-Instruct | Qwen3-TTS-12Hz-1.7B-CustomVoice |
-| **vLLM** | v0.17.0                     | v0.18.0 |
-| **vllm-omni** | commit 199f7832             | v0.18.0rc2 |
-| **CUDA** | 12.9                        | 12.8 |
+| **vLLM** | v0.17.0 | v0.18.0 |
+| **vllm-omni** | commit 199f7832 | v0.18.0rc2 |
+| **CUDA** | 12.9 | 12.8 |
 
 This post walks through each optimization in the same order they are typically enabled in practice, then ends with deployment playbooks for both models.
 
@@ -310,7 +310,68 @@ self._compiled_model_fwd = torch.compile(
 
 `mode="default"` is used instead of `mode="reduce-overhead"` to avoid conflicts with vLLM's own CUDA graph capture on the main Talker model. `dynamic=True` handles the growing sequence length without recompilation.
 
-These optimizations are always-on in the current codebase - all Qwen3-TTS benchmark results in this post include them.
+The historical benchmark results above use re-prefill. It remains the default
+for Qwen3-TTS and the shared Qwen3-Omni predictor.
+
+### Experimental Qwen3-TTS single-GPU throughput profile
+
+The opt-in `qwen3_tts_high_concurrency_mrv2_single_gpu.yaml` profile uses a
+Qwen3-TTS-specific execution path on CUDA with BF16 weights:
+
+- `code_predictor_kv_cache` reuses each layer's keys and values across the 15
+  residual-codebook predictions within one audio frame. The first step writes
+  two positions; subsequent steps write one. Scratch storage is bounded by the
+  configured batch capacity, overwritten for every frame, and never shared
+  between requests across frames. The worker's serialized CUDA stream owns it;
+  it is separate from the Talker's paged KV cache.
+- A short causal/GQA Triton kernel masks unwritten K/V loads. Dynamic-batch
+  `torch.compile` warms the predictor before the runner captures whole-MTP
+  CUDA graphs. This path creates no inner CUDA graphs. Unsupported devices and
+  dtypes retain re-prefill; FP16 retains its existing precision safeguards.
+- `code_predictor_fused_sampling` fuses temperature scaling, top-k filtering,
+  and Gumbel argmax when the runner supplies FP32 uniforms. It preserves top-k
+  ties and uses the supplied randomness. Explicit generator paths retain the
+  existing sampler. Qwen3-Omni does not read either TTS option.
+- `decode_cudnn_benchmark` selects convolution algorithms during Code2Wav
+  warmup and graph capture. The process's cuDNN flags are restored afterwards,
+  including on capture failure. Captured graphs retain the selected algorithms.
+
+- `decode_time_major_conv` keeps codec convolution activations in time-major
+  layout to reduce transposes and copies between layers.
+- `talker_first_audio` completes residual prediction immediately after the
+  first codebook sample and decodes the first frame in the Talker process.
+  The codec advances its state without repeating the delivered frame; the
+  orchestrator orders that frame before subsequent codec chunks. This path
+  requires CUDA, MRV2, asynchronous chunks, a single-process executor, TP/PP 1,
+  and disabled prefix caching. Reference-code requests retain codec delivery.
+  Supported Qwen3-TTS profiles enable it explicitly; other models do not opt in.
+
+The predictor, sampling, convolution-layout and autotuning options default to
+false outside this throughput profile. It uses fixed 1/25-frame chunks,
+72 frames of left context, Talker/codec capacities of 128/64, codec batches
+through 8, and synchronous codec scheduling. Client concurrency is independent
+of those capacities. This profile also sets `cuda_mps: true`; the runtime owns
+a private MPS daemon unless an operator supplies an existing MPS pipe directory.
+Ordinary profiles leave MPS disabled. See
+[experimental MPS deployment](../configuration/stage_configs.md#experimental-mps-deployment)
+for requirements and ownership. Startup includes compilation and graph capture.
+
+```bash
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni \
+  --deploy-config vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2_single_gpu.yaml \
+  --api-server-count 4 --stage-init-timeout 1800 --init-timeout 2400
+```
+
+KV reuse changes GEMM shapes and attention reduction order; cuDNN autotuning
+can also change waveform numerics. This path is not bitwise equivalent to
+re-prefill, and aggregate WER alone cannot establish perceptual or speaker
+similarity equivalence. Validate quality on the intended workload before
+adopting it. Qwen3-Omni keeps its existing predictor and audio-delivery behavior.
+
+For a controlled ablation, copy the profile and disable individual options while keeping capacities, chunking, hardware, client concurrency,
+and warmup identical. Report repeated runs, first-audio latency and failures
+alongside throughput. Warm requests measure a ready service; they do not measure
+cold-start latency.
 
 ---
 

docs/features/sleep_mode.md

@@ -251,8 +251,8 @@ curl -X POST http://localhost:8000/v1/images/generations \
 
 | Method | Arguments | Return Type | Description |
 | :--- | :--- | :--- | :--- |
-| **sleep** | `stage_ids: List[int], level: int` | `List[OmniACK]` | Triggers hibernation for specified stages. |
-| **wake_up** | `stage_ids: List[int]` | `List[OmniACK]` | Reloads weights and re-maps memory. |
+| **sleep** | `stage_ids: List[int], level: int` | `List[OmniACK]` | Triggers hibernation for specified stages. Raises `RuntimeError` if a stage fails; the stages stay marked as sleeping, so `wake_up` brings them back, unless a level-2 sleep already discarded weights on some replica. |
+| **wake_up** | `stage_ids: List[int]` | `List[OmniACK]` | Reloads weights and re-maps memory. Raises `RuntimeError` if a stage fails; that stage stays sleeping and the call can be retried. |
 | **pause_generation** | `mode: str, stage_ids: List[int]` | `None` | Stops admission; with `mode="keep"` also stops diffusion schedulers and returns after their ACK. |
 | **resume_generation** | `stage_ids: List[int]` | `None` | Reopens paused schedulers, then admission. |
 
@@ -261,7 +261,7 @@ curl -X POST http://localhost:8000/v1/images/generations \
 | Field | Type | Description |
 | :--- | :--- | :--- |
 | **task_id** | `str` | Unique identifier for the operation. |
-| **status** | `str` | `SUCCESS` or `ERROR`. |
+| **status** | `str` | `SUCCESS` or `ERROR`. `sleep` and `wake_up` raise instead of returning an `ERROR` ACK, and `/v1/omni/sleep` and `/v1/omni/wakeup` return HTTP 500 with the error. |
 | **stage_id** | `int` | The ID of the stage that responded. |
 | **rank** | `int` | The rank ID within the Tensor Parallel group. |
 | **freed_bytes** | `int` | Actual amount of physical VRAM reclaimed. |

docs/getting_started/installation/README.md

@@ -1,7 +1,7 @@
 # Installation
 
 !!! important
-    vLLM-Omni is released against the matching upstream vLLM major/minor version. The 0.30 development line requires vLLM 0.30.x; the source-install instructions pin vLLM 0.30.0. Published vLLM-Omni 0.28.0 wheels and images still require vLLM 0.28.x.
+    vLLM-Omni is released against the matching upstream vLLM major/minor version. This source checkout targets vLLM 0.31.0. The published vLLM-Omni 0.30.0 wheel instructions use vLLM 0.30.0; choose the installation path that matches your Omni revision.
 
 vLLM-Omni supports the following hardware platforms:
 

docs/getting_started/installation/gpu.md

@@ -34,9 +34,7 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
 
 ### Pre-built wheels
 
-The source-install instructions target vLLM 0.30.0. Until vLLM-Omni 0.30.0 artifacts are published, the CUDA/ROCm pre-built examples below remain on the matching 0.28.0 release pair.
-
-Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, 0.25.0, 0.26.0, and 0.28.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
+Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, 0.25.0, 0.26.0, 0.28.0, and 0.30.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
 
 === "NVIDIA CUDA"
 

docs/getting_started/installation/gpu/cuda.inc.md

@@ -1,11 +1,13 @@
 # --8<-- [start:requirements]
 
-- GPU: compute capability 7.0 or higher (e.g., V100, T4, RTX20xx, A100, L4, H100, etc.)
+- GPU: compute capability 7.5 or higher (e.g., T4, RTX20xx, A100, L4, H100, etc.)
 
 # --8<-- [end:requirements]
 # --8<-- [start:set-up-using-python]
 
-vLLM-Omni depends on the matching major/minor release of vLLM. The 0.30 development line uses vLLM 0.30.x. Published 0.28.0 wheels and images use vLLM 0.28.x.
+vLLM-Omni depends on the matching major/minor release of vLLM. This source
+checkout targets vLLM 0.31.0. The pre-built Omni wheel instructions below install
+the published 0.30.0 release with its matching vLLM dependency.
 
 !!! note
     PyTorch installed via `conda` will statically link `NCCL` library, which can cause issues when vLLM tries to use `NCCL`. See <gh-issue:8420> for more details.
@@ -18,22 +20,22 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new**
 
 #### Installation of vLLM
 
-These pre-built wheel instructions install the published vLLM-Omni 0.28.0 release. For the 0.30 development line, use the source-install instructions below.
+These pre-built wheel instructions install the published vLLM-Omni 0.30.0 release.
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm==0.28.0 --torch-backend=auto
+uv pip install vllm==0.30.0 --torch-backend=auto
 ```
 
 #### Installation of vLLM-Omni
 
 ```bash
-uv pip install vllm-omni==0.28.0
+uv pip install vllm-omni==0.30.0
 ```
 
 To run Gradio demos, also install the optional extras:
 ```bash
-uv pip install 'vllm-omni[demo]==0.28.0'
+uv pip install 'vllm-omni[demo]==0.30.0'
 ```
 
 # --8<-- [end:pre-built-wheels]
@@ -41,13 +43,17 @@ uv pip install 'vllm-omni[demo]==0.28.0'
 # --8<-- [start:build-wheel-from-source]
 
 #### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.30.0 release version of the library
+If you do not need to modify source code of vLLM, install the final 0.31.0
+release wheel from its commit-pinned index:
 
 ```bash
-uv pip install vllm==0.30.0 --torch-backend=auto
+uv pip install vllm==0.31.0 --torch-backend=auto \
+  --extra-index-url https://wheels.vllm.ai/db9527a46873454610df6dbedf79a36d6bf1a7f6
 ```
 
-The 0.30.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The default 0.31.0 wheel contains CUDA 12.9 binaries. If you need a different
+CUDA variant or want to reuse an existing PyTorch installation, build vLLM
+from source instead.
 
 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -68,16 +74,16 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.30.0
+git checkout v0.31.0
 ```
 Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
 ```bash
-#For CUDA 13.0 (the default for v0.30.0; the wheel filename has no `+cu130` suffix)
-export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.30.0/vllm-0.30.0-cp38-abi3-manylinux_2_28_x86_64.whl
+# Default CUDA 12.9 wheel for v0.31.0 on x86_64
+export VLLM_PRECOMPILED_WHEEL_LOCATION=https://wheels.vllm.ai/db9527a46873454610df6dbedf79a36d6bf1a7f6/vllm-0.31.0-cp38-abi3-manylinux_2_28_x86_64.whl
 ```
 Install vllm with command below (If you have no existing PyTorch).
 ```bash
-uv pip install --editable .
+VLLM_USE_PRECOMPILED=1 uv pip install --editable . --torch-backend=auto
 ```
 Install vllm with command below (If you already have PyTorch).
 ```bash
@@ -111,7 +117,7 @@ docker run --runtime nvidia --gpus 2 \
     --env "HF_TOKEN=$HF_TOKEN" \
     -p 8091:8091 \
     --ipc=host \
-    vllm/vllm-omni:v0.28.0 \
+    vllm/vllm-omni:v0.30.0 \
     vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```
 
@@ -128,12 +134,13 @@ docker run --runtime nvidia --gpus 2 \
 DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.cuda -t vllm-omni-cuda .
 ```
 
-If you want to specify the base vLLM version:
+The source build defaults to `vllm/vllm-openai:v0.31.0`. To select that
+upstream release image explicitly:
 
 ```bash
 DOCKER_BUILDKIT=1 docker build \
   -f docker/Dockerfile.cuda \
-  --build-arg BASE_IMAGE=vllm/vllm-openai:v0.30.0 \
+  --build-arg BASE_IMAGE=vllm/vllm-openai:v0.31.0 \
   -t vllm-omni-cuda .
 ```
 

docs/getting_started/installation/gpu/rocm.inc.md

@@ -10,9 +10,9 @@
 
 For ROCm, vLLM-Omni currently recommends the setup steps through Docker Images.
 
-vLLM-Omni depends on the matching major/minor release of vLLM. The 0.30
-development line uses vLLM 0.30.x. Published 0.28.0 wheels and images use vLLM
-0.28.x.
+vLLM-Omni depends on the matching major/minor release of vLLM. This source
+checkout targets vLLM 0.31.0. The pre-built Omni wheel instructions below install
+the published 0.30.0 release with its matching vLLM dependency.
 
 The Dockerfile's `BASE_IMAGE` pin applies only to Docker builds. The
 `vllm-omni` package does not install vLLM as a dependency, so non-Docker source
@@ -23,12 +23,12 @@ installing vLLM-Omni, as shown below.
 
 #### Installation of vLLM
 
-These pre-built wheel instructions install the published vLLM-Omni 0.28.0 release. For the 0.30 development line, use the source-install instructions below.
+These pre-built wheel instructions install the published vLLM-Omni 0.30.0 release.
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 
 ```bash
-uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.28.0/rocm723
+uv pip install vllm==0.30.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.30.0/rocm723
 ```
 
 #### Installation of vLLM-Omni
@@ -37,7 +37,7 @@ uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/roc
 # we need to add --no-build-isolation as the torch
 # is not obtained from pypi, we have to install using the
 # torch installed in our environment
-uv pip install vllm-omni==0.28.0
+uv pip install vllm-omni==0.30.0
 
 # Optional if want to run Qwen3 TTS
 uv pip uninstall onnxruntime # should be removed before we can install onnxruntime-rocm
@@ -50,13 +50,14 @@ uv pip install onnxruntime-rocm
 
 #### Installation of vLLM
 
-If you do not need to modify source code of vLLM, you can directly install the stable 0.30.0 release version of the library
+If you do not need to modify source code of vLLM, install the final 0.31.0
+release wheel for ROCm 7.2.3:
 
 ```bash
-uv pip install vllm==0.30.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.30.0/rocm723
+uv pip install vllm==0.31.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.31.0/rocm723
 ```
 
-The pre-built 0.30.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The pre-built 0.31.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
 
 #### Installation of vLLM-Omni
 
@@ -76,7 +77,7 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.30.0
+git checkout v0.31.0
 python3 -m pip install -r requirements/rocm.txt
 python3 setup.py develop
 ```
@@ -88,7 +89,7 @@ python3 setup.py develop
 #### Build docker image
 
 The source build defaults to the published upstream base image
-`vllm/vllm-openai-rocm:v0.30.0`, aligned with the vLLM release used by CI.
+`vllm/vllm-openai-rocm:v0.31.0`, aligned with this source checkout.
 Older or custom bases must provide the vLLM APIs checked by the Dockerfile's
 image-build canary. This upstream base is distinct from the prebuilt
 `vllm/vllm-omni-rocm` images discussed below; their published-tag availability
@@ -103,7 +104,7 @@ To select the upstream ROCm base explicitly, pass `BASE_IMAGE`:
 ```bash
 DOCKER_BUILDKIT=1 docker build \
   -f docker/Dockerfile.rocm \
-  --build-arg BASE_IMAGE=vllm/vllm-openai-rocm:v0.30.0 \
+  --build-arg BASE_IMAGE=vllm/vllm-openai-rocm:v0.31.0 \
   -t vllm-omni-rocm .
 ```
 
@@ -150,6 +151,8 @@ vllm-omni-rocm
 
 vLLM-Omni offers an official docker image for deployment. These images are built on top of vLLM docker images and available on Docker Hub as [vllm/vllm-omni-rocm](https://hub.docker.com/r/vllm/vllm-omni-rocm/tags). The version of vLLM-Omni indicates which release of vLLM it is based on.
 
+The prebuilt ROCm image is published separately from the release pipeline: `v0.28.0` is the latest tag currently available on Docker Hub, and newer-tag availability is tracked in [#7405](https://github.com/vllm-project/vllm-omni/issues/7405).
+
 #### Launch vLLM-Omni Server
 
 Here's an example deployment command that has been verified on 2 x MI300's:

docs/getting_started/installation/gpu/xpu.inc.md

@@ -1,8 +1,11 @@
+<!-- markdownlint-disable MD001 MD025 -->
+
 # --8<-- [start:requirements]
 
 - GPU: Validated on Intel® Arc™ B-Series.
 
 # --8<-- [end:requirements]
+
 # --8<-- [start:set-up-using-python]
 
 vLLM-Omni currently recommends using the Docker image setup steps below.
@@ -20,12 +23,15 @@ vLLM-Omni currently recommends using the Docker image setup steps below.
 #### Build docker image
 
 ```bash
-DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.xpu -t vllm-omni-xpu --shm-size=4g .
+export VLLM_VERSION=v0.31.0
+DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.xpu -t vllm-omni-xpu --shm-size=4g \
+  --build-arg VLLM_VERSION="$VLLM_VERSION" .
 ```
 
 This layers vLLM-Omni on top of the published `vllm/vllm-openai-xpu:<VLLM_VERSION>`
-base image (v0.30.0 by default), which Docker pulls automatically. To target a different vLLM release,
-pass `--build-arg VLLM_VERSION=<tag>`. If that tag has not been published yet, build
+base image, which Docker pulls automatically. The Dockerfile defaults to
+v0.31.0. To target a different vLLM release, pass
+`--build-arg VLLM_VERSION=<tag>`. If that tag has not been published yet, build
 the base from upstream's own Dockerfile first and point the build at it:
 
 ```bash
@@ -40,7 +46,7 @@ DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.xpu -t vllm-omni-xpu --shm-s
 
 ##### Launch with OpenAI API Server
 
-```
+```bash
 docker run -it -d --shm-size 10g \
   --name {container_name} \
   --net=host \

docs/getting_started/installation/npu/npu.inc.md

@@ -10,25 +10,25 @@ The recommended way to use vLLM-Omni on NPU is through the vLLM-Ascend pre-built
 === "A2"
 
     ```bash
-    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.30.0
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.31.0
     ```
 
 === "A3"
 
     ```bash
-    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.30.0-a3
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.31.0-a3
     ```
 
 === "A5"
 
     ```bash
-    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.30.0-a5
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.31.0-a5
     ```
 
 === "310P (Experimental)"
 
     ```bash
-    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.30.0-310p
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.31.0-310p
     ```
 
 ```bash
@@ -53,7 +53,7 @@ docker run --rm \
 
 # Inside the container, install vLLM-Omni from source
 cd /vllm-workspace
-git clone -b v0.30.0rc1 https://github.com/vllm-project/vllm-omni.git
+git clone -b v0.30.0 https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
 pip install -v -e . --no-build-isolation
 # or VLLM_OMNI_TARGET_DEVICE=npu pip install -v -e .
@@ -71,10 +71,14 @@ We are keeping [issue #886](https://github.com/vllm-project/vllm-omni/issues/886
 
 # --8<-- [start:installation-main]
 
-You can also install vLLM-Omni from the latest main branch if you want to use the latest features or bug fixes. Use the aligned vLLM-Ascend v0.30.0 image described above, and check [issue #886](https://github.com/vllm-project/vllm-omni/issues/886) for the status of the latest vLLM-Omni commit on NPU.
+You can also install vLLM-Omni from source if you want to use the latest features
+or bug fixes. This source checkout targets vLLM 0.31.0 and needs a matching
+vLLM-Ascend runtime. The v0.30.0 images above are for the published Omni v0.30.0
+release. Check [issue #886](https://github.com/vllm-project/vllm-omni/issues/886)
+for aligned versions and NPU validation before selecting a development image.
 
 ```bash
-# Inside the vLLM-Ascend v0.30.0 container, install vLLM-Omni from main
+# Inside a container with matching vLLM and vLLM-Ascend, install Omni from source
 cd /vllm-workspace
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni

docs/getting_started/quickstart.md

@@ -19,17 +19,20 @@ uv venv --python 3.12 --seed
 source .venv/bin/activate
 
 # On CUDA
-uv pip install vllm==0.30.0 --torch-backend=auto
+uv pip install vllm==0.31.0 --torch-backend=auto \
+  --extra-index-url https://wheels.vllm.ai/db9527a46873454610df6dbedf79a36d6bf1a7f6
 
 # On ROCm
-uv pip install vllm==0.30.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.30.0/rocm723
+uv pip install vllm==0.31.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.31.0/rocm723
 
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
 uv pip install -e .
 ```
 
-For additional installation methods — please see the [installation guide](installation/README.md).
+These commands install vLLM 0.31.0 for this source checkout. The CUDA index is
+pinned to the final release commit. For published vLLM-Omni wheels and additional
+installation methods, see the [installation guide](installation/README.md).
 
 !!! note
     It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.

docs/models/seedvr2.md

@@ -0,0 +1,280 @@
+# SeedVR2 3B video restoration
+
+The native `SeedVR2Pipeline` restores an input video using the released 3B FP16
+NaDiT and s8/c16/t4 causal VAE. It uses fixed checkpoint conditioning, CFG=1,
+and one Euler step. Text prompts do not change conditioning.
+
+## Code ownership
+
+The model package groups each computation stage in one file: `nadit.py` owns
+window geometry, routing, RoPE and the DiT; `vae.py` owns the causal VAE and its
+temporal/spatial tiling; `pipeline_seedvr2.py` owns input validation, decoding,
+conditioning, the Euler step and colour transfer. `config.py` owns model limits,
+and `long_video.py` owns the optional HTTP job/window lifecycle. Framework code
+only registers the model, its pre/postprocessors, metadata and HTTP routes.
+
+`color_correction_method` is model-specific: online requests use the existing
+`extra_params` JSON object, and offline sampling uses
+`extra_args={"color_correction_method": "wavelet"}`. The default remains `lab`;
+invalid values are rejected by model-side validation before inference. It is
+not a top-level field in the common video API.
+
+## Model directory
+
+Place these files together, retaining their names:
+
+| File | SHA-256 of the validated release |
+| --- | --- |
+| `seedvr2_ema_3b_fp16.safetensors` | `2fd0e03a3dad24e07086750360727ca437de4ecd456f769856e960ae93e2b304` |
+| `ema_vae_fp16.safetensors` | `20678548f420d98d26f11442d3528f8b8c94e57ee046ef93dbb7633da8612ca1` |
+| `pos_emb.pt` | `fa07a14844314772266b66c3b95deb0027696d8fe7065721263db5176f45d799` |
+
+Add `model_index.json` containing:
+
+```json
+{"_class_name": "SeedVR2Pipeline"}
+```
+
+The loader checks every checkpoint tensor, including RoPE buffers, and rejects
+missing, duplicate, unexpected, or incompatible tensors. Install PyAV for video
+decoding. The validated environment uses PyAV 18.1, PyTorch 2.13/CUDA 13, and L20 GPUs.
+
+## Serve and restore a video
+
+```bash
+vllm serve "$MODEL_DIR" --omni \
+  --model-class-name SeedVR2Pipeline --dtype float16 --enforce-eager \
+  --num-gpus 1 --host 127.0.0.1 --port 8098
+
+curl --fail-with-body http://127.0.0.1:8098/v1/videos/sync \
+  -F 'prompt= ' \
+  -F 'input_references=@input.mp4;type=video/mp4' \
+  -F 'size=224x128' -F 'num_inference_steps=1' \
+  -F 'guidance_scale=1' -F 'seed=7723' \
+  --output restored.mp4
+```
+
+For two-rank window sequence parallelism, expose two GPUs and replace `--num-gpus 1` with:
+
+```bash
+--num-gpus 2 --distributed-executor-backend mp \
+  --stage-overrides '{"0":{"ulysses_degree":2}}'
+```
+
+Use degree 4 and four visible GPUs for SP4. Window-SP alone replicates the VAE.
+To shard VAE activations across the same ranks, use:
+
+```bash
+--vae-use-tiling --num-gpus 2 --distributed-executor-backend mp \
+  --stage-overrides '{"0":{"ulysses_degree":2,"vae_patch_parallel_size":2,"vae_parallel_mode":"spatial_shard_height"}}'
+```
+
+The VAE patch degree must match the window-SP degree.
+For a 16-frame 1280×720 input restored at its original size, use four GPUs,
+VAE tiling, and height sharding with both degrees set to four:
+
+```bash
+CUDA_VISIBLE_DEVICES=0,1,2,3 vllm serve "$MODEL_DIR" --omni \
+  --model-class-name SeedVR2Pipeline --dtype float16 --enforce-eager \
+  --vae-use-tiling --num-gpus 4 --distributed-executor-backend mp \
+  --stage-overrides '{"0":{"ulysses_degree":4,"vae_patch_parallel_size":4,"vae_parallel_mode":"spatial_shard_height"}}' \
+  --host 127.0.0.1 --port 8098
+```
+
+Request `size=1280x720` for this landscape input or `size=720x1280` for a
+portrait input of the same dimensions. Keep the requested output size equal to
+the input size when testing restoration without upscaling.
+
+The multipart API requires a prompt field; a single space supplies a blank
+prompt. Nonblank text is rejected. Omit `fps` to retain the source frame rate;
+an explicit rate must match the source. Output dimensions are explicit multiples
+of 16. Input frames are resized with antialiased bicubic interpolation. The final
+frame is repeated internally to reach 4n+1, and the decoded output is cropped back
+to the original frame count. Five frames remain five; six frames are internally
+padded to nine and return six.
+
+## Colour correction
+
+Restoration reproduces detail faithfully but shifts global colour, so the
+resized input is used to transfer colour back onto the restored frames. This is
+on by default; pass `extra_params={"color_correction_method":"wavelet"}` to choose how:
+
+| Value | Behavior |
+| --- | --- |
+| `lab` (default) | Swaps the lowest frequency band, then histogram-matches CIELAB chroma. Most faithful colour. |
+| `wavelet` | Swaps only the lowest frequency band, keeping every higher band from the restoration. |
+| `adain` | Matches per-channel mean and standard deviation. Cheapest, and corrects only a global tint. |
+| `none` | Skips the transfer and returns raw restored colour. |
+
+```bash
+curl --fail-with-body http://127.0.0.1:8098/v1/videos/sync \
+  -F 'prompt= ' -F 'input_references=@input.mp4;type=video/mp4' \
+  -F 'size=224x128' -F 'num_inference_steps=1' -F 'guidance_scale=1' \
+  -F 'seed=7723' -F 'extra_params={"color_correction_method":"wavelet"}' \
+  --output restored.mp4
+```
+
+`lab` and `wavelet` retain the restored high-frequency detail; `adain` measurably
+softens it because it rescales every frequency. The transfer runs in FP32 after
+the VAE decode and costs a fraction of a second per frame. Unrecognized values
+are rejected with 400 before the clip is admitted. Correction cannot recover
+colour lost to 4:2:0 chroma subsampling in the returned MP4, which dominates the
+remaining deviation once the transfer is applied.
+
+## Request admission
+
+The default 3B limit is 848×480 pixels per input or output frame and 2,035,200
+pixels across the requested output clip. With VAE tiling and window SP of at
+least four ranks whose `vae_patch_parallel_size` matches `ulysses_degree`, the
+per-frame limit rises to 2560×1472 and the clip budget to 18,841,600 pixels.
+Raising the degree never lowers the budget. Both budgets count temporal
+padding to 4n+1 frames: the default admits up to five 848×480 frames or 93
+192×112 frames; the sharded profile admits up to 45 848×480 frames or five
+2560×1472 frames. A 16-frame 1280×720 or 720×1280 clip pads to 17 frames and
+uses 15,667,200 of the sharded clip's 18,841,600 pixels. Both orientations passed
+original-size HTTP restoration on four RTX 5090 GPUs with VAE tiling and height
+sharding. An independent 257-frame cap bounds per-frame decoder work for tiny
+inputs. The other longer combinations above are admission bounds, not completed
+GPU validation. The decoder checks
+declared duration, frame count, and input dimensions when available, then
+enforces the limits as frames arrive. Requests outside these budgets return 400
+before building the resized whole-clip tensor.
+
+The per-frame cap is calibrated for the smallest qualified device. The clip cap
+scales with the smallest visible device's memory: it stays at the calibrated
+five 2560×1472 frames on a 32 GB device and grows on larger ones from a
+per-rank memory model measured at SP4 (see `sharded_budget` in
+`vllm_omni/diffusion/models/seedvr2/pipeline_seedvr2.py`), which keeps 15% of the device
+free and leaves room for allocator fragmentation. It stops at 2,118,057,984
+padded pixels (513 frames at 1536×2688), the largest clip validated on four
+B300 ranks; 80 GB devices get about 1.08 billion and 141 GB or larger devices
+reach the cap. Clips at the cap in both memory regimes (561 frames at 2560×1472
+and 2,049 frames at 768×1344) peaked at 54 GiB of activations per rank. Set
+these before starting the server to override the caps; each must be a positive
+integer, and validating an override on the target hardware is the operator's
+responsibility:
+
+| Variable | Default | Bounds |
+| --- | --- | --- |
+| `VLLM_OMNI_SEEDVR2_SHARDED_FRAME_PIXELS` | 3,768,320 | Pixels per frame on the sharded profile |
+| `VLLM_OMNI_SEEDVR2_SHARDED_CLIP_PIXELS` | 18,841,600, scaled up with device memory | Padded pixels per clip on the sharded profile |
+| `VLLM_OMNI_SEEDVR2_MAX_FRAMES` | 257 | Decoder-work frame cap, all profiles |
+
+The sharded profile applies to any `ulysses_degree` of four or more, so eight
+ranks inherit a budget calibrated on four. Raising these caps trades a 400 for a
+CUDA OOM: the admission check passes and the request fails later inside the DiT
+forward or the VAE decode, which can abort every in-flight request on the engine
+rather than only the oversized one. Before serving with raised caps, restore the
+largest clip they admit once and confirm it completes.
+
+A 362-frame 1536×2688 2x upscale was restored on eight ranks with
+`ulysses_degree=8`, VAE tiling and height sharding, using 4,300,000 for the
+per-frame cap, the memory-scaled clip cap and 2,000 for the frame cap. The long-video
+route sizes its windows from these caps, so the clip cap sets how many frames
+each window covers.
+
+## Long-video restoration
+
+`POST /v1/seedvr2/restore-long` accepts one uploaded 24 FPS video, a blank
+prompt, `size`, `num_frames` (up to 7,200), and optional `loop_input=true` and
+`extra_params={"color_correction_method":"wavelet"}`, which it applies to every window.
+An output frame must fit the sharded per-frame pixel cap. It returns a job ID; poll
+`GET /v1/seedvr2/restore-long/{id}` and download the completed MP4 from
+`GET /v1/seedvr2/restore-long/{id}/content`. `DELETE
+/v1/seedvr2/restore-long/{id}` asks a running job to stop; it settles as
+`cancelled` at the next window boundary. A job whose owning process is gone,
+such as after a server restart, reports `failed` on the next poll.
+
+Jobs live on the API server's local disk, so the route requires
+`--api-server-count 1` and runs one job at a time. Storage is bounded by an
+upload cap and by deleting settled jobs once they age out:
+
+| Variable | Default | Bounds |
+| --- | --- | --- |
+| `VLLM_OMNI_SEEDVR2_LONG_OUTPUT_DIR` | system temp dir | Parent of the job directories |
+| `VLLM_OMNI_SEEDVR2_LONG_MAX_UPLOAD_BYTES` | 8 GiB | Largest accepted upload |
+| `VLLM_OMNI_SEEDVR2_LONG_JOB_TTL_SECONDS` | 3,600 | Age at which a settled job is deleted |
+| `VLLM_OMNI_SEEDVR2_LONG_MAX_WINDOW` | 121 | Longest model window, in frames |
+
+Download a completed job before its TTL expires; the sweep runs on each new
+submission and removes the output with the job directory.
+
+The service sends windows through the existing SeedVR2 endpoint, blends four
+frames at each boundary, and writes one continuous MP4 encoder. Each window is
+the longest 4n+1-frame clip that the sharded clip-pixel cap, the frame cap and
+`VLLM_OMNI_SEEDVR2_LONG_MAX_WINDOW` admit at the output size. Every window pays
+a fixed round trip through the serving stack and recomputes its overlap, and a
+window of four latent frames gets no temporal attention in the DiT, so raise
+the clip cap as far as the device allows. Windows carry source pixels, which
+the model resizes on the device, and cross the endpoint losslessly; only the
+final MP4 is compressed. The next window is restored while the previous one is
+blended and encoded. It repeats source frames and audio only when
+`loop_input=true`. The sharded serving profile above (VAE tiling, VAE height
+sharding, Ulysses degree of at least four) is required. This route keeps a
+window within the whole-clip pixel budget; it does not raise the
+`/v1/videos/sync` 257-frame or pixel limits. The long route
+uses fixed one-step, CFG=1 conditioning and requires `imageio[ffmpeg]` for
+audio muxing. The 7,200-frame 768×1344 case is still undergoing full GPU E2E.
+
+## Temporal and spatial VAE tiling
+
+Add `--vae-use-tiling` to bound VAE intermediate activations along time. The
+encoder processes nine frames first, then eight per chunk; the decoder processes
+two latent frames per chunk. Each causal convolution carries its past inputs
+within that encode/decode call, with temporal stride and first-frame upsampling
+alignment preserved. GroupNorm and attention still see the entire spatial frame.
+There is no overlap blend or independent restart at chunk boundaries.
+
+Five-frame clips use one temporal chunk. On a single GPU, convolutions with
+more than 256 input rows additionally process tiles of 128 output rows with
+exact halos. GroupNorm and bottleneck attention retain full-frame statistics.
+This bounds convolution workspace; full-frame activations and the DiT's
+whole-clip activations still consume memory.
+
+With VAE patch parallelism, each rank owns a band of latent rows and the
+corresponding encoder/decoder rows. Neighbor exchanges supply convolution halos;
+all-reduced centered statistics preserve full-frame GroupNorm. Bottleneck
+attention gathers the low-resolution frame before splitting it again. Encoder
+parameters are gathered before latent sampling, and decoded pixels are gathered
+before returning the video. Unequal bands are supported; fewer latent rows than
+ranks fall back to replicated execution. Width sharding and batch slicing are
+unsupported. FP16 reduction and convolution order can change rounding, so
+parallel outputs are numerically close rather than bitwise identical.
+
+## Current integration scope
+
+| Property | Contract |
+| --- | --- |
+| Weights / dtype | Released 3B DiT and VAE, FP16 |
+| Reference semantics | C0 whole clip, colour correction on by default |
+| Video input | One uploaded file, or offline TCHW RGB floats / PIL frames |
+| Timing | Constant frame rate, increasing PTS; normalize the video origin to zero |
+| Audio | First mono/stereo track, aligned by source PTS, cropped to the video interval, re-encoded as AAC |
+| Randomness | Per-request generator; preserve reference latent strides when sampling noise |
+| Sequence parallelism | Native engine SP1/2/4 correctness verified on five-frame L20 cases |
+| VAE placement | Replicated by default; optional height sharding on the window-SP group |
+| Unsupported | VFR, multichannel audio, 7B, other sampling schedules, quantization, cache acceleration, VAE width sharding / batch slicing, CPU offload, CFG/TP/PP parallelism, compiled execution, LoRA |
+
+Unsupported engine modes are rejected before process hooks and worker creation.
+`ulysses_degree` selects the model-owned SP group. This branch assigns whole
+windows to ranks; the specialized head-sharded path is a dependent change.
+Ring and AllGather-KV are unsupported.
+
+The practical P0 reference's five-frame batching, overlap, and CPU swapping are
+separate execution semantics. Temporal tiling and VAE patch
+parallelism reduce activation peaks but do not bound the whole-clip DiT memory.
+Large output frames may still exceed device capacity.
+
+Invalid inputs return 400. A single-GPU model OOM fails that request; a
+subsequent short request succeeds. With VAE patch parallelism, restart the
+service after an OOM: another rank may still be waiting in a collective. Cancelling an in-progress video job removes it through the
+existing DELETE endpoint; in-flight GPU work may drain before memory is reusable.
+A lost SP worker fails the request and makes health return 503. Restart the
+service to restore availability; automatic rank recovery is not provided.
+
+## Reproducible deployment
+
+See the [RTX 5090 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/ByteDance/SeedVR2-RTX-5090.md) for the
+input/output contract, complete serving command, and media checks. The local
+tests in `tests/diffusion/models/seedvr2/test_seedvr2_e2e.py` exercise the full
+3B pipeline, all colour modes, and HTTP restoration with USP 1 and 8.

docs/models/supported_models.md

@@ -19,9 +19,9 @@ th {
 
 | Architecture | Models | Example HF Models | NVIDIA GPU | AMD GPU | Ascend NPU | Intel GPU | Recipe |
 | -------------- | -------- | ------------------- | ------------ | --------- | ----- | ----------- | -------- |
-| `Qwen3OmniMoeForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen3-Omni.md) |
+| `Qwen3OmniMoeForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen3-Omni.md) |
 | `Qwen2_5OmniForConditionalGeneration` | Qwen2.5-Omni | `Qwen/Qwen2.5-Omni-7B`, `Qwen/Qwen2.5-Omni-3B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `MingFlashOmniForConditionalGeneration` + `MingImagePipeline` | Ming-flash-omni-2.0 (omni-speech + imagegen<sup>1</sup>) | `Jonathan1909/Ming-flash-omni-2.0` | ✅︎ | | | | — |
+| `MingFlashOmniForConditionalGeneration` + `MingImagePipeline` | Ming-flash-omni-2.0 (omni-speech + imagegen<sup>1</sup>) | `Jonathan1909/Ming-flash-omni-2.0` | ✅︎ | | | ✅︎ | — |
 | `BagelForConditionalGeneration` | BAGEL (DiT-only) | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `InternVLAA1Pipeline` | InternVLA-A1 | `InternRobotics/InternVLA-A1-3B` | ✅︎ | ✅︎ | | | — |
 | `Gr00tN1d7Pipeline` | GR00T N1.7 | `nvidia/GR00T-N1.7-3B` | ✅︎ | | | | — |
@@ -29,13 +29,15 @@ th {
 | `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` | ✅︎ | | | ✅︎ | [Published](https://recipes.vllm.ai/Qwen/Qwen-Image) |
 | `QwenImagePipeline` | Qwen-Image-2512 | `Qwen/Qwen-Image-2512` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen-Image-2512.md) |
 | `QwenImageEditPipeline` | Qwen-Image-Edit | `Qwen/Qwen-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `JoyImageEditPipeline` | JoyAI-Image-Edit | `jdopensource/JoyAI-Image-Edit-Diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/JD/JoyAI-Image-Edit.md) |
 | `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2509 | `Qwen/Qwen-Image-Edit-2509` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `QwenImageLayeredPipeline` | Qwen-Image-Layered | `Qwen/Qwen-Image-Layered` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2511 | `Qwen/Qwen-Image-Edit-2511` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `QwenImage21Pipeline` | Qwen-Image-2.1 (T2I + image-conditioned generation, up to 4 condition images) | `Qwen/Qwen-Image-2.1` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen-Image-2.1.md) |
 | `GlmImagePipeline` | GLM-Image | `zai-org/GLM-Image` | ✅︎ | ✅︎ | | | — |
-| `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `Krea2Pipeline` | Krea 2 (Raw + Turbo) | `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | ✅︎ | | | ✅︎ | — |
-| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tongyi-MAI/Z-Image-Turbo.md) |
+| `Krea2Pipeline` | Krea 2 (Raw + Turbo) | `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/krea/Krea-2.md) |
+| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Wan-AI/Wan2.2-T2V.md) |
 | `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `LingBotWorldCausalDMDPipeline` | LingBot-World 2.0 v1 (experimental) | `robbyant/lingbot-world-v2-14b-causal-fast-diffusers` | Experimental | | | | — |
 | `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V, V2V, T2V with sound, action policy | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
@@ -43,45 +45,47 @@ th {
 | `Wan22S2VPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ | | | ✅︎ | — |
-| `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | — |
-| `LTX2TwoStagePipeline` | LTX-2 / LTX-2.3 ordinary two-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` + matching Lightricks LoRA and upsampler | ✅︎ | ✅︎ | | | — |
-| `LTX2DistilledOneStagePipeline` | LTX-2 / LTX-2.3 merged-distilled one-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` | ✅︎ | ✅︎ | | | — |
-| `LTX2DistilledTwoStagePipeline` | LTX-2 / LTX-2.3 merged-distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` + matching Lightricks upsampler | ✅︎ | ✅︎ | | | — |
-| `LTX2Pipeline` | LTX-2.5 Full/SFT one-stage T2V and I2V | `Lightricks/LTX-2.5-Diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.5.md) |
+| `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.md) |
+| `LTX2TwoStagePipeline` | LTX-2 / LTX-2.3 ordinary two-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` + matching Lightricks LoRA and upsampler | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.md) |
+| `LTX2DistilledOneStagePipeline` | LTX-2 / LTX-2.3 merged-distilled one-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.md) |
+| `LTX2DistilledTwoStagePipeline` | LTX-2 / LTX-2.3 merged-distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` + matching Lightricks upsampler | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.md) |
+| `LTX2Pipeline` | LTX-2.5 Full/SFT one-stage T2V and I2V | `Lightricks/LTX-2.5-Diffusers` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.5.md) |
 | `LTX2TwoStagePipeline` | LTX-2.5 Full/SFT two-stage T2V and I2V | `Lightricks/LTX-2.5-Diffusers` + `Lightricks/LTX-2.5` LoRA | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.5.md) |
 | `LTX2DistilledOneStagePipeline` | LTX-2.5 merged-distilled one-stage T2V and I2V | `Lightricks/LTX-2.5-Diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.5.md) |
 | `LTX2DistilledTwoStagePipeline` | LTX-2.5 merged-distilled two-stage T2V and I2V | `Lightricks/LTX-2.5-Diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/LTX/LTX-2.5.md) |
 | `LingBotVideoPipeline` | LingBot-Video dense and MoE T2I, T2V, TI2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | | — |
-| `SanaVideoPipeline` | SANA-Video-2B T2V (native 480p/720p; Diffusers adapter validated at both resolutions) | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers`, `Efficient-Large-Model/SANA-Video_2B_720p_diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/SANA-Video-2B.md) |
+| `SanaVideoPipeline` | SANA-Video-2B T2V (native 480p/720p; Diffusers adapter validated at both resolutions) | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers`, `Efficient-Large-Model/SANA-Video_2B_720p_diffusers` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/SANA-Video-2B.md) |
 | `SanaImageToVideoPipeline` | SANA-Video-2B I2V (native and Diffusers adapter validated at 480p/720p) | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers`, `Efficient-Large-Model/SANA-Video_2B_720p_diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/SANA-Video-2B.md) |
+| `SeedVR2Pipeline` | [SeedVR2 3B](seedvr2.md) video restoration | `ByteDance-Seed/SeedVR2-3B` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/ByteDance/SeedVR2-RTX-5090.md) |
 | `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | ✅︎ | | | [Published](https://recipes.vllm.ai/MiniMaxAI/MiniMax-H3) |
 | `Magi2Pipeline` | MAGI-2 native Preview-stage T2VA and I2VA (272p/540p) | `sand-ai/MAGI-2-preview` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SandAI/MAGI-2-preview-L20X.md) |
 | `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Pi0Pipeline` | π0 (Pi-Zero) VLA | `lerobot/pi0_base` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/lerobot/Pi0.md) |
-| `Pi05Pipeline` | π0.5 VLA | `lerobot/pi05_base` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/lerobot/Pi05.md) |
+| `Pi05Pipeline` | π0.5 VLA | `lerobot/pi05_base` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/lerobot/Pi05.md) |
 | `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | | — |
 | `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `LongCatVideoAvatarPipeline` | LongCat-Video-Avatar-1.5 A2V/AI2V (native single-speaker and multi-speaker AI2V/AVC) | `meituan-longcat/LongCat-Video-Avatar-1.5` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/meituan-longcat/LongCat-Video-Avatar-1.5.md) |
 | `BooguImagePipeline` | Boogu-Image | `Boogu/Boogu-Image-0.1-Base`, `Boogu/Boogu-Image-0.1-Edit`, `Boogu/Boogu-Image-0.1-Edit-Turbo` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Boogu/Boogu-Image.md) |
-| `BooguImageTurboPipeline` | Boogu-Image Turbo distilled T2I | `Boogu/Boogu-Image-0.1-Turbo` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Boogu/Boogu-Image.md) |
+| `BooguImageTurboPipeline` | Boogu-Image Turbo distilled T2I | `Boogu/Boogu-Image-0.1-Turbo` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Boogu/Boogu-Image.md) |
 | `StableDiffusionXLPipeline` | Stable-Diffusion-XL | `stabilityai/stable-diffusion-xl-base-1.0` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ | — |
+| `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/StabilityAI/Stable-Diffusion-3.5.md) |
 | `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ | — |
-| `AuKForConditionalGeneration` | AuK / AuK-Flash (instruction-driven speech generation and editing) | `tencent/AuK`, `tencent/AuK-Flash` (with `Qwen/Qwen2.5-Omni-3B`; assemble with `tools/prepare_auk_checkpoint.py`) | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/AuK-H100.md) |
+| `AuKForConditionalGeneration` | AuK / AuK-Flash (instruction-driven speech generation and editing) | `tencent/AuK`, `tencent/AuK-Flash` (with `Qwen/Qwen2.5-Omni-3B`; assemble with `tools/prepare_auk_checkpoint.py`) | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/AuK-H100.md) |
 | `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | ✅︎ | — |
 | `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | ✅︎ | — |
 | `BreezeForConditionalGeneration` | Breeze-TTS-2 (voice design, voice clone, voice direction, CFG) | `BreezeBlue/Breeze-TTS-2` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/BreezeBlue/Breeze-TTS-2.md) |
-| `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | | — |
+| `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `MammothModa2ForConditionalGeneration` | MammothModa2-Dev (AR-only image understanding) | `bytedance-research/MammothModa2-Dev` | ✅︎ | | | | — |
-| `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | | — |
+| `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/black-forest-labs/FLUX.2-klein.md) |
+| `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `FluxPipeline` | FLUX.1-dev | `black-forest-labs/FLUX.1-dev` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `FluxPipeline` | FLUX.1-schnell | `black-forest-labs/FLUX.1-schnell` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `OmniGen2Pipeline` | OmniGen2 | `OmniGen2/OmniGen2` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` | ✅︎ | ✅︎ | | ✅︎ | — |
-| `MiniMaxMusic3ForConditionalGeneration` | MiniMax Music 3 (text-to-music) | `MiniMaxAI/MiniMax-Music3` | ✅︎ | | | | — |
+| `MiniMaxMusic3ForConditionalGeneration` | MiniMax Music 3 (text-to-music) | `MiniMaxAI/MiniMax-Music3` | ✅︎ | | | ✅︎ | — |
+| `Yue2ForCausalLM` | YuE2-3B (text-to-music with ABC plan) | `m-a-p/YuE2-3B` | ✅︎ | | | ✅︎ | [Recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/m-a-p/YuE2-3B.md) / [Audio-to-score](https://github.com/vllm-project/vllm-omni/blob/main/recipes/m-a-p/SheetSage2-H200.md) |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | | | ✅︎ | [Published](https://recipes.vllm.ai/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice) |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
@@ -91,27 +95,30 @@ th {
 | `MossTTSDelayModel` | MOSS-TTS, MOSS-TTSD, MOSS-SoundEffect, MOSS-VoiceGenerator | `OpenMOSS-Team/MOSS-VoiceGenerator` | ✅︎ | | | | — |
 | `MossTTSRealtime` | MOSS-TTS-Realtime | `OpenMOSS-Team/MOSS-TTS-Realtime` | ✅︎ | | | | — |
 | `HiggsAudioV2ForConditionalGeneration` | Higgs-Audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` | ✅︎ | | | | — |
-| `HiggsMultimodalQwen3ForConditionalGeneration` | Higgs-Audio v3 (TTS) | `bosonai/higgs-audio-v3-tts-4b` | ✅︎ | | | | [Published](https://recipes.vllm.ai/bosonai/higgs-audio-v3-tts-4b) |
+| `HiggsMultimodalQwen3ForConditionalGeneration` | Higgs-Audio v3 (TTS) | `bosonai/higgs-audio-v3-tts-4b` | ✅︎ | | | ✅︎ | [Published](https://recipes.vllm.ai/bosonai/higgs-audio-v3-tts-4b) |
 | `IndexTTS2TalkerForConditionalGeneration` | IndexTTS-2 | `IndexTeam/IndexTTS-2` | ✅︎ | | | | — |
 | `IndexTTS25TalkerForConditionalGeneration` | IndexTTS-2.5 | native `checkpoints/` bundle | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/IndexTeam/IndexTTS-2_5.md) |
 | `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | | — |
 | `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | | — |
-| `Flux2Pipeline` | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | ✅︎ | ✅︎ | | | — |
+| `Flux2Pipeline` | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | ✅︎ | — |
-| `Audio8TTSSlowARForConditionalGeneration` | Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | ✅︎ | | | | — |
-| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only), SenseNova-U1.5 | `SenseNova/SenseNova-U1-8B-MoT`, `sensenova/SenseNova-U1.5-8B-MoT` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SenseNova/SenseNova-U1.5.md) |
+| `Audio8TTSSlowARForConditionalGeneration` | Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | ✅︎ | | | ✅︎ | — |
+| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only), SenseNova-U1.5 | `SenseNova/SenseNova-U1-8B-MoT`, `sensenova/SenseNova-U1.5-8B-MoT` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SenseNova/SenseNova-U1.5.md) |
+| `SenseNovaU1Pipeline` | SenseNova-U1-A3B (MoE) | `sensenova/SenseNova-U1-A3B-MoT` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SenseNova/SenseNova-U1.md#a3b-moe-variant) |
 | `LancePipeline` | Lance | `bytedance-research/Lance` | ✅︎ | | | | — |
-| `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | | — |
-| `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | | — |
+| `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/HunyuanVideo-1.5.md) |
+| `Kandinsky6TI2VAPipeline` | Kandinsky 6 TI2VA | `kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Kandinsky/Kandinsky6-TI2VA.md) |
+| `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/HunyuanVideo-1.5.md) |
 | `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | | — |
 | `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | | — |
 | `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/OpenBMB/MiniCPM-o-4_5.md) |
+| `PersonaPlexTalkerForConditionalGeneration` | PersonaPlex (full duplex, 24 kHz speech in/out over `/v1/realtime?duplex=1`) | `nvidia/personaplex-7b-v1` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/PersonaPlex.md) |
 | `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `GepardTalkerForConditionalGeneration` | Gepard-1.0 | `nineninesix/gepard-1.0` | ✅︎ | | | ✅︎ | — |
 | `HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | | — |
 | `HiDreamO1ImagePipeline` | HiDream-O1-Image | `HiDream-ai/HiDream-O1-Image` | ✅︎ | | | | — |
-| `AnimaPipeline` | Anima | `circlestone-labs/Anima` | ✅︎ | ✅︎ | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/circlestone-labs/Anima.md) |
+| `AnimaPipeline` | Anima | `circlestone-labs/Anima` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/circlestone-labs/Anima.md) |
 
 The `Recipe` column links the corresponding deployment recipe. `Published` is
 preferred when a corresponding `recipes.vllm.ai` page exists; otherwise the

docs/serving/README.md

@@ -144,7 +144,7 @@ interchangeably.
 | ---------- | ---------- | ------------------- | --------- |
 | Incremental text input for speech synthesis | `WS /v1/audio/speech/stream` | Send text events and receive audio | [Streaming Text to Speech](speech_api.md#streaming-text-input-websocket) |
 | Live video understanding | `WS /v1/video/chat/stream` | Send video frames and receive text/audio | [Streaming Video Input](video_stream_api.md) |
-| Turn-based realtime audio | `WS /v1/realtime` | Stream one audio input and receive transcript/audio events | [Realtime Audio](realtime_api.md) |
+| Turn-based realtime audio | `WS /v1/realtime` | Stream one audio input and receive transcript/audio events | [Realtime Audio](realtime_openai_duplex_api.md) |
 | Continuous speech-to-speech interaction | `WS /v1/realtime?duplex=1` (alias `WS /v1/duplex`) | Listen and speak concurrently with session control | [Full Duplex](full_duplex_api.md) |
 | Generated video chunks | `WS /v1/realtime/video` | Start a diffusion request and receive fragmented MP4 | [Streaming Video Output](streaming_video_output_api.md) |
 | Robot policy inference | `WS /v1/realtime/robot/openpi` | Send MessagePack observations and receive action arrays | [OpenPI Robot Policy](openpi_api.md) |
@@ -154,18 +154,16 @@ All six routes are model- or configuration-dependent. In particular,
 pipeline declares a `duplex_plugin` and its deploy configuration sets
 `session_mode: duplex` (with `session_mode: turn` the same model boots the
 ordinary turn-based serving stack instead). On a duplex server a stock
-Realtime client needs no vendor query parameter; `duplex=0` selects the
-turn-based Realtime handler, which a duplex server does not mount, so that
-connection is refused with `Realtime API is not available`. Such a server
-serves the websocket route plus `POST /v1/chat/completions`, `/v1/models`
-and `/health`, and no other turn-based HTTP route. The chat route is the
-ordinary chat service running on the duplex engine: a request is a
-turn-based generation on the same stages, served alongside the live
-websocket sessions; it opens no duplex session and holds no
-`duplex_session.max_sessions` slot -- see [Full Duplex](full_duplex_api.md).
-On a server that is not duplex, an explicit `?duplex=1` is refused rather
-than answered by the turn-based handler, so a client that requested duplex
-never silently gets the other protocol.
+Realtime client needs no vendor query parameter; `duplex=0` opts out of native
+duplex and selects the turn-based Realtime handler. It runs generation on the
+same stages, alongside live websocket sessions, and opens no duplex session
+or consumes a `duplex_session.max_sessions` slot -- see
+[Full Duplex](full_duplex_api.md).
+Such a server serves the websocket route plus `POST /v1/chat/completions`,
+`/v1/models` and `/health`, and no other turn-based HTTP route. On a server
+that is not duplex, an explicit `?duplex=1` is refused rather than answered by
+the turn-based handler, so a client that requested duplex never silently gets
+the other protocol.
 
 ## Related Endpoints
 

docs/serving/full_duplex_api.md

@@ -5,7 +5,7 @@ continue receiving speech while producing speech. It adds persistent session
 state, model-specific turn policy, overlap handling, playback acknowledgement,
 and optional session resume.
 
-Full duplex is distinct from the turn-based [Realtime Audio API](realtime_api.md).
+Full duplex is distinct from the turn-based [Realtime Audio API](realtime_openai_duplex_api.md).
 
 This page is the endpoint overview. For the complete wire contract, the
 `vllm_omni.clients.duplex.DuplexClient` Python library, and the per-model

docs/serving/realtime_duplex_api.md

@@ -54,7 +54,7 @@ vllm-omni serve openbmb/MiniCPM-o-4_5 \
 ```
 
 `vllm_omni/deploy/minicpmo_4_5.yaml` declares `session_mode: duplex` and
-`duplex_session.max_sessions: 4`. Because the MiniCPM-o 4.5 pipeline declares a
+`duplex_session.max_sessions: 16`. Because the MiniCPM-o 4.5 pipeline declares a
 `duplex_plugin`, `vllm-omni serve` runs it through `DuplexOmni`: the server
 mounts `ws://<host>:8099/v1/realtime?duplex=1` (this page; `ws://<host>:8099/v1/duplex`
 is an alias of the same route), `POST /v1/chat/completions`, `/v1/models` and
@@ -250,6 +250,40 @@ call; transport failures raise `DuplexConnectionError`; a session that ends
 while you are awaiting it raises `DuplexSessionClosedError`. All three derive
 from `DuplexClientError`.
 
+### Output limits and slow consumers
+
+Deployment settings under `duplex_session` bound each session's pending
+output: `max_pending_output_bytes_per_session` defaults to 2 MiB and
+`max_pending_output_events_per_session` to 512 events. Bytes count compact
+Realtime JSON, including base64 audio, not decoded audio. The limits apply
+to both WebSocket delivery and direct Python handles, not to replay journals
+or client playback queues.
+
+Exceeding a limit fails the active response without committing its partial
+text to history, then closes only that session. If overflow occurs while a
+completion is being queued, its not-yet-queued `response.done` becomes failed
+with the same identity and output, and its response history entry is removed.
+An ending already accepted by the buffer is not replaced or followed by a
+second ending. Open a new session to continue; the closed session cannot be
+resumed. The notifications use these fields:
+
+| Event | Field values |
+| --- | --- |
+| `error` | `error.type: "rate_limit_error"`, `error.code: "output_backpressure"` |
+| `response.done` (active response or not-yet-queued ending) | `response.status: "failed"`, `response.status_details: {"type": "failed", "reason": "output_backpressure"}` |
+| `session.closed` | `reason: "output_backpressure"` |
+
+Queued audio for the affected response is removed. Previously queued non-audio
+events retain their order, but newly emitted ordinary events, including
+audio/text, content-part and output-item completion markers, are suppressed.
+Clients must therefore handle closure without waiting for every completion
+marker. Errors and response endings have an extra 64 KiB / eight-event reserve;
+if that also fills, either notification may be omitted. Closure has its own
+independent slot. A closure notification exceeding 4 KiB is compacted to
+`reason: "close_details_exceed_output_limit"` with empty details, preserving
+the event type and identity. One event held by the consumer is outside the
+queued budget; audio already sent still requires the client to stop playback.
+
 ### Measure latency
 
 `EventCollector` accumulates events for assertions and metrics:
@@ -315,6 +349,8 @@ single-consumer async iterator that ends after `session.closed` /
 `session.expired`; rejected commands come back as `ErrorEvent`s on it.
 Every event renders the wire JSON with `to_realtime()`, so anything written
 against the WebSocket protocol works unchanged on the typed stream.
+If opening fails, the abandoned handle is closed. Late engine events do not
+reopen a locally closed handle.
 
 `vllm_omni.clients.inline_duplex.InlineDuplexClient` wraps a `DuplexOmni`
 behind the `DuplexClient` API, so the same application code runs in-process
@@ -384,9 +420,9 @@ with no OpenAI counterpart.
 
 The event vocabulary is uniform, but several surfaces are gated by the
 `capabilities` object the server returns in `session.created`; a client must
-branch on those flags rather than on the model name. MiniCPM-o 4.5 is the
-only model on the plugin contract today; the other two columns record what
-their integrations advertise once the follow-up PRs port them:
+branch on those flags rather than on the model name. MiniCPM-o 4.5 and
+PersonaPlex are on the plugin contract today; the Nemotron VoiceChat column
+records what its integration advertises once the follow-up PR ports it:
 
 | Capability | MiniCPM-o 4.5 | PersonaPlex | Nemotron VoiceChat | Gated surface |
 | --- | --- | --- | --- | --- |
@@ -401,7 +437,11 @@ their integrations advertise once the follow-up PRs port them:
 Everything else in the catalogue — session lifecycle, heartbeat and event
 acknowledgement, append/commit/clear, the response envelope, playback
 acknowledgement, and the error envelope — behaves identically for every
-model.
+model. Two PersonaPlex specifics follow from its capabilities rather than from
+special-casing: a model with `supports_client_commit=false` auto-responds
+without `extra_body.auto_response`, and `response.cancel` /
+`output_audio_buffer.clear` restart its conversation context (a new Stage 0
+request replays the voice/persona prefill).
 
 ### Compatibility with the OpenAI Realtime protocol
 
@@ -569,7 +609,7 @@ formats (`pcm16`, `pcm_s16le`, `s16le`, `pcm_f32le`, `g711_ulaw`,
 | `session.created` | 2 | Session opened. OpenAI name; adds `attachment_generation`, `resume_token` and vLLM-Omni keys inside `session` (`id` is the server-allocated session id, `epoch`, `turn_id`, `playback`, `capabilities`, ...). |
 | `session.updated` | 2 | Echo of the effective session config after every `session.update` (same extended `session` object). |
 | `session.heartbeat_ack` | 3 | Reply to `session.heartbeat`. |
-| `session.closed` | 3 | Last event on the socket, emitted once the engine released the session; `reason` ∈ `client_close`, `disconnect`, `timeout`, `transport_error`, `shutdown`, ... |
+| `session.closed` | 3 | Last event on the socket, emitted once the engine released the session; `reason` ∈ `client_close`, `disconnect`, `timeout`, `transport_error`, `shutdown`, `output_backpressure`, `close_details_exceed_output_limit`, ... |
 | `session.resumed` | 3 | Resume accepted; carries the new `attachment_generation` and rotated `resume_token`; journaled events are replayed after it. |
 | `session.replaced` | 3 | Sent to the superseded socket when another socket resumes the session. |
 | `session.expired` | 3 | Engine lease reaped (`disconnect_grace_expired`, `idle_ttl_expired`); socket closes after it. |

docs/serving/realtime_openai_duplex_api.md

@@ -1,33 +1,31 @@
 # Realtime Audio WebSocket API
 
-Use `WS /v1/realtime` to stream PCM audio into a compatible omni model and
-receive incremental transcription and synthesized response audio. This is a
-turn-based streaming API: it starts one generation over an audio stream and
-finishes after the client closes that input stream.
-
-For a model that listens while it is already speaking, use the
-[Full-Duplex API](full_duplex_api.md) instead.
+As a companion to the vLLM-Omni-proprietary [fullduplex `/v1/realtime`](full_duplex_api.md),
+supported models also provide a `/v1/realtime` handler supporting
+a substantial subset of the OpenAI `/v1/realtime` API supporting
+audio streaming, client-driven VAD, and tool calling
 
 ## Quick Start
 
-Start a compatible model. The currently documented example uses Qwen3-Omni:
+The endpoint is currently only supported for Qwen3-Omni. To start:
+
+Start vLLM-Omni with port `8000` bound to an address reachable from Docker:
 
 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8000 --host 0.0.0.0
 ```
 
-Run the provided client with a mono, 16-bit PCM, 16 kHz WAV file:
+Then start the docker compose:
 
 ```bash
-python examples/online_serving/qwen3_omni/openai_realtime_client.py \
-  --url ws://localhost:8091/v1/realtime \
-  --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
-  --input-wav input_16k_mono.wav \
-  --output-wav response.wav \
-  --output-text response.txt
+cd examples/online_serving/qwen3_omni/fullduplex-livekit-frontend
+VLLM_OMNI_HOST=$VLLM_OMNI_HOST docker compose up --build
 ```
 
-The client dependency is `websockets`.
+Where `VLLM_OMNI_HOST` is a network address. If vLLM-Omni is running on localhost, use
+`host.docker.internal`
+
+Open [http://localhost:3000](http://localhost:3000) and begin chatting.
 
 ## Protocol
 
@@ -37,10 +35,9 @@ event is base64-encoded raw PCM16.
 | Direction | Event | Purpose |
 | ----------- | ------- | --------- |
 | Server to client | `session.created` | Confirms that the WebSocket connection is ready |
-| Client to server | `session.update` | Selects and validates the served model |
-| Client to server | `input_audio_buffer.commit` with `final: false` | Starts generation over the incoming stream |
+| Client to server | `session.update` | Selects and validates the served model. Configures system prompt and tools |
 | Client to server | `input_audio_buffer.append` | Appends base64 PCM16 audio |
-| Client to server | `input_audio_buffer.commit` with `final: true` | Marks the end of input |
+| Client to server | `input_audio_buffer.commit` | Starts generation over the incoming stream |
 | Server to client | `transcription.delta` | Carries incremental response text |
 | Server to client | `transcription.done` | Carries final text and token usage |
 | Server to client | `response.output_audio.delta` | Carries incremental PCM16 response audio |
@@ -50,15 +47,15 @@ event is base64-encoded raw PCM16.
 A minimal client sends events in this order:
 
 ```json
-{"type":"session.update","model":"Qwen/Qwen3-Omni-30B-A3B-Instruct"}
-{"type":"input_audio_buffer.commit","final":false}
-{"type":"input_audio_buffer.append","audio":"<base64-pcm16>"}
-{"type":"input_audio_buffer.commit","final":true}
+{"type":"input_audio_buffer.append","audio":"<base64-pcm16>"},
+{"type":"input_audio_buffer.commit"}
 ```
 
-The initial non-final commit intentionally starts generation before all audio
-has arrived. Continue sending `input_audio_buffer.append` events while the
-engine is consuming the stream.
+This OpenAI-compatible streaming path is distinct from the vLLM-Omni-proprietary
+lane and as such is reached via different configuration. The vLLM-Omni-proprietary
+path must be specified explicitly via "duplex=1|true|on" as the WS query parameter
+or via an explicit configuration in the `deploy_config`. The OpenAI path simply replaces
+the default `/v1/realtime` handler in all supported models.
 
 ## Audio Handling
 
@@ -67,8 +64,6 @@ engine is consuming the stream.
 - Read `sample_rate_hz` from each audio event instead of assuming an output
   rate. Qwen3-Omni output is typically 24 kHz.
 - Concatenate audio deltas in receive order to construct the output waveform.
-- Set `input_audio_buffer.commit.final` to `true` after the last input chunk so
-  the server can terminate the streaming request.
 
 ## Availability and Limitations
 
@@ -76,7 +71,8 @@ The path is registered on the unified API server, but it is usable only when
 the loaded pipeline implements realtime audio input and produces compatible
 audio output. Unsupported deployments return an `error` event. The endpoint
 does not provide duplex session resume, playback acknowledgement, overlap
-policy, or barge-in controls.
+policy, or barge-in controls internally. Those must be driven by the client application
+via the relevant WS events (e.g. `response.cancel`, `conversation.item.truncate`)
 
 See the [Qwen3-Omni online serving example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/qwen3_omni)
 for concurrency options, chunk pacing, and per-delta audio debugging.

docs/serving/speech_api.md

@@ -183,6 +183,16 @@ emit these headers.
 **Raw audio stream** (`stream_format="audio"`). Streams raw audio bytes (PCM or
 WAV) as they are decoded.
 
+`pcm` is 16-bit signed little-endian, interleaved, at the model's native sample
+rate and channel count (24 kHz mono for most models; MOSS-TTS-Local v1.5 emits
+48 kHz stereo). Raw streams and non-streaming responses state the format in
+response headers:
+
+| Header | Description |
+| --- | --- |
+| `x-audio-sample-rate` | Sample rate of the returned audio in Hz (after any `sample_rate` resampling). |
+| `x-audio-channels` | Channel count of the returned audio. |
+
 Both streaming modes (`stream_format="audio"` and `"sse"`) require
 `response_format="pcm"` or `"wav"`, and `speed` must be `1.0` (or omitted).
 
@@ -193,9 +203,13 @@ Each event has an `event:` line and a JSON `data:` line:
 - `speech.audio.delta` — a base64 audio chunk:
 
     ```json
-    { "type": "speech.audio.delta", "audio": "<base64>", "response_format": "pcm" }
+    { "type": "speech.audio.delta", "audio": "<base64>", "response_format": "pcm",
+      "sample_rate": 24000, "channels": 1 }
     ```
 
+    `sample_rate` and `channels` extend the OpenAI event and describe the
+    decoded chunk.
+
 - `speech.audio.done` — terminal event, carrying token `usage`:
 
     ```json
@@ -340,7 +354,7 @@ Server -> Client:
 
 | Message | Description |
 | --------- | ------------- |
-| `{"type": "audio.start", "utterance_index": 0, "sentence_index": 0, "sentence_text": "...", "format": "pcm", "sample_rate": 24000}` | Audio generation starting for the buffered input |
+| `{"type": "audio.start", "utterance_index": 0, "sentence_index": 0, "sentence_text": "...", "format": "pcm", "sample_rate": 24000, "channels": 1}` | Audio generation starting for the buffered input. With `stream_audio=true` it is sent with the first audio chunk, and for `pcm` it carries that audio's `sample_rate` and `channels` (omitted when no audio was produced). |
 | Binary frame | Raw audio bytes (one or more PCM chunks when `stream_audio=true`) |
 | `{"type": "audio.done", "utterance_index": 0, "sentence_index": 0, "total_bytes": 96000, "error": false}` | Audio complete for the buffered input |
 | `{"type": "session.done", "utterance_index": 0, "total_sentences": N}` | Flushed utterance complete |
@@ -1029,3 +1043,64 @@ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
     --enforce-eager \
     --uvicorn-log-level debug
 ```
+
+### Reusing reference audio
+
+The voice upload API and cache settings below are shared serving features.
+For repeated voices, upload the reference once and reuse its name. The example
+uses MOSS-TTS Local: uploading stores the audio, and the first speech request
+encodes it. MOSS-TTS Local can then reuse cached reference codes without resolving
+the waveform while the speaker cache entry remains resident. After a server
+restart or speaker-cache eviction, another speech request warms the codes again.
+Reference processing and cache-hit behavior for other models depend on their
+serving adapters.
+
+```bash
+curl --fail http://localhost:8123/v1/audio/voices \
+  -F 'name=my-speaker' \
+  -F 'consent=user_consent_id' \
+  -F 'audio_sample=@/path/to/reference.wav'
+
+# Run once to warm the reference codes, then reuse the same voice.
+curl --fail http://localhost:8123/v1/audio/speech \
+  -H 'Content-Type: application/json' \
+  -d '{"model":"OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5","voice":"my-speaker","input":"Hello, world.","response_format":"wav"}' \
+  --output speech.wav
+```
+
+If you have the reference transcript, upload it with `-F 'ref_text=...'`.
+MOSS-TTS Local uses a reference transcript for continuation conditioning; omitting
+it uses reference-audio conditioning instead. Keep this choice consistent when
+comparing benchmarks. Supplying inline `ref_audio` overrides the uploaded voice
+and does not use the named-voice cache shortcut.
+
+The separate decoded-waveform resolve cache defaults to **4 GiB and 2048 entries
+per speech server instance**. These are LRU limits, not preallocated memory or a
+limit on total server RSS. Waveforms retain their source sampling rate and are
+stored as owned, contiguous float32 arrays in **CPU memory**. The byte limit
+counts numeric buffers (four bytes per mono sample), not the encoded file size
+or GPU memory.
+Either limit can cause eviction. Configure the API-process caches in the deploy
+YAML passed to `--deploy-config` (alongside `stages`, not inside a stage):
+
+```yaml
+speech_cache:
+  resolve_max_bytes: 4294967296
+  resolve_max_entries: 2048
+  speaker_max_bytes: 536870912
+```
+
+All values must be non-negative integers. Either resolve limit set to `0`
+disables waveform storage; `speaker_max_bytes: 0` disables speaker artifact
+storage. Invalid values fail configuration loading. `base_config` inheritance
+merges this section by field, preserving unspecified values from the base.
+Restart the service to apply changes. The effective settings are logged at startup.
+
+The speaker budget defaults to **512 MiB** and counts cached tensor bytes,
+not Python metadata or total process RSS. It configures the API process's
+speaker cache. MOSS reference codes are cached as **CPU int32 tensors**; this
+budget does not reserve GPU memory or include the reference encoder's GPU
+weights and working memory. The shared cache itself does not move tensors
+between devices; other adapters determine the device of their cached artifacts.
+Separate model-worker caches are not controlled by this section. The process-wide singleton rejects a
+conflicting explicit budget instead of silently ignoring it.

docs/serving/video_stream_api.md

@@ -16,6 +16,10 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \
     --trust-remote-code
 ```
 
+To enable [incremental prefill](#incremental-prefill), set
+`enable_prefix_caching: true` on stage 0 (the thinker) in the deploy config. Otherwise
+the server falls back to rebuilding the prompt per query.
+
 ### Run the Example Client
 
 ```bash
@@ -54,7 +58,7 @@ WebSocket /v1/video/chat/stream
 
 When buffered frames include client-supplied `frame_id` values, the server emits `video.frames.consumed` after the first engine output. Its `frame_ids`, `frames`, and `latest_pts_ms` describe the images included in that query's prompt.
 
-Queries stride-sample the buffered frames, keeping the last frame, then exclude frames already known to have failed decoding. Excluded frames are not replaced with other buffered frames. Frames whose background decoding has not finished remain eligible through the image URL path.
+Queries stride-sample the buffered frames, keeping the last frame, then exclude frames already known to have failed decoding. With incremental prefill active, every buffered frame is included instead of stride-sampling, with the same decode-failure exclusion. Excluded frames are not replaced with other buffered frames. The incremental path waits for the whole window to finish decoding. The legacy path waits only for its sampled positions, so an unselected frame still decoding does not delay the query.
 
 **Bugfix compatibility note:** `video.frames.consumed` now excludes known decode failures that were already excluded from the prompt. Older versions could report those frames and their timestamps as consumed. The event name and fields are unchanged; an empty selection reports empty lists and `latest_pts_ms: null`.
 
@@ -64,10 +68,10 @@ Queries stride-sample the buffered frames, keeping the last frame, then exclude
 | ------- | ------ | --------- | ------------- |
 | `model` | string or null | null | Optional model name. Usually omitted because the server hosts one model. |
 | `modalities` | list[string] | `["text", "audio"]` | Output modalities. Use `["text"]`, `["audio"]`, or both. |
-| `num_frames` | integer, 1-128 | `4` | Number of buffered frames sampled for each query. |
-| `max_frames` | integer, 1-256 | `50` | Maximum retained frame buffer size. Oldest frames are evicted first. |
+| `num_frames` | integer, 1-128 | `4` | Number of buffered frames sampled for each query. Legacy path only: with incremental prefill active, every buffered frame is submitted and frame density is the client's responsibility (clients push discrete frames at their own rate). |
+| `max_frames` | integer, 1-256 | `50` | Maximum retained frame buffer size. Below the cap both paths keep every frame. Legacy evicts the oldest frame once full. Incremental prefill drops the oldest half at once, then appends until full again. |
 | `system_prompt` | string or null | null | Optional custom system prompt. |
-| `use_audio_in_video` | bool | `true` | Include streamed audio chunks in multimodal video understanding when audio is present. |
+| `use_audio_in_video` | bool | `true` | Incremental prefill: forwarded on warmup and every query when this is true, so multimodal hashes stay aligned. Legacy: forwarded only when this query has input audio. |
 | `sampling_params_list` | list or null | null | Optional per-stage parameter dictionaries. Each provided entry replaces that stage's deployment sampling settings. |
 | `enable_frame_filter` | bool | `true` | Enable EVS near-duplicate frame filtering. |
 | `frame_filter_threshold` | float, 0.0-1.0 | `0.95` | EVS similarity threshold. Higher keeps more frames; lower drops more near-duplicates. |
@@ -107,9 +111,42 @@ The server accepts these legacy field names and rewrites them before validation.
 
 EVS compares downsampled frames and drops near-duplicate frames before they enter the session frame buffer. `frame_filter_threshold` controls retention: higher values are more permissive and keep more frames; lower values are more aggressive and drop more similar frames.
 
+## Incremental Prefill
+
+Server-side only (stage-0 `enable_prefix_caching: true` and `modalities: ["text"]`).
+There is no `session.config` flag. Audio-output sessions stay on the legacy path.
+
+Input is unchanged: `video.frame` and `audio.chunk` are separate buffers. Warmup
+prefills `history + frames` as they arrive (`max_tokens=1`). When
+`use_audio_in_video` is true, warmup and query both pass that processor kwarg.
+`video.query` is the same prompt plus optional input audio and the question text,
+so the warmed vision prefix cache is reused and the query pays for the audio/text
+suffix. All buffered frames are
+submitted in arrival order (`num_frames` subsample is legacy-only). Once the buffer
+hits `max_frames`, the oldest half is dropped in one step and a warning is logged;
+until then the buffer matches the legacy path. After a turn,
+history compresses to the last two text messages and the next warmup follows the
+new prefix.
+
+### Legacy path (incremental prefill inactive)
+
+Stage-0 prefix caching is off, or `modalities` includes `"audio"`: no warmup,
+frames are re-sampled up to `num_frames` each query, and the query waits only
+for those sampled frames to decode. Prompt shape is the same
+(`image_pil` frames, optional trailing `input_audio`, text). Both paths keep the
+frame buffer across queries (`max_frames`) and compress history the same way.
+
 ## Known Limitations
 
-- Session KV reuse and incremental prefill are not implemented in this PR. Each `video.query` rebuilds the model prompt from the retained frame and audio buffers.
+- Past `max_frames`, incremental prefill keeps a suffix of the newest frames, not a resample of the whole stream. Rebuilding that longer window is left to a follow-up.
+- Incremental prefill applies only to text-only sessions with stage-0 prefix caching enabled
+  on `WS /v1/video/chat/stream`. It does not accelerate `/v1/realtime` camera input, which
+  uses duplex conversation items rather than `video.frame` / `video.query`. Audio-output
+  sessions on this endpoint rebuild the prompt each query.
+- Stage-0 prefix caching is safe for audio-output requests: on a hit, the omni prefix cache
+  restores the thinker hidden states the talker needs. A preempted request that resumes on a
+  hit does not get them back, so size the deployment to avoid preemption; see
+  [Prefix Caching](../design/feature/prefix_caching.md).
 - Back-to-back short replies can still expose an engine-layer scheduler race. The PR notes an observed workaround of at least 200 ms idle between turns when clients repeatedly see idle timeouts.
 - If the audio buffer exceeds the server limit, the server emits `Audio buffer overflow` and clears the currently buffered audio for the session.
 - The API is intended for Qwen3-Omni streaming video understanding; other models may not support the same multimodal processor arguments.

docs/serving/videos_api.md

@@ -237,13 +237,22 @@ At least one mask is required. A nontrivial `video_noise_mask` requires
 `source_audio` or a `source_video` with an audio stream. Mask values are in
 `[0, 1]`: `0` preserves the source, `1` regenerates it, and fractional values
 blend the two behaviors. Exact all-one masks are no-ops and do not require a
-source. Source uploads without a mask are rejected. Masks may be a JSON scalar
-or arrays matching the H3 latent/token grid; pixel-resolution masks must be
-resized or pooled by the client before upload.
+source. Source uploads without a mask are rejected. Masks may be a JSON scalar,
+an array matching the H3 latent/token grid, or (video only) a frame-space
+array that the server resizes to the latent grid.
 
 For an aligned output of `F` frames at `W x H`, the video latent grid is
 `[Tv, H/16, W/16]`, where `Tv = 2 + 5 * ((F - 5) / 17)`. The video mask may be
-a scalar, a flat token vector, `[Tv, H/32, W/32]`, or the full latent grid. For
+a scalar, a flat token vector, `[Tv, H/32, W/32]`, the full latent grid, a
+spatial `[h, w]` mask applied to every frame, or a frame-space `[T, h, w]` mask
+with one slice per 24 fps output frame. A shape that matches the token or full
+latent grid is always read as that grid. Frame-space masks are area-resized to
+`[H/16, W/16]`; along time, a mask shorter than `F` is padded with its last
+slice (as the source video is) and a longer one is truncated, then each latent
+takes the maximum over the frames the causal video VAE folds into it, so any
+regenerated frame regenerates its latent. Because the 8 MiB limit below applies
+to the JSON file, clients should area-downsample frame-space masks by 16
+spatially before upload. For
 the model input, timestep, and velocity, a full-grid mask is max-pooled over
 each 2x2 spatial token and fractional values are rounded upward to 1/256
 levels. The final x0 restore uses the original, unquantized mask, so full-grid
@@ -335,6 +344,13 @@ window and output frame count stay unchanged. Wan S2V keeps its existing
 per-clip behavior by default. Zero, negative, fractional, boolean, string, and
 null values are rejected when pre-encoding is enabled.
 
+The transfer ring applies backpressure before another D2H copy when both of
+its two slots are occupied or the combined pending uint8 payload would exceed
+256 MiB. This byte bound keeps large resolutions and uneven final chunks from
+turning a fixed item count into unexpectedly large memory growth. A native VAE
+chunk larger than 256 MiB is admitted only when the ring is otherwise empty,
+so supported chunk shapes cannot deadlock.
+
 `preencode_mp4` applies to the complete-MP4 response paths only. The
 `/v1/realtime/video` WebSocket endpoint rejects it, because that path already
 overlaps encoding through its own incremental fragmented-MP4 encoder. Wan also

docs/user_guide/diffusion/cpu_offload.md

@@ -42,6 +42,7 @@ higher-priority strategy is selected.
 | --- | --- | --- | :---: | :---: | :---: | --- |
 | Flux2Pipeline | `black-forest-labs/FLUX.2-dev` | `Flux2Transformer2DModel` | yes | yes | — | `transformer_blocks`, `single_transformer_blocks` |
 | LongCatImagePipeline | `meituan-longcat/LongCat-Image` | `LongCatImageTransformer2DModel` | — | yes | — | `transformer_blocks`, `single_transformer_blocks` |
+| JoyImageEditPipeline | `jdopensource/JoyAI-Image-Edit-Diffusers` | `JoyImageEditTransformer3DModel` | yes | yes | — | `double_blocks` |
 | NextStep11Pipeline | `stepfun-ai/NextStep-1.1` | `NextStepModel` | — | yes | — | `layers` |
 | OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | — | yes | — | `transformer` |
 | QwenImagePipeline | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | yes | yes | — | `transformer_blocks` |

docs/user_guide/diffusion/device_side_video_postprocess.md

@@ -70,3 +70,42 @@ as safety checks or audio/video packaging.
 
 See [RFC #6541](https://github.com/vllm-project/vllm-omni/issues/6541) for the
 contract, lifecycle, batching rules, and migration plan.
+
+## Experimental registered-SHM video transport
+
+`video_output_transport.enable_registered_shm` is a separate, default-off option
+for CUDA request-mode workers. It copies eligible video tensors directly into
+CUDA-registered shared memory and returns borrowed CPU views, removing the
+intermediate pinned-to-SHM and SHM-to-consumer host copies. The D2H transfer still
+occurs.
+
+`video_output_transport.enable_borrowed_frames` independently enables borrowed
+RGB AVFrames in the Videos API MP4 response encoder, removing the third host
+copy into PyAV frame buffers. It also defaults to false. Eligible uint8 RGB
+frames are wrapped with `VideoFrame.from_numpy_buffer`; other dtypes/layouts use
+the existing encoder path. Color conversion, compression and muxing still occur.
+
+Enable it through the same CLI, Python or deploy-stage configuration:
+
+```bash
+--video-output-transport '{"enable_registered_shm": true, "enable_borrowed_frames": true}'
+```
+
+Registered SHM applies only to prepared typed video media (currently WAN2.2) and
+explicitly marked legacy video outputs (currently MiniMax-H3). H3 marks the video
+entry of its `(video, audio)` output; audio, trajectory data, images and unmarked
+legacy outputs keep their existing transport. Device-side postprocessing is
+independent and does not need to be enabled for registered SHM.
+Pre-encoded MP4 outputs bypass raw-frame transport and API-side encoding.
+
+The existing SHM routing threshold is unchanged: the tensor or its backing
+storage must exceed 1,000,000 bytes. This is not a measured performance crossover.
+SHM is allocated and registered on every transfer, without pooling; small outputs
+can regress and registration latency can vary. Benchmark representative output
+sizes before opting in. CPU outputs, non-CUDA backends, and synchronous/step-mode
+transfers keep their existing path.
+
+Borrowed views keep the unlinked mapping alive until their last user releases
+it. CUDA registration/copy failures fail the request; if CUDA cleanup cannot be
+confirmed, the worker retains the mapping for safety and must be restarted before
+further registered-SHM transfers.

docs/user_guide/diffusion/parallelism/vae_parallelism.md

@@ -1,12 +1,12 @@
 # VAE Parallelism Guide
 
-
 ## Table of Content
 
 - [Overview](#overview)
 - [Quick Start](#quick-start)
 - [Example Script](#example-script)
 - [Configuration Parameters](#configuration-parameters)
+- [Batch Parallel Decode](#batch-parallel-decode)
 - [Best Practices](#best-practices)
 - [Troubleshooting](#troubleshooting)
 - [Summary](#summary)
@@ -15,23 +15,22 @@
 
 ## Overview
 
-VAE parallelism distributes VAE (Variational AutoEncoder) decode/encode work across multiple GPUs. This guide covers VAE patch/tile parallelism, which splits latent space into spatial tiles or patches, and Wan spatial-shard decode, which shards decoder feature maps along height or width.
+VAE parallelism distributes VAE (Variational AutoEncoder) decode/encode work across multiple GPUs. This guide covers patch/tile parallelism, batch parallel decode for image VAEs, and Wan spatial-shard decode. Patch/tile parallelism divides images into spatial regions; batch parallel decode assigns complete images to ranks.
 
 This is particularly useful for:
+
 - **High-resolution image generation** where VAE decode can become a memory bottleneck
 - **Memory-constrained environments** where the VAE decode activation peak exceeds available VRAM
 - **Multi-GPU setups** where you want to leverage distributed resources for the VAE stage
 
 See supported models list in [Supported Models](../../diffusion_features.md#supported-models).
 
-
 VAE patch parallelism uses two strategies based on image size:
 
-| Strategy | Use Case | How It Works | Overlap Handling | Output Quality |
-|----------|----------|--------------|------------------|----------------|
-| **Tiled Decode** | Large images (triggers VAE tiling) | Distributes existing VAE tiling computation across ranks. Each rank decodes a subset of overlapping tiles. | Uses VAE's native `blend_v` and `blend_h` functions to seamlessly merge overlapping regions | Bit-identical (same logic as single-GPU tiling) |
-| **Patch Decode** | Small images (no VAE tiling) | Splits latent into spatial patches with halos. Each rank decodes one patch with boundary context. | Halo regions provide edge context; core regions are directly stitched without blending | Near-identical (diff < 0.5%, visually imperceptible) |
-
+| Strategy         | Use Case                           | How It Works                                                                                               | Overlap Handling                                                                            | Output Quality                                       |
+| ---------------- | ---------------------------------- | ---------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- | ---------------------------------------------------- |
+| **Tiled Decode** | Large images (triggers VAE tiling) | Distributes existing VAE tiling computation across ranks. Each rank decodes a subset of overlapping tiles. | Uses VAE's native `blend_v` and `blend_h` functions to seamlessly merge overlapping regions | Bit-identical (same logic as single-GPU tiling)      |
+| **Patch Decode** | Small images (no VAE tiling)       | Splits latent into spatial patches with halos. Each rank decodes one patch with boundary context.          | Halo regions provide edge context; core regions are directly stitched without blending      | Near-identical (diff < 0.5%, visually imperceptible) |
 
 VAE Patch Parallelism **reuses the DiT process group** (`dit_group`) and does not initialize a separate ProcessGroup. This means:
 
@@ -110,19 +109,80 @@ vllm serve Tongyi-MAI/Z-Image-Turbo --omni --port 8091 \
 
 In `DiffusionParallelConfig`:
 
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `vae_patch_parallel_size` | int | 1 | Number of GPUs for VAE patch/tile parallelism. Set to 2 or higher to enable. Should typically match `tensor_parallel_size` as they share the same process group. |
-| `vae_parallel_mode` | str | `"tile"` | VAE parallel decode strategy: `"tile"` (default tile/patch parallel decode), `"spatial_shard_height"`, or `"spatial_shard_width"` (spatially-sharded decode, Wan only). See [Spatially-Sharded Decode](#spatially-sharded-decode-wan). |
+| Parameter                 | Type   | Default   | Description                                                                                                                                                       |
+| ------------------------- | ------ | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `vae_patch_parallel_size` | int    | 1         | Number of ranks requested for VAE parallelism, including batch mode. Set to 2 or higher to enable. Batch mode caps this degree at the existing worker group size. |
+| `vae_parallel_mode`       | str    | `"tile"`  | `"tile"` for tile/patch decode, `"batch"` for complete-image batch decode, or `"spatial_shard_height"` / `"spatial_shard_width"` for Wan spatial-shard decode.    |
 
 Additional requirements:
 
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `vae_use_tiling` | bool | False | Must be set to `True` when using VAE patch parallelism. |
+| Parameter        | Type   | Default   | Description                                             |
+| ---------------- | ------ | --------- | ------------------------------------------------------- |
+| `vae_use_tiling` | bool   | False     | Must be set to `True` when using VAE patch parallelism. |
 
 !!! note "Automatic VAE Tiling"
-    When `vae_patch_parallel_size > 1` and the model has a distributed VAE (`DistributedVaeMixin`), the system automatically sets `vae_use_tiling=True` if not already enabled.
+    When `vae_patch_parallel_size > 1` and the model has a distributed VAE (`DistributedVaeMixin`), tile and spatial-shard modes automatically enable `vae_use_tiling`. Batch mode preserves the requested tiling setting.
+
+---
+
+## Batch Parallel Decode
+
+Set `vae_parallel_mode="batch"` to distribute complete images across ranks during
+VAE decode. This mode supports pipelines that load `DistributedAutoencoderKL`
+or `DistributedAutoencoderKLFlux2`, including FLUX.2-dev and FLUX.2 Klein. Klein
+selects the distributed VAE only for batch mode; its default path is unchanged.
+Selecting this mode for an
+unsupported VAE, including Wan, raises an error at model initialization.
+
+For a pipeline already using two tensor-parallel workers, select batch decode
+with these Python settings:
+
+```python
+parallel_config = DiffusionParallelConfig(
+    tensor_parallel_size=2,
+    vae_patch_parallel_size=2,
+    vae_parallel_mode="batch",
+)
+sampling_params = OmniDiffusionSamplingParams(num_outputs_per_prompt=2)
+```
+
+The corresponding serving flags are `--vae-patch-parallel-size 2
+--vae-parallel-mode batch`. Keep the existing DiT parallelism configuration;
+the VAE degree does not allocate extra workers. Send a request for multiple
+images to use the batch split.
+
+Each rank decodes a contiguous part of the image batch through its native VAE
+decoder. Results are reassembled in the original image order and made available
+on every worker. With two images and four workers, two workers decode one image
+each. With five images and four workers, the assigned batch sizes are 2, 1, 1,
+and 1. Workers without images still participate in the required communication.
+
+**Constraints and behavior:**
+
+- Batch mode and distributed tile/patch mode are mutually exclusive. Encoding
+  uses the native local path, including native tiling when enabled, rather than
+  distributing spatial tiles across workers.
+- Batch size one, VAE degree one, or a single worker uses native decode on each
+  worker. Batch mode does not automatically switch to distributed tile/patch
+  decode for a single image, even with `vae_use_tiling=True`. To distribute a
+  single image across workers, select `vae_parallel_mode="tile"` on a pipeline
+  that supports distributed tile/patch decoding.
+- Data parallelism, pipeline parallelism, and CFG parallelism must each have
+  degree one because this implementation communicates over the worker WORLD
+  group. Tensor and sequence parallelism may supply the shared workers.
+- Batch mode does not force tiling. Explicit native VAE tiling and slicing
+  settings remain in effect within each rank's assigned batch and on the
+  single-image fallback. Native tiling splits an image locally on one worker;
+  it does not add cross-worker tile parallelism to batch mode.
+- It does not merge independent serving requests. The VAE must receive a batch
+  containing multiple images, for example through `num_outputs_per_prompt`.
+
+Splitting the batch can reduce per-rank decoder work and activation memory, but
+collecting decoded images adds communication and buffers. Measure complete
+request latency and memory for the intended model, image count, and resolution.
+This feature has no universal speedup or memory-saving guarantee. Changing the
+decoder batch shape can also change floating-point rounding; validate generated
+outputs when selecting a deployment configuration.
 
 ---
 
@@ -194,7 +254,8 @@ python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
 ### Common Issue 1: Model Not Support VAE Patch Parallel
 
 **Symptoms**:
-```
+
+```text
 WARNING: vae_patch_parallel_size=2 is set but VAE patch parallelism is NOT enabled for xxxPipeline; ignoring.
 ```
 
@@ -217,7 +278,6 @@ if vae_pp_size > 1 and not is_distributed_vae:
 
 2. To add support for a new model, implement `DistributedVaeMixin` on its VAE class (contributions are welcome).
 
-
 ### Common Issue 2: `vae_patch_parallel_size` Exceeds DiT Process Group Size
 
 **Symptoms**: Shows warning message, and vae patch parallel size is resized to DiT process group size
@@ -227,6 +287,7 @@ if vae_pp_size > 1 and not is_distributed_vae:
 **Recommendation**: Always set `vae_patch_parallel_size` to be no greater than your DiT process group size.
 
 Note that the size of DiT process group size equals to:
+
 ```text
 dit_parallel_size = data_parallel_size
                   × cfg_parallel_size
@@ -235,7 +296,8 @@ dit_parallel_size = data_parallel_size
                   × tensor_parallel_size
 
 ```
-_sequence_parallel_size = ulysses_degree × ring_degree_
+
+`sequence_parallel_size = ulysses_degree × ring_degree`
 
 ---
 

docs/user_guide/diffusion/vae_fast_path.md

@@ -7,7 +7,8 @@ convolutional network whose reference implementation in diffusers moves every
 activation through memory several times per layer (normalization, activation,
 causal padding, feature-cache bookkeeping, shortcut upsampling). vLLM-Omni
 installs a fast path on every loaded Wan VAE that fuses this data movement into
-a handful of Triton kernels while leaving the convolutions themselves untouched.
+a handful of Triton kernels. The opt-in `channels_last` level also specializes
+first-frame convolutions that have no history.
 
 ## Levels
 
@@ -15,9 +16,9 @@ The fast path is controlled by `--vae-fast-path` (engine argument
 `vae_fast_path`, deploy-config key `vae_fast_path`):
 
 | Level | Default | Output vs. diffusers | What it does |
-|---|---|---|---|
+| --- | --- | --- | --- |
 | `lossless` | yes | bit-identical | Fused RMSNorm epilogue, fused causal-conv input assembly and cache refresh, fused shortcut upsampling and residual adds (with the neighbouring convolution biases folded in), single-pass nearest 2x upsampling, preallocated output assembly. |
-| `channels_last` | no | within tolerance (PSNR typically > 60 dB) | Everything in `lossless`, plus decoder convolution weights converted to channels-last memory format so cuDNN picks its channels-last kernels, and a single-pass channels-last RMSNorm+SiLU kernel that also absorbs the bias of the preceding `conv1`. |
+| `channels_last` | no | within tolerance (PSNR typically > 60 dB) | Everything in `lossless`, plus channels-last convolution weights, a single-pass channels-last RMSNorm+SiLU kernel that also absorbs the preceding `conv1` bias, and Conv2d for supported single-frame causal convolutions without history. |
 | `off` | no | bit-identical | Reference diffusers decoder. |
 
 ```bash
@@ -61,6 +62,12 @@ stages:
 - The `channels_last` level changes the order in which cuDNN accumulates
   convolutions, so outputs differ from the reference in the last bits. Use
   `lossless` when bitwise reproducibility against diffusers matters.
+- For a supported causal convolution with one input frame and no history,
+  the temporal input is `[0, 0, x]`. The `channels_last` level uses Conv2d with
+  the last temporal weight slice, preserving the convolution formula for
+  finite inputs and the existing cache update. Calls with history retain
+  Conv3d. The weight slice is taken on each call, so weight updates and device
+  moves do not leave a stale packed copy.
 - For VAEs kept in fp32, cuDNN's channels-last convolution algorithms run in
   TF32 under PyTorch's default `torch.backends.cudnn.allow_tf32 = True`, which
   dominates the difference to the reference (about 1e-3). Set
@@ -73,10 +80,10 @@ Cosmos3-Nano VAE, 1280x720 x 189 frames, bf16, one GB200 GPU, `bench_wan_vae_dec
 (best of 2 runs after warmup):
 
 | `--vae-fast-path` | Decode time | Speedup | Output vs. `off` |
-|---|-------------|---------|---|
-| `off` | 6.00 s      | 1.00x   | reference |
-| `lossless` | 3.26 s      | 1.84x   | bit-identical |
-| `channels_last` | 2.51 s      | 2.39x   | PSNR 62.6 dB, max abs diff 4.6e-2 |
+| --- | ------------- | --------- | --- |
+| `off` | 6.00 s | 1.00x | reference |
+| `lossless` | 3.26 s | 1.84x | bit-identical |
+| `channels_last` | 2.51 s | 2.39x | PSNR 62.6 dB, max abs diff 4.6e-2 |
 
 Peak decode memory was unchanged (about 13 GiB): it is set by the largest
 activations and the cuDNN workspace, not by the output assembly.
@@ -93,6 +100,22 @@ python benchmarks/diffusion/bench_wan_vae_decode.py --model nvidia/Cosmos3-Nano
     --size 1280x720 --frames 189 --fast-path off,lossless,channels_last --profile
 ```
 
+For a single-GPU comparison that isolates first-frame Conv2d within
+`channels_last`, use `--first-frame-ablation`. It alternates the two paths,
+disables cuDNN TF32 and autotuning, and prints JSON with every timing sample,
+summary statistics, peak allocated memory and output comparisons. Per-call
+weight packing is included in the measured decode time.
+
+```bash
+python benchmarks/diffusion/bench_wan_vae_decode.py \
+    --model Wan-AI/Wan2.1-T2V-1.3B-Diffusers --size 832x480 --frames 1 \
+    --dtype bf16 --first-frame-ablation --warmup 60 --iters 50 --seed 0
+```
+
+The default input is a seeded latent tensor. `--latents /path/to/latents.pt`
+can instead load a saved VAE input after latent mean/std scaling; the benchmark
+selects the temporal prefix requested by `--frames` and checks its shape.
+
 The same script benchmarks multi-GPU VAE decode when launched with `torchrun`;
 the decode is timed across all ranks and rank 0 reports:
 

docs/user_guide/diffusion_features.md

@@ -152,9 +152,10 @@ The following tables show which models support each feature:
 | **HiDream-O1-Image**     |     ❌     |     ✅      |           ❌           |       ❌        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **HunyuanImage3**        |     ❌     |     ✅      |           ❌           |       ❌        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ✅        |        ✅*        |
 | **Krea 2**               |     ❌     |     ✅      |           ❌           |       ❌        |         ❌         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
+| **JoyAI-Image-Edit**    |     ❌     |     ❌      |           ❌           |       ✅        |         ❌         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **LongCat-Image**        |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **LongCat-Image-Edit**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
-| **MammothModa2(T2I)**    |     ❌     |     ✅      |           ❌           |       ❌        |         ❌         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
+| **MammothModa2(T2I)**    |     ✅     |     ✅      |           ❌           |       ❌        |         ❌         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **Nextstep_1(T2I)**      |     ❓     |     ❓      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **OmniGen2**             |     ❌     |     ✅      |           ✅           |       ❌        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **Ovis-Image**           |     ❌     |     ✅      |           ❌           |       ✅        |         ❌         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
@@ -163,7 +164,10 @@ The following tables show which models support each feature:
 | **Qwen-Image-Edit**      |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Qwen-Image-Edit-2509** |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |        ✅ (decode)         |          ✅           |       ❌        |        ❌         |
 | **Qwen-Image-Layered**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
-| **SenseNova-U1 / U1.5**  |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **Qwen-Image-2.1**       |     ❌     |     ❌      |           ❓           |       ✅        |         ✅         |          ❌          |   ❓    |             ❓             |      ✅ (decode)      |       ✅        |        ✅         |
+| **SenseNova-U1**         |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **SenseNova-U1.5**       |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ✅        |        ❌         |
+| **SenseNova-U1-A3B**     |     ❌     |     ❌      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **Stable-Diffusion-XL**  |     ❌     |     ❌      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Stable-Diffusion3.5**  |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Z-Image**              |     ✅     |     ✅      |           ✅           |       ❓        |   ✅ (TP=2 only)   |          ❌          |   ✅    |             ❌             |      ✅ (decode)      |       ✅        |        ❌         |
@@ -179,6 +183,8 @@ The following tables show which models support each feature:
 > 5. HunyuanImage3 supports step execution. Multi-request step execution requires `TORCH_SDPA`; see [Diffusion Execution Modes](diffusion/execution_modes.md#step-execution).
 > 6. BAGEL step execution supports image generation with `bagel.yaml`, `bagel_think.yaml`, and `bagel_single_stage.yaml`; two-stage Thinker execution and explicit single-stage text output remain on their existing complete-request paths. Image requests require `num_inference_steps >= 2`. BAGEL step execution cannot currently be combined with sequence parallelism or a diffusion cache backend; see [Diffusion Execution Modes](diffusion/execution_modes.md#step-execution).
 > 7. MammothModa2 runs its DiT stage on the diffusion runner (`StageExecutionType.DIFFUSION`); Cache-DiT is enabled through the standard diffusion-stage knobs on the stage entry of the deploy YAML (`cache_backend: cache_dit`, plus optional `cache_config` / `enable_cache_dit_summary`). The runner installs the backend at startup and the pipeline adopts it per request. Only the repeated main-layer stack is cached; requests with `text_guidance_scale = 1.0` bypass the cache hooks.
+> 8. Qwen-Image-2.1 (`QwenImage21Pipeline`) handles both text-to-image and image-conditioned generation (up to 4 condition images) and supports LoRA through `QwenImageLoraLoaderMixin`. Ulysses sequence parallelism is wired and ring attention is not supported, but multi-GPU padding e2e and a parity number are not in this PR's H200 results, so SP stays ❓. Cache acceleration (TeaCache / Cache-DiT) is unavailable; its built-in KV cache for the text/condition prefix is a separate feature from those cache backends. Online FP8 is documented in the quantization guide with H200 numbers.
+> 9. SenseNova-U1-A3B: `--cfg-parallel-size 2` and TP=2 both complete. CFG-P covers t2i 2-branch (`cond`+`uncond`), img2img 2-branch (`cond`+`img_cond` when `img_cfg_scale=1`), and img2img 3-branch (`cond`+`img_cond`+`uncond` when `img_cfg_scale` is neither 1 nor equal to `cfg_scale`; 3 branches on 2 ranks dispatch as `[[0, 2], [1]]`). Cache-DiT does not: the adapter is declared on `SenseNovaU1Model`, but discovery walks `_dit_modules` (`language_model` / `SenseNovaU1ForCausalLM`) and raises. Layerwise CPU offload is a no-op for the same reason — that class has no `_layerwise_offload_blocks_attrs`, so the 72.2 GiB expert weights stay resident.
 
 ### VideoGen
 
@@ -193,6 +199,7 @@ The following tables show which models support each feature:
 |  **SANA-Video-2B T2V I2V**   |     ❌     |     ❌      |   ✅ (`--usp`, frame-sharded)   |       ✅       |  ✅ (TP=2 only)   |         ❌         |   ❌   |            ❌             |          ❌          |       ❌       |        ❌        |
 | **Helios**                   |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ✅*        |
 | **HunyuanVideo-1.5 T2V I2V** |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
+| **Kandinsky 6 TI2VA**        | ✅* (uncalibrated) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ✅ (FP8) | ✅ (1 request) |
 | **Cosmos3**                  |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
 | **LongCat-Video-Avatar-1.5** |     ❌     |     ❌      |           ❌           |       ❌        |         ❌         |         ❌         |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **MiniMax-H3**               | ✅ (FL2VA) |     ✅      |           ✅           |       ❌        |       ✅ (DiT/TE)  |         ❌         |   ✅    |             ✅             |       ✅ (tile)       |      ✅ (DiT)      |        ❌         |

docs/user_guide/examples/offline_inference/image_to_image.md

@@ -3,7 +3,7 @@
 Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/image_to_image>.
 
 
-This example edits an input image with `Qwen/Qwen-Image-Edit` using the `image_edit.py` CLI.
+This example edits an input image with `Qwen/Qwen-Image-Edit` and `JoyAI-Image-Edit` using the `image_edit.py` CLI.
 
 ## Local CLI Usage
 
@@ -26,6 +26,27 @@ python image_edit.py \
   --cfg-scale 4.0
 ```
 
+### JoyAI-Image-Edit
+
+JoyAI-Image-Edit uses the same `image_edit.py` CLI and supports one input
+image per request. Requested dimensions are snapped to the nearest supported
+Joy bucket; for square outputs, use `1024x1024`.
+
+```bash
+python image_edit.py \
+  --model jdopensource/JoyAI-Image-Edit-Diffusers \
+  --image input.png \
+  --prompt "Change the background to a clean studio while preserving the subject." \
+  --height 1024 \
+  --width 1024 \
+  --num-inference-steps 50 \
+  --guidance-scale 4.0 \
+  --output output_joyai_edit.png
+```
+
+JoyAI defaults to guidance scale `4.0`; pass `--guidance-scale` explicitly when
+you need a different value.
+
 ### Multiple Image Editing (Qwen-Image-Edit-2509)
 
 For multiple image inputs, use `Qwen/Qwen-Image-Edit-2509` or  `Qwen/Qwen-Image-Edit-2511`:
@@ -43,7 +64,7 @@ python image_edit.py \
 
 Key arguments:
 
-- `--model`: model name or path. Use `Qwen/Qwen-Image-Edit-2509` or later for multiple image support.
+- `--model`: model name or path. Use `Qwen/Qwen-Image-Edit-2509` or later for multiple image support. `jdopensource/JoyAI-Image-Edit-Diffusers` supports one input image.
 - `--image`: path(s) to the source image(s) (PNG/JPG, converted to RGB). Can specify multiple images.
 - `--prompt` / `--negative-prompt`: text description (string).
 - `--cfg-scale`: true classifier-free guidance scale (default: 4.0). Classifier-free guidance is enabled by setting cfg_scale > 1 and providing a negative_prompt. Higher guidance scale encourages images closely linked to the text prompt, usually at the expense of lower image quality.

docs/user_guide/examples/offline_inference/text_to_speech.md

@@ -130,7 +130,7 @@ python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
 ### Notes
 
 - Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform.
-- Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config <path>` to override.
+- H200 automatically selects `cosyvoice3_packed_streaming_optimized_standard.yaml` when the CUDA MPS control tool is available; other devices and environments use `cosyvoice3.yaml`. See the [serving defaults and tradeoffs](../online_serving/text_to_speech.md#cosyvoice3). Pass `--deploy-config cosyvoice3.yaml` for the previous profile or `--deploy-config <path>` to override. The example uses the resolved profile's sampling parameters.
 - `async_chunk: true` is the default; pass `--no-async-chunk` to switch to the legacy synchronous path.
 
 ---

docs/user_guide/examples/online_serving/image_to_image.md

@@ -3,10 +3,14 @@
 Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/image_to_image>.
 
 
-This example demonstrates how to deploy Qwen-Image-Edit model for online image editing service using vLLM-Omni.
+This example demonstrates how to deploy image-to-image models for online image editing service using vLLM-Omni.
+
+Supported models include Qwen-Image-Edit, BAGEL, JoyAI-Image-Edit, and other image-to-image pipelines.
 
 For **multi-image** input editing, use **Qwen-Image-Edit-2509** (QwenImageEditPlusPipeline) and send multiple images in the user message content.
 
+For **JoyAI-Image-Edit**, use `jdopensource/JoyAI-Image-Edit-Diffusers` and send exactly one input image.
+
 ## Start Server
 
 ### Basic Start
@@ -24,6 +28,12 @@ vllm serve Qwen/Qwen-Image-Edit --omni --port 8092
 vllm serve Qwen/Qwen-Image-Edit-2509 --omni --port 8092
 ```
 
+### JoyAI-Image-Edit
+
+```bash
+vllm serve jdopensource/JoyAI-Image-Edit-Diffusers --omni --port 8092
+```
+
 ### Start with Parameters
 
 
@@ -76,6 +86,11 @@ curl -s http://localhost:8092/v1/chat/completions \
   | cut -d',' -f2 | base64 -d > output.png
 ```
 
+For JoyAI-Image-Edit, set classifier-free guidance with `guidance_scale`
+in `extra_body` (default: `4.0`). `true_cfg_scale` remains supported as an
+alias; if both are provided, their values must match. Values at or below `1.0`
+disable CFG.
+
 ### Method 2: Using OpenAI Python SDK
 
 ```python

docs/user_guide/examples/online_serving/text_to_speech.md

@@ -16,6 +16,7 @@ For the full list of supported architectures across all modalities, see
 
 | Model | HuggingFace repo | Voice cloning | Streaming | Voice presets / upload | Gradio demo |
 | --- | --- | --- | --- | --- | --- |
+| AuK / AuK-Flash | assembled local bundle | ✓ (`ref_audio`) | HTTP audio after full generation | — | — |
 | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | — |
 | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
 | Gepard-1.0 | `nineninesix/gepard-1.0` | — (zero-shot default voice) | ✓ (PCM / WAV stream) | `default` only | — |
@@ -99,6 +100,41 @@ For full request-shape documentation (all parameters, response formats, error co
 
 ---
 
+## AuK / AuK-Flash
+
+AuK serves speech generation and reference-voice synthesis at 24 kHz. Assemble a local bundle before serving:
+
+```bash
+python tools/prepare_auk_checkpoint.py \
+    --auk-dir ckpts/AuK --qwen-dir ckpts/Qwen2.5-Omni-3B --out ckpts/auk-omni
+MODEL=ckpts/auk-omni bash examples/online_serving/text_to_speech/auk/run_server.sh
+```
+
+AuK accepts the complete instruction after you fill in the [local recipe template](../../../../recipes/Tencent/AuK-H100.md#supported-model-contract) or the [official AuK cookbook](https://github.com/Tencent-Hunyuan/AuK/blob/main/docs/COOKBOOK.md). Pass it in `instructions` and leave `input` empty. For instruct TTS, fill in `Generate speech based on the following description: "{voice description}". The content to speak is: "{text}".`; for zero-shot TTS, editing, enhancement and separation, use the corresponding complete instruction from the cookbook. Set `duration_seconds` for text-only generation; with `ref_audio`, omitting it keeps the source length.
+
+For compatibility with clients that can only send `input`, AuK treats an input-only request as the complete instruction and logs a recommendation to move it to `instructions` with `input=""`.
+
+`extra_params` accepts `num_inference_steps`, `guidance_scale`, `sway`, `t_grid`, and `vae_sample`; `seed` is a top-level field.
+
+```bash
+python examples/online_serving/text_to_speech/auk/speech_client.py \
+    --model ckpts/auk-omni \
+    --instructions 'Generate speech based on the following description: "A calm young woman speaking warmly and slowly.". The content to speak is: "Welcome back, how was your day?".' \
+    --duration-seconds 3 --seed 7
+```
+
+Complete-instruction editing also does not need a separate text argument:
+
+```bash
+python examples/online_serving/text_to_speech/auk/speech_client.py \
+    --model ckpts/auk-omni --instructions "Keep pure speech voice, remove noise and reverberation." \
+    --ref-audio noisy.wav --seed 7 --output clean.wav
+```
+
+The client accepts local reference files, URLs, and data URLs. `--stream` writes signed 16-bit 24 kHz PCM after the complete waveform is ready. See `speech_client.py --help` for editing and sampling options.
+
+---
+
 ## CosyVoice3
 
 2-stage TTS (`talker` + flow-matching `code2wav`) at 24 kHz. Voice cloning only — every request needs `ref_audio` + `ref_text`; there are no built-in voice presets.
@@ -119,16 +155,22 @@ vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 --omni --port 8091 --trust-remot
 ./cosyvoice3/run_server.sh
 ```
 
-Streaming is on by default via `async_chunk: true` in `vllm_omni/deploy/cosyvoice3.yaml`. Pass `--no-async-chunk` (or `NO_ASYNC_CHUNK=1 ./cosyvoice3/run_server.sh`) for the legacy synchronous path.
+Streaming is on by default. On CUDA Hopper GPUs with at least 140 GiB of device memory (H200), serving automatically selects `cosyvoice3_packed_streaming_optimized_standard.yaml` when `nvidia-cuda-mps-control` is available on `PATH`: AR32 / codec32, standard sampling, packed Flow, cached ISTFT, HiFT decode graphs, and local CUDA MPS. No `--hf-overrides` is needed. Other devices and environments without the MPS control tool use `cosyvoice3.yaml` with RAS sampling and AR8 / codec8.
 
-Cross-request Stage-1 flow batching is opt-in. Enable it in the server environment when concurrent requests should share a flow-estimator call:
+The optimized profile is not a universal hardware configuration. Its packed kernels are Hopper-only; the talker's Model Runner V2 is unsupported on NPU/XPU, and CUDA MPS requires a local NVIDIA GPU and its control tool. Larger batches, 25-frame chunks, and graph allocations also require per-device validation on smaller or slower GPUs. Automatic selection retains the generic profile outside the validated device class; an explicit deploy configuration takes precedence.
+
+The H200 default targets streaming throughput at C32/C64. It changes sampling from RAS to standard (temperature 0.7, top-p 0.8, top-k 20, repetition penalty 1.21); output lengths and quality can differ. HiFT captures a recurring exact shape on its third use, which adds latency to that request, and its graph pool needs memory headroom beyond the engine reservations. This profile is not a claim of optimal first-audio latency at low concurrency. Pass `--deploy-config cosyvoice3.yaml` to retain the previous configuration, or `--deploy-config <path>` to select another profile.
+
+Pass `--no-async-chunk` (or `NO_ASYNC_CHUNK=1 ./cosyvoice3/run_server.sh`) for the legacy synchronous path; use `--deploy-config cosyvoice3.yaml` alongside it when selecting that path on H200.
+
+Cross-request Stage-1 flow batching is enabled in the H200 profile. For the generic configuration, enable it in the server environment when concurrent requests should share a flow-estimator call:
 
 ```bash
 export COSYVOICE3_BATCH_FLOW=1
 vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 --omni --port 8091 --trust-remote-code
 ```
 
-Batching preserves output lengths and streaming cache alignment, but the different GEMM shapes can produce small waveform differences compared with processing each request separately. Leave `COSYVOICE3_BATCH_FLOW` unset (or set it to `0`) when request-independent numerical behavior is required. Set `COSYVOICE3_BATCH_FLOW_DEBUG=1` to log the observed group-size distribution and enable detailed Stage-1 profiler scopes; diagnostics are disabled by default to avoid per-step profiling overhead.
+Batching preserves output lengths and streaming cache alignment, but the different GEMM shapes can produce small waveform differences compared with processing each request separately. Use `--deploy-config cosyvoice3.yaml` and leave `COSYVOICE3_BATCH_FLOW` unset (or set it to `0`) when request-independent numerical behavior is required. Set `COSYVOICE3_BATCH_FLOW_DEBUG=1` to log the observed group-size distribution and enable detailed Stage-1 profiler scopes; diagnostics are disabled by default to avoid per-step profiling overhead.
 
 ### CLI client
 
@@ -158,7 +200,7 @@ The client supports `--api-base`, `--model`, `--text`, `--ref-audio`, `--ref-tex
 ### Notes
 
 - Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform.
-- Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config <path>` to override.
+- Deploy config auto-loads the device-specific CosyVoice3 default described above. Pass `--deploy-config <path>` to override.
 - For offline inference and the end-to-end script, see the [offline CosyVoice3 section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#cosyvoice3).
 
 ---

docs/user_guide/quantization/fp8.md

@@ -1,4 +1,4 @@
-# FP8 Quantization
+# FP8 Quantization
 
 ## Overview
 
@@ -17,7 +17,7 @@ in deep DiT blocks.
 ## Hardware Support
 
 | Device | Support |
-|--------|---------|
+| -------- | --------- |
 | NVIDIA Blackwell GPU (SM 100+) | ✅ |
 | NVIDIA Ada/Hopper GPU (SM 89+) | ✅ |
 | NVIDIA Ampere GPU (SM 80+) | ✅ |
@@ -93,8 +93,9 @@ warmup_quack_fp8([(14040, 2048, 6144), (14040, 2048, 2048)])
 ### Diffusion Models
 
 | Model | HF models | Online | Pre-calibrated | Recommendation | `ignored_layers` | Text-Encoder quantization |
-|-------|-----------|:-------:|:------:|----------------|------------------|------------------|
+| ------- | ----------- | :-------: | :------: | ---------------- | ------------------ | ------------------ |
 | Qwen-Image | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes | Yes | Skip sensitive image-stream MLPs when quality regresses | `img_mlp` | |
+| Qwen-Image-2.1 | `Qwen/Qwen-Image-2.1` | Yes | Not validated | Skip image-stream MLPs for best fidelity; all-layer FP8 is loadable. On H200 (1024×1024, teapot prompt) DiT FP8-all is ~30.4 GB / 6.06 s vs BF16 ~36.9 GB / 7.34 s (~33 dB vs BF16). Text encoder FP8 covers the language model only — the vision tower and LM head stay BF16 | `img_mlp` | ✅︎ (language model only) |
 | Wan2.2 | Wan2.2 diffusion pipelines | Not validated | Not validated | Validate against BF16 before documenting as supported | TBD | |
 | LTX-2 | `Lightricks/LTX-2`, `rootonchair/LTX-2-19b-distilled` | Yes | Not validated | Transformer only; use dynamic phase LoRA for ordinary two-stage | None | |
 | LTX-2.3 | `diffusers/LTX-2.3-Diffusers`, `diffusers/LTX-2.3-Distilled-Diffusers` | Yes | Not validated | Transformer only; use dynamic phase LoRA for ordinary two-stage | None | |
@@ -106,23 +107,24 @@ warmup_quack_fp8([(14040, 2048, 6144), (14040, 2048, 2048)])
 | HunyuanVideo-1.5 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `720p_t2v`, `480p_i2v` | Yes | Yes | All layers | None | |
 | Cosmos3 | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | Yes | Not validated | All layers | None | |
 | MiniMax-H3 | `MiniMaxAI/MiniMax-H3` (`FL2VA` / `Ref2VA`) | Yes | Not validated | `quantization="fp8"` quantizes eligible DiT and text-encoder linears; mixed-precision input/output heads stay FP32 | None | ✅︎ |
+| SenseNova-U1.5 | `sensenova/SenseNova-U1.5-8B-MoT` | Yes | Not validated | UND/GEN language-model linears only | None | |
 
 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
 
-| Model | Scope | Format | Status |
-|-------|-------|--------|--------|
+| Model      | Scope                        | Format                                   | Status                              |
+| ---------- | ---------------------------- | ---------------------------------------- | ----------------------------------- |
 | Qwen3-Omni | Thinker language-model stage | [ModelOpt](modelopt.md) `quant_algo=FP8` | Tested for thinker memory reduction |
-| Qwen3-TTS | TTS language-model stage | Checkpoint config | Not validated |
+| Qwen3-TTS  | TTS language-model stage     | Checkpoint config                        | Not validated                       |
 
 Audio encoder, vision encoder, talker, and code2wav stay in BF16 unless a
 model-specific guide says otherwise.
 
 ### Multi-Stage Diffusion Model (BAGEL, GLM-Image)
 
-| Model | Scope | Status | Notes |
-|-------|-------|--------|-------|
-| BAGEL | Stage-specific transformer or DiT module | Not validated | Route FP8 to the intended stage before enabling |
-| GLM-Image | Stage-specific transformer or DiT module | Not validated | Validate quality against BF16 baseline |
+| Model     | Scope                                    | Status        | Notes                                           |
+| --------- | ---------------------------------------- | ------------- | ----------------------------------------------- |
+| BAGEL     | Stage-specific transformer or DiT module | Not validated | Route FP8 to the intended stage before enabling |
+| GLM-Image | Stage-specific transformer or DiT module | Not validated | Validate quality against BF16 baseline          |
 
 ## Configuration
 
@@ -161,7 +163,7 @@ For a pipeline that exposes both a transformer and a quantization-aware text
 encoder, the scope is:
 
 | Configuration | Transformer | Text encoder | Components without supported quantizable layers |
-|---------------|-------------|--------------|-------------------------------------------------|
+| --------------- | ------------- | -------------- | ------------------------------------------------- |
 | `quantization="fp8"` | FP8 | FP8 | checkpoint precision |
 | `{"transformer": {"method": "fp8"}}` | FP8 | checkpoint precision | checkpoint precision |
 | `{"text_encoder": {"method": "fp8"}}` | checkpoint precision | FP8 | checkpoint precision |
@@ -210,7 +212,7 @@ does not fit the workload.
 ## Parameters
 
 | Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
+| ----------- | ------ | --------- | ------------- |
 | `method` | str | - | Quantization method (`"fp8"`) |
 | `ignored_layers` | list[str] | `[]` | Layer name patterns to keep in BF16 |
 | `activation_scheme` | str | `"dynamic"` | `"dynamic"` selects online activation scaling, or `"static"` when scales are available |

examples/offline_inference/image_to_image/image_edit.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
 """
 Example script for image editing with OmniGen2.
@@ -39,6 +39,20 @@ Usage (multiple images):
         --cfg-scale 4.0 \
         --guidance-scale 1.0
 
+Usage (JoyAI-Image-Edit, single image):
+    python image_edit.py \
+        --model jdopensource/JoyAI-Image-Edit-Diffusers \
+        --image input.png \
+        --prompt "Change the background to a clean studio while preserving the subject." \
+        --height 1024 \
+        --width 1024 \
+        --num-inference-steps 50 \
+        --guidance-scale 4.0 \
+        --output output_joyai_edit.png
+
+    Note: JoyAI-Image-Edit snaps requested dimensions to the nearest supported
+    Joy bucket; for square outputs, use 1024x1024.
+
 Usage (with cache-dit acceleration):
     python image_edit.py \
         --image input.png \
@@ -209,14 +223,14 @@ def parse_profiler_config(value: str) -> dict[str, Any]:
 
 
 def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Edit an image with Qwen-Image-Edit.")
+    parser = argparse.ArgumentParser(description="Edit images with vLLM-Omni.")
     parser.add_argument(
         "--model",
         default="Qwen/Qwen-Image-Edit",
         help=(
             "Diffusion model name or local path. "
-            "For multiple image inputs, use Qwen/Qwen-Image-Edit-2509 or Qwen/Qwen-Image-Edit-2511"
-            "which supports QwenImageEditPlusPipeline."
+            "For multiple image inputs, use Qwen/Qwen-Image-Edit-2509, "
+            "Qwen/Qwen-Image-Edit-2511, or Qwen/Qwen-Image-2.1."
         ),
     )
     parser.add_argument(
@@ -270,9 +284,10 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--cfg-scale",
         type=float,
-        default=4.0,
+        default=None,
         help=(
-            "True classifier-free guidance scale (default: 4.0). Guidance scale as defined in Classifier-Free "
+            "True classifier-free guidance scale. When omitted, the pipeline uses its model-specific default. "
+            "Guidance scale as defined in Classifier-Free "
             "Diffusion Guidance. Classifier-free guidance is enabled by setting cfg_scale > 1 and providing "
             "a negative_prompt. Higher guidance scale encourages images closely linked to the text prompt, "
             "usually at the expense of lower image quality."
@@ -281,9 +296,9 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--guidance-scale",
         type=float,
-        default=1.0,
+        default=None,
         help=(
-            "Guidance scale for guidance-distilled models (default: 1.0, disabled). "
+            "Guidance scale for guidance-distilled models. When omitted, the pipeline uses its model-specific default. "
             "Unlike classifier-free guidance (--cfg-scale), guidance-distilled models take the guidance scale "
             "directly as an input parameter. Enabled when guidance_scale > 1. Ignored when not using guidance-distilled models."
         ),
@@ -368,7 +383,7 @@ def parse_args() -> argparse.Namespace:
         "--color-format",
         type=str,
         default="RGB",
-        help="For Qwen-Image-Layered, set to RGBA.",
+        help="For Qwen-Image-Layered or Qwen-Image-2.1, set to RGBA to preserve transparency.",
     )
 
     # Cache-DiT specific parameters
@@ -523,6 +538,12 @@ def parse_args() -> argparse.Namespace:
             "silently masking with a possibly-wrong prompt."
         ),
     )
+    parser.add_argument(
+        "--init-timeout",
+        type=int,
+        default=600,
+        help="Overall pipeline initialization timeout in seconds.",
+    )
     return parser.parse_args()
 
 
@@ -593,6 +614,7 @@ def main():
         enable_cpu_offload=args.enable_cpu_offload,
         enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
         profiler_config=args.profiler_config,
+        init_timeout=args.init_timeout,
     )
     if args.enforce_eager is not None:
         omni_kwargs["enforce_eager"] = args.enforce_eager
@@ -647,6 +669,7 @@ def main():
 
     diffusion_params = OmniDiffusionSamplingParams(
         generator=generator,
+        seed=args.seed,
         true_cfg_scale=args.cfg_scale,
         guidance_scale=args.guidance_scale,
         guidance_scale_2=args.guidance_scale_2,

examples/offline_inference/image_to_image/image_to_image.md

@@ -1,6 +1,6 @@
 # Image-To-Image
 
-This example edits an input image with `Qwen/Qwen-Image-Edit` using the `image_edit.py` CLI.
+This example edits an input image with `Qwen/Qwen-Image-Edit` and `JoyAI-Image-Edit` using the `image_edit.py` CLI.
 
 ## Local CLI Usage
 
@@ -23,6 +23,27 @@ python image_edit.py \
   --cfg-scale 4.0
 ```
 
+### JoyAI-Image-Edit
+
+JoyAI-Image-Edit uses the same `image_edit.py` CLI and supports one input
+image per request. Requested dimensions are snapped to the nearest supported Joy
+bucket; for square outputs, use 1024x1024.
+
+```bash
+python image_edit.py \
+  --model jdopensource/JoyAI-Image-Edit-Diffusers \
+  --image input.png \
+  --prompt "Change the background to a clean studio while preserving the subject." \
+  --height 1024 \
+  --width 1024 \
+  --num-inference-steps 50 \
+  --guidance-scale 4.0 \
+  --output output_joyai_edit.png
+```
+
+JoyAI defaults to guidance scale `4.0`; pass `--guidance-scale` explicitly when
+you need a different value.
+
 ### Multiple Image Editing (Qwen-Image-Edit-2509)
 
 For multiple image inputs, use `Qwen/Qwen-Image-Edit-2509` or  `Qwen/Qwen-Image-Edit-2511`:
@@ -40,7 +61,7 @@ python image_edit.py \
 
 Key arguments:
 
-- `--model`: model name or path. Use `Qwen/Qwen-Image-Edit-2509` or later for multiple image support.
+- `--model`: model name or path. Use `Qwen/Qwen-Image-Edit-2509` or later for multiple image support. `jdopensource/JoyAI-Image-Edit-Diffusers` supports one input image.
 - `--image`: path(s) to the source image(s) (PNG/JPG, converted to RGB). Can specify multiple images.
 - `--prompt` / `--negative-prompt`: text description (string).
 - `--cfg-scale`: true classifier-free guidance scale (default: 4.0). Classifier-free guidance is enabled by setting cfg_scale > 1 and providing a negative_prompt. Higher guidance scale encourages images closely linked to the text prompt, usually at the expense of lower image quality.