fix: preserve direct execution of the review verifier

.buildkite/amd/scripts/prewarm_cosmos3_aiter.py

@@ -1,78 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
-
-"""Prebuild the BF16 AITER MHA modules used by Cosmos3 on ROCm."""
-
-from __future__ import annotations
-
-import os
-from pathlib import Path
-
-import torch
-
-from vllm_omni.diffusion.attention.backends.utils import fa
-from vllm_omni.platforms import current_omni_platform
-
-ATTENTION_VARIANTS = (
-    (True, "mha_fwd_bf16_nbias_mask_nlse_ndropout_nqscale"),
-    (False, "mha_fwd_bf16_nbias_nmask_nlse_ndropout_nqscale"),
-)
-
-
-def main() -> None:
-    if not current_omni_platform.is_rocm():
-        raise RuntimeError("The Cosmos3 AITER prewarm must run on ROCm")
-    if fa.flash_attn_func is None:
-        raise RuntimeError("ROCm FlashAttention/AITER is unavailable")
-
-    cache_dir_value = os.environ.get("AITER_JIT_DIR")
-    if not cache_dir_value:
-        raise RuntimeError("AITER_JIT_DIR must identify the job-local cache")
-    cache_dir = Path(cache_dir_value)
-
-    device = torch.device(current_omni_platform.device_type)
-    dtype = torch.bfloat16
-    batch_size = 1
-    sequence_length = 64
-    num_heads = 8
-    head_dim = 64
-
-    torch.manual_seed(42)
-    query = torch.randn(
-        batch_size,
-        sequence_length,
-        num_heads,
-        head_dim,
-        device=device,
-        dtype=dtype,
-    )
-    key = torch.randn_like(query)
-    value = torch.randn_like(query)
-
-    for causal, expected_module in ATTENTION_VARIANTS:
-        output = fa.flash_attn_func(
-            query,
-            key,
-            value,
-            dropout_p=0.0,
-            softmax_scale=head_dim**-0.5,
-            causal=causal,
-        )
-        if isinstance(output, tuple):
-            output = output[0]
-        current_omni_platform.synchronize()
-
-        if output.shape != query.shape:
-            raise RuntimeError(f"Unexpected AITER output shape: {output.shape}")
-        if not torch.isfinite(output).all():
-            raise RuntimeError("AITER prewarm produced non-finite output")
-
-        module_path = cache_dir / f"{expected_module}.so"
-        if not module_path.is_file():
-            raise RuntimeError(f"Expected AITER module was not built: {module_path}")
-        print(f"Prebuilt Cosmos3 AITER module: {module_path}", flush=True)
-
-
-if __name__ == "__main__":
-    main()

.buildkite/amd/test-amd-merge.yml

@@ -37,22 +37,24 @@ steps:
       mirror_hardwares: [amdproduction]
       grade: Blocking
       timeout_in_minutes: 45
-      # Buildkite's status-based rule cannot prove every exit 134 is a GPU hang.
-      # It retries 134 once; the Kubernetes job retry starts a fresh pod.
-      # Assertions (1), timeouts (124), and OOMs (137) remain non-retryable.
-      retry:
-        automatic:
-          - exit_status: 134
-            limit: 1
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
         # Keep multi-GPU tests out of this one-GPU pool even when their module
         # also carries the cpu marker for unit tests.
-        # Keep π0.5 CPU units out of these shards: CUDA READY/MERGE already run
-        # the same suite mandatorily, and AMD covers it non-blocking in nightly.
         - "pytest -sv tests/diffusion -m 'core_model and cpu and not (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)' --ignore=tests/diffusion/cache/test_teacache_extractors.py --ignore=tests/diffusion/models/pi05/test_pi05_units.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+    # This module takes about 22 minutes on MI300 and was consistently assigned to shard 4,
+    # pushing that shard to the 45-minute timeout. Keep its module-scoped fixture in one lane.
+    - label: "Simple · Pi0.5 CPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 45
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/diffusion/models/pi05/test_pi05_units.py -m 'core_model and cpu'"
 
     - label: "Diffusion · AuK VAE GPU Test"
       agent_pool: mi300_1
@@ -89,25 +91,6 @@ steps:
 
 - group: ":card_index_dividers: Diffusion Test"
   steps:
-    # AR paged attention is real GPU work, including cold AITER compilation.
-    # Keep its complete numerical matrix out of the CPU-marked Simple shards.
-    - label: "ROCm · AR Diffusion Paged Attention GPU Test"
-      agent_pool: mi300_1
-      depends_on: amd-build
-      mirror_hardwares: [amdproduction]
-      grade: Blocking
-      timeout_in_minutes: 20
-      commands:
-        - export VLLM_ROCM_USE_AITER=1
-        - export VLLM_OMNI_AR_FA_REQUIRED=1
-        - export AITER_JIT_DIR="/tmp/vllm-omni-ar-aiter-$$BUILDKITE_JOB_ID"
-        - export TORCH_EXTENSIONS_DIR="/tmp/vllm-omni-ar-torch-extensions-$$BUILDKITE_JOB_ID"
-        - mkdir -p "$$AITER_JIT_DIR" "$$TORCH_EXTENSIONS_DIR"
-        - >-
-          timeout --signal=TERM --kill-after=2m 15m
-          pytest -sv tests/diffusion/ar_diffusion/test_paged_attention.py
-          -m 'core_model and rocm and MI325 and cards_1' --run-level core_model
-
     - label: "Diffusion · LoRA Test"
       agent_pool: mi300_1
       depends_on: amd-build
@@ -126,7 +109,6 @@ steps:
       commands:
         - export VLLM_LOGGING_LEVEL=DEBUG
         - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        - export DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA
         - pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
         - >-
           pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py

.buildkite/amd/test-amd-nightly.yml

@@ -82,89 +82,3 @@ steps:
               limit: 1
         commands:
           - pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"
-
-  # π0.5 CPU units are mandatory on CUDA READY/MERGE. On AMD they exercise the
-  # ROCm container/import path with CPU tensors only, so keep them off the
-  # blocking READY/MERGE critical path and retain non-blocking nightly coverage.
-  - group: ":robot_face: Pi0.5 CPU Nightly"
-    steps:
-      - label: "Simple · Pi0.5 CPU Test"
-        agent_pool: mi300_1
-        mirror_hardwares: [amdproduction]
-        grade: NonBlocking
-        timeout_in_minutes: 60
-        commands:
-          - export VLLM_ROCM_USE_AITER=0
-          - "pytest -sv tests/diffusion/models/pi05/test_pi05_units.py -m 'core_model and cpu'"
-
-  - group: ":full_moon: LingBot-Video MI300 Nightly"
-    steps:
-      - label: "LingBot-Video Dense CFG-off Smoke"
-        agent_pool: mi300_1
-        mirror_hardwares: [amdproduction]
-        grade: NonBlocking
-        timeout_in_minutes: 90
-        artifact_paths:
-          - artifacts/rocm-lingbot-nightly/**/*
-        commands:
-          - pytest --collect-only -q tests/e2e/online_serving/test_lingbot_video_expansion.py::test_cfg_off -m "full_model and diffusion and rocm and MI325 and cards_1" --run-level full_model
-          - pytest -s -v tests/e2e/online_serving/test_lingbot_video_expansion.py::test_cfg_off -m "full_model and diffusion and rocm and MI325 and cards_1" --run-level full_model --junitxml=/tmp/lingbot-pytest.xml
-          - |
-            artifact_dir="$${BUILDKITE_BUILD_CHECKOUT_PATH:?}/artifacts/rocm-lingbot-nightly"
-            mkdir -p "$$artifact_dir"
-            cp -v /tmp/lingbot-pytest.xml "$$artifact_dir/"
-
-  - group: ":full_moon: Cosmos3 MI300 Nightly"
-    steps:
-      - label: "Cosmos3 T2I Function"
-        agent_pool: mi300_1
-        mirror_hardwares: [amdproduction]
-        grade: NonBlocking
-        timeout_in_minutes: 90
-        artifact_paths:
-          - artifacts/rocm-cosmos3-nightly/**/*
-        commands:
-          # Cosmos3 uses both masked and unmasked BF16 AITER MHA modules. Build
-          # them before server startup, then reuse the same job-local cache
-          # during the smoke test.
-          - export VLLM_ROCM_USE_AITER=1
-          - export AITER_JIT_DIR="/tmp/vllm-omni-cosmos3-aiter-$$BUILDKITE_JOB_ID"
-          - export TORCH_EXTENSIONS_DIR="/tmp/vllm-omni-cosmos3-torch-extensions-$$BUILDKITE_JOB_ID"
-          - mkdir -p "$$AITER_JIT_DIR" "$$TORCH_EXTENSIONS_DIR"
-          - pytest --collect-only -q tests/e2e/online_serving/test_cosmos3.py::test_text_to_image_001 -m "core_model and rocm and MI325 and cards_1" --run-level full_model
-          - >-
-            timeout --signal=TERM --kill-after=2m 55m
-            python .buildkite/amd/scripts/prewarm_cosmos3_aiter.py
-          - >-
-            timeout --signal=TERM --kill-after=2m 25m
-            pytest -s -v
-            tests/e2e/online_serving/test_cosmos3.py::test_text_to_image_001
-            -m "core_model and rocm and MI325 and cards_1"
-            --run-level full_model
-            --junitxml=/tmp/cosmos3-pytest.xml
-          - |
-            artifact_dir="$${BUILDKITE_BUILD_CHECKOUT_PATH:?}/artifacts/rocm-cosmos3-nightly"
-            mkdir -p "$$artifact_dir"
-            cp -v /tmp/cosmos3-pytest.xml "$$artifact_dir/"
-
-  - group: ":full_moon: HunyuanImage3 MI300 Nightly"
-    steps:
-      - label: "HunyuanImage3 Offline Pixel Accuracy"
-        agent_pool: mi300_4
-        mirror_hardwares: [amdproduction]
-        grade: NonBlocking
-        timeout_in_minutes: 180
-        env:
-          HUNYUAN_IMAGE3_MODEL: "tencent/HunyuanImage-3.0-Instruct"
-          HUNYUAN_IMAGE3_DEVICES: "0,1,2,3"
-          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
-        artifact_paths:
-          - artifacts/rocm-hunyuanimage3/**/*
-        commands:
-          - pytest --collect-only -q tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py::test_hunyuan_image3_pixel_accuracy_offline -m "full_model and rocm and MI325 and cards_4" --run-level full_model
-          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py::test_hunyuan_image3_pixel_accuracy_offline -m "full_model and rocm and MI325 and cards_4" --run-level full_model --junitxml=/tmp/hunyuanimage3-pytest.xml
-          - |
-            artifact_dir="$${BUILDKITE_BUILD_CHECKOUT_PATH:?}/artifacts/rocm-hunyuanimage3"
-            mkdir -p "$$artifact_dir"
-            cp -v tests/e2e/accuracy/artifacts/HunyuanImage-3_0-Instruct/*.png "$$artifact_dir/"
-            cp -v /tmp/hunyuanimage3-pytest.xml "$$artifact_dir/"

.buildkite/amd/test-amd-ready.yml

@@ -37,22 +37,24 @@ steps:
       mirror_hardwares: [amdproduction]
       grade: Blocking
       timeout_in_minutes: 45
-      # Buildkite's status-based rule cannot prove every exit 134 is a GPU hang.
-      # It retries 134 once; the Kubernetes job retry starts a fresh pod.
-      # Assertions (1), timeouts (124), and OOMs (137) remain non-retryable.
-      retry:
-        automatic:
-          - exit_status: 134
-            limit: 1
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
         # Keep multi-GPU tests out of this one-GPU pool even when their module
         # also carries the cpu marker for unit tests.
-        # Keep π0.5 CPU units out of these shards: CUDA READY/MERGE already run
-        # the same suite mandatorily, and AMD covers it non-blocking in nightly.
         - "pytest -sv tests/diffusion -m 'core_model and cpu and not (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)' --ignore=tests/diffusion/cache/test_teacache_extractors.py --ignore=tests/diffusion/models/pi05/test_pi05_units.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+    # This module takes about 22 minutes on MI300 and was consistently assigned to shard 4,
+    # pushing that shard to the 45-minute timeout. Keep its module-scoped fixture in one lane.
+    - label: "Simple · Pi0.5 CPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 45
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/diffusion/models/pi05/test_pi05_units.py -m 'core_model and cpu'"
 
     - label: "Diffusion · AuK VAE GPU Test"
       agent_pool: mi300_1
@@ -126,34 +128,6 @@ steps:
           tests/diffusion/attention/
           -m 'core_model and rocm and MI325 and cards_1'
           --run-level "core_model"
-        # MammothModa2 exercises the same two AITER MHA modules built by the
-        # preceding suite. Keep its default-backend run in this job so a cold
-        # worker compiles each module once instead of timing out the generic
-        # model job.
-        - >-
-          timeout --signal=TERM --kill-after=2m 10m
-          pytest -s -v -n 1
-          tests/diffusion/models/mammoth_moda2/test_dit_attention_cuda.py
-          --run-level "core_model"
-
-    # AR paged attention is real GPU work, including cold AITER compilation.
-    # Keep its complete numerical matrix out of the CPU-marked Simple shards.
-    - label: "ROCm · AR Diffusion Paged Attention GPU Test"
-      agent_pool: mi300_1
-      depends_on: amd-build
-      mirror_hardwares: [amdproduction]
-      grade: Blocking
-      timeout_in_minutes: 20
-      commands:
-        - export VLLM_ROCM_USE_AITER=1
-        - export VLLM_OMNI_AR_FA_REQUIRED=1
-        - export AITER_JIT_DIR="/tmp/vllm-omni-ar-aiter-$$BUILDKITE_JOB_ID"
-        - export TORCH_EXTENSIONS_DIR="/tmp/vllm-omni-ar-torch-extensions-$$BUILDKITE_JOB_ID"
-        - mkdir -p "$$AITER_JIT_DIR" "$$TORCH_EXTENSIONS_DIR"
-        - >-
-          timeout --signal=TERM --kill-after=2m 15m
-          pytest -sv tests/diffusion/ar_diffusion/test_paged_attention.py
-          -m 'core_model and rocm and MI325 and cards_1' --run-level core_model
 
     - label: "Diffusion · Batch Test"
       agent_pool: mi300_1
@@ -223,17 +197,9 @@ steps:
         - >-
           timeout 15m pytest -sv tests/diffusion/models/
           --ignore=tests/diffusion/models/auk/test_auk_vae_fastpath.py
-          --ignore=tests/diffusion/models/mammoth_moda2/test_dit_attention_cuda.py
           -m 'core_model and cuda and not
           (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
           --run-level "core_model"
-        # Keep the model-specific numerical coverage blocking without paying
-        # the default ROCm backend's cold AITER compilation cost in this lane.
-        - >-
-          timeout 10m env DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA
-          pytest -sv
-          tests/diffusion/models/mammoth_moda2/test_dit_attention_cuda.py
-          --run-level "core_model"
 
     - label: "Diffusion · Wan22 Test"
       agent_pool: mi300_1
@@ -260,7 +226,7 @@ steps:
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
 
-- label: "ROCm · Model Executor GPU Coverage (R2-01)"
+- label: "ROCm · Engine & Model Executor GPU Coverage (R2-01)"
   agent_pool: mi300_1
   depends_on: amd-build
   mirror_hardwares: [amdproduction]
@@ -273,17 +239,18 @@ steps:
     # artifacts relative to its checkout directory.
     - export R2_01_ARTIFACT_DIR="$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/rocm-r2-01"
     - mkdir -p "$$R2_01_ARTIFACT_DIR"
-    # The dedicated Engine Test below owns the only selected tests/engine case;
-    # do not duplicate its model startup in this time-bounded coverage lane.
     # The CUDA ready job also covers tests/worker/test_batched_omni_output.py.
-    # Keep the initial ROCm burn-in scoped to tests/model_executor/.
+    # Keep the initial ROCm burn-in scoped to the requested directories.
     # Verbose pytest output and JUnit retain node IDs without a second
     # collection-only pass over the same GPU-backed test tree.
+    # The CUDA/L4-only CosyVoice3 ragged-flow parity test cold-JITs multiple
+    # AITER kernels on ROCm and is covered by the CUDA ready lane.
     - >-
       pytest -s -v -ra
-      tests/model_executor/
-      -m 'core_model and cuda and not nvidia_only and not
+      tests/engine/ tests/model_executor/
+      -m 'core_model and cuda and not
       (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
+      -k 'not test_code2wav_streaming_batch_matches_ragged_flow_numerics'
       --run-level "core_model"
       --durations=0
       --junitxml="$$R2_01_ARTIFACT_DIR/pytest.xml"

.buildkite/common/ci_source_file_dependencies.yml

@@ -16,9 +16,6 @@ x-qwen3-tts: &qwen3_tts
   - vllm_omni/worker/gpu_model_runner.py
   - vllm_omni/model_executor/models/qwen3_tts/
   - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-  - vllm_omni/model_executor/models/common/fused_sampling.py
-  - vllm_omni/model_executor/models/common/short_kv_attention.py
-  - vllm_omni/model_executor/models/common/talker_first_audio.py
   - vllm_omni/model_executor/models/common/snake_activation.py
   - vllm_omni/model_executor/models/common/alias_free_activation.py
   - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
@@ -88,16 +85,11 @@ x-qwen3-omni: &qwen3_omni
   - vllm_omni/entrypoints/openai/serving_chat.py
   - vllm_omni/model_executor/models/qwen3_omni/
   - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-  - vllm_omni/model_executor/models/common/fused_sampling.py
-  - vllm_omni/model_executor/models/common/short_kv_attention.py
-  - vllm_omni/model_executor/models/common/talker_first_audio.py
   - vllm_omni/model_executor/models/common/snake_activation.py
   - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
   - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
   - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
   - vllm_omni/deploy/qwen3_omni_moe.yaml
-  - vllm_omni/deploy/qwen3_omni_moe_mrv2.yaml
-  - vllm_omni/deploy/qwen3_omni_moe_mrv2_h200.yaml
   - vllm_omni/deploy/qwen3_omni_moe_thinking.yaml
 
 x-wan22: &wan22
@@ -115,10 +107,6 @@ x-qwen-image: &qwen_image
   - vllm_omni/diffusion/models/dmd2/
   - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py
 
-x-joy-image: &joy_image
-  - vllm_omni/diffusion/models/joy_image/
-  - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_wan.py
-
 x-hunyuan-image3: &hunyuan_image3
   - vllm_omni/diffusion/models/hunyuan_image3/
   - vllm_omni/model_executor/models/hunyuan_image3/
@@ -153,10 +141,6 @@ x-lingbot-video: &lingbot_video
 x-lingbot-world: &lingbot_world
   - vllm_omni/diffusion/models/lingbot_world/
 
-x-kandinsky6: &kandinsky6
-  - vllm_omni/diffusion/models/kandinsky6/
-  - vllm_omni/model_extras/kandinsky6.py
-
 
 source_file_dependencies:
   # Hardware presets and shared test helpers. Not a job key — do not attach this
@@ -391,12 +375,6 @@ source_file_dependencies:
     - tests/e2e/accuracy/test_qwen_image.py
     - tests/e2e/accuracy/test_diffusers_backend_similarity.py
 
-  diffusion_joy_image_accuracy:
-    - *joy_image
-    - tests/diffusion/models/joy_image/
-    - tests/e2e/accuracy/test_joy_image_edit.py
-    - recipes/JD/JoyAI-Image-Edit.md
-
   diffusion_qwen_image_perf:
     - *qwen_image
     - tests/dfx/perf/scripts/run_diffusion_benchmark.py
@@ -505,11 +483,6 @@ source_file_dependencies:
     - tests/dfx/perf/scripts/run_benchmark.py
     - tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json
 
-  diffusion_kandinsky6_function:
-    - *kandinsky6
-    - tests/e2e/offline_inference/test_kandinsky6.py
-    - tests/e2e/online_serving/test_kandinsky6.py
-
 
   # Invalid-param jobs: model business code only, plus the reliability scripts.
   reliability_invalid_param_h100:

.buildkite/common/scripts/upload_pipeline.py

@@ -102,7 +102,6 @@ CI_SOURCE_FILE_DEPENDENCIES_PATH = ROOT / ".buildkite/common/ci_source_file_depe
 SOURCE_FILTER_FALLBACK_KEY = "source_filter_fallback"
 CUDA_HF_TOKEN_ENV = "VLLM_CI_HF_TOKEN"
 CUDA_HF_TOKEN_EXPORT = f'if [ -n "$${{{CUDA_HF_TOKEN_ENV}:-}}" ]; then export HF_TOKEN="$${{{CUDA_HF_TOKEN_ENV}}}"; fi'
-CUDA_RUNTIME_CHECK = "python3 .buildkite/cuda/scripts/check_vllm_runtime.py"
 
 # Bootstrap Buildkite ``if`` expressions.
 # ``*_MAIN_IF``: main + env schedule. ``*_LABEL_IF``: PR label (and/or composed with MAIN).
@@ -621,11 +620,11 @@ def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any] | None:
     if any(preset_name == chip or preset_name.startswith(f"{chip}_") for chip in _cuda_mirror_chips()):
         commands = merged.get("commands")
         if isinstance(commands, list):
-            merged["commands"] = [CUDA_HF_TOKEN_EXPORT, CUDA_RUNTIME_CHECK, *commands]
+            merged["commands"] = [CUDA_HF_TOKEN_EXPORT, *commands]
         elif commands is None:
-            merged["commands"] = [CUDA_HF_TOKEN_EXPORT, CUDA_RUNTIME_CHECK]
+            merged["commands"] = [CUDA_HF_TOKEN_EXPORT]
         else:
-            merged["commands"] = [CUDA_HF_TOKEN_EXPORT, CUDA_RUNTIME_CHECK, commands]
+            merged["commands"] = [CUDA_HF_TOKEN_EXPORT, commands]
     # Preset retry (K8S_RETRY on l4_*) must not clobber a step that opted out.
     if "retry" in step:
         merged["retry"] = step["retry"]

.buildkite/cuda/scripts/check_vllm_runtime.py

@@ -1,38 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
-"""Verify the pinned vLLM and compiled CUDA imports on a GPU CI worker."""
-
-from __future__ import annotations
-
-import importlib
-import re
-from pathlib import Path
-
-
-def main() -> None:
-    dockerfile = Path(__file__).resolve().parents[3] / "docker" / "Dockerfile.ci"
-    match = re.search(
-        r"^ARG VLLM_VERSION=([0-9]+\.[0-9]+\.[0-9]+)$",
-        dockerfile.read_text(),
-        re.MULTILINE,
-    )
-    if match is None:
-        raise RuntimeError(f"Missing vLLM release version in {dockerfile}")
-    expected_version = match.group(1)
-
-    import torch
-    import vllm
-
-    assert vllm.__version__ == expected_version, vllm.__version__
-    assert torch.cuda.is_available(), "vLLM runtime check requires a GPU worker"
-    for module in ("flashinfer", "vllm._C_stable_libtorch"):
-        importlib.import_module(module)
-    print(
-        f"Verified vLLM CUDA runtime: vllm={vllm.__version__} "
-        f"expected={expected_version} torch={torch.__version__} "
-        f"cuda={torch.version.cuda} device={torch.cuda.get_device_name()}"
-    )
-
-
-if __name__ == "__main__":
-    main()

.buildkite/cuda/test-merge.yml

@@ -38,17 +38,6 @@ steps:
   - group: ":card_index_dividers: Diffusion Test"
     depends_on: upload-merge-pipeline
     steps:
-      - label: "Diffusion · AR Paged Attention GPU Test"
-        timeout_in_minutes: 20
-        commands:
-          - export VLLM_OMNI_AR_FA_REQUIRED=1
-          - >-
-            timeout 15m pytest -sv
-            tests/diffusion/ar_diffusion/test_paged_attention.py
-            -m 'core_model and cuda and L4 and cards_1'
-            --run-level "core_model"
-        mirror_hardwares: l4_1
-
       - label: "Diffusion · LoRA Test"
         timeout_in_minutes: 30
         commands:
@@ -135,30 +124,6 @@ steps:
           - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py -m 'advanced_model and cuda' --run-level 'advanced_model'
         mirror_hardwares: h100_2
 
-      - label: "Omni · Qwen3-Omni Async-Chunk SHM Leak"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen3_omni_shm_leak.py
-          - tests/helpers/runtime.py
-          - tests/helpers/stage_config.py
-          - vllm_omni/core/sched/omni_ar_scheduler.py
-          - vllm_omni/distributed/omni_connectors/connectors/shm_connector.py
-          - vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py
-          - vllm_omni/engine/orchestrator.py
-          - vllm_omni/engine/stage_engine_core_proc.py
-          - vllm_omni/engine/stage_pool.py
-          - vllm_omni/entrypoints/openai/serving_chat.py
-          - vllm_omni/model_executor/models/qwen3_omni/
-          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-          - vllm_omni/model_executor/models/common/snake_activation.py
-          - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
-          - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
-          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
-          - vllm_omni/deploy/qwen3_omni_moe.yaml
-        timeout_in_minutes: 40
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_shm_leak.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        mirror_hardwares: h100_2
-
       - label: "Omni · MiniCPM-o 4.5 Test"
         source_file_dependencies: omni_minicpmo_4_5_function
         timeout_in_minutes: 50
@@ -189,15 +154,6 @@ steps:
             '
         mirror_hardwares: h100_1
 
-      - label: "Diffusion · Kandinsky6 Test"
-        source_file_dependencies: diffusion_kandinsky6_function
-        commands:
-          - |
-            timeout 40m bash -c "
-              pytest -s -v tests/e2e/offline_inference/test_kandinsky6.py tests/e2e/online_serving/test_kandinsky6.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
-            "
-        mirror_hardwares: h100_1
-
       - label: "Diffusion · Wan22 Test"
         source_file_dependencies: diffusion_wan22_function
         commands:

.buildkite/cuda/test-nightly.yml

@@ -48,8 +48,6 @@ steps:
       - label: ":full_moon: Omni · Doc Test with H100 · 2-GPU"
         source_file_dependencies: omni_qwen3_omni_doc
         timeout_in_minutes: 90
-        artifact_paths:
-          - audio_*.wav
         commands:
           - pytest -s -v tests/examples/ -m "full_model and omni and H100 and B200 and cards_2" --run-level "full_model"
 
@@ -158,11 +156,6 @@ steps:
       - label: ":full_moon: TTS · Function Test with L4"
         source_file_dependencies: tts_qwen3_tts_function
         timeout_in_minutes: 120
-        env:
-          VLLM_OMNI_FAILED_SPEECH_AUDIO_DIR: .
-        artifact_paths:
-          - failed_speech_*.wav
-          - failed_speech_*.json
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
           - export VLLM_USE_DEEP_GEMM="0"
@@ -268,12 +261,6 @@ steps:
           - pytest -s -v tests/e2e/accuracy/test_qwen_image.py -m "H100 and B200 and cards_1" --run-level full_model
           - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m "full_model and H100 and B200 and cards_1" --run-level full_model
 
-      - label: ":full_moon: JoyAI-Image-Edit · Accuracy Test"
-        source_file_dependencies: diffusion_joy_image_accuracy
-        timeout_in_minutes: 180
-        commands:
-          - pytest -s -v tests/e2e/accuracy/test_joy_image_edit.py -m "full_model and diffusion and H100 and B200 and cards_1" --run-level full_model
-
       - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
         source_file_dependencies: diffusion_hunyuan_image3_accuracy
         timeout_in_minutes: 180

.buildkite/cuda/test-ready.yml

@@ -25,16 +25,6 @@ steps:
           - "pytest -sv tests/model_executor/models/cosyvoice3/test_cosyvoice3_full_batch.py -k packed -m 'core_model and cuda' --run-level=core_model"
         mirror_hardwares: h100_1
 
-      - label: "Simple · MRV2 FA3 Graph Bounds Test"
-        timeout_in_minutes: 15
-        source_file_dependencies:
-          - tests/worker_v2/test_capture_query_bound_gpu.py
-          - tests/worker_v2/test_decode_query_bound_gpu.py
-          - vllm_omni/worker_v2/model_states/omni_model_state.py
-        commands:
-          - "pytest -sv tests/worker_v2/test_capture_query_bound_gpu.py tests/worker_v2/test_decode_query_bound_gpu.py -m 'core_model and cuda and H100' --run-level=core_model"
-        mirror_hardwares: h100_1
-
       - label: "Simple · Diffusion Test"
         timeout_in_minutes: 30
         commands:
@@ -105,7 +95,7 @@ steps:
 
       - label: "Diffusion · Other Test"
         commands:
-          - timeout 15m pytest -sv tests/diffusion/*.py tests/diffusion/ar_diffusion tests/diffusion/layers/test_adalayernorm_*.py -m 'core_model and cuda' --run-level "core_model"
+          - timeout 15m pytest -sv tests/diffusion/*.py tests/diffusion/ar_diffusion -m 'core_model and cuda' --run-level "core_model"
         mirror_hardwares: l4_1
 
   - group: ":card_index_dividers: Tiny Model Tests [Single GPU]"
@@ -122,7 +112,7 @@ steps:
     commands:
       - |
         timeout 15m bash -c "
-                  pytest -sv tests/engine/ tests/model_executor/ tests/worker/test_batched_omni_output.py tests/worker/test_gpu_generation_async_output.py tests/worker_v2/ -m 'core_model and cuda' --run-level 'core_model'
+                  pytest -sv tests/engine/ tests/model_executor/ tests/worker/test_batched_omni_output.py tests/worker_v2/ -m 'core_model and cuda' --run-level 'core_model'
         "
     mirror_hardwares: l4_1
 

.buildkite/intel/pipeline-intel.yml

@@ -10,7 +10,7 @@ steps:
           DOCKER_BUILDKIT: "1"
           # Buildkite will automatically replace this with the actual commit hash
           VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.31.0"
+          VLLM_VERSION: "v0.30.0"
         priority: 100
         timeout_in_minutes: 60
         soft_fail: false

.claude/skills/review-pr/SKILL.md

@@ -9,6 +9,15 @@ Review like a maintainer: direct, selective, and focused on issues that CI does
 not prove. Prefer a few high-confidence findings over exhaustive commentary.
 Zero findings is a valid result.
 
+## Target-version evidence
+
+Resolve Python/dependency compatibility, pytest selectors and hardware needs
+from the frozen head's metadata, requirements, CI, and fixtures. Run levels such
+as `core_model` do not imply CPU-only execution. Read the target's accepted
+module contracts; draft or deferred architecture is not a merge requirement.
+Recheck only affected assumptions when reviewing another release. Historical
+coverage does not establish compatibility with a new snapshot.
+
 ## Quality contract
 
 Make every finding:
@@ -103,6 +112,12 @@ If docs and live code disagree, verify the code/tests and report the drift.
 | [examples-policy.md](../precheck-pr/references/examples-policy.md) | The PR adds, copies, or renames Python under `examples/`; apply the canonical policy shared with `precheck-pr`. |
 | [find-simplifications](../find-simplifications/SKILL.md) | Every review; run a diff-scoped subtraction and simplification pass after correctness blockers. |
 
+### Batch sessions and review helpers
+
+For a requested batch session, unanswered review replies, or inline-coordinate
+validation, read [batch-review.md](references/process/batch-review.md). The
+repository owns the helper scripts and their offline regression tests.
+
 ### Delivery and reviewer coordination
 
 | Reference | Read when |
@@ -110,6 +125,15 @@ If docs and live code disagree, verify the code/tests and report the drift.
 | [maintainer-style-study.md](references/delivery/maintainer-style-study.md) | Findings are ready for concise maintainer-style delivery. |
 | [review-requests.md](references/delivery/review-requests.md) | The user asks to identify, suggest, request, or ping code-owner reviewers. |
 
+## Optional review workers
+
+For independent defect discovery on a large diff, or when the user selects
+`auto`, `codex`, `claude`, or `none`, read
+[reviewer-backends.md](references/process/reviewer-backends.md). Pass immutable
+base/head SHAs and a bounded read-only scope. The parent owns domain routing,
+verification, deduplication, severity, and any separately authorized posting.
+Missing optional tooling falls back to direct review without blocking it.
+
 ## Workflow
 
 ### 1. Freeze and report the snapshot

.claude/skills/review-pr/references/checks/test-quality-evaluation.md

@@ -20,11 +20,15 @@ were reverted or broken. Check:
   tolerances are deterministic or explicitly controlled;
 - normal, invalid, boundary, feature-off, failure/cancellation, and regression
   paths are covered where the diff changes them;
-- run-level and domain markers place hardware/model tests in the intended CI
-  lane without silently skipping the contract.
+- read the target's pytest configuration, collection hooks, hardware helpers,
+  and CI together: run-level/domain/backend/SKU/card-count selectors must place
+  tests in the intended lane. `core_model` alone is not CPU-only; use explicit
+  hardware selection such as `core_model and cpu` only when supported by the
+  target. Zero selected or skipped accelerator tests are not validation.
 
-Map source symbols to tests with bounded `rg` searches; do not assume the test
-tree mirrors production paths.
+Map source symbols to tests with bounded `rg` searches and inspect the target's
+test-writing guide. Components may mirror source while model E2E and feature
+suites live elsewhere; imports, fixtures, and assertions establish coverage.
 
 ## Runtime check
 

.claude/skills/review-pr/references/process/batch-review.md

@@ -0,0 +1,50 @@
+# Requested batch review and helper tools
+
+Use only for a requested batch session, review follow-up, or proposed inline
+comments. Commands below run from this skill directory, require `gh`, `jq`,
+Python, and Bash, and only read GitHub. Resolve the connected reviewer's login;
+do not infer it from examples or repository ownership.
+
+This is generic candidate discovery, not a daily eligibility policy. If the
+user's session specifies eligibility labels, a changed head since prior review,
+a daily cap, or another selection rule, build that eligible set from current
+metadata first. Do not let the legacy selector's own-PR/already-reviewed filters
+silently override it: select directly when those filters would discard an
+explicitly eligible re-review. User-selected PRs do not need to pass the helper.
+
+1. Check outstanding replies first when the session includes follow-up:
+   `bash scripts/check_replies.sh --reviewer <login> --days 14`.
+   Reopen each returned thread and inspect its current resolution before
+   deciding whether a response is needed. A reply can already be addressed by
+   code or another maintainer; do not send an automatic acknowledgement.
+2. Select candidates with
+   `bash scripts/select_prs.sh --reviewer <login> --days 7 --limit 5`.
+   This preserves the existing selector: open recent PRs, excluding own,
+   draft/WIP/"don't merge", and already-reviewed PRs, with low-review-count
+   candidates first. It does not filter ordinary docs/CI PRs or prove
+   mergeability, CI success, or complete search coverage.
+3. Freeze and review each selected PR using the normal execution contract.
+   Keep CI conclusions separate from conflicts/mergeability. Match depth to
+   changed behavior, not candidate rank; no quota requires manufacturing
+   findings. Search/review APIs can be limited or fail; inspect raw status when
+   coverage matters rather than treating a zero count as a verified clean state.
+4. Before separately authorized inline posting, validate the proposed payload:
+   `printf '%s' "$REVIEW_JSON" | bash scripts/verify_line_numbers.sh <PR>`.
+   Input is an object with a `comments` array containing `path`, `line`,
+   optional `side` (default `RIGHT`), and optional same-side `start_line`.
+   It checks actual diff hunks, including deletion/rename paths. It does not
+   verify the finding or submit it. Re-read base/head before and after its
+   live diff fetch; require equality with the reviewed snapshot and use that
+   head as `commit_id` for any authorized post. Otherwise refresh the review
+   and line mapping first.
+5. Preserve a short session record of pinned heads, findings, validation gaps,
+   and already-posted comment links so a repeat pass reviews only the delta.
+
+Run helper regression tests without project dependencies or GitHub access:
+
+```bash
+python tests/test_helpers.py
+```
+
+The helpers are bounded triage aids, not a scheduler. They do not authorize
+follow-up comments, review events, code changes, or recurring monitoring.

.claude/skills/review-pr/references/process/reviewer-backends.md

@@ -0,0 +1,80 @@
+# Reviewer Backends
+
+Use this reference when delegating defect discovery to Codex, Claude, or another reviewer. The parent `review-pr` workflow remains responsible for gates, domain routing, verification, prioritization, and any GitHub writes.
+
+## Backend Selection
+
+| Choice | Behavior |
+|--------|----------|
+| `auto` | Use one available reviewer backend; fall back to direct review |
+| `codex` | Use an available Codex review agent or skill |
+| `claude` | Use an available Claude review agent or CLI |
+| `none` | Review directly without delegation |
+
+Honor an explicit user choice. Do not block when the requested backend is unavailable; report the fallback locally and continue directly unless the user required that backend specifically.
+
+Use one backend for ordinary code changes. Use at most two reviewers for large or high-risk changes, either with non-overlapping path scopes or as independent passes. Do not run multiple reviewers merely to vote on the same conclusion.
+
+## Common Read-Only Handoff
+
+Give every backend the same concrete target:
+
+```yaml
+repository: /absolute/path/to/vllm-omni
+base_ref: origin/main
+base_sha: <resolved-base-sha>
+head_sha: <captured-pr-head-sha>
+merge_base_sha: <resolved-merge-base-sha>
+scope: complete-diff | path-partition
+paths: []
+focus: []
+constraints:
+  read_only: true
+  no_github_writes: true
+  no_delegation: true
+output:
+  finding: "[P0-P3] Imperative title — path:line"
+  exhaustive: true
+  include_assessment: true
+  include_test_gaps: true
+  include_residual_risks: true
+```
+
+Capture immutable SHAs before delegation. Ask the reviewer to:
+
+1. Read the applicable `AGENTS.md`.
+2. Inspect the complete assigned diff and enough surrounding code to understand it.
+3. Check relevant tests and call sites.
+4. Report only concrete regressions introduced by the change.
+5. Cite the smallest changed-line range that demonstrates each finding.
+6. Continue through the full assigned scope after finding the first issue.
+
+The worker must not modify files, create commits, push branches, post comments, submit a review event, or delegate again.
+
+## Backend Adapters
+
+### Codex
+
+Use an available Codex review agent or review skill; do not assume that a named system skill is installed. Give it the common handoff plus selected vLLM-Omni focus areas. Use it for a complete change or bounded path partition, not for a one-function context lookup.
+
+### Claude
+
+Pass the same handoff and output format to the available Claude review surface. Do not assume a particular command, model, or installation path. Explicitly include the read-only and no-GitHub-write constraints in the prompt.
+
+### Direct fallback
+
+If no backend is available, the parent performs the same checks. Keep the same evidence threshold and comment budget. Do not mention missing tooling in the GitHub review unless the user asks.
+
+## Aggregate Results
+
+The parent reviewer must:
+
+1. Re-read the PR head SHA. If it changed, discard stale line mappings and rerun affected scopes.
+2. Verify each finding against code, tests, and the captured diff.
+3. Drop speculative, pre-existing, intentional, or style-only findings.
+4. Deduplicate by root cause and affected call path rather than by line alone.
+5. Classify `blocking` independently from `P0`-`P3`; priority alone does not determine the verdict.
+6. Apply the vLLM-Omni comment budget after collecting the complete result.
+7. Preserve material omitted findings, test gaps, and residual risks in the local report.
+
+Only the parent may turn selected findings into GitHub comments, and only with user authorization.

.claude/skills/review-pr/scripts/check_replies.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+# check_replies.sh — Find unanswered contributor replies to your review comments.
+#
+# Usage: ./check_replies.sh --reviewer lishunyang12 [--days 14]
+#
+# Scans PRs you've reviewed for threads where a contributor replied
+# but you haven't responded yet. Outputs actionable list.
+
+set -euo pipefail
+
+REPO="vllm-project/vllm-omni"
+REVIEWER=""
+DAYS=14
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --reviewer) REVIEWER="$2"; shift 2 ;;
+    --days)     DAYS="$2"; shift 2 ;;
+    *) echo "Unknown arg: $1" >&2; exit 1 ;;
+  esac
+done
+
+if [ -z "$REVIEWER" ]; then
+  echo "Usage: $0 --reviewer <github_login>" >&2
+  exit 1
+fi
+
+SINCE=$(date -d "-${DAYS} days" +%Y-%m-%d 2>/dev/null || date -v-${DAYS}d +%Y-%m-%d)
+
+echo "Scanning for unanswered replies to ${REVIEWER} (last ${DAYS} days)..." >&2
+
+# Find PRs the reviewer has commented on recently
+REVIEWED_PRS=$(gh api "search/issues?q=repo:${REPO}+reviewed-by:${REVIEWER}+is:pr+is:open+updated:>=${SINCE}&per_page=50" \
+  --jq '.items[].number' 2>/dev/null)
+
+UNANSWERED=0
+
+for PR_NUM in $REVIEWED_PRS; do
+  # Get all review comments on this PR
+  COMMENTS=$(gh api "repos/${REPO}/pulls/${PR_NUM}/comments" \
+    --jq '[.[] | {id, user: .user.login, body: .body[:100], in_reply_to_id, created_at, path, url: .html_url}]' 2>/dev/null)
+
+  # Find threads: reviewer's comments that got a reply, but reviewer didn't reply back
+  # 1. Get reviewer's comment IDs
+  REVIEWER_IDS=$(echo "$COMMENTS" | jq -r --arg r "$REVIEWER" '[.[] | select(.user == $r) | .id]')
+
+  # 2. Find replies to reviewer's comments (from others)
+  REPLIES=$(echo "$COMMENTS" | jq -c --arg r "$REVIEWER" --argjson rids "$REVIEWER_IDS" '
+    [.[] |
+      select(.user != $r) |
+      select(.in_reply_to_id != null) |
+      select([.in_reply_to_id] | inside($rids))
+    ]
+  ')
+
+  REPLY_COUNT=$(echo "$REPLIES" | jq 'length')
+  if [ "$REPLY_COUNT" -eq 0 ]; then
+    continue
+  fi
+
+  # 3. For each reply, check if reviewer replied after it
+  while IFS= read -r reply; do
+    THREAD_ID=$(echo "$reply" | jq -r '.in_reply_to_id')
+    REPLY_DATE=$(echo "$reply" | jq -r '.created_at')
+    REPLY_USER=$(echo "$reply" | jq -r '.user')
+    REPLY_URL=$(echo "$reply" | jq -r '.url')
+
+    # Check if reviewer replied after this reply
+    FOLLOWUP=$(echo "$COMMENTS" | jq --arg r "$REVIEWER" --arg date "$REPLY_DATE" --argjson tid "$THREAD_ID" '
+      [.[] | select(.user == $r) | select(.in_reply_to_id == $tid) | select(.created_at > $date)] | length
+    ')
+
+    if [ "$FOLLOWUP" -eq 0 ]; then
+      REPLY_BODY=$(echo "$reply" | jq -r '.body')
+      echo "PR #${PR_NUM} | @${REPLY_USER} replied | ${REPLY_URL}"
+      echo "  > ${REPLY_BODY}"
+      echo ""
+      UNANSWERED=$((UNANSWERED + 1))
+    fi
+  done < <(echo "$REPLIES" | jq -c '.[]')
+done
+
+if [ "$UNANSWERED" -eq 0 ]; then
+  echo "No unanswered replies found." >&2
+fi

.claude/skills/review-pr/scripts/select_prs.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+# select_prs.sh — Pick reviewable PRs from vllm-omni for a review session.
+#
+# Usage: ./select_prs.sh [--days 7] [--limit 5] [--reviewer lishunyang12]
+#
+# Filters:
+#   - Excludes your own PRs (--reviewer)
+#   - Excludes WIP / Draft / "Don't merge" PRs
+#   - Excludes pure docs/CI-only merged PRs
+#   - Excludes PRs you already reviewed
+#   - Prioritizes zero-review PRs
+#
+# Output: JSON array of {number, title, author, createdAt, reviewCount}
+
+set -euo pipefail
+
+REPO="vllm-project/vllm-omni"
+DAYS=7
+LIMIT=5
+REVIEWER=""
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --days)    DAYS="$2"; shift 2 ;;
+    --limit)   LIMIT="$2"; shift 2 ;;
+    --reviewer) REVIEWER="$2"; shift 2 ;;
+    *) echo "Unknown arg: $1" >&2; exit 1 ;;
+  esac
+done
+
+SINCE=$(date -d "-${DAYS} days" +%Y-%m-%d 2>/dev/null || date -v-${DAYS}d +%Y-%m-%d)
+
+echo "Fetching open PRs from last ${DAYS} days..." >&2
+
+# Fetch all open PRs from the time range
+ALL_PRS=$(gh pr list --repo "$REPO" --state open --limit 200 \
+  --json number,title,author,createdAt,isDraft \
+  --search "created:>=${SINCE}")
+
+# Filter in a single jq pass
+FILTERED=$(echo "$ALL_PRS" | jq --arg reviewer "$REVIEWER" '
+  [.[] |
+    # Skip own PRs
+    select(.author.login != $reviewer) |
+    # Skip drafts
+    select(.isDraft == false) |
+    # Skip WIP / Dont merge
+    select(.title | test("\\[WIP\\]|\\[Draft\\]|Don.t[ ]+merge|\\[Don.t\\]"; "i") | not)
+  ]
+')
+
+# For each remaining PR, check review count and whether reviewer already reviewed
+echo "Checking review status for $(echo "$FILTERED" | jq length) PRs..." >&2
+
+echo "$FILTERED" | jq -c '.[]' | while read -r pr; do
+  NUM=$(echo "$pr" | jq -r '.number')
+
+  # Get review count
+  REVIEW_COUNT=$(gh api "repos/${REPO}/pulls/${NUM}/reviews" --jq 'length' 2>/dev/null || echo "0")
+
+  # Check if reviewer already reviewed
+  if [ -n "$REVIEWER" ]; then
+    ALREADY=$(gh api "repos/${REPO}/pulls/${NUM}/reviews" \
+      --jq "[.[] | select(.user.login==\"${REVIEWER}\")] | length" 2>/dev/null || echo "0")
+    if [ "$ALREADY" -gt 0 ]; then
+      continue
+    fi
+  fi
+
+  echo "$pr" | jq --argjson rc "$REVIEW_COUNT" '. + {reviewCount: $rc}'
+done | jq -s "sort_by(.reviewCount) | .[:${LIMIT}]"

.claude/skills/review-pr/scripts/verify_line_numbers.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Check review comment coordinates against the actual lines of a PR diff."""
+
+import argparse
+import ast
+import json
+import subprocess
+import sys
+
+
+def parse_hunk(text):
+    """Parse a unified-diff hunk header without a runtime dependency."""
+    parts = text.split(" ", 4)
+    if len(parts) < 4 or parts[0] != "@@" or not parts[3].startswith("@@"):
+        return None
+    bounds = []
+    for value, sign in zip(parts[1:3], ("-", "+")):
+        if not value.startswith(sign):
+            return None
+        numbers = value[1:].split(",")
+        if len(numbers) not in (1, 2) or not all(number.isdecimal() for number in numbers):
+            return None
+        bounds.extend((numbers[0], numbers[1] if len(numbers) == 2 else None))
+    return tuple(bounds)
+
+
+def diff_path(value):
+    """Decode a git path, including C-quoted UTF-8 bytes."""
+    if value.startswith('"'):
+        value = ast.literal_eval(value).encode("latin1").decode("utf-8")
+    return None if value == "/dev/null" else value[2:]
+
+
+def diff_lines(diff):
+    """Map each file/side to commentable lines, preserving rename/delete paths."""
+    lines = {}
+    old_path = new_path = None
+    old_remaining = new_remaining = 0
+    for text in diff.splitlines():
+        if text.startswith("diff --git "):
+            old_path = new_path = None
+            old_remaining = new_remaining = 0
+        elif hunk := parse_hunk(text):
+            old_line, old_count, new_line, new_count = hunk
+            old_line, new_line = int(old_line), int(new_line)
+            old_remaining = int(old_count) if old_count is not None else 1
+            new_remaining = int(new_count) if new_count is not None else 1
+        elif old_remaining or new_remaining:
+            # A hunk's content can itself start with --- or +++.
+            path = new_path or old_path
+            if text.startswith((" ", "-")):
+                lines.setdefault((path, "LEFT"), set()).add(old_line)
+                old_line += 1
+                old_remaining -= 1
+            if text.startswith((" ", "+")):
+                lines.setdefault((path, "RIGHT"), set()).add(new_line)
+                new_line += 1
+                new_remaining -= 1
+        elif text.startswith("--- "):
+            old_path = diff_path(text[4:])
+        elif text.startswith("+++ "):
+            new_path = diff_path(text[4:])
+    return lines
+
+
+def validate_comments(comments, lines):
+    """Validate single-line and same-side multiline review comments."""
+    valid = True
+    for comment in comments:
+        path = comment["path"]
+        side = comment.get("side", "RIGHT")
+        line = comment["line"]
+        start = comment.get("start_line", line)
+        start_side = comment.get("start_side", side)
+        available = lines.get((path, side), set())
+        if (
+            type(line) is not int
+            or type(start) is not int
+            or start < 1
+            or line < start
+            or start_side != side
+            or line - start + 1 > len(available)
+            or not all(number in available for number in range(start, line + 1))
+        ):
+            print(
+                f"ERROR: {path}:{start}-{line} ({side}) is outside diff lines",
+                file=sys.stderr,
+            )
+            valid = False
+        else:
+            print(f"OK: {path}:{line} ({side})", file=sys.stderr)
+    return valid
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("pr_number", type=int)
+    args = parser.parse_args()
+    try:
+        review = json.load(sys.stdin)
+        comments = review["comments"]
+        if not isinstance(comments, list):
+            raise TypeError("comments must be an array")
+        result = subprocess.run(
+            [
+                "gh",
+                "pr",
+                "diff",
+                str(args.pr_number),
+                "--repo",
+                "vllm-project/vllm-omni",
+            ],
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+        return 0 if validate_comments(comments, diff_lines(result.stdout)) else 1
+    except subprocess.CalledProcessError as error:
+        print(error.stderr, file=sys.stderr, end="")
+        return 1
+    except (OSError, ValueError, KeyError, TypeError, SyntaxError) as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.claude/skills/review-pr/scripts/verify_line_numbers.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+# Usage: echo '<review_json>' | ./verify_line_numbers.sh <pr_number>
+# Requires Python 3 and gh. Exits nonzero for invalid comments or fetch failures.
+set -euo pipefail
+
+exec python3 "$(dirname "${BASH_SOURCE[0]}")/verify_line_numbers.py" "$@"

.claude/skills/review-pr/tests/test_helpers.py

@@ -0,0 +1,227 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Exercise repository review helpers without network access."""
+
+import json
+import os
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+REVIEW = ROOT / "scripts"
+
+
+class HelperTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temporary.cleanup)
+        self.root = Path(self.temporary.name)
+        self.bin = self.root / "bin"
+        self.bin.mkdir()
+        gh = self.bin / "gh"
+        gh.write_text("""#!/usr/bin/env python3
+import json, os, sys
+from pathlib import Path
+args = sys.argv[1:]
+root = Path(os.environ['FIXTURES'])
+if os.environ.get('FAIL_GH'):
+    print('GitHub unavailable', file=sys.stderr)
+    raise SystemExit(1)
+if args[:2] == ['pr', 'list']:
+    print((root / 'prs.json').read_text())
+elif args[:2] == ['pr', 'diff']:
+    print((root / 'diff').read_text(), end='')
+elif args[0] == 'api' and args[1].startswith('search/issues?'):
+    print('1')
+elif args[0] == 'api' and args[1].endswith('/comments'):
+    print((root / 'comments.json').read_text())
+elif args[0] == 'api' and args[1].endswith('/reviews'):
+    print('0')
+else:
+    raise SystemExit('Unexpected gh arguments: ' + repr(args))
+""")
+        gh.chmod(0o755)
+        self.env = dict(
+            os.environ,
+            PATH=f"{self.bin}{os.pathsep}{os.environ['PATH']}",
+            FIXTURES=str(self.root),
+            PYTHONDONTWRITEBYTECODE="1",
+        )
+
+    def run_script(self, script, *args, data=None):
+        return subprocess.run(
+            ["bash", str(script), *args],
+            input=data,
+            env=self.env,
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+
+    def test_select_filters_drafts_wip_and_own_prs(self):
+        prs = [
+            {"number": i, "title": title, "author": {"login": author}, "isDraft": draft}
+            for i, (title, author, draft) in enumerate(
+                [
+                    ("Fix bug", "author", False),
+                    ("[WIP] Feature", "author", False),
+                    ("[Draft] Feature", "author", False),
+                    ("Don't merge yet", "author", False),
+                    ("Feature", "reviewer", False),
+                    ("Feature", "author", True),
+                ],
+                1,
+            )
+        ]
+        (self.root / "prs.json").write_text(json.dumps(prs))
+        result = self.run_script(REVIEW / "select_prs.sh", "--reviewer", "reviewer")
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual([pr["number"] for pr in json.loads(result.stdout)], [1])
+
+    def verify(self, comments, diff=None):
+        (self.root / "diff").write_text(
+            diff
+            or (
+                "diff --git a/src/example.py b/src/example.py\n"
+                "--- a/src/example.py\n+++ b/src/example.py\n"
+                "@@ -10,2 +20,2 @@\n-old\n+new\n context\n"
+            )
+        )
+        return self.run_script(
+            REVIEW / "verify_line_numbers.sh",
+            "1",
+            data=json.dumps({"comments": comments}),
+        )
+
+    def test_verify_valid_nested_path_and_both_sides(self):
+        result = self.verify(
+            [
+                {"path": "src/example.py", "line": line, "side": side, "body": "Check"}
+                for side, line in [("RIGHT", 20), ("RIGHT", 21), ("LEFT", 10)]
+            ]
+        )
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(result.stderr.count("OK:"), 3)
+
+    def test_verify_rejects_outside_hunk_or_missing_file(self):
+        for path, line in [("src/example.py", 999), ("missing.py", 20)]:
+            with self.subTest(path=path):
+                result = self.verify([{"path": path, "line": line, "body": "Check"}])
+                self.assertNotEqual(result.returncode, 0)
+
+    def test_verify_deleted_file_on_left(self):
+        diff = "diff --git a/gone.py b/gone.py\n--- a/gone.py\n+++ /dev/null\n@@ -1 +0,0 @@\n-old\n"
+        result = self.verify([{"path": "gone.py", "line": 1, "side": "LEFT", "body": "Check"}], diff)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        result = self.verify([{"path": "gone.py", "line": 1, "side": "RIGHT", "body": "Check"}], diff)
+        self.assertNotEqual(result.returncode, 0)
+
+    def test_verify_new_file_with_explicit_zero_old_count(self):
+        diff = "diff --git a/new.py b/new.py\n--- /dev/null\n+++ b/new.py\n@@ -0,0 +1,2 @@\n+one\n+two\n"
+        result = self.verify([{"path": "new.py", "line": 2, "side": "RIGHT", "body": "Check"}], diff)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        result = self.verify([{"path": "new.py", "line": 1, "side": "LEFT", "body": "Check"}], diff)
+        self.assertNotEqual(result.returncode, 0)
+
+    def test_verify_checks_multiline_comment_start(self):
+        result = self.verify(
+            [
+                {
+                    "path": "src/example.py",
+                    "start_line": 1,
+                    "start_side": "RIGHT",
+                    "line": 20,
+                    "side": "RIGHT",
+                    "body": "Check",
+                }
+            ]
+        )
+        self.assertNotEqual(result.returncode, 0)
+
+    def test_verify_multiple_files_hunks_and_renamed_path(self):
+        diff = (
+            "diff --git a/old.py b/new.py\n--- a/old.py\n+++ b/new.py\n"
+            "@@ -1 +1 @@\n-old\n+new\n"
+            "@@ -10 +10 @@\n-old\n+new\n"
+            "diff --git a/second.py b/second.py\n--- a/second.py\n+++ b/second.py\n"
+            "@@ -1 +1 @@\n-old\n+new\n"
+        )
+        result = self.verify(
+            [
+                {"path": path, "line": line, "body": "Check"}
+                for path, line in [("new.py", 1), ("new.py", 10), ("second.py", 1)]
+            ],
+            diff,
+        )
+        self.assertEqual(result.returncode, 0, result.stderr)
+        result = self.verify([{"path": "second.py", "line": 10, "body": "Check"}], diff)
+        self.assertNotEqual(result.returncode, 0)
+
+    def test_verify_quoted_path_and_header_like_content(self):
+        diff = (
+            'diff --git "a/a\\tb.py" "b/a\\tb.py"\n'
+            '--- "a/a\\tb.py"\n+++ "b/a\\tb.py"\n'
+            "@@ -1 +1 @@\n--- content\n+++ content\n"
+        )
+        result = self.verify([{"path": "a\tb.py", "line": 1, "body": "Check"}], diff)
+        self.assertEqual(result.returncode, 0, result.stderr)
+
+    def test_verify_rejects_malformed_input_and_fetch_failure(self):
+        for data in ("not json", '{"comments": null}'):
+            result = self.run_script(REVIEW / "verify_line_numbers.sh", "1", data=data)
+            self.assertNotEqual(result.returncode, 0)
+        self.env["FAIL_GH"] = "1"
+        result = self.verify([{"path": "src/example.py", "line": 20, "body": "Check"}])
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("GitHub unavailable", result.stderr)
+
+    def replies(self, answered):
+        comments = [
+            {
+                "id": 100,
+                "user": "reviewer",
+                "body": "Please fix",
+                "in_reply_to_id": None,
+                "created_at": "2026-09-01T00:00:00Z",
+                "url": "root",
+            },
+            {
+                "id": 101,
+                "user": "author",
+                "body": "Fixed",
+                "in_reply_to_id": 100,
+                "created_at": "2026-09-02T00:00:00Z",
+                "url": "reply",
+            },
+        ]
+        if answered:
+            comments.append(
+                {
+                    "id": 102,
+                    "user": "reviewer",
+                    "body": "Thanks",
+                    "in_reply_to_id": 100,
+                    "created_at": "2026-09-03T00:00:00Z",
+                    "url": "answer",
+                }
+            )
+        (self.root / "comments.json").write_text(json.dumps(comments))
+        return self.run_script(REVIEW / "check_replies.sh", "--reviewer", "reviewer")
+
+    def test_replies_ignores_answered_thread(self):
+        result = self.replies(answered=True)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(result.stdout, "")
+        self.assertIn("No unanswered replies found", result.stderr)
+
+    def test_replies_reports_unanswered_without_false_clean_summary(self):
+        result = self.replies(answered=False)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn("@author replied", result.stdout)
+        self.assertNotIn("No unanswered replies found", result.stderr)
+
+
+if __name__ == "__main__":
+    unittest.main()

apps/ComfyUI-vLLM-Omni/README.md

@@ -271,10 +271,10 @@ The [WF-05 template](example_workflows/vLLM-Omni%20MiniMax-H3%20Latent%20Mask%20
 Connect a **Latent Mask Editing** node to **Generate Video → latent_edit** to edit a source clip instead of generating from scratch. It uploads the source media and serializes the video/audio noise masks the MiniMax H3 API accepts:
 
 - `source_video` / `source_audio` — the media to edit.
-- `video_mask` — a ComfyUI mask image; `0` preserves a region, `1` regenerates it, fractional values blend. A 2D mask `[H, W]` is applied to every frame; a 3D mask `[T, H, W]` is treated as a temporal mask (one slice per frame) for continuation or extension. The client area-downsamples it by 16 to keep the upload small, and the server resizes it to the video latent grid.
+- `video_mask` — a ComfyUI mask image; `0` preserves a region, `1` regenerates it, fractional values blend. A 2D mask `[H, W]` is applied to every frame; a 3D mask `[T, H, W]` is treated as a temporal mask (one slice per frame) for continuation or extension. It is resized to the video latent grid.
 - `audio_mask` — a scalar in `[0, 1]`; `0` keeps the source audio, `1` regenerates it, fractional values blend.
 
-The node requires at least one mask; the server then enforces the cross-field rules (a source requires a matching mask, and a non-trivial mask requires its source). This node only forwards inputs to the server; the served model must declare latent-mask editing support (MiniMax-H3).
+A non-trivial mask requires its matching source, and a source without a mask is rejected. This node only forwards inputs to the server; the served model must declare latent-mask editing support (MiniMax-H3).
 
 ### TTS (e.g., Qwen TTS series)
 

apps/ComfyUI-vLLM-Omni/__init__.py

@@ -36,6 +36,7 @@ from .comfyui_vllm_omni.nodes import (
 
 # A dictionary that contains all nodes you want to export with their names
 NODE_CLASS_MAPPINGS = {
+    "VLLMOmniMiniMaxH3TemporalMask": VLLMOmniMiniMaxH3TemporalMask,
     # === Generation ===
     "VLLMOmniGenerateImage": VLLMOmniGenerateImage,
     "VLLMOmniGenerateVideo": VLLMOmniGenerateVideo,
@@ -45,7 +46,6 @@ NODE_CLASS_MAPPINGS = {
     "VLLMOmniVoiceClone": VLLMOmniVoiceClone,
     "VLLMOmniVideoReferences": VLLMOmniVideoReferences,
     "VLLMOmniLatentMaskEditing": VLLMOmniLatentMaskEditing,
-    "VLLMOmniMiniMaxH3TemporalMask": VLLMOmniMiniMaxH3TemporalMask,
     # === Params ===
     "VLLMOmniARSampling": VLLMOmniARSampling,
     "VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling,
@@ -59,6 +59,7 @@ NODE_CLASS_MAPPINGS = {
 
 # A dictionary that contains the friendly/humanly readable titles for the nodes
 NODE_DISPLAY_NAME_MAPPINGS = {
+    "VLLMOmniMiniMaxH3TemporalMask": "MiniMax-H3 Temporal Mask",
     # === Generation ===
     "VLLMOmniGenerateImage": "Generate Image",
     "VLLMOmniGenerateVideo": "Generate Video",
@@ -68,7 +69,6 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "VLLMOmniVoiceClone": "TTS Voice Cloning",
     "VLLMOmniVideoReferences": "Video References",
     "VLLMOmniLatentMaskEditing": "Latent Mask Editing",
-    "VLLMOmniMiniMaxH3TemporalMask": "MiniMax-H3 Temporal Mask",
     # === Params ===
     "VLLMOmniARSampling": "AR Sampling Params",
     "VLLMOmniDiffusionSampling": "Diffusion Sampling Params",

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py

@@ -8,7 +8,7 @@ import torch
 from comfy_api.input import AudioInput, VideoInput
 
 from .utils.api_client import VLLMOmniClient
-from .utils.latent_mask import _align_frame_count
+from .utils.latent_mask import _align_frame_count, _video_latent_t
 from .utils.logger import get_logger
 from .utils.models import lookup_model_spec
 from .utils.types import (
@@ -1133,13 +1133,13 @@ class VLLMOmniMiniMaxH3TemporalMask:
         else:
             raise ValueError(f"Unknown temporal mask mode: {mode}")
         available = min(frames, math.floor(boundary * 24 + 1e-8))
-        # Snap the preserved prefix to whole VAE clips (5 + 17n frames) so every
-        # latent it maps to is fully preserved.
         prefix = 0 if available < 5 else 5 + 17 * ((available - 5) // 17)
-        # One slice per output frame; the server pools frames to the latent grid.
-        mask = torch.ones(frames, 1, 1)
-        mask[:prefix] = 0
+        preserved = _video_latent_t(prefix) if prefix else 0
+        total = _video_latent_t(frames)
+        mask = torch.ones(total, 1, 1)
+        mask[:preserved] = 0
         indices = torch.arange(frames, device=images.device)
         source_indices = (indices * (source_fps / 24)).floor().long().clamp(max=images.shape[0] - 1)
         preview_images = images.index_select(0, source_indices)
-        return mask, 24.0, preview_images, mask
+        preview_mask = mask.index_select(0, torch.arange(frames) * total // frames)
+        return mask, 24.0, preview_images, preview_mask

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py

@@ -30,7 +30,7 @@ from .format import (
     video_to_base64,
     video_to_bytes,
 )
-from .latent_mask import scalar_mask_to_json, video_mask_to_json
+from .latent_mask import scalar_mask_to_json, video_mask_to_grid_json
 from .logger import get_logger, pretty_printer
 from .models import lookup_model_spec
 from .types import (
@@ -388,15 +388,6 @@ class VLLMOmniClient:
                 content_type="image/png",
             )
 
-        if len(keyframe_images) == 2:
-            for image_filename, image in keyframe_images:
-                form.add_field(
-                    "input_references",
-                    image_tensor_to_png_bytes(image, image_filename),
-                    filename=image_filename,
-                    content_type="image/png",
-                )
-
         # === latent-mask editing (MiniMax H3) ===
         if latent_edit is not None:
             source_video = latent_edit.get("source_video")
@@ -407,16 +398,21 @@ class VLLMOmniClient:
             if video_mask is None and audio_mask is None:
                 raise ValueError("Latent-mask editing requires at least one mask.")
 
+            video_mask_trivial = video_mask is None or bool((video_mask == 1.0).all().item())
+            audio_mask_trivial = audio_mask is None or audio_mask == 1.0
+            if not video_mask_trivial and source_video is None:
+                raise ValueError("A non-trivial video mask requires a source video.")
+            if not audio_mask_trivial and source_audio is None and source_video is None:
+                raise ValueError("A non-trivial audio mask requires a source audio or a source video with audio.")
+
             if source_video is not None:
                 form.add_field(
                     "source_video",
-                    video_to_bytes(source_video),
+                    video_to_bytes(source_video, "source.mp4"),
                     filename="source.mp4",
                     content_type="video/mp4",
                 )
             if source_audio is not None:
-                # The filename extension selects the audio codec (mp3), so it is
-                # meaningful even though aiohttp uses the explicit filename below.
                 form.add_field(
                     "source_audio",
                     audio_to_bytes(source_audio, "source_audio.mp3"),
@@ -424,7 +420,7 @@ class VLLMOmniClient:
                     content_type="audio/mpeg",
                 )
             if video_mask is not None:
-                mask_json = video_mask_to_json(video_mask)
+                mask_json = video_mask_to_grid_json(video_mask, width=width, height=height, num_frames=num_frames)
                 form.add_field(
                     "video_noise_mask",
                     mask_json.encode("utf-8"),
@@ -439,6 +435,15 @@ class VLLMOmniClient:
                     content_type="application/json",
                 )
 
+        if len(keyframe_images) == 2:
+            for image_filename, image in keyframe_images:
+                form.add_field(
+                    "input_references",
+                    image_tensor_to_png_bytes(image, image_filename),
+                    filename=image_filename,
+                    content_type="image/png",
+                )
+
         # === model specific params. Either use a specialized builder, or add flattened fields as-is ===
         if model_params is not None:
             model_params = dict(model_params)

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py

@@ -1,25 +1,11 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
-"""Client-side serialization of MiniMax-H3 latent-edit masks.
-
-A video mask is sent in frame space — a 2D spatial ``[H, W]`` (applied to every
-frame) or a 3D ``[T, H, W]`` (one slice per frame) — and the server resolves it
-to the latent grid. An audio mask is a scalar applied to all time steps.
-"""
-
 import json
-import math
 
 import torch
 import torch.nn.functional as F
 
-# The server rejects mask JSON above 8 MiB, so the spatial axes are
-# area-downsampled by the VAE spatial stride before upload. This depends only on
-# the stride, not on the H3 shape lattice, which the server still owns.
-_VAE_SPATIAL_STRIDE = 16
-_MASK_DECIMALS = 4
-
 
 def scalar_mask_to_json(value: float) -> str:
     if not 0.0 <= value <= 1.0:
@@ -27,27 +13,6 @@ def scalar_mask_to_json(value: float) -> str:
     return str(value)
 
 
-def video_mask_to_json(mask: torch.Tensor) -> str:
-    """Serialize a frame-space video mask to JSON for the ``video_noise_mask`` field.
-
-    ``mask`` is a 2D spatial mask or a 3D frame-space mask. Its spatial axes are
-    area-downsampled by the VAE stride; the server resizes the result to the
-    latent grid, so the client neither floors the canvas nor maps frames to
-    latents.
-    """
-    if mask.ndim not in (2, 3):
-        raise ValueError(f"expected a 2D or 3D mask tensor, got {mask.ndim}D")
-    frames = mask.unsqueeze(0) if mask.ndim == 2 else mask
-    size = tuple(max(1, math.ceil(dim / _VAE_SPATIAL_STRIDE)) for dim in frames.shape[1:])
-    frames = F.interpolate(frames.unsqueeze(1).float(), size=size, mode="area").squeeze(1)
-    if mask.ndim == 2:
-        frames = frames.squeeze(0)
-    # Round in float64 so tolist() emits short decimals rather than float32 noise.
-    return json.dumps(frames.double().round(decimals=_MASK_DECIMALS).tolist(), separators=(",", ":"))
-
-
-# The temporal-mask node snaps its preserve boundary to the H3 frame lattice;
-# this mirrors the server shape planner.
 def _align_frame_count(frame_count: int) -> int:
     if frame_count <= 0:
         return 1
@@ -55,3 +20,32 @@ def _align_frame_count(frame_count: int) -> int:
     while current % 17 != 5:
         current += 1
     return current
+
+
+def _video_latent_t(frame_count: int) -> int:
+    if frame_count <= 5:
+        return 2
+    return ((int(frame_count) - 5) // 17) * 5 + 2
+
+
+def video_mask_to_grid(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> torch.Tensor:
+    tv = _video_latent_t(_align_frame_count(num_frames))
+    height = int(height) // 32 * 32
+    width = int(width) // 32 * 32
+    gh, gw = height // 16, width // 16
+
+    if mask.ndim == 2:
+        mask = mask.unsqueeze(0)
+    elif mask.ndim != 3:
+        raise ValueError(f"expected a 2D or 3D mask tensor, got {mask.ndim}D")
+
+    grid = F.interpolate(mask.unsqueeze(1).float(), size=(gh, gw), mode="area").squeeze(1)
+    if grid.shape[0] == 1:
+        grid = grid.expand(tv, gh, gw)
+    elif grid.shape[0] != tv:
+        grid = F.interpolate(grid.unsqueeze(0).unsqueeze(0), size=(tv, gh, gw), mode="nearest").squeeze(0).squeeze(0)
+    return grid
+
+
+def video_mask_to_grid_json(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> str:
+    return json.dumps(video_mask_to_grid(mask, width=width, height=height, num_frames=num_frames).tolist())

apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md

@@ -12,6 +12,6 @@ Edit a source video with MiniMax-H3 using spatial or temporal masks. The workflo
 Open **vLLM-Omni MiniMax-H3 Latent Mask Editing.json** from the example workflows and set the Generate Video URL/model for the case you want to run.
 
 1. Upload your source video in Load Video.
-2. Adjust the case's mask. For removal and inpainting, set Mask Canvas to the video dimensions, Mask Rectangle to the region size, and Combine Masks x/y to its position, or replace that chain with Load Image (as Mask) for an arbitrary-shape mask. For continuation and extension, set MiniMax-H3 Temporal Mask's mode and duration; continuation also uses `preserve_fraction`. Match the duration in Generate Video, and use a duration longer than the source for extension. Mask values are `0` to preserve and `1` to regenerate.
+2. Adjust the case's mask. For removal and inpainting, set Mask Canvas to the video dimensions, Mask Rectangle to the region size, and Combine Masks x/y to its position. For continuation and extension, set MiniMax-H3 Temporal Mask's mode and duration; continuation also uses `preserve_fraction`. Match the duration in Generate Video, and use a duration longer than the source for extension. Mask values are `0` to preserve and `1` to regenerate.
 3. Adjust the prompt to describe the desired content. Set the output dimensions and duration in Generate Video. If editing audio, update the sound description and audio mask (`0` preserves, `1` regenerates the whole clip).
 4. Select the case's Mask Preview Save Video node and choose **Execute to selected output nodes** to preview the mask as a red overlay without inference, or select its Result Save Video node to generate the edited video. Running the entire workflow generates all four cases.

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json

@@ -0,0 +1,499 @@
+{
+  "id": "latent-mask-editing-7380",
+  "revision": 0,
+  "last_node_id": 7,
+  "last_link_id": 5,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "MarkdownNote",
+      "pos": [
+        40,
+        20
+      ],
+      "size": [
+        420,
+        210
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Latent-mask editing server prerequisite",
+      "properties": {},
+      "widgets_values": [
+        "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends."
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 2,
+      "type": "LoadVideo",
+      "pos": [
+        40,
+        280
+      ],
+      "size": [
+        420,
+        120
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "file",
+          "name": "file",
+          "type": "COMBO",
+          "widget": {
+            "name": "file"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        ""
+      ]
+    },
+    {
+      "id": 3,
+      "type": "LoadImageMask",
+      "pos": [
+        40,
+        460
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image",
+          "type": "STRING",
+          "widget": {
+            "name": "image"
+          },
+          "link": null
+        },
+        {
+          "name": "channel",
+          "type": "STRING",
+          "widget": {
+            "name": "channel"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "mask",
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadImageMask"
+      },
+      "widgets_values": [
+        "",
+        "red"
+      ]
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniLatentMaskEditing",
+      "pos": [
+        560,
+        260
+      ],
+      "size": [
+        420,
+        300
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "source_video",
+          "name": "source_video",
+          "type": "VIDEO",
+          "link": 1,
+          "shape": 7
+        },
+        {
+          "localized_name": "source_audio",
+          "name": "source_audio",
+          "type": "AUDIO",
+          "link": null
+        },
+        {
+          "localized_name": "video_mask",
+          "name": "video_mask",
+          "type": "MASK",
+          "link": 2,
+          "shape": 7
+        },
+        {
+          "localized_name": "audio_mask",
+          "name": "audio_mask",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_mask"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniLatentMaskEditing"
+      },
+      "widgets_values": [
+        0.5
+      ]
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        40,
+        680
+      ],
+      "size": [
+        420,
+        140
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "audio_flow_shift",
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "flow_shift",
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video params",
+          "name": "video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ]
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        1080,
+        200
+      ],
+      "size": [
+        540,
+        680
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "frame",
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "references",
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "sampling_params",
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": null
+        },
+        {
+          "localized_name": "lora",
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "localized_name": "model_params",
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 4,
+          "shape": 7
+        },
+        {
+          "localized_name": "fast_h3",
+          "name": "fast_h3",
+          "type": "FASTH3_DEPLOYMENT",
+          "link": null
+        },
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "link": 3,
+          "shape": 7
+        },
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "prompt",
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "negative_prompt",
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "fps",
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "duration",
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "Partially restyle the clip and soundtrack",
+        "",
+        160,
+        120,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 7,
+      "type": "SaveVideo",
+      "pos": [
+        1700,
+        300
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "link": 5,
+          "shape": 7
+        },
+        {
+          "localized_name": "filename_prefix",
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "format",
+          "name": "format",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "vllm-omni/latent-edit",
+        "mp4"
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      2,
+      0,
+      4,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      3,
+      0,
+      4,
+      2,
+      "MASK"
+    ],
+    [
+      3,
+      4,
+      0,
+      6,
+      6,
+      "LATENT_MASK_EDITING"
+    ],
+    [
+      4,
+      5,
+      0,
+      6,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      5,
+      6,
+      0,
+      7,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json

@@ -0,0 +1,560 @@
+{
+  "last_node_id": 9,
+  "last_link_id": 7,
+  "nodes": [
+    {
+      "id": 2,
+      "type": "LoadVideo",
+      "pos": [
+        40,
+        100
+      ],
+      "size": [
+        420,
+        120
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "file",
+          "type": "STRING",
+          "widget": {
+            "name": "file"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        ""
+      ]
+    },
+    {
+      "id": 3,
+      "type": "SolidMask",
+      "pos": [
+        40,
+        300
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "value",
+          "type": "FLOAT",
+          "widget": {
+            "name": "value"
+          },
+          "link": null
+        },
+        {
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SolidMask"
+      },
+      "widgets_values": [
+        0.0,
+        160,
+        120
+      ]
+    },
+    {
+      "id": 4,
+      "type": "SolidMask",
+      "pos": [
+        40,
+        500
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "value",
+          "type": "FLOAT",
+          "widget": {
+            "name": "value"
+          },
+          "link": null
+        },
+        {
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SolidMask"
+      },
+      "widgets_values": [
+        1.0,
+        160,
+        120
+      ]
+    },
+    {
+      "id": 5,
+      "type": "BatchMasksNode",
+      "pos": [
+        250,
+        380
+      ],
+      "size": [
+        200,
+        100
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "masks.mask0",
+          "type": "MASK",
+          "link": 2,
+          "slot_index": 0
+        },
+        {
+          "name": "masks.mask1",
+          "type": "MASK",
+          "link": 3,
+          "slot_index": 1
+        }
+      ],
+      "outputs": [
+        {
+          "name": "MASK",
+          "type": "MASK",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "BatchMasksNode"
+      },
+      "widgets_values": []
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniLatentMaskEditing",
+      "pos": [
+        450,
+        180
+      ],
+      "size": [
+        400,
+        120
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "source_video",
+          "type": "VIDEO",
+          "link": 1
+        },
+        {
+          "name": "source_audio",
+          "type": "AUDIO",
+          "link": null
+        },
+        {
+          "name": "video_mask",
+          "type": "MASK",
+          "link": 4
+        },
+        {
+          "name": "audio_mask",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_mask"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniLatentMaskEditing"
+      },
+      "widgets_values": [
+        -1.0
+      ]
+    },
+    {
+      "id": 7,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        40,
+        700
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            6
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ]
+    },
+    {
+      "id": 8,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        700,
+        180
+      ],
+      "size": [
+        450,
+        380
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null
+        },
+        {
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null
+        },
+        {
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": null
+        },
+        {
+          "name": "lora",
+          "type": "LORA",
+          "link": null
+        },
+        {
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 6
+        },
+        {
+          "name": "fast_h3",
+          "type": "FAST_H3",
+          "link": null
+        },
+        {
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "link": 5
+        },
+        {
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            7
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "temporal mask: regenerate the ending",
+        "",
+        160,
+        120,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 9,
+      "type": "SaveVideo",
+      "pos": [
+        900,
+        180
+      ],
+      "size": [
+        420,
+        200
+      ],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "link": 7
+        },
+        {
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "name": "format",
+          "type": "STRING",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "temporal_mask",
+        "mp4"
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      2,
+      0,
+      6,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      3,
+      0,
+      5,
+      0,
+      "MASK"
+    ],
+    [
+      3,
+      4,
+      0,
+      5,
+      1,
+      "MASK"
+    ],
+    [
+      4,
+      5,
+      0,
+      6,
+      2,
+      "MASK"
+    ],
+    [
+      5,
+      6,
+      0,
+      8,
+      6,
+      "LATENT_MASK_EDITING"
+    ],
+    [
+      6,
+      7,
+      0,
+      8,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      7,
+      8,
+      0,
+      9,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json

@@ -0,0 +1,511 @@
+{
+  "id": "latent-mask-editing-7380",
+  "revision": 0,
+  "last_node_id": 7,
+  "last_link_id": 5,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "MarkdownNote",
+      "pos": [
+        40,
+        20
+      ],
+      "size": [
+        420,
+        210
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Latent-mask editing server prerequisite",
+      "properties": {},
+      "widgets_values": [
+        "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends."
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 2,
+      "type": "LoadVideo",
+      "pos": [
+        40,
+        280
+      ],
+      "size": [
+        420,
+        120
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "file",
+          "name": "file",
+          "type": "COMBO",
+          "widget": {
+            "name": "file"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        ""
+      ]
+    },
+    {
+      "id": 3,
+      "type": "SolidMask",
+      "pos": [
+        40,
+        460
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "value",
+          "name": "value",
+          "type": "FLOAT",
+          "widget": {
+            "name": "value"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "mask",
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SolidMask"
+      },
+      "widgets_values": [
+        1.0,
+        160,
+        120
+      ]
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniLatentMaskEditing",
+      "pos": [
+        560,
+        260
+      ],
+      "size": [
+        420,
+        300
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "source_video",
+          "name": "source_video",
+          "type": "VIDEO",
+          "link": 1,
+          "shape": 7
+        },
+        {
+          "localized_name": "source_audio",
+          "name": "source_audio",
+          "type": "AUDIO",
+          "link": null
+        },
+        {
+          "localized_name": "video_mask",
+          "name": "video_mask",
+          "type": "MASK",
+          "link": 2,
+          "shape": 7
+        },
+        {
+          "localized_name": "audio_mask",
+          "name": "audio_mask",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_mask"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniLatentMaskEditing"
+      },
+      "widgets_values": [
+        0.5
+      ]
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        40,
+        680
+      ],
+      "size": [
+        420,
+        140
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "audio_flow_shift",
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "flow_shift",
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video params",
+          "name": "video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ]
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        1080,
+        200
+      ],
+      "size": [
+        540,
+        680
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "frame",
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "references",
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "sampling_params",
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": null
+        },
+        {
+          "localized_name": "lora",
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "localized_name": "model_params",
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 4,
+          "shape": 7
+        },
+        {
+          "localized_name": "fast_h3",
+          "name": "fast_h3",
+          "type": "FASTH3_DEPLOYMENT",
+          "link": null
+        },
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "link": 3,
+          "shape": 7
+        },
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "prompt",
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "negative_prompt",
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "fps",
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "duration",
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "Partially restyle the clip and soundtrack",
+        "",
+        160,
+        120,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 7,
+      "type": "SaveVideo",
+      "pos": [
+        1700,
+        300
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "link": 5,
+          "shape": 7
+        },
+        {
+          "localized_name": "filename_prefix",
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "format",
+          "name": "format",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "vllm-omni/latent-edit",
+        "mp4"
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      2,
+      0,
+      4,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      3,
+      0,
+      4,
+      2,
+      "MASK"
+    ],
+    [
+      3,
+      4,
+      0,
+      6,
+      6,
+      "LATENT_MASK_EDITING"
+    ],
+    [
+      4,
+      5,
+      0,
+      6,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      5,
+      6,
+      0,
+      7,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json

@@ -2293,10 +2293,10 @@
       "title": "Usage Guide",
       "properties": {},
       "widgets_values": [
-        "Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. For an arbitrary-shape spatial mask, replace a case's Mask Rectangle chain with Load Image (as Mask). Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference."
+        "Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference."
       ],
       "widgets_values_named": {
-        "text": "Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. For an arbitrary-shape spatial mask, replace a case's Mask Rectangle chain with Load Image (as Mask). Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference."
+        "text": "Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference."
       },
       "color": "#432",
       "bgcolor": "#000"
@@ -2823,7 +2823,7 @@
         "Node name for S&R": "VLLMOmniGenerateVideo"
       },
       "widgets_values": [
-        "http://127.0.0.1:8000/v1",
+        "http://127.0.0.1:8001/v1",
         "MiniMaxAI/MiniMax-H3",
         "integrated_multimodal_description: The masked area is a flat, seamless continuation of the surrounding wall, matching its color, texture, and lighting. The wall surface remains consistent throughout the video.\n\noverall_soundscape: \n\nnon_diegetic_music: ",
         "",
@@ -2833,7 +2833,7 @@
         5
       ],
       "widgets_values_named": {
-        "url": "http://127.0.0.1:8000/v1",
+        "url": "http://127.0.0.1:8001/v1",
         "model": "MiniMaxAI/MiniMax-H3",
         "prompt": "integrated_multimodal_description: The masked area is a flat, seamless continuation of the surrounding wall, matching its color, texture, and lighting. The wall surface remains consistent throughout the video.\n\noverall_soundscape: \n\nnon_diegetic_music: ",
         "negative_prompt": "",
@@ -3677,7 +3677,7 @@
         "Node name for S&R": "VLLMOmniGenerateVideo"
       },
       "widgets_values": [
-        "http://127.0.0.1:8000/v1",
+        "http://127.0.0.1:8001/v1",
         "MiniMaxAI/MiniMax-H3",
         "integrated_multimodal_description: A small orange cat rests curled up in the masked area, gently swaying its tail from side to side. Its appearance matches the surrounding anime style, lighting, and shadows.\n\noverall_soundscape:\n\nnon_diegetic_music: ",
         "",
@@ -3687,7 +3687,7 @@
         5
       ],
       "widgets_values_named": {
-        "url": "http://127.0.0.1:8000/v1",
+        "url": "http://127.0.0.1:8001/v1",
         "model": "MiniMaxAI/MiniMax-H3",
         "prompt": "integrated_multimodal_description: A small orange cat rests curled up in the masked area, gently swaying its tail from side to side. Its appearance matches the surrounding anime style, lighting, and shadows.\n\noverall_soundscape:\n\nnon_diegetic_music: ",
         "negative_prompt": "",
@@ -3916,7 +3916,7 @@
         "Node name for S&R": "VLLMOmniGenerateVideo"
       },
       "widgets_values": [
-        "http://127.0.0.1:8000/v1",
+        "http://127.0.0.1:8001/v1",
         "MiniMaxAI/MiniMax-H3",
         "integrated_multimodal_description: The sky turns completely overcast, with the sun hidden and no direct sunlight, as torrential rain pours over the school entrance in dense visible streaks and splashes on the wet pavement. Distant lightning flashes in the sky behind the school buildings.\n\noverall_soundscape: Loud, steady rainfall, splashing water, and gusts of wind.\n\nnon_diegetic_music: None.",
         "",
@@ -3926,7 +3926,7 @@
         5
       ],
       "widgets_values_named": {
-        "url": "http://127.0.0.1:8000/v1",
+        "url": "http://127.0.0.1:8001/v1",
         "model": "MiniMaxAI/MiniMax-H3",
         "prompt": "integrated_multimodal_description: The sky turns completely overcast, with the sun hidden and no direct sunlight, as torrential rain pours over the school entrance in dense visible streaks and splashes on the wet pavement. Distant lightning flashes in the sky behind the school buildings.\n\noverall_soundscape: Loud, steady rainfall, splashing water, and gusts of wind.\n\nnon_diegetic_music: None.",
         "negative_prompt": "",
@@ -4363,7 +4363,7 @@
         "Node name for S&R": "VLLMOmniGenerateVideo"
       },
       "widgets_values": [
-        "http://127.0.0.1:8000/v1",
+        "http://127.0.0.1:8001/v1",
         "MiniMaxAI/MiniMax-H3",
         "integrated_multimodal_description: Cut to the same school on a summer night beneath a deep blue sky dotted with stars. A firework rises from behind the school buildings, leaving a glowing trail; the camera slowly tilts upward to follow it into the sky, ending as it bursts into a large bloom of colorful sparks.\n\noverall_soundscape: Soft summer insect sounds, followed by the firework rising with a whistle and bursting with a distant boom and crackling sparks.\n\nnon_diegetic_music: N/A",
         "",
@@ -4373,7 +4373,7 @@
         10
       ],
       "widgets_values_named": {
-        "url": "http://127.0.0.1:8000/v1",
+        "url": "http://127.0.0.1:8001/v1",
         "model": "MiniMaxAI/MiniMax-H3",
         "prompt": "integrated_multimodal_description: Cut to the same school on a summer night beneath a deep blue sky dotted with stars. A firework rises from behind the school buildings, leaving a glowing trail; the camera slowly tilts upward to follow it into the sky, ending as it bursts into a large bloom of colorful sparks.\n\noverall_soundscape: Soft summer insect sounds, followed by the firework rising with a whistle and bursting with a distant boom and crackling sparks.\n\nnon_diegetic_music: N/A",
         "negative_prompt": "",

benchmarks/diffusion/bench_wan_vae_decode.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 """Benchmark and exactness check for the Wan VAE decoder fast path.
 
 Loads a diffusers Wan VAE (by default the Cosmos3 one), decodes seeded latents
@@ -30,13 +30,10 @@ return the full video on rank 0 only, so equality and PSNR are computed there.
 from __future__ import annotations
 
 import argparse
-import json
 import math
 import os
-import statistics
 import time
 from collections.abc import Iterable
-from unittest.mock import patch
 
 import torch
 import torch.distributed as dist
@@ -85,10 +82,6 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--profile", action="store_true", help="Print a torch.profiler CUDA kernel table")
     parser.add_argument("--profile-rows", type=int, default=25)
     parser.add_argument("--save-output", default=None, help="Save the `off` decode output to this .pt path")
-    parser.add_argument(
-        "--first-frame-ablation", action="store_true", help="Compare channels_last with/without first-frame Conv2d"
-    )
-    parser.add_argument("--latents", help="For ablation: saved VAE-input tensor (.pt), after latent mean/std scaling")
     parser.add_argument(
         "--vae-patch-parallel-size",
         type=int,
@@ -261,99 +254,8 @@ def run_level(
     return output.detach(), stats
 
 
-@torch.inference_mode()
-def run_first_frame_ablation(args: argparse.Namespace, dtype: torch.dtype, device: torch.device) -> None:
-    """Same-model ABBA comparison; disable only the first-frame helper in the reference arm."""
-    from vllm_omni.diffusion.distributed.autoencoders.wan_vae_fastpath import forwards
-
-    if args.iters < 2 or args.warmup < 0:
-        raise ValueError("Ablation requires --iters >= 2 and --warmup >= 0")
-    vae = load_vae(args, dtype, device, parallel=False)
-    report = install_wan_vae_fastpath(vae, level="channels_last")
-    if not report.installed:
-        raise RuntimeError(f"channels_last installation failed: {report.reason}")
-    latents = make_latents(vae, args, dtype, device)
-    if args.latents:
-        saved = torch.load(args.latents, map_location="cpu", weights_only=True)
-        saved = saved[:, :, : latents.shape[2]]
-        if saved.shape != latents.shape:
-            raise ValueError(f"Saved latent shape {saved.shape} does not match {latents.shape}")
-        latents = saved.to(device=device, dtype=dtype).contiguous()
-    helpers = {"baseline": lambda conv, x, bias: None, "candidate": forwards._first_frame_conv2d}
-    optimized_calls = 0
-
-    def counted(conv, x, bias):
-        nonlocal optimized_calls
-        result = helpers["candidate"](conv, x, bias)
-        optimized_calls += result is not None
-        return result
-
-    with patch.object(forwards, "_first_frame_conv2d", counted):
-        decode(vae, latents)
-        sync()
-    if optimized_calls == 0:
-        raise RuntimeError("The selected workload did not exercise first-frame Conv2d")
-    measurements = []
-    outputs: dict[str, list[torch.Tensor]] = {arm: [] for arm in helpers}
-    for _ in range(args.warmup):
-        for helper in helpers.values():
-            with patch.object(forwards, "_first_frame_conv2d", helper):
-                decode(vae, latents)
-                sync()
-    for iteration in range(args.iters):
-        # Alternating AB and BA pairs give ABBA order without dropping odd runs.
-        arms = ("baseline", "candidate") if iteration % 2 == 0 else ("candidate", "baseline")
-        for arm in arms:
-            with patch.object(forwards, "_first_frame_conv2d", helpers[arm]):
-                sync()
-                torch.accelerator.reset_peak_memory_stats()
-                start = time.perf_counter()
-                output = decode(vae, latents)
-                sync()
-                elapsed = (time.perf_counter() - start) * 1000
-                peak = torch.accelerator.max_memory_allocated()
-            measurements.append({"arm": arm, "wall_ms": elapsed, "peak_allocated_bytes": peak})
-            if len(outputs[arm]) < 2:
-                outputs[arm].append(output.cpu())
-            del output
-    baseline, candidate = outputs["baseline"][0], outputs["candidate"][0]
-    difference = candidate.float() - baseline.float()
-    stats = {}
-    for arm in helpers:
-        times = [row["wall_ms"] for row in measurements if row["arm"] == arm]
-        stats[arm] = {
-            "median_ms": statistics.median(times),
-            "mean_ms": statistics.mean(times),
-            "std_ms": statistics.pstdev(times),
-            "min_ms": min(times),
-            "max_ms": max(times),
-        }
-    result = {
-        "scope": "VAE decode; same-head channels_last first-frame Conv2d ablation",
-        "args": vars(args),
-        "torch": torch.__version__,
-        "cuda": torch.version.cuda,
-        "baseline": stats["baseline"],
-        "candidate": stats["candidate"],
-        "latency_reduction": 1 - stats["candidate"]["median_ms"] / stats["baseline"]["median_ms"],
-        "optimized_calls_per_decode": optimized_calls,
-        "outputs_finite": bool(torch.isfinite(baseline).all() and torch.isfinite(candidate).all()),
-        "self_repeat_equal": {arm: torch.equal(*values) for arm, values in outputs.items()},
-        "candidate_vs_baseline": {
-            "torch_equal": torch.equal(candidate, baseline),
-            "max_abs": difference.abs().max().item(),
-            "relative_l2": (difference.norm() / baseline.float().norm().clamp_min(1e-30)).item(),
-            "psnr_db": psnr(candidate, baseline),
-        },
-        "measurements": measurements,
-    }
-    print("FIRST_FRAME_ABLATION " + json.dumps(result))
-
-
 def main() -> None:
     args = parse_args()
-    if args.latents and not args.first_frame_ablation:
-        raise SystemExit("--latents requires --first-frame-ablation")
     if not torch.cuda.is_available():
         raise SystemExit("CUDA device required")
     rank, world_size = init_distributed()
@@ -365,12 +267,6 @@ def main() -> None:
         )
     device = torch.device("cuda", torch.accelerator.current_device_index()) if world_size > 1 else torch.device("cuda")
     dtype = DTYPES[args.dtype]
-    if args.first_frame_ablation:
-        if world_size != 1 or parallel:
-            raise SystemExit("--first-frame-ablation requires one GPU")
-        with torch.backends.cudnn.flags(benchmark=False, allow_tf32=False):
-            run_first_frame_ablation(args, dtype, device)
-        return
     levels: Iterable[str] = [level.strip() for level in args.fast_path.split(",") if level.strip()]
     for level in levels:
         if level not in VAE_FAST_PATH_LEVELS:

benchmarks/diffusion/benchmark_mammoth_moda2_qk_norm_rope.py

@@ -1,262 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
-
-"""Benchmark MammothModa2's Q/K RMSNorm + adjacent-pair real RoPE.
-
-Example:
-
-    CUDA_VISIBLE_DEVICES=0 python \
-      benchmarks/diffusion/benchmark_mammoth_moda2_qk_norm_rope.py \
-      --tokens 512 4173 8346 --warmup 50 --iters 200
-"""
-
-from __future__ import annotations
-
-import argparse
-import importlib.metadata
-import json
-import os
-import statistics
-from collections.abc import Callable
-
-import torch
-
-from vllm_omni.diffusion.layers.fused_qk_norm_rope import (
-    _fused_cuda_supported,
-    fused_qk_norm_rope,
-)
-from vllm_omni.diffusion.layers.norm import RMSNorm
-from vllm_omni.diffusion.models.mammoth_moda2.rope_real import (
-    apply_real_rotary_emb,
-)
-
-_HEAD_DIM = 120
-_Q_HEADS = 21
-_KV_HEADS = 7
-_EPS = 1e-5
-
-
-def _parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--tokens", nargs="+", type=int, default=[512, 4173, 8346])
-    parser.add_argument("--warmup", type=int, default=50)
-    parser.add_argument("--iters", type=int, default=200)
-    parser.add_argument("--rounds", type=int, default=3)
-    parser.add_argument("--attention-sequences", nargs="*", type=int, default=[])
-    return parser.parse_args()
-
-
-def _version(distribution: str) -> str | None:
-    try:
-        return importlib.metadata.version(distribution)
-    except importlib.metadata.PackageNotFoundError:
-        return None
-
-
-def _measure(
-    fn: Callable[[], object],
-    warmup: int,
-    iters: int,
-) -> dict[str, float]:
-    for _ in range(warmup):
-        fn()
-    torch.accelerator.synchronize()
-
-    torch.accelerator.reset_peak_memory_stats()
-    baseline_bytes = torch.accelerator.memory_allocated()
-    samples = []
-    for _ in range(iters):
-        start = torch.cuda.Event(enable_timing=True)
-        end = torch.cuda.Event(enable_timing=True)
-        start.record()
-        fn()
-        end.record()
-        end.synchronize()
-        samples.append(float(start.elapsed_time(end)))
-
-    ordered = sorted(samples)
-    return {
-        "median_ms": statistics.median(ordered),
-        "mean_ms": statistics.mean(ordered),
-        "stddev_ms": statistics.pstdev(ordered),
-        "min_ms": ordered[0],
-        "max_ms": ordered[-1],
-        "p90_ms": ordered[min(len(ordered) - 1, int(len(ordered) * 0.9))],
-        "extra_peak_mib": (torch.accelerator.max_memory_allocated() - baseline_bytes) / 2**20,
-    }
-
-
-@torch.inference_mode()
-def _run_shape(tokens: int, warmup: int, iters: int, *, fused_first: bool = False) -> dict[str, object]:
-    dtype = torch.bfloat16
-    device = torch.device("cuda")
-    torch.manual_seed(42)
-    query = torch.randn(tokens, _Q_HEADS, _HEAD_DIM, device=device, dtype=dtype)
-    key = torch.randn(tokens, _KV_HEADS, _HEAD_DIM, device=device, dtype=dtype)
-    norm_q = RMSNorm(_HEAD_DIM, eps=_EPS).to(device=device, dtype=dtype)
-    norm_k = RMSNorm(_HEAD_DIM, eps=_EPS).to(device=device, dtype=dtype)
-    norm_q.weight.normal_(mean=1.0, std=0.2)
-    norm_k.weight.normal_(mean=1.0, std=0.2)
-    angles = torch.rand(tokens, _HEAD_DIM // 2, device=device) * (2 * torch.pi)
-    cos = angles.cos().repeat_interleave(2, dim=-1).to(dtype)
-    sin = angles.sin().repeat_interleave(2, dim=-1).to(dtype)
-
-    def native() -> tuple[torch.Tensor, torch.Tensor]:
-        q_out = apply_real_rotary_emb(norm_q(query).unsqueeze(0), cos.unsqueeze(0), sin.unsqueeze(0)).squeeze(0)
-        k_out = apply_real_rotary_emb(norm_k(key).unsqueeze(0), cos.unsqueeze(0), sin.unsqueeze(0)).squeeze(0)
-        return q_out, k_out
-
-    def fused() -> tuple[torch.Tensor, torch.Tensor]:
-        rope_table = torch.cat((cos[..., 0::2], sin[..., 0::2]), dim=-1)
-        return fused_qk_norm_rope(
-            query,
-            key,
-            norm_q.weight,
-            norm_k.weight,
-            rope_table,
-            _EPS,
-            head_dim=_HEAD_DIM,
-            rotary_dim=_HEAD_DIM,
-            interleaved=True,
-        )
-
-    expected_q, expected_k = native()
-    actual_q, actual_k = fused()
-    errors = {
-        "q_max_abs": (actual_q.float() - expected_q.float()).abs().max().item(),
-        "q_mean_abs": (actual_q.float() - expected_q.float()).abs().mean().item(),
-        "k_max_abs": (actual_k.float() - expected_k.float()).abs().max().item(),
-        "k_mean_abs": (actual_k.float() - expected_k.float()).abs().mean().item(),
-    }
-    torch.testing.assert_close(actual_q, expected_q, atol=0.0625, rtol=0.02)
-    torch.testing.assert_close(actual_k, expected_k, atol=0.0625, rtol=0.02)
-
-    functions = {"native": native, "fused": fused}
-    order = ("fused", "native") if fused_first else ("native", "fused")
-    measured = {name: _measure(functions[name], warmup, iters) for name in order}
-    native_stats, fused_stats = measured["native"], measured["fused"]
-    return {
-        "tokens": tokens,
-        "native": native_stats,
-        "fused_including_rope_pack": fused_stats,
-        "median_speedup": native_stats["median_ms"] / fused_stats["median_ms"],
-        "errors": errors,
-    }
-
-
-@torch.inference_mode()
-def _run_attention(seq: int, warmup: int, iters: int, *, fused_first: bool = False) -> dict[str, object]:
-    from vllm_omni.diffusion.models.mammoth_moda2.mammothmoda2_dit_model import TransformerBlock
-
-    torch.manual_seed(42)
-    block = (
-        TransformerBlock(
-            2520,
-            _Q_HEADS,
-            _KV_HEADS,
-            multiple_of=256,
-            ffn_dim_multiplier=1.0,
-            norm_eps=_EPS,
-            rope_repeats_pairs=True,
-        )
-        .to(device="cuda", dtype=torch.bfloat16)
-        .eval()
-    )
-    block.attn.norm_q.weight.normal_(1.0, 0.2)
-    block.attn.norm_k.weight.normal_(1.0, 0.2)
-    hidden = torch.randn(2, seq, 2520, device="cuda", dtype=torch.bfloat16)
-    mask = torch.ones(2, seq, device="cuda", dtype=torch.bool)
-    mask[0, seq - min(300, seq // 2) :] = False
-    mask[1, seq - min(17, seq // 2) :] = False
-    angles = torch.rand(2, seq, _HEAD_DIM // 2, device="cuda") * (2 * torch.pi)
-    # The production producer supplies FP32 tables; each example has its own positions.
-    rotary = angles.cos().repeat_interleave(2, dim=-1), angles.sin().repeat_interleave(2, dim=-1)
-    env = "VLLM_OMNI_FUSED_QK_NORM_ROPE_MIN_TOKENS"
-    previous = os.environ.get(env)
-
-    def run():
-        return block.attn(hidden, hidden, attention_mask=mask, image_rotary_emb=rotary)
-
-    try:
-        outputs = {}
-        timings = {}
-        order = ("fused", "native") if fused_first else ("native", "fused")
-        for name in order:
-            os.environ[env] = "0" if name == "fused" else str(2 * seq + 1)
-            outputs[name] = run()
-            timings[name] = _measure(run, warmup, iters)
-        diff = (outputs["native"].float() - outputs["fused"].float()).abs()[mask]
-        max_diff, mean_diff = diff.max().item(), diff.mean().item()
-        assert max_diff < 2e-2 and mean_diff < 1e-3, (max_diff, mean_diff)
-        assert torch.count_nonzero(outputs["fused"][~mask]) == 0
-        return {
-            "sequence": seq,
-            "batch": 2,
-            "backend": block.attn.omni_attn.attn_backend.__name__,
-            "native": timings["native"],
-            "fused": timings["fused"],
-            "median_speedup": timings["native"]["median_ms"] / timings["fused"]["median_ms"],
-            "max_abs_diff": max_diff,
-            "mean_abs_diff": mean_diff,
-        }
-    finally:
-        if previous is None:
-            os.environ.pop(env, None)
-        else:
-            os.environ[env] = previous
-
-
-def main() -> None:
-    args = _parse_args()
-    if (
-        args.rounds < 1
-        or args.iters < 1
-        or args.warmup < 0
-        or any(value < 1 for value in [*args.tokens, *args.attention_sequences])
-    ):
-        raise ValueError("rounds, iters and sequence lengths must be positive; warmup must be nonnegative")
-    if not torch.cuda.is_available():
-        raise RuntimeError("This benchmark requires CUDA")
-
-    probe_q = torch.empty(1, _Q_HEADS, _HEAD_DIM, device="cuda", dtype=torch.bfloat16)
-    probe_k = torch.empty(1, _KV_HEADS, _HEAD_DIM, device="cuda", dtype=torch.bfloat16)
-    if not _fused_cuda_supported(probe_q, probe_k, _HEAD_DIM, _HEAD_DIM, interleaved=True):
-        raise RuntimeError("The fused CUDA QK norm/RoPE path is unavailable")
-
-    result = {
-        "environment": {
-            "gpu": torch.cuda.get_device_name(),
-            "torch": torch.__version__,
-            "cuda": torch.version.cuda,
-            "triton": _version("triton"),
-            "vllm": _version("vllm"),
-            "vllm_omni": _version("vllm-omni"),
-        },
-        "config": {
-            "dtype": "bfloat16",
-            "q_heads": _Q_HEADS,
-            "kv_heads": _KV_HEADS,
-            "head_dim": _HEAD_DIM,
-            "warmup": args.warmup,
-            "iters": args.iters,
-            "rounds": args.rounds,
-            "inference_mode": True,
-            "alternating_order": True,
-            "norm_weights": "normal(mean=1.0, std=0.2)",
-        },
-        "results": [
-            {"round": round_id, **_run_shape(tokens, args.warmup, args.iters, fused_first=bool(round_id % 2))}
-            for round_id in range(args.rounds)
-            for tokens in args.tokens
-        ],
-        "attention_results": [
-            {"round": round_id, **_run_attention(seq, args.warmup, args.iters, fused_first=bool(round_id % 2))}
-            for round_id in range(args.rounds)
-            for seq in args.attention_sequences
-        ],
-    }
-    print(json.dumps(result, indent=2))
-
-
-if __name__ == "__main__":
-    main()

benchmarks/diffusion/performance_dashboard/mammoth_moda2_serving_performance.md

@@ -1,209 +0,0 @@
-# MammothModa2-Preview Serving Performance Dashboard
-
-This document describes how to deploy and benchmark **bytedance-research/MammothModa2-Preview** (T2I) using vLLM-Omni. It includes service startup configuration, benchmark methodology, dataset settings, and performance results.
-
-MammothModa2 is a two-stage AR→DiT text-to-image pipeline: stage 0 is an autoregressive LLM that encodes the prompt into a visual-token grid, and stage 1 is a generation-LLM DiT that renders the final image (`final_output_type: image`). Unlike single-stage diffusion pipelines, `/v1/images/generations` requests traverse both stages, and the DiT stage ran with `max_num_seqs: 1` in the then-default deploy config, so concurrent requests serialized on the DiT stage while the AR stage batched (request-level DiT batching landed later in #7476).
-
-> **Legacy-topology baseline**: all numbers in this dashboard were measured on the legacy topology, where the MammothModa2 DiT stage runs as an LLM-generation stage (`stage_type: llm`) built from `VllmConfig`, served through the LLM-typed image-stage compatibility path. #7134 migrates this pipeline to the shared diffusion runtime (DiT registered as a standard `DIFFUSION` stage built from `OmniDiffusionConfig`). These results should be treated as the pre-migration baseline and are expected to be rerun after #7134 lands; they are not directly comparable with post-migration numbers.
-
----
-
-# 1. Overview
-
-This document covers:
-
-* Service launch configuration
-* Benchmark scripts and usage
-* Dataset and workload settings
-* Performance measurement results
-* Reproducibility guidelines
-
----
-
-# 2. Test Environment
-
-| Component | Specification |
-|------------|----------------|
-| GPU | NVIDIA A800-SXM4-80GB (1x) |
-| CPU | 18 cores |
-| Model weights | `bytedance-research/MammothModa2-Preview` |
-| Deploy config | `vllm_omni/deploy/mammoth_moda2.yaml` (bundled) |
-| Stage 0 (AR) | `gpu_memory_utilization: 0.5`, `enforce_eager: true`, `max_num_seqs: 100` |
-| Stage 1 (DiT) | `gpu_memory_utilization: 0.3`, `enforce_eager: true`, `max_num_seqs: 1` |
-| Sampling params (per request) | `text_guidance_scale: 9.0`, `cfg_range: [0.0, 1.0]`, `num_inference_steps: 50` sent via `--extra-body` / `--num-inference-steps` |
-
----
-
-# 3. Service Launch Configuration
-
-## 3.1 Basic Serving Command
-
-```bash
-vllm-omni serve bytedance-research/MammothModa2-Preview --omni \
-    --port 8091 \
-    --trust-remote-code
-```
-
-When serving from a local weights directory (no `model_index.json`), pass the bundled deploy config explicitly so the `mammoth_moda2` pipeline is selected:
-
-```bash
-vllm-omni serve /path/to/MammothModa2-Preview --omni \
-    --port 8091 \
-    --trust-remote-code \
-    --deploy-config vllm_omni/deploy/mammoth_moda2.yaml
-```
-
-## 3.2 Key Parameters
-
-| Parameter | Description |
-|-----------|-------------|
-| `--deploy-config` | Path to the deploy YAML; defaults to the bundled `mammoth_moda2.yaml` for the HF checkpoint |
-| `size` (request) | `WxH`, e.g. `1024x1024`; routed to the AR grid and the DiT `target_w/target_h` |
-| `num_inference_steps` (request) | DiT sampling steps (default 50) |
-| `guidance_scale` (request) | Text guidance scale (default 9.0) |
-
-Note: the benchmark script's `--warmup-num-inference-steps` shortcut is not usable for this pipeline — a reduced step count truncates the AR stage's visual-token output and crashes the DiT stage. Use `--warmup-requests 0` and warm up with a real request instead.
-
----
-
-# 4. Benchmark Script
-
-## 4.1 Benchmark Entry
-
-```bash
-python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --base-url http://localhost:8091 \
-    --model bytedance-research/MammothModa2-Preview \
-    --endpoint /v1/images/generations \
-    --task t2i \
-    --dataset random \
-    --num-prompts 8 \
-    --width <W> --height <H> \
-    --num-inference-steps <S> \
-    --extra-body '{"text_guidance_scale": 9.0, "cfg_range": [0.0, 1.0]}' \
-    --max-concurrency <C> \
-    --warmup-requests 0 \
-    --seed 142 \
-    --output-file <OUT>.json
-```
-
-## 4.2 Dataset
-
-`random` synthetic T2I prompts, fixed seed 142, 8 prompts per configuration (4 for 2048x2048 due to per-request latency).
-
----
-
-# 5. Performance Results
-
-All requests succeeded (0 failures across the sweep). Latency is end-to-end per request (AR + DiT + VAE decode + base64 response).
-
-> Note: these are legacy-topology baseline results (see the callout in the overview); rerun after the #7134 diffusion-runtime migration before comparing across topologies.
-
-## 5.1 Latency
-
-| Resolution | Steps | Concurrency | Mean (s) | P50 (s) | P95 (s) | P99 (s) |
-|------------|-------|-------------|----------|---------|---------|---------|
-| 512x512    | 20    | 1           | 21.8     | 21.7    | 22.1    | 22.2    |
-| 512x512    | 20    | 2           | 24.8     | 25.2    | 25.6    | 25.8    |
-| 512x512    | 20    | 4           | 28.5     | 29.2    | 31.1    | 31.8    |
-| 1024x1024  | 50    | 1           | 96.3     | 95.8    | 99.8    | 100.8   |
-| 1024x1024  | 50    | 2           | 121.5    | 125.4   | 127.9   | 128.5   |
-| 1024x1024  | 50    | 4           | 150.9    | 157.6   | 178.6   | 186.3   |
-| 2048x2048  | 50    | 1           | 271.9    | 271.6   | 273.6   | 273.8   |
-
-## 5.2 Throughput
-
-| Resolution | Steps | Concurrency | Throughput (img/s) |
-|------------|-------|-------------|--------------------|
-| 512x512    | 20    | 1           | 0.0459             |
-| 512x512    | 20    | 2           | 0.0798             |
-| 512x512    | 20    | 4           | 0.1329             |
-| 1024x1024  | 50    | 1           | 0.0104             |
-| 1024x1024  | 50    | 2           | 0.0162             |
-| 1024x1024  | 50    | 4           | 0.0238             |
-| 2048x2048  | 50    | 1           | 0.0037             |
-
-## 5.3 Stage Breakdown (concurrency 1, from server logs)
-
-Per-stage wall time from the server's `[OmniTiming]` records across the §5.1/§5.4/§5.5 sweeps (n = 18/9/5 per row; at concurrency 1 stage wall time equals `stage_gen_time_ms` in the engine stats table):
-
-| Resolution | Steps | AR stage (s) | DiT stage (s) | E2E (s) | AR share | AR output tokens | AR ms/token |
-|------------|-------|--------------|---------------|---------|----------|------------------|-------------|
-| 512x512    | 20    | 20.3         | 1.7           | 22.0    | 92%      | 1,057            | 19.2        |
-| 1024x1024  | 50    | 78.9         | 17.2          | 96.1    | 82%      | 4,161            | 19.0        |
-| 2048x2048  | 50    | 156.3        | 114.8         | 271.2   | 58%      | 8,154            | 19.2        |
-
-Stage 0 is the AR prompt encoder (TTFT 82 ms at 1024x1024); stage 1 is the DiT renderer. Three observations:
-
-1. **AR per-token cost is constant (~19 ms/token) across resolutions**, so AR time is fully predictable: the AR stage decodes a fixed visual grid whose length scales with resolution (1,057 → 4,161 → 8,154 tokens), giving `AR ≈ tokens × 19 ms`. This is also why §5.4's step sweep is sub-linear — the AR token count is step-invariant.
-2. **DiT time scales super-linearly with pixels**: a 16x pixel increase (512 → 2048) multiplies DiT time by 67 (1.7 → 114.8 s).
-3. **The dominant stage flips with resolution**: AR is 92% of E2E at 512x512 (DiT optimization is nearly pointless there) but 58% at 2048x2048, where both stages matter.
-
-At concurrency > 1, per-stage wall times additionally include intra-stage queueing (each stage serializes at `max_num_seqs: 1`), so the c2/c4 rows in §5.1 cannot be decomposed additively from this table.
-
-## 5.4 Inference-Step Sensitivity (1024x1024 / 512x512, concurrency 1)
-
-Steps 25 at 1024x1024 (half the default 50) and steps 25 at 512x512, same protocol as §5.1:
-
-| Resolution | Steps | Concurrency | Mean (s) | P50 (s) | P95 (s) | P99 (s) |
-|------------|-------|-------------|----------|---------|---------|---------|
-| 1024x1024  | 25    | 1           | 85.2     | 85.3    | 87.8    | 88.7    |
-| 1024x1024  | 25    | 4           | 113.7    | 115.8   | 126.1   | 129.4   |
-| 512x512    | 25    | 1           | 21.9     | 21.8    | 22.2    | 22.3    |
-
-Halving DiT steps does not halve end-to-end latency (96.3 → 85.2 s at 1024): the AR stage emits a fixed count of visual tokens (4,161 at 1024x1024) regardless of step count, so its ~79 s contribution (§5.3) is step-invariant; only the DiT sampling time scales with steps. At 512x512, steps 20 → 25 adds ~0.1 s per step (21.8 → 21.9 s).
-
-## 5.5 Prompt-Length Sensitivity (long vs short prompts)
-
-A ~430-word long prompt (repeated via `--random-request-config '[{"weight":1,"prompt":"..."}]'`) against the default short synthetic prompt, same seed and settings:
-
-| Resolution | Steps | Concurrency | Prompt | Mean (s) | P99 (s) | Throughput (img/s) |
-|------------|-------|-------------|--------|----------|---------|---------------------|
-| 1024x1024  | 50    | 1           | short  | 96.3     | 100.8   | 0.0104              |
-| 1024x1024  | 50    | 1           | long   | 93.6     | 94.4    | 0.0107              |
-| 1024x1024  | 50    | 4           | short  | 150.9    | 186.3   | 0.0238              |
-| 1024x1024  | 50    | 4           | long   | 141.7    | 174.2   | 0.0255              |
-| 512x512    | 20    | 1           | short  | 21.8     | 22.2    | 0.0459              |
-| 512x512    | 20    | 1           | long   | 22.2     | 22.5    | 0.0450              |
-
-Prompt length has no material effect on latency (the long-prompt rows are within run-to-run variance and even slightly faster with tighter P99 tails). This confirms the AR stage's cost is dominated by decoding the fixed 4,161-token visual grid, not by prompt encoding: the KV prefill of a few hundred extra text tokens is negligible against 4,161 decode steps.
-
-## 5.6 Peak GPU Memory (per-stage process, nvidia-smi polling at 1 Hz)
-
-Sampled during the §5.4/§5.5 sweeps (stage processes identified by PID; both stages share the single A800):
-
-| Configuration (worst case) | stage0_ar (MB) | stage1_dit (MB) |
-|----------------------------|----------------|-----------------|
-| 1024x1024, 50 steps, c4    | 39,290         | 11,104          |
-| all other configs          | 39,194         | 11,102–11,104   |
-
-Memory is essentially flat across the sweep: the AR stage grows by only ~96 MB at concurrency 4 and the DiT stage by 2 MB, because both stages pre-allocate from `gpu_memory_utilization` budgets (0.5 / 0.3) at startup. Combined steady-state usage ≈ 50.4 GiB of the 80 GiB device.
-
-## 5.7 DiT-Stage Component Attribution (pipeline profiler)
-
-With `enable_diffusion_pipeline_profiler: true` in the stage-1 deploy config (`MammothModa2DiTPipeline` targets: `gen_transformer.forward`, `gen_vae.decode`, `gen_image_condition_refiner.forward`), three warm 1024x1024 / 50-step requests (profiler adds a `synchronize` per call, so timings are indicative, not throughput-grade):
-
-| Component | Per request (s) | Share of DiT pipeline |
-|-----------|-----------------|-----------------------|
-| `gen_transformer.forward` (100 calls: 50 steps x 2 passes) | 14.01–14.07 | 98.6% |
-| `gen_vae.decode` (1 call) | 0.148 | 1.0% |
-| `gen_image_condition_refiner.forward` (1 call) | 0.005 | 0.03% |
-| whole `MammothModa2DiTPipeline.forward` | 14.18–14.27 | — |
-
-The DiT latency is ~99% transformer sampling; VAE decode costs only ~0.15 s per image at 1024x1024 and is not a bottleneck. Per-step transformer time is ~140 ms (two forward passes per step: ~2.8 s nominal x CFG), consistent with §5.3's 17.2 s image-TTFT figure once base64 encoding and stage handoff are included.
-
-## 5.8 Endpoint Fix Verification
-
-Before the serving fix, `/v1/images/generations` returned HTTP 503 ("No diffusion stage found in multi-stage pipeline") because the pipeline's image-output stage is a generation-LLM DiT rather than a `diffusion`-typed stage. After the fix all requests return HTTP 200 with a valid base64 image (verified across every sweep configuration above).
-
----
-
-# 6. Reproducibility
-
-1. Start the server as in §3.1 and wait for `Application startup complete`.
-2. Warm up with one real request (any size) — do not use reduced-step warmups.
-3. Run the sweep commands from §4.1 for each row in §5.
-4. Server-side per-stage timings are printed in the engine stats table (`stage_gen_time_ms`, `output_unit_count`) at `--log-stats` level info.
-5. New rows use the same seed (`--seed 142`), `--num-prompts 8`, and `--warmup-requests 0` protocol as §4.1; when saving results with `--output-file`, name them `<res>_s<steps>_c<conc>[_long].json` to keep runs comparable (steps-25, long-prompt, peak-memory and profiler runs).
-6. To reproduce §5.6 peak-memory numbers, poll `nvidia-smi --query-gpu=memory.used --format=csv -l 1` per stage PID while a sweep is in flight (e.g. `nvidia-smi pmon` / `--query-compute-apps`).
-7. To reproduce §5.7, copy the bundled deploy config, set `enable_diffusion_pipeline_profiler: true` on the stage-1 entry, restart, and grep `[DiffusionPipelineProfiler]` lines from the server log; timings include a `torch.cuda.synchronize` per call.

docker/Dockerfile.ci

@@ -1,7 +1,10 @@
-# Use the official release image with its matching CUDA/PyTorch stack.
-ARG VLLM_VERSION=0.31.0
+# Target vLLM version: v0.30.0 (tag @ ced6857afa0e).
+# Use the matching release image and its coherent CUDA/PyTorch/FlashInfer
+# runtime. Do not reinstall a commit wheel or repair dependencies on top of
+# an older base image at a release boundary. For a future unreleased target,
+# restore an explicit wheel pin and validate its complete runtime stack.
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai
-ARG VLLM_BASE_TAG=v${VLLM_VERSION}
+ARG VLLM_BASE_TAG=v0.30.0
 
 # We use registry-based layer caching for the dependencies,
 # where the tag is determined based on the hash of the contents
@@ -9,7 +12,6 @@ ARG VLLM_BASE_TAG=v${VLLM_VERSION}
 # everything except the actual project dir, since that should not
 # change much across branches.
 FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG} AS deps
-ARG VLLM_VERSION
 # Keep APP_DIR outside /workspace so k8s Buildkite workspace mounts cannot
 # shadow the editable install target (#5364). Docker WORKDIR creates /opt if
 # needed.
@@ -34,17 +36,12 @@ RUN apt-get update && \
 # for CI
 COPY pyproject.toml setup.py ./
 COPY requirements/ requirements/
-# Scope the override to Omni; dependency source builds need their own versions.
-ENV VLLM_OMNI_VERSION_OVERRIDE=0.0.0
+ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0
 # Editable install: deps + project metadata are cached in this layer, while
 # the package itself is linked to APP_DIR (populated later by COPY --link).
 # Do NOT use a non-editable install here — that copies an empty stub into
 # site-packages and breaks ``import vllm_omni.entrypoints``.
 RUN uv pip install --system -e ".[dev]"
-# Fail the image build if dependency resolution replaced the selected vLLM.
-# The CPU builder has no NVIDIA driver; CUDA imports are checked at the start
-# of each GPU test job by .buildkite/cuda/scripts/check_vllm_runtime.py.
-RUN python3 -c "import vllm; assert vllm.__version__ == '${VLLM_VERSION}', vllm.__version__"
 
 # The final stage just copies the working dir on top of the
 # dependencies image. This uses --link so that we can skip

docker/Dockerfile.ci.dockerignore

@@ -1,6 +1,6 @@
 # NOTE: this is largely taken from vLLM
 
-# Exclude .git for CI builds — version is set via VLLM_OMNI_VERSION_OVERRIDE
+# Exclude .git for CI builds — version is set via SETUPTOOLS_SCM_PRETEND_VERSION
 .git
 .venv
 /build

docker/Dockerfile.cuda

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai:v0.31.0
+ARG BASE_IMAGE=vllm/vllm-openai:v0.30.0
 FROM ${BASE_IMAGE}
 
 ARG COMMON_WORKDIR=/app

docker/Dockerfile.npu

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_VERSION=v0.31.0
+ARG VLLM_ASCEND_VERSION=v0.30.0
 ARG VLLM_ASCEND_DEVICE_SUFFIX=
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_VERSION}${VLLM_ASCEND_DEVICE_SUFFIX}
 

docker/Dockerfile.npu.ci

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_VERSION=v0.31.0
+ARG VLLM_ASCEND_VERSION=v0.30.0
 ARG VLLM_ASCEND_DEVICE_SUFFIX=
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_VERSION}${VLLM_ASCEND_DEVICE_SUFFIX}
 

docker/Dockerfile.rocm

@@ -1,7 +1,7 @@
 # Keep the ROCm runtime aligned with the vLLM release targeted by
 # docker/Dockerfile.ci. A stale base image can build successfully while failing
 # at runtime when vLLM-Omni imports APIs added by the newer vLLM release.
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.31.0
+ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.30.0
 
 # Keep the dependency-heavy portion independent from the source tree. The
 # registry cache is keyed on these inputs, while the final source copy can be
@@ -12,11 +12,11 @@ FROM ${BASE_IMAGE} AS deps
 # BASE_IMAGE pins the default prebuilt runtime when USE_NIGHTLY_BUILD=0.
 # VLLM_VERSION_OR_COMMIT_HASH is consumed only by the opt-in source rebuild when
 # USE_NIGHTLY_BUILD=1. Keep both aligned with docker/Dockerfile.ci's
-# release version. This is enforced by tests/buildkite/test_rocm_dockerfile.py. An
+# VLLM_BASE_TAG. This is enforced by tests/buildkite/test_rocm_dockerfile.py. An
 # intentional custom source pin must update that regression in the same change,
 # and should return to the stable release as soon as one is available.
 ARG USE_NIGHTLY_BUILD=0
-ARG VLLM_VERSION_OR_COMMIT_HASH=v0.31.0
+ARG VLLM_VERSION_OR_COMMIT_HASH=v0.30.0
 ARG ARG_PYTORCH_ROCM_ARCH
 ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}}
 
@@ -75,8 +75,7 @@ WORKDIR ${COMMON_WORKDIR}/vllm-omni
 RUN mkdir -p vllm_omni && touch vllm_omni/__init__.py
 COPY pyproject.toml setup.py ./
 COPY requirements/ requirements/
-# Scope the override to Omni; dependency source builds need their own versions.
-ENV VLLM_OMNI_VERSION_OVERRIDE=0.0.0
+ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0
 
 RUN uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" \
         --no-cache-dir -e ".[dev]" --no-build-isolation && \

docker/Dockerfile.xpu

@@ -1,4 +1,4 @@
-# Target vLLM version: v0.31.0. Keep VLLM_VERSION aligned with the target vLLM
+# Target vLLM version: v0.30.0. Keep VLLM_VERSION aligned with the target vLLM
 # release on every rebase.
 #
 # This file layers vLLM-Omni on top of the published upstream XPU image, which
@@ -18,7 +18,7 @@
 #
 # .buildkite/intel/scripts/run-xpu-test.sh automates that pull-then-build fallback.
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai-xpu
-ARG VLLM_VERSION=v0.31.0
+ARG VLLM_VERSION=v0.30.0
 ARG VLLM_BASE=${VLLM_BASE_IMAGE}:${VLLM_VERSION}
 
 FROM ${VLLM_BASE} AS vllm-omni

docs/configuration/README.md

@@ -2,7 +2,7 @@
 
 This section lists the most common options for running vLLM-Omni.
 
-For options within a vLLM Engine, please refer to the [vLLM 0.31 configuration guide](https://docs.vllm.ai/en/v0.31.0/configuration/index.html).
+For options within a vLLM Engine, please refer to the [vLLM 0.30 configuration guide](https://docs.vllm.ai/en/v0.30.0/configuration/index.html).
 
 Each model defines fixed topology in a registered `PipelineConfig` and runtime
 overrides in a deploy YAML.

docs/configuration/environment_variables.md

@@ -130,7 +130,7 @@ settings.
 
 | Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
 | --- | --- | --- | --- | --- |
-| `VLLM_OMNI_FUSED_QK_NORM_ROPE_MIN_TOKENS` | Non-negative integer; unset or empty means each consumer's own default (Boogu-Image: `2048`; MammothModa2: `0`) | Token gate of the shared `fused_qk_norm_rope` op: a rotary table is packed for the fused kernel only when it spans at least this many positions (`B*S`); read at forward time | Environment-only. `0` = always fuse; a very large value disables the fused path. Any other value raises `ValueError` on the first forward. The crossover is host-dependent; re-benchmark before overriding. | Experimental performance control |
+| `VLLM_OMNI_FUSED_QK_NORM_ROPE_MIN_TOKENS` | Non-negative integer; unset or empty means each consumer's own default (Boogu-Image: `2048`) | Token gate of the shared `fused_qk_norm_rope` op: a rotary table is packed for the fused kernel only when it spans at least this many positions (`B*S`); read at forward time | Environment-only. `0` = always fuse; a very large value disables the fused path. Any other value raises `ValueError` on the first forward. The crossover is host-dependent; re-benchmark before overriding. | Experimental performance control |
 | `VLLM_OMNI_SKIP_NVFP4_NAN_CLAMP` | Boolean truthy spellings: `1`, `true`, `yes`, `on`; default false | ModelOpt NVFP4 compatibility patch; read when `vllm_omni.patch` imports | Environment-only escape hatch. Any other value means false. Set only to diagnose the upstream NaN-scale issue. | Diagnostic and temporary |
 | `VLLM_OMNI_USE_QUACK_FP8` | Boolean truthy spellings: `1`, `true`, `yes`, `on`; unset means hardware auto-detection | FP8 scaled matrix multiplication; evaluated when quack capability is selected | A set value overrides auto-detection. Any non-truthy value forces quack off. If quack cannot load, vLLM-Omni warns and falls back to FlashInfer. | Experimental performance control |
 
@@ -186,7 +186,7 @@ their keys only.
 ## Inherited vLLM variables
 
 vLLM-Omni also reads variables through its aligned vLLM dependency. Refer to
-the [vLLM 0.31 environment-variable reference](https://docs.vllm.ai/en/v0.31.0/configuration/env_vars.html)
+the [vLLM 0.30 environment-variable reference](https://docs.vllm.ai/en/v0.30.0/configuration/env_vars.html)
 for their definitions. This includes vLLM launch, cache, logging, plugin, ROCm,
 XPU, ModelScope, and FlashInfer workspace settings.
 
@@ -214,7 +214,7 @@ collection, examples, bug reports, or logs.
 
 ## Model-specific variables
 
-The inventory includes 93 variables read by a single model or pipeline family. They are
+The inventory includes 66 variables read by a single model or pipeline family. They are
 not listed as public usage options here because doing so would turn implementation
 escape hatches into an accidental compatibility contract.
 
@@ -223,10 +223,10 @@ Every audited model-specific name has a migration disposition in the
 
 | Disposition | Count | Required outcome |
 | --- | ---: | --- |
-| Promote | 64 | Move a stable setting into typed stage or model configuration. |
+| Promote | 38 | Move a stable setting into typed stage or model configuration. |
 | Request scope | 5 | Move request-varying behavior into a declared request-option schema. |
 | External | 0 | Retain only when a supported third-party library owns the contract. |
-| Internalize | 17 | Keep a debug or diagnostic switch out of public documentation and configuration. |
+| Internalize | 16 | Keep a debug or diagnostic switch out of public documentation and configuration. |
 | Deprecate/remove | 7 | Remove a compatibility escape hatch that has no continuing contract. |
 
 The disposition is a migration target, not a statement that the existing

docs/configuration/hsdp_pre_sharded_loading.md

@@ -1,22 +0,0 @@
-# Pre-sharded HSDP loading
-
-Use `--use-hsdp --hsdp-shard-size 8 --hsdp-weight-load-strategy pre_sharded`
-to shard transformer parameters before loading checkpoint values. Set the shard
-size to the available GPU count (or configure replication explicitly).
-The default strategy, `full`, retains the existing load-then-shard behavior.
-The same `hsdp_weight_load_strategy` option is available through Python and stage
-engine configuration.
-
-The pre-sharded strategy reads rank-local slices from ordinary Hugging Face
-safetensors checkpoints into FSDP-owned storage. No converted checkpoint is
-required. Cosmos 3 and Cosmos 3 Edge transformer blocks release their initialized
-parameter storage to meta as they are constructed; nonpersistent buffers retain
-their values.
-Encoders and VAEs follow their existing loading paths.
-
-This strategy requires the default diffusion pipeline loader, complete dedicated
-transformer checkpoint sources, and supported checkpoint key mappings. Tensor
-layouts must match the runtime layout; tensor transforms are unsupported.
-Checkpoint tensor dtypes must match runtime tensor dtypes at preflight; dtype conversion is unsupported.
-Quantization and LoRA are unsupported. Every worker needs checkpoint access.
-`num_weight_load_threads` controls the safetensors reader's threads per rank.

docs/configuration/stage_configs.md

@@ -35,7 +35,7 @@ Common `StagePipelineConfig` fields include:
 | `final_output` / `final_output_type` | Whether the stage produces a user-visible output and its modality. |
 | `owns_tokenizer` | Whether this stage owns the pipeline tokenizer. |
 | `model_arch`, `hf_config_name` | Stage-specific model architecture and nested HF config selector. |
-| `engine_output_type` | Canonical runtime output type: `text`, `image`, `audio`, `latent`, or `token_ids`. |
+| `engine_output_type` | Runtime output representation such as `text`, `latent`, or `audio`. |
 | `custom_process_input_func` | Processor applied to this stage's incoming payload. |
 | `custom_process_next_stage_input_func` | Processor used for full-payload handoff to the next stage. |
 | `async_chunk_process_next_stage_input_func` | Processor used for async chunk handoff. |
@@ -45,50 +45,6 @@ To add or change topology, define and register a new pipeline variant. Use
 deploy YAML only for runtime placement, resource sizing, connectors, and other
 deployment overrides.
 
-### Engine output types
-
-`engine_output_type` is parsed when the LLM stage's output processor is built
-during service initialization. Invalid values fail startup before requests
-are accepted; engine replicas may already have been initialized at that point
-and are cleaned up on failure.
-
-Use the exact lowercase names listed above. Combinations such as `text+audio`
-or `text,audio` remain supported, with every component canonical. Omitted or
-`None` values retain the default behavior (`TEXT` in the output processor).
-Empty strings, whitespace, case variants, and aliases are rejected.
-
-For existing custom pipelines or direct callers, replace old aliases explicitly:
-
-| Old value | Canonical value |
-| ------- | ------- |
-| `speech`, `wav`, `waveform` | `audio` |
-| `images`, `pixel_values`, `pixels` | `image` |
-| `latents` | `latent` |
-| `tokens` | `text` for text output, or `token_ids` for discrete IDs |
-
-`token_ids` is a separate type, not a `text` alias. It describes discrete
-vocabulary or codebook indices, distinct from continuous latent representations.
-Existing models using `latent` retain their historical payload contracts.
-GLM-Image's AR stage keeps `engine_output_type="token_ids"`; generated IDs still
-travel through `cumulative_token_ids`, and source-image IDs through
-`ids.prior_image`.
-
-Bare primary tensors and generic producer keys (`model_outputs` or `hidden`)
-use the declared type as their default output key, including `token_ids`.
-The normalizer does not inspect or convert their dtype. Distinct auxiliary
-outputs must use explicit field names: `model_outputs` and `hidden` both map
-to the same default key and must not represent two independent outputs.
-GLM-Image disables auxiliary hidden output for its default GPU execution.
-Some runner paths still attach it; that field is then named `token_ids` even
-though it contains hidden states. The GLM-Image bridge ignores this auxiliary
-field and reads generated IDs from `cumulative_token_ids` instead.
-
-Token-ID tensors use the existing dimension-0 accumulation default, subject to
-registered per-key overrides, and remain available after DELTA emissions.
-This is separate from the existing accumulation of `cumulative_token_ids`.
-`final_output_type` and Diffusion's `output_type` are separate settings and are
-unchanged by these rules.
-
 ## Deploy configuration schema
 
 The new deploy schema lives under `vllm_omni/deploy/` and is paired with a frozen `PipelineConfig` registered by the model's `pipeline.py`. Each deploy YAML has these top-level fields:
@@ -122,44 +78,7 @@ explicit CLI or stage override, or the downstream vLLM engine default.
 Note: for the diffusion path, an omitted `distributed_executor_backend` selects
 `uni` on a single GPU (in-process worker, no MessageQueue / `/dev/shm` output
 segments) and `mp` when `num_gpus > 1`. Set `mp` explicitly to keep a worker
-subprocess on one GPU. Set `ray` for Ray-managed single- or multi-node
-diffusion workers. `external_launcher` is not supported yet.
-
-The diffusion `ray` backend runs one worker actor per GPU and supports workers
-spanning Ray nodes. Enable it in the deploy YAML:
-
-```yaml
-distributed_executor_backend: ray
-```
-
-The executor reuses Ray when already initialized; otherwise it calls
-`ray.init()`, which handles cluster discovery or local startup and respects
-`RAY_ADDRESS`. There is no diffusion-specific address setting. Ray must be
-installed on every node.
-
-Ray diffusion workers receive the driver's `VLLM_*`, `OMNI_*`, `DIFFUSION_*`,
-`NCCL_*`, `TORCH_NCCL_*`, `UCX_*`, and Hugging Face environment settings,
-plus `PYTHONPATH`, `CUDA_LAUNCH_BLOCKING`, and `OMP_NUM_THREADS`.
-Explicit stage `runtime.env` entries are also forwarded and take precedence.
-Device visibility, rank/rendezvous settings, worker host addresses, and `RAY_*`
-variables are excluded so each worker retains its own assignment and identity.
-
-For multiple nodes, start the cluster before launching Omni. For example,
-with head IP `10.0.0.1`:
-
-```bash
-# On the head node:
-ray start --head --node-ip-address=10.0.0.1 --port=6379
-
-# On each additional worker node:
-ray start --address=10.0.0.1:6379
-
-# On the host launching Omni (MODEL and my_deploy.yaml are placeholders):
-RAY_ADDRESS=10.0.0.1:6379 vllm serve MODEL --omni --deploy-config my_deploy.yaml
-```
-
-Omni creates actors on the resources of that cluster; it does not start Ray
-on additional machines.
+subprocess on one GPU. `ray` / `external_launcher` are not fully supported yet.
 
 ### Stage-level runner selection
 
@@ -400,9 +319,7 @@ ROCm and MUSA. Select one of these deployment profiles:
 | `qwen3_tts_mrv2.yaml` | V2 | B1 | Explicit MRV2 profile (same runner selection as the default) |
 | `qwen3_tts_high_concurrency_mrv2.yaml` | V2 | B1, B2 | Opt-in throughput tuning |
 | `qwen3_tts_high_concurrency_mrv2_b4.yaml` | V2 | B1, B2, B3, B4 | Experimental throughput / buffered playback |
-| `qwen3_tts_high_concurrency_mrv2_single_gpu.yaml` | V2 | B1–B8 | Experimental two-stage deployment on one GPU, with MPS |
 | `qwen3_tts_high_concurrency.yaml` | V1 | Existing defaults | V1 high-concurrency control |
-| `qwen3_tts_fused_single_gpu.yaml` | V2 | N/A (in-Talker decoder) | Opt-in single-stage CUDA pipeline on one GPU |
 
 ```bash
 # MRV2 is the default; pass a copy with `model_runner: v1` to force V1.
@@ -433,73 +350,12 @@ supply `mtp_sampling_params` and `get_mtp_seed(sampling_params)` for model-local
 (with `mtp_sample_steps` and `mtp_sample_vocab_size`) as needed. Qwen3-TTS retains
 its existing `talker_mtp` entry point for V1.
 
-### Selecting one or two Qwen3-TTS stages
-
-Choose the deployment profile explicitly with `--deploy-config`. The default
-`qwen3_tts` pipeline retains two stages: the Talker publishes codec chunks to
-Code2Wav, which produces audio. Its stage devices can be placed on the same
-GPU or on separate GPUs. `qwen3_tts_fused` is a separate, opt-in pipeline for
-one CUDA GPU: the Talker owns the stateful codec decoder and emits PCM directly,
-without a Code2Wav engine or stage connector.
-
-```bash
-# Two stages on one GPU; the two-stage pipeline remains independently selectable.
-vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni \
-  --deploy-config vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2_single_gpu.yaml
-
-# One stage on one GPU; no inter-stage transport or Code2Wav scheduling.
-vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni \
-  --deploy-config vllm_omni/deploy/qwen3_tts_fused_single_gpu.yaml
-```
-
-The single-stage profile requires CUDA, V2, asynchronous chunks, TP=PP=1,
-an in-process worker and disabled prefix caching. Select the full profile;
-adding `talker_stream_decode: true` to a two-stage Talker is rejected because
-its latent output belongs to the Code2Wav input contract. Single-stage options
-live in `stages[].additional_config`; two-stage transport and first-frame
-options live in `connectors.*.extra`.
-
-The single-stage profile defaults `talker_stream_first_audio` to `false` for
-throughput. To opt into a separate first-frame delivery queue, save this
-overlay beside the bundled deployment files and pass its path to
-`--deploy-config`:
-
-```yaml
-base_config: qwen3_tts_fused_single_gpu.yaml
-stages:
-  - stage_id: 0
-    additional_config:
-      talker_stream_decode: true
-      talker_stream_first_audio: true
-      ref_code_context_frames: 72
-      code_predictor_kv_cache: true
-      code_predictor_fused_sampling: true
-      code_predictor_fused: true
-```
-
-This queue uses PCM from the single-stage decoder. The two-stage
-`talker_first_audio` option below uses its own first-frame decoder and does
-not require single-stage execution. The two options cannot be enabled
-together. Single-stage prefix-cache reset with
-`reset_running_requests=True` returns `False` while requests are active;
-wait for completion or abort them before retrying.
-
-Reference-cloning context is selected independently by each profile. The
-single-stage profile explicitly uses 72 reference frames, matching the
-`qwen3_tts.yaml` context boundary. A shorter
-`additional_config.ref_code_context_frames` (for example 25) can change
-timbre continuity even when WER is similar;
-the throughput measurements for CustomVoice do not qualify Base voice
-cloning quality. Reference layout, grouping and priming are model hooks;
-the shared worker does not select Qwen's context length or codebook layout.
-
 ### Qwen3-TTS first-frame delivery and rollback
 
 The standard `qwen3_tts.yaml`, `qwen3_tts_high_concurrency.yaml`,
 `qwen3_tts_mrv2.yaml` and `qwen3_tts_high_concurrency_mrv2.yaml` profiles
 enable the `talker_first_audio` connector option, as does the experimental
-two-stage `qwen3_tts_high_concurrency_mrv2_single_gpu.yaml` profile.
-This changes the default CUDA MRV2 streaming path:
+single-GPU profile. This changes the default CUDA MRV2 streaming path:
 residual prediction runs eagerly after the Talker sample, the Talker loads
 an additional first-frame decoder with its weights and CUDA graphs, and the
 orchestrator orders its audio before subsequent Code2Wav chunks.
@@ -595,55 +451,3 @@ for that comparison; this switch only controls automatic MPS management.
 
 MPS does not reserve a GPU. Use only assigned GPUs, explicitly place stages on
 the intended GPU, and warm the complete pipeline before measuring performance.
-
-## MOSS-TTS Local 1.5 with Model Runner V2
-
-`OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5` defaults to CUDA MRV2 using
-the shared runtime introduced for Qwen3-TTS. To select the C64 profile without
-MPS explicitly:
-
-```bash
-vllm serve OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 --omni \
-  --deploy-config vllm_omni/deploy/moss_tts_local_mrv2.yaml
-```
-
-This profile inherits the batching, codec graph buckets and 1-frame/15-frame
-chunk geometry from `moss_tts_local_v1.yaml`, and selects V2 for both the Local
-Talker and codec stages. The Local depth predictor exposes the MRV2 `mtp`
-capabilities while retaining its V1 `talker_mtp` implementation, sampling
-defaults and explicit request-seed handling. Codec chunks use the native data
-plane; the internal Talker retains its final-only orchestrator output policy.
-
-On CUDA, the profile bounds stage-0 prefill work to 512 tokens per iteration.
-The codec retains its own CUDA Graph capture/replay, including batch buckets
-through 64, but disables Inductor compilation (`compilation_config.mode: 0`)
-of the stateful decoder to avoid lengthy compilation at startup. This is
-independent of `enforce_eager`; the profile keeps `enforce_eager: false`.
-It explicitly selects `cudagraph_mode: FULL`, which works without Inductor;
-the default `FULL_AND_PIECEWISE` would otherwise normalize to `NONE` and clear
-the codec's capture buckets when compilation is disabled.
-
-For sustained C128 serving on a large-memory CUDA GPU, select the
-separate `moss_tts_local_mrv2_high_concurrency.yaml` profile. It uses 128
-stream slots per stage, a 32 GiB Talker KV budget, and the codec's
-`triton_slot` backend with Inductor compilation and codec-owned CUDA graphs.
-The bounded KV budget leaves room for codec state and graphs; it does not
-guarantee that 128 maximum-length prompts fit simultaneously. See the
-[MOSS recipe](gh-file:recipes/OpenMOSS/MOSS-TTS.md#local-15-mrv2-and-slot-attention)
-for activation, backend comparisons, memory requirements and benchmark commands.
-
-Omitting `--deploy-config` selects the C128 system profile on CUDA GPUs with
-at least 140 GiB memory and MPS available. Smaller GPUs or a failed memory
-query use C64 with utilization-based memory budgets; missing MPS selects
-C64 without MPS. Selecting `moss_tts_local.yaml` explicitly uses C64 with MPS.
-Use `moss_tts_local_v1.yaml` for the previous V1 deployment.
-NPU, XPU, ROCm and MUSA overrides retain V1. These profiles do not enable
-MRV2 for MOSS Delay, Realtime or Nano. Local 1.5 outputs 48 kHz stereo audio;
-set `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=48000` and
-`VLLM_OMNI_BENCH_AUDIO_CHANNELS=2` when benchmarking raw PCM.
-
-Event-driven orchestration remains independently selectable with
-`VLLM_OMNI_EVENT_DRIVEN_ORCH=0` or `1`. Keep the runner and deployment identical
-when comparing these modes. Model-runner selection does not change the
-orchestration default. MPS, codec dispatch and graph settings belong to the
-deployment profile; the automatic CUDA default selects MPS when available.

docs/design/feature/attention_backend_selection.md

@@ -4,9 +4,6 @@ This document defines the selection and extension contract for diffusion
 attention backends. User-facing backend choices, installation, and tuning are
 in the [attention backend guides](../../user_guide/diffusion/attention_backends.md).
 
-The [execution contract PoC](attention_execution_contract_poc.md) describes the
-experimental post-selection capability and lifecycle contract from RFC #7226.
-
 ## Scope
 
 The contract applies to DiT and other diffusion attention layers. It is

docs/design/feature/attention_execution_contract_poc.md

@@ -1,154 +0,0 @@
-# Diffusion attention execution contract PoC
-
-This prototype accompanies [RFC #7226](https://github.com/vllm-project/vllm-omni/issues/7226)
-and extends the [attention selection design](attention_backend_selection.md).
-It demonstrates path-specific capabilities with dense BF16 FA4, NPU/ROCm
-routing examples, and tensor-state lifetime with a test-only attention module.
-
-## Execution contract
-
-`ExecutionContext` describes the requested execution path. `ExecutionPathResult`
-reports its identity, support status and reason, and compilation mode.
-`AttentionBackend.resolve_capabilities()` provides conservative pre-construction
-results. After initialization, `Attention.resolve_execution_path()` supplies the
-active parallel, paged-KV, and HSDP context; the backend combines it with the
-selected kernel and normalized metadata.
-
-Resolution runs outside compiled execution. FA4 reports `SUPPORTED` and
-`CUSTOM_OP` for dense, noncausal BF16 without parallel or HSDP boundaries when
-its kernel accepts the head dimensions. Dimension validation delegates to FA4's
-architecture-specific rules through `backends/utils/fa.py`. Kernel rejections
-report `UNSUPPORTED` with an actionable reason. Missing private validators and
-other unmigrated paths report `UNMIGRATED` with advisory `EAGER_ONLY` defaults.
-`requested_support()` checks a fullgraph request; it does not enforce selection
-or change existing execution.
-
-FA4 execution uses an opaque custom op. Its fake output preserves Q's batch,
-sequence, and head count and uses V's head dimension. Output is contiguous even
-when Q is noncontiguous. Metadata normalization is
-shared with dispatch. Producers must update published mask semantics when masks
-change; unpublished masks remain runtime-dependent to avoid synchronization.
-Callers resolve again when execution metadata changes.
-
-## NPU and ROCm worked examples
-
-The same `FLASH_ATTN` backend resolves MindIE paths on NPU and AITER paths on
-ROCm. Both examples report `UNMIGRATED`: their route is inspected, while hardware
-correctness and compilation remain unvalidated. `EAGER_ONLY` is an advisory
-default, not a measured compiler limitation. Existing execution is unchanged.
-
-| Input | NPU / MindIE | ROCm / AITER |
-| --- | --- | --- |
-| Dense | `npu_dense` | `rocm_dense` |
-| Padding mask | `npu_masked`: expand the key mask to `[B, 1, Q, K]` | `rocm_masked_varlen`: unpad, call varlen attention, then restore rows |
-| Packed inputs | Opt-in `[real, pad]` contract; choose varlen or prefix-KV slicing from the environment | Forward complete packed-document boundaries to varlen attention |
-| Incomplete packing | Rebuild a mask when possible; otherwise reject before the kernel | Shared CUDA-like normalization rejects incomplete metadata |
-| Unpublished mask semantics | Route to the masked wrapper | Keep resolution runtime-dependent without reading mask values |
-
-NPU resolution reuses `_resolve_packed_seq_npu`; ROCm reuses the metadata
-normalization used by dispatch. AITER identity comes from initialization rather
-than caller-provided context. Quantized, paged, piecewise, and parallel paths
-remain outside these worked examples.
-
-The new L1 tests use CPU tensors and replace only vendor kernel calls. They
-check path identity, forwarded masks/boundaries, fallback rejection, and
-conservative fullgraph-request handling. They do not test vendor numerics or
-compiler support. Existing L1 CI already collects this test file.
-
-```bash
-python -m pytest tests/diffusion/attention/test_flash_attn.py \
-  -k 'npu_contract or rocm_contract' -m 'core_model and cpu' -q
-```
-
-### Hardware validation required before migration
-
-On an Ascend/MindIE or ROCm/AITER runner, record device, driver, PyTorch, vendor
-library, and compiler versions, then validate the corresponding rows above:
-
-1. Compare eager outputs with FP32 SDPA for dense and padding-mask inputs,
-   including unequal Q/K lengths. Compare valid query rows; do not assume both
-   platforms define padded-query outputs identically.
-2. Compare packed outputs with separate per-document references. On NPU, cover
-   both varlen and laser prefix-slicing modes, plus mask reconstruction and the
-   missing-mask error. A padding fallback does not establish multi-document
-   isolation. On ROCm, cover multiple real documents.
-3. Check output shape, dtype, device, finite values, and unchanged inputs.
-   Start BF16 comparisons at `atol=rtol=1e-2` and have the platform maintainer
-   confirm the tolerance against its reference tests.
-4. Run `torch.compile(fullgraph=True, dynamic=True)` with the platform's supported
-   compiler over multiple sequence lengths. Record graph breaks and recompiles;
-   choose `TRACEABLE`, `CUSTOM_OP`, or verified `EAGER_ONLY` from that evidence.
-   A custom-op path additionally needs schema/fake checks and repeated replay.
-
-Promote only the paths validated on that platform. These hardware checks have
-not been run locally; ROCm/NPU fullgraph support is not claimed.
-
-## State lifetime example
-
-`test_attention_state_lifetime.py` prepares an owned query-scale tensor eagerly
-and passes it explicitly to an opaque attention op. The compiled module retains
-its state after the caller drops its reference and releases ownership when the
-compiled callable is deleted. Equivalent instances reuse a graph with their own
-state values.
-
-This example tests ownership and release, dynamic replay across instances,
-custom-op schema/fake behavior, and Inductor execution. It adds no production
-backend or state registry. Planning keys, caching, fallback-policy declarations,
-opaque handles, CUDA graph capture, and TRTLLM integration are deferred.
-
-## Validation
-
-```bash
-python -m pytest tests/diffusion/attention/test_flash_attn_compile.py \
-  tests/diffusion/attention/test_flash_attn.py \
-  tests/diffusion/attention/test_attention_capabilities.py \
-  tests/diffusion/attention/test_attention_state_lifetime.py -q -rs
-```
-
-The focused suite covers capability decisions and metadata changes, existing
-padding/mask regressions, tensor-state lifetime, and real FA4 fullgraph execution
-against FP32 SDPA and eager FA4. Representative Q/K and V dimensions are
-(32, 32), (64, 64), (80, 48), (192, 128), and (256, 256), with batches 1 and 2
-and sequence lengths up to 1,024. Invalid dimensions are checked against actual
-kernel errors. These samples exercise the contract rather than define support.
-BF16 comparisons use `atol=rtol=1e-2`; `torch.library.opcheck` checks schema and
-fake-output correctness.
-
-Single-graph reuse is asserted in the tensor-state test. The FA4 numerical tests
-retain compiler caches across shapes within each case but do not require one
-graph across different batch sizes, Q/K length equality, or head dimensions.
-
-Tested environment: GB300, PyTorch `2.13.0+cu130`, FA4 `4.0.0b18`, CUTLASS DSL
-`4.6.2` with CUDA 13 libraries, Quack `0.6.4`, and TVM FFI `0.1.11`.
-Real-kernel tests require CUDA and CuTe FA4; their availability is checked at runtime.
-
-Before the rebase, the focused suite passed all 97 tests without skips, including
-11 CPU NPU/ROCm routing tests. After rebasing onto upstream `e3be42e05`, the
-expanded suite reports 99 passed and 11 failed in the local vLLM 0.28 environment.
-All 11 failures are import errors: upstream now requires `compute_layout_strides`
-from vLLM, which this environment lacks. Both noncontiguous FA4 regression cases
-and real-kernel schema/fake checks pass. The full suite must be rerun with the
-vLLM version required by upstream. NPU/ROCm device numerics and compilation remain
-unvalidated. Pre-commit passes with the CI hook skips. Upstream PyTorch/CUTLASS
-warnings remain.
-
-## TRTLLM dense execution
-
-`TRTLLM_ATTN` declares `SUPPORTED` / `CUSTOM_OP` for noncausal dense BF16 on
-B200/GB200 (SM100) and B300/GB300 (SM103), head dimension 128, and equal Q/K/V
-head counts, without parallel, paged-KV, piecewise, or HSDP boundaries. Pre-construction, SAGE, skip-softmax, packed,
-and other unverified paths remain `UNMIGRATED`. Resolution and dispatch share
-metadata validation; workspace mutation is explicit in the custom-op schema.
-
-CPU contract tests cover fullgraph replay and schema/fake consistency using a
-substituted dispatcher. Real-kernel validation requires Blackwell with FlashInfer:
-
-```bash
-python -m pytest tests/diffusion/attention/test_trtllm_attn.py \
-  -k dense_contract_fullgraph_matches_sdpa -q -rs
-```
-
-These hardware tests compare eager/compiled output with FP32 SDPA and check
-schema/fake agreement. The TRTLLM and contract suite passed on Blackwell before the final architecture
-restriction was added: 61 passed, no skips (19 dependency deprecation warnings).
-The SM100/SM103 restriction is additionally covered by CPU contract tests.

docs/design/feature/sequence_parallel.md

@@ -281,10 +281,6 @@ NOTE: be careful to test adequately when refactoring classes that take this styl
 | `expected_dims` | int \| None | Expected tensor rank for validation (optional) |
 | `split_output` | bool | `False`: shard **input** params; `True`: shard **output** tensors |
 | `auto_pad` | bool | Auto-pad if sequence not divisible by world_size (default: `False`) |
-| `clone_shard` | bool | Give multi-rank shards their own contiguous storage (default: `False`); single-rank inputs are unchanged |
-
-With `clone_shard=True`, memory savings require callers to drop their references to the full tensor.
-Other references to the same storage also keep the full allocation alive.
 
 **SequenceParallelOutput Parameters:**
 

docs/design/fullduplex.md

@@ -220,7 +220,6 @@ vllm_omni/
 │       │                            DUPLEX_REALTIME_CAPABILITIES, duplex_response_format
 │       ├── events.py                re-export shim (DuplexEvent = RealtimeEvent) + the runner's
 │       │                            epoch-filter sets DOMAIN_TERMINAL_EVENTS / MODEL_OUTPUT_EVENTS
-│       ├── delivery.py              DuplexOutputBuffer (bounded output shared with the session handle)
 │       ├── realtime_events.py       RealtimeProjectionState: internal event -> typed events
 │       ├── messages.py              queue envelopes (Open/Close/Resume/Touch/Command/Result/Event), DuplexSessionError
 │       ├── config.py                DuplexSessionConfig, DuplexCapabilities, ResponseCreateOptions
@@ -285,8 +284,7 @@ command  handle.submit(DuplexCommand)
          -> engine.submit_command_async -> DuplexSessionCommandMessage on the request queue [one-way]
          -> DuplexSessionManager.dispatch: unknown_session / input_backpressure checks, then runner mailbox
 output   DuplexOrchestrator._intercept_stage_output -> runner.on_stage_output -> mailbox -> typed events
-         -> session's bounded output buffer -> handle.events(); closure and errors without a live session
-            use output_sink (DuplexSessionEventMessage) -> DuplexOmni._route_engine_message
+         -> output_sink (DuplexSessionEventMessage) -> DuplexOmni._route_engine_message -> handle.events()
 detach   DuplexOmni.detach_session(expected_lease_generation) -> touch(DETACH): engine-owned disconnect
          grace, refused for a lease newer than the one the caller held; expiry -> SessionExpired
 resume   DuplexOmni.resume_session(expected_lease_generation) -> lease CAS keyed by the control id (a replay
@@ -305,9 +303,7 @@ reap     DuplexSessionManager.reaper_loop: idle TTL / disconnect grace expiry, c
 
 ### Concurrency and ordering model of the runner
 
-The runner's session state is owned by the orchestrator asyncio loop and
-needs no lock. The output buffer shared with the caller thread uses a lock
-for queue updates and the final validity check before delivery.
+Everything below runs on the orchestrator asyncio loop; there is no lock.
 
 - **Single writer per session.** `DuplexEngineSession` is mutated only by its
   runner. Inputs reach the runner through one mailbox in this order: client
@@ -336,15 +332,13 @@ for queue updates and the final validity check before delivery.
   on one loop in program order, nothing can be emitted for an old epoch after
   its terminal event; `runner.emit()` still drops a terminal carrying a stale
   epoch and stamps `epoch` on every event for clients that filter
-  defensively. In this experimental follow-up, public output goes directly to
-  one bounded buffer shared by the engine and `DuplexSessionHandle`. Accepted
-  cancellation removes queued `AudioDelta` events for that response through
-  the cancelled epoch. Text increments, audio/transcript completion markers,
-  other responses and normal completion retain FIFO order. The consumer also
-  checks an event already taken from the buffer: WebSocket delivery repeats
-  that check under the connection lock, before sequencing and journaling.
-  Already sequenced replay entries are unchanged; already delivered audio
-  still requires the client to stop its own playback.
+  defensively. This is a statement about order, not about latency: the engine
+  output queue and each `DuplexSessionHandle` outbox are unbounded and FIFO,
+  so a cancellation is delivered behind whatever audio was already emitted for
+  the response it cancels. A client that must stop quickly cancels its own
+  playback on `response.done` / `audio.cancelled` rather than waiting for the
+  stream to drain. Bounding those buffers and letting an accepted invalidation
+  skip undelivered media is left to the follow-up RFC.
 - **Backpressure before the mailbox.** `DuplexSessionManager.dispatch`
   checks `max_pending_input_bytes_per_session` and reserves a pending turn for
   `Commit` before the put; a rejected command is answered with
@@ -355,46 +349,6 @@ for queue updates and the final validity check before delivery.
   `ResponseDone.stage_metrics` and `metadata.vllm_omni.stage_metrics` on the
   wire keep their meaning.
 
-### Experimental output limits
-
-`max_pending_output_bytes_per_session` defaults to 2 MiB and
-`max_pending_output_events_per_session` to 512. The byte limit counts compact
-Realtime JSON, including base64 audio, not decoded PCM or Python object
-overhead. Audio is already base64-encoded by the producer; counting its length
-avoids serializing or decoding that large field again. Neither default has
-been tuned from a performance study.
-
-Errors and response endings have an additional 64 KiB / eight-event reserve
-and retain FIFO order. One final session-closure notification has an independent
-slot, so an exhausted reserve cannot prevent closure or break delivery to
-other sessions. If that reserve also fills, an error or response-ending
-notification may be omitted; the final session-closure notification still
-has its independent slot. Oversized close details (over 4 KiB with generated identity)
-are replaced by a compact reason, preserving terminal type and identity.
-Late writes after closure are ignored; a locally ended iterator is not reopened
-by a late terminal, and a failed open closes the abandoned handle. Events other
-than errors are not forwarded once their session's buffer has been removed.
-One event held by the consumer is outside the queued budget.
-
-The overflow policy fails the active response without committing its partial
-text to history and closes only the affected session. If overflow occurs while
-a completion batch is being delivered, its not-yet-queued `ResponseDone` is
-converted to failure with its identity and output preserved; any history entry
-for that response is removed. An ending already accepted by the buffer is not
-replaced or followed by a second ending. After overflow, newly
-emitted ordinary events are suppressed, including text/audio completion,
-content-part and output-item completion markers. Previously queued non-audio
-events retain FIFO order; error and failed response-ending notifications may
-use the reserve before the independent closure notification. This differs from
-accepted cancellation, which removes only matching audio. See the
-[public error and closure fields](../serving/realtime_duplex_api.md#output-limits-and-slow-consumers).
-Response-only recovery is not implemented. These limits do not bound the runner's raw stage
-output mailbox, replay journal, or client playback queues. The shared buffer
-requires the current same-process, separate-thread engine arrangement; it is
-not a cross-process delivery protocol. Direct handle iteration and WebSocket
-delivery share buffer validity, but end-to-end client behavior still needs
-independent real-model verification.
-
 ## Orchestrator seams
 
 `OrchestratorBase` keeps the generic machinery (request handler skeleton,

docs/design/index.md

@@ -68,7 +68,6 @@ guides list selectable backends, platform defaults, installation, and tuning.
 The design contracts separate selection mechanics from backend algorithms:
 
 - [Attention Backend Selection](feature/attention_backend_selection.md)
-- [Attention Execution Contract PoC](feature/attention_execution_contract_poc.md)
 - [Skip-Softmax](feature/skip_softmax.md)
 
 #### CPU offloading
@@ -117,5 +116,4 @@ The pre-#5137 pages are preserved in the
 [legacy module archive](module/archive/README.md) for historical reference and
 are not active design contracts.
 
-- [Qwen3-Omni MRv2 performance](qwen3_omni_mrv2_performance.md)
 - [MiniCPM-o 4.5 turn-mode MRv2 performance](minicpm_o45_mrv2_performance.md)

docs/design/minicpm_o45_mrv2_performance.md

@@ -12,25 +12,11 @@ require new end-to-end performance measurements after the mainline integration.
 Previously reported numbers do not describe this revised codec backend.
 
 The generic MRv2 profile and the opt-in native V1 duplex H200 profile enable
-ordinary TF32 for CFM DiT GEMMs within Code2Wav forward/capture only
-(`torch.backends.cuda.matmul.allow_tf32` on dense QKV/MLP, Triton
-`input_precision="tf32"` on tiled attention). This is not compensated TF32x3.
-The previous process matmul policy is restored afterwards. cuDNN's TF32 policy
-is independent. HiFT stays IEEE FP32. TF32 changes rounding.
+TF32 matmuls within Code2Wav forward/capture only; the previous process policy
+is restored afterwards. cuDNN's TF32 policy is independent. TF32 changes rounding.
 Duplex keeps the mainline V1 session path. Turn results do not establish duplex
 performance or interruption correctness.
 
 The shared asynchronous output snapshot and batched Talker preprocessing also
 apply to V1 when async chunking/scheduling is enabled. Default V1 behavior must
 therefore be included in end-to-end regression validation.
-
-CFM graph execution defaults to the fused DiT body on CUDA when TF32 is
-allowed. NVIDIA SM80+ uses tiled TF32 attention; older CUDA devices use the
-fused body's SDPA fallback. Unsupported DiT layouts or non-FP32 attention
-caches retain the original body. Other platforms retain their existing default.
-Set `cfm_fused_body: false` in connector `extra` to restore the original body.
-Disabling TF32 with `token2wav_allow_tf32: false` or
-`MINICPMO_CODE2WAV_TF32=off` also disables the default fusion choice; an explicit
-`cfm_fused_body: true` overrides that choice and may use TF32 attention.
-Slot pooling and row-offset merging remain opt-in. Fusion changes floating-point
-rounding and does not promise identical waveforms.

docs/design/minicpmo_encoder_cuda_graph.md

@@ -1,61 +0,0 @@
-# MiniCPM-o 4.5 input encoder CUDA graphs
-
-The vision transformer and stateless audio encoder reuse vLLM's encoder graph
-manager for exact input shapes. Each encoder retains at most four graphs by
-default, and captures on the second call with the same shape and CUDA stream.
-Startup profiling can consume slots. Graphs are never evicted: once the cache
-is full, other shapes run eagerly for the lifetime of the model.
-
-Use `--hf-overrides` to configure both encoders:
-
-```json
-{
-  "encoder_cuda_graph": true,
-  "encoder_cuda_graph_max_graphs": 4,
-  "encoder_cuda_graph_min_capture_calls": 2,
-  "encoder_cuda_graph_min_free_bytes": 1073741824,
-  "encoder_cuda_graph_share_pools": true
-}
-```
-
-`max_graphs` is a nonnegative integer per encoder; zero disables capture.
-`min_capture_calls` is an integer of at least two. Increasing it reduces capture
-of short-lived shapes. Admission history is bounded to four times `max_graphs`,
-so an evicted history entry must accumulate its calls again. Increasing the
-graph cap can improve coverage but retains more GPU memory. Existing graphs
-are not replaced, avoiding repeated capture costs when traffic changes.
-
-By default, graphs in the same encoder on the same device and replay stream
-share a memory pool and capture stream. Different encoders and replay streams
-remain isolated. Graph inputs and output buffers are allocated outside capture;
-only transient capture allocations share the pool. Returned outputs remain
-cloned. A late capture waits for earlier replay work and output clones before
-reusing the pool. Set `encoder_cuda_graph_share_pools` to `false` for an A/B
-comparison with a separate pool and capture stream per graph.
-
-The graph count is **not a byte limit**. Sharing pools reuses intermediate
-storage but does not eliminate per-shape static input and output buffers.
-Before a new capture, the adapter checks device-free memory against
-`min_free_bytes`, a nonnegative integer (default: 1 GiB; zero disables the check).
-Below this floor the call runs eagerly, and a later call may try admission
-again. Existing graphs can still replay. This check is conservative about
-reusable allocator memory and does not predict capture allocations, reserve
-memory, or guarantee that capture fits. Size the floor for the deployment;
-disable encoder graphs when their additional memory cannot be accommodated.
-
-Each adapter exposes `get_cumulative_stats()`: graph hits, total misses, hit
-rate, graph count, and separate ineligible, capacity, warmup and memory misses.
-These include calls rejected before the upstream manager, so a full cache's
-lost coverage is visible. Vision uses `vpm._encoder_graph`; audio uses
-`_audio_encoder_graph` on the model. The counters are per adapter, not global.
-
-Capture failures propagate the original exception and make that adapter
-unusable. Subsequent calls raise without retrying capture or running CUDA eager
-work. Restart the worker, optionally with encoder graphs disabled. This is a
-fatal contract, not automatic recovery: a failed CUDA capture can leave the
-device context unusable. `capture_failures` records this terminal state.
-
-`--enforce-eager` always disables these graphs. CPU, grad/autocast, stateful
-streaming audio and the other model-specific eager fallbacks remain unchanged.
-The adapter clones graph outputs through upstream postprocessing so retained
-embeddings survive later replays.

docs/design/module/ar_runtime.md

@@ -47,29 +47,6 @@ adapter, not injected through unrelated scheduler state.
 
 **Rule:** Workers and model runners MUST NOT implement cross-stage routing.
 
-## Token-only output payloads
-
-An AR stage whose downstream decoder consumes sampled tokens rather than
-hidden states can set `omni_pooler_payload_include_hidden = False`. Hidden
-states remain available for logits and sampling; prefill conditioning and
-other required multimodal outputs still cross the stage boundary.
-
-A chunk processor that needs IDs on steps without a tensor payload declares
-`requires_token_updates = True`. The scheduler then enqueues nonempty sampled
-IDs while preserving final-stage exclusions and the processor's EOF handling.
-Processors without this declaration retain their tensor-payload trigger.
-
-When hidden and multimodal payloads are absent, the shared runner skips empty
-per-request payload construction. Prefix-cache merges and pending model
-postprocess retain the normal path. This policy is independent of whether
-output materialization runs inline or in the background.
-
-Inline construction consumes query offsets, scheduled-token counts and
-scheduler metadata before the next execution can reuse them. Background
-construction snapshots those inputs. Both paths reuse the request IDs already
-copied by upstream bookkeeping; connector collection stays on the main thread
-because it can run a tensor-parallel collective.
-
 ## Safe-change guide
 
 Test request lifecycle, abort, cache state, and every affected worker execution

docs/design/module/diffusion/diffusion_runtime.md

@@ -36,7 +36,6 @@ validation_paths:
   - tests/diffusion/test_diffusion_worker.py
   - tests/diffusion/test_multiproc_engine_concurrency.py
   - tests/diffusion/test_uniproc_executor.py
-  - tests/diffusion/test_ray_executor.py
   - tests/diffusion/test_result_pump.py
   - tests/diffusion/test_async_output_worker.py
   - tests/diffusion/test_diffusion_ipc.py
@@ -97,10 +96,8 @@ The executor then chooses how the worker runs:
   RPC.
 - **`mp`** (default when `num_gpus > 1`, or when set explicitly):
   `MultiprocDiffusionExecutor` starts one `WorkerProc` per device.
-- **`ray`** (explicit): `RayDiffusionExecutor` starts one GPU actor per worker
-  across one or more Ray nodes.
 
-In all backends, each worker owns its device, distributed state,
+Either way, each worker owns its device, distributed state,
 `DiffusionWorker`, `DiffusionModelRunner`, and pipeline instance.
 
 ```mermaid
@@ -111,7 +108,7 @@ flowchart TB
         stageProc["StageDiffusionProc<br/>stage transport and lifecycle"]
         engine["DiffusionEngine<br/>admission and control loop"]
         scheduler["RequestScheduler or StepScheduler<br/>request state and policy"]
-        executor["DiffusionExecutor<br/>uni, mp, or ray backend"]
+        executor["DiffusionExecutor<br/>uni or mp backend"]
         streams["Per-request output streams"]
 
         stageProc --> engine
@@ -135,8 +132,6 @@ flowchart TB
     client -.->|"inline: direct call"| engine
     executor -->|"mp: broadcast / result queues"| workerProc
     executor -.->|"uni: in-process call"| worker
-    executor -->|"ray: actor RPC"| rayActor["RayDiffusionWorkerWrapper"]
-    rayActor --> worker
     streams -->|"process-backed"| stageProc
     stageProc -->|"ZMQ results"| client
     streams -.->|"inline"| client
@@ -159,7 +154,6 @@ flowchart TB
 | `DiffusionExecutor` | Execution backend contract, worker RPC, health, shutdown | Admission and request-state transitions |
 | `UniProcDiffusionExecutor` | Single in-process worker for `num_gpus == 1`; no IPC or async output pump | Multi-GPU execution |
 | `MultiprocDiffusionExecutor` | Worker processes, shared-memory message queues, result dispatch, worker monitoring | In-process single-GPU path |
-| `RayDiffusionExecutor` | GPU actors, placement groups, remote RPC, actor monitoring | Local worker subprocesses and shared-memory IPC |
 | `WorkerProc` | One `mp` worker process's IPC loop and reply rules | Scheduling policy; unused by `uni` |
 | `DiffusionWorker` | Device/distributed setup, LoRA activation, profiling, sleep/wake, runner delegation | Request admission |
 | `DiffusionModelRunner` | Pipeline loading, request-local model state, cache/compile setup, request or step execution | Queueing and cross-stage routing |
@@ -225,12 +219,12 @@ uses fused `execute_model_batch()` and requires the pipeline to declare
 request-batch support.
 
 **Distributed layerwise offload with AllGather (DLO DP concurrency)** is a
-separate dispatch path supported by `mp` and `ray`. It activates when
+separate multiproc dispatch path. It activates when
 `data_parallel_size > 1`, `enable_distributed_layerwise_offload` is set, and
 `dlo_use_allgather` is true. The engine then sets `dp_concurrent = True` and
 raises `scheduler.max_num_running_reqs` to `dp_size`, overriding the
 `max_num_seqs` value from `initialize()`. The scheduler still emits a
-multi-request `DiffusionSchedulerOutput`; these executors route that
+multi-request `DiffusionSchedulerOutput`; the multiproc executor routes that
 wave through `execute_request()` instead of fused `execute_model_batch()`.
 
 Optional admission coalescing can fill a `dp_size` wave: when
@@ -239,7 +233,7 @@ the first schedule of a wave (under `dp_concurrent`, the stable window is
 `min(0.3s, wait/2)`). With the default `request_batch_max_wait_ms == 0`,
 there is no wait.
 
-Before dispatch, both backends reject waves
+Before dispatch, `MultiprocDiffusionExecutor.execute_request()` rejects waves
 whose requests differ in sampling-parameter compatibility or `extra_args`
 (AllGather requires every DP rank to follow the same forward schedule). Each
 worker rank then picks one envelope from the wave:
@@ -303,9 +297,8 @@ head-of-line blocking.
 | --- | --- | --- |
 | `uni` | Default for `num_gpus == 1`, or set explicitly | One in-process worker |
 | `mp` | Default for `num_gpus > 1`, or set explicitly | One worker process per device |
-| `ray` | Set explicitly | One GPU actor per worker across Ray nodes |
 
-The external-launcher diffusion backend is not implemented. A custom
+Ray and external-launcher diffusion backends are not implemented. A custom
 `DiffusionExecutor` subclass or import path is also accepted.
 
 ### Uniproc backend
@@ -344,27 +337,11 @@ result path and does not start those pumps.
 See [Async diffusion output](../../feature/async_diffusion_output.md) for the
 multiproc request-mode timeline.
 
-### Ray backend
-
-`RayDiffusionExecutor` reuses initialized Ray or calls `ray.init()` (respecting
-`RAY_ADDRESS`). It reuses the current placement group or reserves one GPU
-bundle per worker with `PACK` placement. Ray owns GPU placement; PyTorch
-owns distributed collectives through a shared TCP rendezvous. Global ranks
-are grouped by host, while each actor uses local device 0.
-
-Request batches, step execution, and DLO DP waves use actor RPCs with
-rank-aware replies. Returned tensors move to CPU before Ray serialization;
-this backend does not use the multiproc async output pumps. Driver inference
-settings and explicit stage `runtime.env` are forwarded, excluding worker
-identity and device assignment. See [Stage configuration](../../../configuration/stage_configs.md)
-for cluster setup and environment details.
-
 ## Worker and runner boundary
 
 On the `mp` path, `WorkerProc` receives messages and calls methods through
 `WorkerWrapperBase`. On the `uni` path, the executor owns that wrapper
-directly. Ray actors also construct `WorkerWrapperBase`. In all cases,
-`DiffusionWorker` handles the parts tied to a device:
+directly. In both cases, `DiffusionWorker` handles the parts tied to a device:
 distributed initialization, model-runner construction, LoRA activation,
 profiling, and memory sleep/wake. It delegates actual model work to
 `DiffusionModelRunner`.
@@ -422,10 +399,6 @@ streams with an error, closes scheduler state, and shuts down the executor.
   async-output futures as failed, then clears them. Completed async outputs
   cached for later `wait_output_ready()` calls are dropped with the executor
   object.
-- **`ray`**: the executor gives actors up to five seconds to shut down, then
-  kills them and removes placement groups it owns. Actor death is monitored
-  even while idle; actor errors and RPC timeouts fail the group and notify the
-  stage. The stage client allows subprocess cleanup before signal fallback.
 - **`uni`**: the executor shuts down the in-process worker, drops the final
   model reference, and empties the accelerator cache so a later engine can reuse
   the device.
@@ -434,7 +407,7 @@ streams with an error, closes scheduler state, and shuts down the executor.
 
     Do not continue using a worker group after a fatal collective timeout or
     unexpected worker exit. Distributed state may be incomplete. The
-    `mp` and `ray` executors deliberately fail closed so the stage can restart.
+    multiprocess executor deliberately fails closed so the stage can restart.
     The uniproc executor fails closed when the accelerator context itself is
     poisoned.
 
@@ -444,11 +417,10 @@ streams with an error, closes scheduler state, and shuts down the executor.
   `diffusion_model_runner_cls` configuration still wins.
 - Tests and custom engine integrations may inject a `BaseScheduler` subclass.
   `SchedulerInterface` remains only as a deprecated compatibility name.
-- `distributed_executor_backend` may be `"uni"`, `"mp"`, `"ray"`, a custom `DiffusionExecutor`
+- `distributed_executor_backend` may be `"uni"`, `"mp"`, a custom `DiffusionExecutor`
   subclass, or an import path. A backend must preserve scheduler output,
   request identity, health, and cleanup contracts. Use `"mp"` when you need
-  local process isolation or multi-GPU, or `"ray"` for workers across nodes;
-  `"uni"` is the single-GPU default.
+  process isolation or multi-GPU; `"uni"` is the single-GPU default.
 - Worker extensions go through `WorkerWrapperBase`. They add worker methods;
   they do not become a second scheduler or request lifecycle.
 
@@ -494,7 +466,7 @@ Test the smallest affected slice, then cover its neighboring boundary:
 | --- | --- |
 | Admission or state transitions | `test_diffusion_scheduler.py`, including duplicate IDs, compatibility, abort, and missing output |
 | Engine loop or output delivery | `test_diffusion_engine.py`, `test_diffusion_engine_cleanup.py`, `test_diffusion_engine_rpc_routing.py` |
-| Executor or IPC | `test_multiproc_engine_concurrency.py`, `test_uniproc_executor.py`, `test_ray_executor.py`, `test_result_pump.py`, `test_diffusion_ipc.py`, `test_async_output_worker.py` |
+| Executor or IPC | `test_multiproc_engine_concurrency.py`, `test_uniproc_executor.py`, `test_result_pump.py`, `test_diffusion_ipc.py`, `test_async_output_worker.py` |
 | Worker or runner | `test_diffusion_worker.py`, `test_diffusion_model_runner.py` |
 | Stage boundary | `test_stage_diffusion_proc.py`, `test_inline_stage_diffusion_client.py` |
 | Warmup or streaming | `test_diffusion_engine_dummy_run.py`, `test_diffusion_streaming_output.py` |

docs/design/qwen3_omni_mrv2_performance.md

@@ -1,22 +0,0 @@
-# Qwen3-Omni MRv2 performance
-
-This model adaptation builds on #8184's existing stage runtime and output
-contracts. It adds Talker CPU metadata parsing, sampled-token embedding
-handoff, an optional first-frame decoder, length-grouped Code2Wav graph
-buckets, and opt-in incremental short-KV code prediction. The H200 profile
-also enables conservative fused sampling and bounded cuDNN algorithm search.
-The shared predictor keeps the upstream re-prefill helper and its default
-sampling behavior; short KV is explicitly selected by the deployment profile.
-
-Use `vllm_omni/deploy/qwen3_omni_moe_mrv2_h200.yaml` for the experimental H200
-profile. Preserve a control with `code_predictor_fused_sampling=false` and
-`codec_cudnn_benchmark=false` when C64 P99 is the primary requirement.
-cuDNN search restores process settings on exit and has startup cost. Kernel
-arithmetic is not bitwise-equivalent; the first-frame decoder uses additional
-model memory. Full-duplex performance is not established by turn-mode tests.
-
-First-frame delivery requires `talker_first_audio: true` in connector extra,
-a one-frame initial codec chunk, CUDA, a local single-rank Talker and prefix caching disabled.
-Other deployments retain the regular Code2Wav path.
-`codec_fused_snake` explicitly opts CUDA profiles into fused decoder blocks;
-the base V1 profile retains its original blocks and indexed text cache.

docs/getting_started/installation/README.md

@@ -1,7 +1,7 @@
 # Installation
 
 !!! important
-    vLLM-Omni is released against the matching upstream vLLM major/minor version. This source checkout targets vLLM 0.31.0. The published vLLM-Omni 0.30.0 wheel instructions use vLLM 0.30.0; choose the installation path that matches your Omni revision.
+    vLLM-Omni is released against the matching upstream vLLM major/minor version. The vLLM-Omni 0.30.x release line uses vLLM 0.30.x; the source-install instructions pin vLLM 0.30.0.
 
 vLLM-Omni supports the following hardware platforms:
 

docs/getting_started/installation/gpu/cuda.inc.md

@@ -1,13 +1,11 @@
 # --8<-- [start:requirements]
 
-- GPU: compute capability 7.5 or higher (e.g., T4, RTX20xx, A100, L4, H100, etc.)
+- GPU: compute capability 7.0 or higher (e.g., V100, T4, RTX20xx, A100, L4, H100, etc.)
 
 # --8<-- [end:requirements]
 # --8<-- [start:set-up-using-python]
 
-vLLM-Omni depends on the matching major/minor release of vLLM. This source
-checkout targets vLLM 0.31.0. The pre-built Omni wheel instructions below install
-the published 0.30.0 release with its matching vLLM dependency.
+vLLM-Omni depends on the matching major/minor release of vLLM. The vLLM-Omni 0.30.x release line uses vLLM 0.30.x.
 
 !!! note
     PyTorch installed via `conda` will statically link `NCCL` library, which can cause issues when vLLM tries to use `NCCL`. See <gh-issue:8420> for more details.
@@ -43,17 +41,13 @@ uv pip install 'vllm-omni[demo]==0.30.0'
 # --8<-- [start:build-wheel-from-source]
 
 #### Installation of vLLM
-If you do not need to modify source code of vLLM, install the final 0.31.0
-release wheel from its commit-pinned index:
+If you do not need to modify source code of vLLM, you can directly install the stable 0.30.0 release version of the library
 
 ```bash
-uv pip install vllm==0.31.0 --torch-backend=auto \
-  --extra-index-url https://wheels.vllm.ai/db9527a46873454610df6dbedf79a36d6bf1a7f6
+uv pip install vllm==0.30.0 --torch-backend=auto
 ```
 
-The default 0.31.0 wheel contains CUDA 12.9 binaries. If you need a different
-CUDA variant or want to reuse an existing PyTorch installation, build vLLM
-from source instead.
+The 0.30.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
 
 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -74,16 +68,16 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.31.0
+git checkout v0.30.0
 ```
 Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
 ```bash
-# Default CUDA 12.9 wheel for v0.31.0 on x86_64
-export VLLM_PRECOMPILED_WHEEL_LOCATION=https://wheels.vllm.ai/db9527a46873454610df6dbedf79a36d6bf1a7f6/vllm-0.31.0-cp38-abi3-manylinux_2_28_x86_64.whl
+#For CUDA 13.0 (the default for v0.30.0; the wheel filename has no `+cu130` suffix)
+export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.30.0/vllm-0.30.0-cp38-abi3-manylinux_2_28_x86_64.whl
 ```
 Install vllm with command below (If you have no existing PyTorch).
 ```bash
-VLLM_USE_PRECOMPILED=1 uv pip install --editable . --torch-backend=auto
+uv pip install --editable .
 ```
 Install vllm with command below (If you already have PyTorch).
 ```bash
@@ -134,13 +128,12 @@ docker run --runtime nvidia --gpus 2 \
 DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.cuda -t vllm-omni-cuda .
 ```
 
-The source build defaults to `vllm/vllm-openai:v0.31.0`. To select that
-upstream release image explicitly:
+If you want to specify the base vLLM version:
 
 ```bash
 DOCKER_BUILDKIT=1 docker build \
   -f docker/Dockerfile.cuda \
-  --build-arg BASE_IMAGE=vllm/vllm-openai:v0.31.0 \
+  --build-arg BASE_IMAGE=vllm/vllm-openai:v0.30.0 \
   -t vllm-omni-cuda .
 ```
 

docs/getting_started/installation/gpu/rocm.inc.md

@@ -10,9 +10,8 @@
 
 For ROCm, vLLM-Omni currently recommends the setup steps through Docker Images.
 
-vLLM-Omni depends on the matching major/minor release of vLLM. This source
-checkout targets vLLM 0.31.0. The pre-built Omni wheel instructions below install
-the published 0.30.0 release with its matching vLLM dependency.
+vLLM-Omni depends on the matching major/minor release of vLLM. The vLLM-Omni
+0.30.x release line uses vLLM 0.30.x.
 
 The Dockerfile's `BASE_IMAGE` pin applies only to Docker builds. The
 `vllm-omni` package does not install vLLM as a dependency, so non-Docker source
@@ -50,14 +49,13 @@ uv pip install onnxruntime-rocm
 
 #### Installation of vLLM
 
-If you do not need to modify source code of vLLM, install the final 0.31.0
-release wheel for ROCm 7.2.3:
+If you do not need to modify source code of vLLM, you can directly install the stable 0.30.0 release version of the library
 
 ```bash
-uv pip install vllm==0.31.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.31.0/rocm723
+uv pip install vllm==0.30.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.30.0/rocm723
 ```
 
-The pre-built 0.31.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The pre-built 0.30.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
 
 #### Installation of vLLM-Omni
 
@@ -77,7 +75,7 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.31.0
+git checkout v0.30.0
 python3 -m pip install -r requirements/rocm.txt
 python3 setup.py develop
 ```
@@ -89,7 +87,7 @@ python3 setup.py develop
 #### Build docker image
 
 The source build defaults to the published upstream base image
-`vllm/vllm-openai-rocm:v0.31.0`, aligned with this source checkout.
+`vllm/vllm-openai-rocm:v0.30.0`, aligned with the vLLM release used by CI.
 Older or custom bases must provide the vLLM APIs checked by the Dockerfile's
 image-build canary. This upstream base is distinct from the prebuilt
 `vllm/vllm-omni-rocm` images discussed below; their published-tag availability
@@ -104,7 +102,7 @@ To select the upstream ROCm base explicitly, pass `BASE_IMAGE`:
 ```bash
 DOCKER_BUILDKIT=1 docker build \
   -f docker/Dockerfile.rocm \
-  --build-arg BASE_IMAGE=vllm/vllm-openai-rocm:v0.31.0 \
+  --build-arg BASE_IMAGE=vllm/vllm-openai-rocm:v0.30.0 \
   -t vllm-omni-rocm .
 ```
 

docs/getting_started/installation/gpu/xpu.inc.md

@@ -1,11 +1,8 @@
-<!-- markdownlint-disable MD001 MD025 -->
-
 # --8<-- [start:requirements]
 
 - GPU: Validated on Intel® Arc™ B-Series.
 
 # --8<-- [end:requirements]
-
 # --8<-- [start:set-up-using-python]
 
 vLLM-Omni currently recommends using the Docker image setup steps below.
@@ -23,15 +20,12 @@ vLLM-Omni currently recommends using the Docker image setup steps below.
 #### Build docker image
 
 ```bash
-export VLLM_VERSION=v0.31.0
-DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.xpu -t vllm-omni-xpu --shm-size=4g \
-  --build-arg VLLM_VERSION="$VLLM_VERSION" .
+DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.xpu -t vllm-omni-xpu --shm-size=4g .
 ```
 
 This layers vLLM-Omni on top of the published `vllm/vllm-openai-xpu:<VLLM_VERSION>`
-base image, which Docker pulls automatically. The Dockerfile defaults to
-v0.31.0. To target a different vLLM release, pass
-`--build-arg VLLM_VERSION=<tag>`. If that tag has not been published yet, build
+base image (v0.30.0 by default), which Docker pulls automatically. To target a different vLLM release,
+pass `--build-arg VLLM_VERSION=<tag>`. If that tag has not been published yet, build
 the base from upstream's own Dockerfile first and point the build at it:
 
 ```bash
@@ -46,7 +40,7 @@ DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.xpu -t vllm-omni-xpu --shm-s
 
 ##### Launch with OpenAI API Server
 
-```bash
+```
 docker run -it -d --shm-size 10g \
   --name {container_name} \
   --net=host \

docs/getting_started/installation/npu/npu.inc.md

@@ -10,25 +10,25 @@ The recommended way to use vLLM-Omni on NPU is through the vLLM-Ascend pre-built
 === "A2"
 
     ```bash
-    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.31.0
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.30.0
     ```
 
 === "A3"
 
     ```bash
-    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.31.0-a3
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.30.0-a3
     ```
 
 === "A5"
 
     ```bash
-    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.31.0-a5
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.30.0-a5
     ```
 
 === "310P (Experimental)"
 
     ```bash
-    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.31.0-310p
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.30.0-310p
     ```
 
 ```bash
@@ -71,14 +71,10 @@ We are keeping [issue #886](https://github.com/vllm-project/vllm-omni/issues/886
 
 # --8<-- [start:installation-main]
 
-You can also install vLLM-Omni from source if you want to use the latest features
-or bug fixes. This source checkout targets vLLM 0.31.0 and needs a matching
-vLLM-Ascend runtime. The v0.30.0 images above are for the published Omni v0.30.0
-release. Check [issue #886](https://github.com/vllm-project/vllm-omni/issues/886)
-for aligned versions and NPU validation before selecting a development image.
+You can also install vLLM-Omni from the latest main branch if you want to use the latest features or bug fixes. Use the aligned vLLM-Ascend v0.30.0 image described above, and check [issue #886](https://github.com/vllm-project/vllm-omni/issues/886) for the status of the latest vLLM-Omni commit on NPU.
 
 ```bash
-# Inside a container with matching vLLM and vLLM-Ascend, install Omni from source
+# Inside the vLLM-Ascend v0.30.0 container, install vLLM-Omni from main
 cd /vllm-workspace
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni

docs/getting_started/quickstart.md

@@ -19,20 +19,17 @@ uv venv --python 3.12 --seed
 source .venv/bin/activate
 
 # On CUDA
-uv pip install vllm==0.31.0 --torch-backend=auto \
-  --extra-index-url https://wheels.vllm.ai/db9527a46873454610df6dbedf79a36d6bf1a7f6
+uv pip install vllm==0.30.0 --torch-backend=auto
 
 # On ROCm
-uv pip install vllm==0.31.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.31.0/rocm723
+uv pip install vllm==0.30.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.30.0/rocm723
 
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
 uv pip install -e .
 ```
 
-These commands install vLLM 0.31.0 for this source checkout. The CUDA index is
-pinned to the final release commit. For published vLLM-Omni wheels and additional
-installation methods, see the [installation guide](installation/README.md).
+For additional installation methods — please see the [installation guide](installation/README.md).
 
 !!! note
     It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.

docs/models/supported_models.md

@@ -29,11 +29,9 @@ th {
 | `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` | ✅︎ | | | ✅︎ | [Published](https://recipes.vllm.ai/Qwen/Qwen-Image) |
 | `QwenImagePipeline` | Qwen-Image-2512 | `Qwen/Qwen-Image-2512` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen-Image-2512.md) |
 | `QwenImageEditPipeline` | Qwen-Image-Edit | `Qwen/Qwen-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `JoyImageEditPipeline` | JoyAI-Image-Edit | `jdopensource/JoyAI-Image-Edit-Diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/JD/JoyAI-Image-Edit.md) |
 | `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2509 | `Qwen/Qwen-Image-Edit-2509` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `QwenImageLayeredPipeline` | Qwen-Image-Layered | `Qwen/Qwen-Image-Layered` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2511 | `Qwen/Qwen-Image-Edit-2511` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `QwenImage21Pipeline` | Qwen-Image-2.1 (T2I + image-conditioned generation, up to 4 condition images) | `Qwen/Qwen-Image-2.1` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen-Image-2.1.md) |
 | `GlmImagePipeline` | GLM-Image | `zai-org/GLM-Image` | ✅︎ | ✅︎ | | | — |
 | `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tongyi-MAI/Z-Image-Turbo.md) |
 | `Krea2Pipeline` | Krea 2 (Raw + Turbo) | `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/krea/Krea-2.md) |
@@ -56,12 +54,12 @@ th {
 | `LingBotVideoPipeline` | LingBot-Video dense and MoE T2I, T2V, TI2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | | — |
 | `SanaVideoPipeline` | SANA-Video-2B T2V (native 480p/720p; Diffusers adapter validated at both resolutions) | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers`, `Efficient-Large-Model/SANA-Video_2B_720p_diffusers` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/SANA-Video-2B.md) |
 | `SanaImageToVideoPipeline` | SANA-Video-2B I2V (native and Diffusers adapter validated at 480p/720p) | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers`, `Efficient-Large-Model/SANA-Video_2B_720p_diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/SANA-Video-2B.md) |
-| `SeedVR2Pipeline` | [SeedVR2 3B](seedvr2.md) video restoration | `ByteDance-Seed/SeedVR2-3B` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/ByteDance/SeedVR2-RTX-5090.md) |
+| `SeedVR2Pipeline` | [SeedVR2 3B](seedvr2.md) video restoration | `ByteDance-Seed/SeedVR2-3B` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/ByteDance/SeedVR2-RTX-5090.md) |
 | `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | ✅︎ | | | [Published](https://recipes.vllm.ai/MiniMaxAI/MiniMax-H3) |
 | `Magi2Pipeline` | MAGI-2 native Preview-stage T2VA and I2VA (272p/540p) | `sand-ai/MAGI-2-preview` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SandAI/MAGI-2-preview-L20X.md) |
 | `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Pi0Pipeline` | π0 (Pi-Zero) VLA | `lerobot/pi0_base` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/lerobot/Pi0.md) |
-| `Pi05Pipeline` | π0.5 VLA | `lerobot/pi05_base` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/lerobot/Pi05.md) |
+| `Pi05Pipeline` | π0.5 VLA | `lerobot/pi05_base` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/lerobot/Pi05.md) |
 | `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | | — |
 | `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
@@ -72,7 +70,7 @@ th {
 | `StableDiffusionXLPipeline` | Stable-Diffusion-XL | `stabilityai/stable-diffusion-xl-base-1.0` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/StabilityAI/Stable-Diffusion-3.5.md) |
 | `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ | — |
-| `AuKForConditionalGeneration` | AuK / AuK-Flash (instruction-driven speech generation and editing) | `tencent/AuK`, `tencent/AuK-Flash` (with `Qwen/Qwen2.5-Omni-3B`; assemble with `tools/prepare_auk_checkpoint.py`) | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/AuK-H100.md) |
+| `AuKForConditionalGeneration` | AuK / AuK-Flash (instruction-driven speech generation and editing) | `tencent/AuK`, `tencent/AuK-Flash` (with `Qwen/Qwen2.5-Omni-3B`; assemble with `tools/prepare_auk_checkpoint.py`) | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/AuK-H100.md) |
 | `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | ✅︎ | — |
 | `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | ✅︎ | — |
 | `BreezeForConditionalGeneration` | Breeze-TTS-2 (voice design, voice clone, voice direction, CFG) | `BreezeBlue/Breeze-TTS-2` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/BreezeBlue/Breeze-TTS-2.md) |
@@ -84,8 +82,8 @@ th {
 | `FluxPipeline` | FLUX.1-schnell | `black-forest-labs/FLUX.1-schnell` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `OmniGen2Pipeline` | OmniGen2 | `OmniGen2/OmniGen2` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` | ✅︎ | ✅︎ | | ✅︎ | — |
-| `MiniMaxMusic3ForConditionalGeneration` | MiniMax Music 3 (text-to-music) | `MiniMaxAI/MiniMax-Music3` | ✅︎ | | | ✅︎ | — |
-| `Yue2ForCausalLM` | YuE2-3B (text-to-music with ABC plan) | `m-a-p/YuE2-3B` | ✅︎ | | | ✅︎ | [Recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/m-a-p/YuE2-3B.md) / [Audio-to-score](https://github.com/vllm-project/vllm-omni/blob/main/recipes/m-a-p/SheetSage2-H200.md) |
+| `MiniMaxMusic3ForConditionalGeneration` | MiniMax Music 3 (text-to-music) | `MiniMaxAI/MiniMax-Music3` | ✅︎ | | | | — |
+| `Yue2ForCausalLM` | YuE2-3B (text-to-music with ABC plan) | `m-a-p/YuE2-3B` | ✅︎ | | | | — |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | | | ✅︎ | [Published](https://recipes.vllm.ai/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice) |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
@@ -95,7 +93,7 @@ th {
 | `MossTTSDelayModel` | MOSS-TTS, MOSS-TTSD, MOSS-SoundEffect, MOSS-VoiceGenerator | `OpenMOSS-Team/MOSS-VoiceGenerator` | ✅︎ | | | | — |
 | `MossTTSRealtime` | MOSS-TTS-Realtime | `OpenMOSS-Team/MOSS-TTS-Realtime` | ✅︎ | | | | — |
 | `HiggsAudioV2ForConditionalGeneration` | Higgs-Audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` | ✅︎ | | | | — |
-| `HiggsMultimodalQwen3ForConditionalGeneration` | Higgs-Audio v3 (TTS) | `bosonai/higgs-audio-v3-tts-4b` | ✅︎ | | | ✅︎ | [Published](https://recipes.vllm.ai/bosonai/higgs-audio-v3-tts-4b) |
+| `HiggsMultimodalQwen3ForConditionalGeneration` | Higgs-Audio v3 (TTS) | `bosonai/higgs-audio-v3-tts-4b` | ✅︎ | | | | [Published](https://recipes.vllm.ai/bosonai/higgs-audio-v3-tts-4b) |
 | `IndexTTS2TalkerForConditionalGeneration` | IndexTTS-2 | `IndexTeam/IndexTTS-2` | ✅︎ | | | | — |
 | `IndexTTS25TalkerForConditionalGeneration` | IndexTTS-2.5 | native `checkpoints/` bundle | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/IndexTeam/IndexTTS-2_5.md) |
 | `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ | — |
@@ -108,7 +106,6 @@ th {
 | `SenseNovaU1Pipeline` | SenseNova-U1-A3B (MoE) | `sensenova/SenseNova-U1-A3B-MoT` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SenseNova/SenseNova-U1.md#a3b-moe-variant) |
 | `LancePipeline` | Lance | `bytedance-research/Lance` | ✅︎ | | | | — |
 | `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/HunyuanVideo-1.5.md) |
-| `Kandinsky6TI2VAPipeline` | Kandinsky 6 TI2VA | `kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Kandinsky/Kandinsky6-TI2VA.md) |
 | `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/HunyuanVideo-1.5.md) |
 | `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | | — |
 | `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | | — |
@@ -118,7 +115,7 @@ th {
 | `GepardTalkerForConditionalGeneration` | Gepard-1.0 | `nineninesix/gepard-1.0` | ✅︎ | | | ✅︎ | — |
 | `HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | | — |
 | `HiDreamO1ImagePipeline` | HiDream-O1-Image | `HiDream-ai/HiDream-O1-Image` | ✅︎ | | | | — |
-| `AnimaPipeline` | Anima | `circlestone-labs/Anima` | ✅︎ | ✅︎ | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/circlestone-labs/Anima.md) |
+| `AnimaPipeline` | Anima | `circlestone-labs/Anima` | ✅︎ | ✅︎ | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/circlestone-labs/Anima.md) |
 
 The `Recipe` column links the corresponding deployment recipe. `Published` is
 preferred when a corresponding `recipes.vllm.ai` page exists; otherwise the

docs/serving/README.md

@@ -154,16 +154,18 @@ All six routes are model- or configuration-dependent. In particular,
 pipeline declares a `duplex_plugin` and its deploy configuration sets
 `session_mode: duplex` (with `session_mode: turn` the same model boots the
 ordinary turn-based serving stack instead). On a duplex server a stock
-Realtime client needs no vendor query parameter; `duplex=0` opts out of native
-duplex and selects the turn-based Realtime handler. It runs generation on the
-same stages, alongside live websocket sessions, and opens no duplex session
-or consumes a `duplex_session.max_sessions` slot -- see
-[Full Duplex](full_duplex_api.md).
-Such a server serves the websocket route plus `POST /v1/chat/completions`,
-`/v1/models` and `/health`, and no other turn-based HTTP route. On a server
-that is not duplex, an explicit `?duplex=1` is refused rather than answered by
-the turn-based handler, so a client that requested duplex never silently gets
-the other protocol.
+Realtime client needs no vendor query parameter; `duplex=0` selects the
+turn-based Realtime handler, which a duplex server does not mount, so that
+connection is refused with `Realtime API is not available`. Such a server
+serves the websocket route plus `POST /v1/chat/completions`, `/v1/models`
+and `/health`, and no other turn-based HTTP route. The chat route is the
+ordinary chat service running on the duplex engine: a request is a
+turn-based generation on the same stages, served alongside the live
+websocket sessions; it opens no duplex session and holds no
+`duplex_session.max_sessions` slot -- see [Full Duplex](full_duplex_api.md).
+On a server that is not duplex, an explicit `?duplex=1` is refused rather
+than answered by the turn-based handler, so a client that requested duplex
+never silently gets the other protocol.
 
 ## Related Endpoints
 

docs/serving/realtime_duplex_api.md

@@ -250,40 +250,6 @@ call; transport failures raise `DuplexConnectionError`; a session that ends
 while you are awaiting it raises `DuplexSessionClosedError`. All three derive
 from `DuplexClientError`.
 
-### Output limits and slow consumers
-
-Deployment settings under `duplex_session` bound each session's pending
-output: `max_pending_output_bytes_per_session` defaults to 2 MiB and
-`max_pending_output_events_per_session` to 512 events. Bytes count compact
-Realtime JSON, including base64 audio, not decoded audio. The limits apply
-to both WebSocket delivery and direct Python handles, not to replay journals
-or client playback queues.
-
-Exceeding a limit fails the active response without committing its partial
-text to history, then closes only that session. If overflow occurs while a
-completion is being queued, its not-yet-queued `response.done` becomes failed
-with the same identity and output, and its response history entry is removed.
-An ending already accepted by the buffer is not replaced or followed by a
-second ending. Open a new session to continue; the closed session cannot be
-resumed. The notifications use these fields:
-
-| Event | Field values |
-| --- | --- |
-| `error` | `error.type: "rate_limit_error"`, `error.code: "output_backpressure"` |
-| `response.done` (active response or not-yet-queued ending) | `response.status: "failed"`, `response.status_details: {"type": "failed", "reason": "output_backpressure"}` |
-| `session.closed` | `reason: "output_backpressure"` |
-
-Queued audio for the affected response is removed. Previously queued non-audio
-events retain their order, but newly emitted ordinary events, including
-audio/text, content-part and output-item completion markers, are suppressed.
-Clients must therefore handle closure without waiting for every completion
-marker. Errors and response endings have an extra 64 KiB / eight-event reserve;
-if that also fills, either notification may be omitted. Closure has its own
-independent slot. A closure notification exceeding 4 KiB is compacted to
-`reason: "close_details_exceed_output_limit"` with empty details, preserving
-the event type and identity. One event held by the consumer is outside the
-queued budget; audio already sent still requires the client to stop playback.
-
 ### Measure latency
 
 `EventCollector` accumulates events for assertions and metrics:
@@ -349,8 +315,6 @@ single-consumer async iterator that ends after `session.closed` /
 `session.expired`; rejected commands come back as `ErrorEvent`s on it.
 Every event renders the wire JSON with `to_realtime()`, so anything written
 against the WebSocket protocol works unchanged on the typed stream.
-If opening fails, the abandoned handle is closed. Late engine events do not
-reopen a locally closed handle.
 
 `vllm_omni.clients.inline_duplex.InlineDuplexClient` wraps a `DuplexOmni`
 behind the `DuplexClient` API, so the same application code runs in-process
@@ -609,7 +573,7 @@ formats (`pcm16`, `pcm_s16le`, `s16le`, `pcm_f32le`, `g711_ulaw`,
 | `session.created` | 2 | Session opened. OpenAI name; adds `attachment_generation`, `resume_token` and vLLM-Omni keys inside `session` (`id` is the server-allocated session id, `epoch`, `turn_id`, `playback`, `capabilities`, ...). |
 | `session.updated` | 2 | Echo of the effective session config after every `session.update` (same extended `session` object). |
 | `session.heartbeat_ack` | 3 | Reply to `session.heartbeat`. |
-| `session.closed` | 3 | Last event on the socket, emitted once the engine released the session; `reason` ∈ `client_close`, `disconnect`, `timeout`, `transport_error`, `shutdown`, `output_backpressure`, `close_details_exceed_output_limit`, ... |
+| `session.closed` | 3 | Last event on the socket, emitted once the engine released the session; `reason` ∈ `client_close`, `disconnect`, `timeout`, `transport_error`, `shutdown`, ... |
 | `session.resumed` | 3 | Resume accepted; carries the new `attachment_generation` and rotated `resume_token`; journaled events are replayed after it. |
 | `session.replaced` | 3 | Sent to the superseded socket when another socket resumes the session. |
 | `session.expired` | 3 | Engine lease reaped (`disconnect_grace_expired`, `idle_ttl_expired`); socket closes after it. |

docs/serving/speech_api.md

@@ -183,16 +183,6 @@ emit these headers.
 **Raw audio stream** (`stream_format="audio"`). Streams raw audio bytes (PCM or
 WAV) as they are decoded.
 
-`pcm` is 16-bit signed little-endian, interleaved, at the model's native sample
-rate and channel count (24 kHz mono for most models; MOSS-TTS-Local v1.5 emits
-48 kHz stereo). Raw streams and non-streaming responses state the format in
-response headers:
-
-| Header | Description |
-| --- | --- |
-| `x-audio-sample-rate` | Sample rate of the returned audio in Hz (after any `sample_rate` resampling). |
-| `x-audio-channels` | Channel count of the returned audio. |
-
 Both streaming modes (`stream_format="audio"` and `"sse"`) require
 `response_format="pcm"` or `"wav"`, and `speed` must be `1.0` (or omitted).
 
@@ -203,13 +193,9 @@ Each event has an `event:` line and a JSON `data:` line:
 - `speech.audio.delta` — a base64 audio chunk:
 
     ```json
-    { "type": "speech.audio.delta", "audio": "<base64>", "response_format": "pcm",
-      "sample_rate": 24000, "channels": 1 }
+    { "type": "speech.audio.delta", "audio": "<base64>", "response_format": "pcm" }
     ```
 
-    `sample_rate` and `channels` extend the OpenAI event and describe the
-    decoded chunk.
-
 - `speech.audio.done` — terminal event, carrying token `usage`:
 
     ```json
@@ -354,7 +340,7 @@ Server -> Client:
 
 | Message | Description |
 | --------- | ------------- |
-| `{"type": "audio.start", "utterance_index": 0, "sentence_index": 0, "sentence_text": "...", "format": "pcm", "sample_rate": 24000, "channels": 1}` | Audio generation starting for the buffered input. With `stream_audio=true` it is sent with the first audio chunk, and for `pcm` it carries that audio's `sample_rate` and `channels` (omitted when no audio was produced). |
+| `{"type": "audio.start", "utterance_index": 0, "sentence_index": 0, "sentence_text": "...", "format": "pcm", "sample_rate": 24000}` | Audio generation starting for the buffered input |
 | Binary frame | Raw audio bytes (one or more PCM chunks when `stream_audio=true`) |
 | `{"type": "audio.done", "utterance_index": 0, "sentence_index": 0, "total_bytes": 96000, "error": false}` | Audio complete for the buffered input |
 | `{"type": "session.done", "utterance_index": 0, "total_sentences": N}` | Flushed utterance complete |

docs/serving/video_stream_api.md

@@ -16,10 +16,6 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \
     --trust-remote-code
 ```
 
-To enable [incremental prefill](#incremental-prefill), set
-`enable_prefix_caching: true` on stage 0 (the thinker) in the deploy config. Otherwise
-the server falls back to rebuilding the prompt per query.
-
 ### Run the Example Client
 
 ```bash
@@ -58,7 +54,7 @@ WebSocket /v1/video/chat/stream
 
 When buffered frames include client-supplied `frame_id` values, the server emits `video.frames.consumed` after the first engine output. Its `frame_ids`, `frames`, and `latest_pts_ms` describe the images included in that query's prompt.
 
-Queries stride-sample the buffered frames, keeping the last frame, then exclude frames already known to have failed decoding. With incremental prefill active, every buffered frame is included instead of stride-sampling, with the same decode-failure exclusion. Excluded frames are not replaced with other buffered frames. The incremental path waits for the whole window to finish decoding. The legacy path waits only for its sampled positions, so an unselected frame still decoding does not delay the query.
+Queries stride-sample the buffered frames, keeping the last frame, then exclude frames already known to have failed decoding. Excluded frames are not replaced with other buffered frames. Frames whose background decoding has not finished remain eligible through the image URL path.
 
 **Bugfix compatibility note:** `video.frames.consumed` now excludes known decode failures that were already excluded from the prompt. Older versions could report those frames and their timestamps as consumed. The event name and fields are unchanged; an empty selection reports empty lists and `latest_pts_ms: null`.
 
@@ -68,10 +64,10 @@ Queries stride-sample the buffered frames, keeping the last frame, then exclude
 | ------- | ------ | --------- | ------------- |
 | `model` | string or null | null | Optional model name. Usually omitted because the server hosts one model. |
 | `modalities` | list[string] | `["text", "audio"]` | Output modalities. Use `["text"]`, `["audio"]`, or both. |
-| `num_frames` | integer, 1-128 | `4` | Number of buffered frames sampled for each query. Legacy path only: with incremental prefill active, every buffered frame is submitted and frame density is the client's responsibility (clients push discrete frames at their own rate). |
-| `max_frames` | integer, 1-256 | `50` | Maximum retained frame buffer size. Below the cap both paths keep every frame. Legacy evicts the oldest frame once full. Incremental prefill drops the oldest half at once, then appends until full again. |
+| `num_frames` | integer, 1-128 | `4` | Number of buffered frames sampled for each query. |
+| `max_frames` | integer, 1-256 | `50` | Maximum retained frame buffer size. Oldest frames are evicted first. |
 | `system_prompt` | string or null | null | Optional custom system prompt. |
-| `use_audio_in_video` | bool | `true` | Incremental prefill: forwarded on warmup and every query when this is true, so multimodal hashes stay aligned. Legacy: forwarded only when this query has input audio. |
+| `use_audio_in_video` | bool | `true` | Include streamed audio chunks in multimodal video understanding when audio is present. |
 | `sampling_params_list` | list or null | null | Optional per-stage parameter dictionaries. Each provided entry replaces that stage's deployment sampling settings. |
 | `enable_frame_filter` | bool | `true` | Enable EVS near-duplicate frame filtering. |
 | `frame_filter_threshold` | float, 0.0-1.0 | `0.95` | EVS similarity threshold. Higher keeps more frames; lower drops more near-duplicates. |
@@ -111,42 +107,9 @@ The server accepts these legacy field names and rewrites them before validation.
 
 EVS compares downsampled frames and drops near-duplicate frames before they enter the session frame buffer. `frame_filter_threshold` controls retention: higher values are more permissive and keep more frames; lower values are more aggressive and drop more similar frames.
 
-## Incremental Prefill
-
-Server-side only (stage-0 `enable_prefix_caching: true` and `modalities: ["text"]`).
-There is no `session.config` flag. Audio-output sessions stay on the legacy path.
-
-Input is unchanged: `video.frame` and `audio.chunk` are separate buffers. Warmup
-prefills `history + frames` as they arrive (`max_tokens=1`). When
-`use_audio_in_video` is true, warmup and query both pass that processor kwarg.
-`video.query` is the same prompt plus optional input audio and the question text,
-so the warmed vision prefix cache is reused and the query pays for the audio/text
-suffix. All buffered frames are
-submitted in arrival order (`num_frames` subsample is legacy-only). Once the buffer
-hits `max_frames`, the oldest half is dropped in one step and a warning is logged;
-until then the buffer matches the legacy path. After a turn,
-history compresses to the last two text messages and the next warmup follows the
-new prefix.
-
-### Legacy path (incremental prefill inactive)
-
-Stage-0 prefix caching is off, or `modalities` includes `"audio"`: no warmup,
-frames are re-sampled up to `num_frames` each query, and the query waits only
-for those sampled frames to decode. Prompt shape is the same
-(`image_pil` frames, optional trailing `input_audio`, text). Both paths keep the
-frame buffer across queries (`max_frames`) and compress history the same way.
-
 ## Known Limitations
 
-- Past `max_frames`, incremental prefill keeps a suffix of the newest frames, not a resample of the whole stream. Rebuilding that longer window is left to a follow-up.
-- Incremental prefill applies only to text-only sessions with stage-0 prefix caching enabled
-  on `WS /v1/video/chat/stream`. It does not accelerate `/v1/realtime` camera input, which
-  uses duplex conversation items rather than `video.frame` / `video.query`. Audio-output
-  sessions on this endpoint rebuild the prompt each query.
-- Stage-0 prefix caching is safe for audio-output requests: on a hit, the omni prefix cache
-  restores the thinker hidden states the talker needs. A preempted request that resumes on a
-  hit does not get them back, so size the deployment to avoid preemption; see
-  [Prefix Caching](../design/feature/prefix_caching.md).
+- Session KV reuse and incremental prefill are not implemented in this PR. Each `video.query` rebuilds the model prompt from the retained frame and audio buffers.
 - Back-to-back short replies can still expose an engine-layer scheduler race. The PR notes an observed workaround of at least 200 ms idle between turns when clients repeatedly see idle timeouts.
 - If the audio buffer exceeds the server limit, the server emits `Audio buffer overflow` and clears the currently buffered audio for the session.
 - The API is intended for Qwen3-Omni streaming video understanding; other models may not support the same multimodal processor arguments.

docs/serving/videos_api.md

@@ -237,22 +237,13 @@ At least one mask is required. A nontrivial `video_noise_mask` requires
 `source_audio` or a `source_video` with an audio stream. Mask values are in
 `[0, 1]`: `0` preserves the source, `1` regenerates it, and fractional values
 blend the two behaviors. Exact all-one masks are no-ops and do not require a
-source. Source uploads without a mask are rejected. Masks may be a JSON scalar,
-an array matching the H3 latent/token grid, or (video only) a frame-space
-array that the server resizes to the latent grid.
+source. Source uploads without a mask are rejected. Masks may be a JSON scalar
+or arrays matching the H3 latent/token grid; pixel-resolution masks must be
+resized or pooled by the client before upload.
 
 For an aligned output of `F` frames at `W x H`, the video latent grid is
 `[Tv, H/16, W/16]`, where `Tv = 2 + 5 * ((F - 5) / 17)`. The video mask may be
-a scalar, a flat token vector, `[Tv, H/32, W/32]`, the full latent grid, a
-spatial `[h, w]` mask applied to every frame, or a frame-space `[T, h, w]` mask
-with one slice per 24 fps output frame. A shape that matches the token or full
-latent grid is always read as that grid. Frame-space masks are area-resized to
-`[H/16, W/16]`; along time, a mask shorter than `F` is padded with its last
-slice (as the source video is) and a longer one is truncated, then each latent
-takes the maximum over the frames the causal video VAE folds into it, so any
-regenerated frame regenerates its latent. Because the 8 MiB limit below applies
-to the JSON file, clients should area-downsample frame-space masks by 16
-spatially before upload. For
+a scalar, a flat token vector, `[Tv, H/32, W/32]`, or the full latent grid. For
 the model input, timestep, and velocity, a full-grid mask is max-pooled over
 each 2x2 spatial token and fractional values are rounded upward to 1/256
 levels. The final x0 restore uses the original, unquantized mask, so full-grid

docs/user_guide/diffusion/cpu_offload.md

@@ -42,7 +42,6 @@ higher-priority strategy is selected.
 | --- | --- | --- | :---: | :---: | :---: | --- |
 | Flux2Pipeline | `black-forest-labs/FLUX.2-dev` | `Flux2Transformer2DModel` | yes | yes | — | `transformer_blocks`, `single_transformer_blocks` |
 | LongCatImagePipeline | `meituan-longcat/LongCat-Image` | `LongCatImageTransformer2DModel` | — | yes | — | `transformer_blocks`, `single_transformer_blocks` |
-| JoyImageEditPipeline | `jdopensource/JoyAI-Image-Edit-Diffusers` | `JoyImageEditTransformer3DModel` | yes | yes | — | `double_blocks` |
 | NextStep11Pipeline | `stepfun-ai/NextStep-1.1` | `NextStepModel` | — | yes | — | `layers` |
 | OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | — | yes | — | `transformer` |
 | QwenImagePipeline | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | yes | yes | — | `transformer_blocks` |

docs/user_guide/diffusion/parallelism/vae_parallelism.md

@@ -1,12 +1,12 @@
 # VAE Parallelism Guide
 
+
 ## Table of Content
 
 - [Overview](#overview)
 - [Quick Start](#quick-start)
 - [Example Script](#example-script)
 - [Configuration Parameters](#configuration-parameters)
-- [Batch Parallel Decode](#batch-parallel-decode)
 - [Best Practices](#best-practices)
 - [Troubleshooting](#troubleshooting)
 - [Summary](#summary)
@@ -15,22 +15,23 @@
 
 ## Overview
 
-VAE parallelism distributes VAE (Variational AutoEncoder) decode/encode work across multiple GPUs. This guide covers patch/tile parallelism, batch parallel decode for image VAEs, and Wan spatial-shard decode. Patch/tile parallelism divides images into spatial regions; batch parallel decode assigns complete images to ranks.
+VAE parallelism distributes VAE (Variational AutoEncoder) decode/encode work across multiple GPUs. This guide covers VAE patch/tile parallelism, which splits latent space into spatial tiles or patches, and Wan spatial-shard decode, which shards decoder feature maps along height or width.
 
 This is particularly useful for:
-
 - **High-resolution image generation** where VAE decode can become a memory bottleneck
 - **Memory-constrained environments** where the VAE decode activation peak exceeds available VRAM
 - **Multi-GPU setups** where you want to leverage distributed resources for the VAE stage
 
 See supported models list in [Supported Models](../../diffusion_features.md#supported-models).
 
+
 VAE patch parallelism uses two strategies based on image size:
 
-| Strategy         | Use Case                           | How It Works                                                                                               | Overlap Handling                                                                            | Output Quality                                       |
-| ---------------- | ---------------------------------- | ---------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- | ---------------------------------------------------- |
-| **Tiled Decode** | Large images (triggers VAE tiling) | Distributes existing VAE tiling computation across ranks. Each rank decodes a subset of overlapping tiles. | Uses VAE's native `blend_v` and `blend_h` functions to seamlessly merge overlapping regions | Bit-identical (same logic as single-GPU tiling)      |
-| **Patch Decode** | Small images (no VAE tiling)       | Splits latent into spatial patches with halos. Each rank decodes one patch with boundary context.          | Halo regions provide edge context; core regions are directly stitched without blending      | Near-identical (diff < 0.5%, visually imperceptible) |
+| Strategy | Use Case | How It Works | Overlap Handling | Output Quality |
+|----------|----------|--------------|------------------|----------------|
+| **Tiled Decode** | Large images (triggers VAE tiling) | Distributes existing VAE tiling computation across ranks. Each rank decodes a subset of overlapping tiles. | Uses VAE's native `blend_v` and `blend_h` functions to seamlessly merge overlapping regions | Bit-identical (same logic as single-GPU tiling) |
+| **Patch Decode** | Small images (no VAE tiling) | Splits latent into spatial patches with halos. Each rank decodes one patch with boundary context. | Halo regions provide edge context; core regions are directly stitched without blending | Near-identical (diff < 0.5%, visually imperceptible) |
+
 
 VAE Patch Parallelism **reuses the DiT process group** (`dit_group`) and does not initialize a separate ProcessGroup. This means:
 
@@ -109,80 +110,19 @@ vllm serve Tongyi-MAI/Z-Image-Turbo --omni --port 8091 \
 
 In `DiffusionParallelConfig`:
 
-| Parameter                 | Type   | Default   | Description                                                                                                                                                       |
-| ------------------------- | ------ | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `vae_patch_parallel_size` | int    | 1         | Number of ranks requested for VAE parallelism, including batch mode. Set to 2 or higher to enable. Batch mode caps this degree at the existing worker group size. |
-| `vae_parallel_mode`       | str    | `"tile"`  | `"tile"` for tile/patch decode, `"batch"` for complete-image batch decode, or `"spatial_shard_height"` / `"spatial_shard_width"` for Wan spatial-shard decode.    |
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `vae_patch_parallel_size` | int | 1 | Number of GPUs for VAE patch/tile parallelism. Set to 2 or higher to enable. Should typically match `tensor_parallel_size` as they share the same process group. |
+| `vae_parallel_mode` | str | `"tile"` | VAE parallel decode strategy: `"tile"` (default tile/patch parallel decode), `"spatial_shard_height"`, or `"spatial_shard_width"` (spatially-sharded decode, Wan only). See [Spatially-Sharded Decode](#spatially-sharded-decode-wan). |
 
 Additional requirements:
 
-| Parameter        | Type   | Default   | Description                                             |
-| ---------------- | ------ | --------- | ------------------------------------------------------- |
-| `vae_use_tiling` | bool   | False     | Must be set to `True` when using VAE patch parallelism. |
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `vae_use_tiling` | bool | False | Must be set to `True` when using VAE patch parallelism. |
 
 !!! note "Automatic VAE Tiling"
-    When `vae_patch_parallel_size > 1` and the model has a distributed VAE (`DistributedVaeMixin`), tile and spatial-shard modes automatically enable `vae_use_tiling`. Batch mode preserves the requested tiling setting.
-
----
-
-## Batch Parallel Decode
-
-Set `vae_parallel_mode="batch"` to distribute complete images across ranks during
-VAE decode. This mode supports pipelines that load `DistributedAutoencoderKL`
-or `DistributedAutoencoderKLFlux2`, including FLUX.2-dev and FLUX.2 Klein. Klein
-selects the distributed VAE only for batch mode; its default path is unchanged.
-Selecting this mode for an
-unsupported VAE, including Wan, raises an error at model initialization.
-
-For a pipeline already using two tensor-parallel workers, select batch decode
-with these Python settings:
-
-```python
-parallel_config = DiffusionParallelConfig(
-    tensor_parallel_size=2,
-    vae_patch_parallel_size=2,
-    vae_parallel_mode="batch",
-)
-sampling_params = OmniDiffusionSamplingParams(num_outputs_per_prompt=2)
-```
-
-The corresponding serving flags are `--vae-patch-parallel-size 2
---vae-parallel-mode batch`. Keep the existing DiT parallelism configuration;
-the VAE degree does not allocate extra workers. Send a request for multiple
-images to use the batch split.
-
-Each rank decodes a contiguous part of the image batch through its native VAE
-decoder. Results are reassembled in the original image order and made available
-on every worker. With two images and four workers, two workers decode one image
-each. With five images and four workers, the assigned batch sizes are 2, 1, 1,
-and 1. Workers without images still participate in the required communication.
-
-**Constraints and behavior:**
-
-- Batch mode and distributed tile/patch mode are mutually exclusive. Encoding
-  uses the native local path, including native tiling when enabled, rather than
-  distributing spatial tiles across workers.
-- Batch size one, VAE degree one, or a single worker uses native decode on each
-  worker. Batch mode does not automatically switch to distributed tile/patch
-  decode for a single image, even with `vae_use_tiling=True`. To distribute a
-  single image across workers, select `vae_parallel_mode="tile"` on a pipeline
-  that supports distributed tile/patch decoding.
-- Data parallelism, pipeline parallelism, and CFG parallelism must each have
-  degree one because this implementation communicates over the worker WORLD
-  group. Tensor and sequence parallelism may supply the shared workers.
-- Batch mode does not force tiling. Explicit native VAE tiling and slicing
-  settings remain in effect within each rank's assigned batch and on the
-  single-image fallback. Native tiling splits an image locally on one worker;
-  it does not add cross-worker tile parallelism to batch mode.
-- It does not merge independent serving requests. The VAE must receive a batch
-  containing multiple images, for example through `num_outputs_per_prompt`.
-
-Splitting the batch can reduce per-rank decoder work and activation memory, but
-collecting decoded images adds communication and buffers. Measure complete
-request latency and memory for the intended model, image count, and resolution.
-This feature has no universal speedup or memory-saving guarantee. Changing the
-decoder batch shape can also change floating-point rounding; validate generated
-outputs when selecting a deployment configuration.
+    When `vae_patch_parallel_size > 1` and the model has a distributed VAE (`DistributedVaeMixin`), the system automatically sets `vae_use_tiling=True` if not already enabled.
 
 ---
 
@@ -254,8 +194,7 @@ python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
 ### Common Issue 1: Model Not Support VAE Patch Parallel
 
 **Symptoms**:
-
-```text
+```
 WARNING: vae_patch_parallel_size=2 is set but VAE patch parallelism is NOT enabled for xxxPipeline; ignoring.
 ```
 
@@ -278,6 +217,7 @@ if vae_pp_size > 1 and not is_distributed_vae:
 
 2. To add support for a new model, implement `DistributedVaeMixin` on its VAE class (contributions are welcome).
 
+
 ### Common Issue 2: `vae_patch_parallel_size` Exceeds DiT Process Group Size
 
 **Symptoms**: Shows warning message, and vae patch parallel size is resized to DiT process group size
@@ -287,7 +227,6 @@ if vae_pp_size > 1 and not is_distributed_vae:
 **Recommendation**: Always set `vae_patch_parallel_size` to be no greater than your DiT process group size.
 
 Note that the size of DiT process group size equals to:
-
 ```text
 dit_parallel_size = data_parallel_size
                   × cfg_parallel_size
@@ -296,8 +235,7 @@ dit_parallel_size = data_parallel_size
                   × tensor_parallel_size
 
 ```
-
-`sequence_parallel_size = ulysses_degree × ring_degree`
+_sequence_parallel_size = ulysses_degree × ring_degree_
 
 ---
 

docs/user_guide/diffusion/vae_fast_path.md

@@ -7,8 +7,7 @@ convolutional network whose reference implementation in diffusers moves every
 activation through memory several times per layer (normalization, activation,
 causal padding, feature-cache bookkeeping, shortcut upsampling). vLLM-Omni
 installs a fast path on every loaded Wan VAE that fuses this data movement into
-a handful of Triton kernels. The opt-in `channels_last` level also specializes
-first-frame convolutions that have no history.
+a handful of Triton kernels while leaving the convolutions themselves untouched.
 
 ## Levels
 
@@ -16,9 +15,9 @@ The fast path is controlled by `--vae-fast-path` (engine argument
 `vae_fast_path`, deploy-config key `vae_fast_path`):
 
 | Level | Default | Output vs. diffusers | What it does |
-| --- | --- | --- | --- |
+|---|---|---|---|
 | `lossless` | yes | bit-identical | Fused RMSNorm epilogue, fused causal-conv input assembly and cache refresh, fused shortcut upsampling and residual adds (with the neighbouring convolution biases folded in), single-pass nearest 2x upsampling, preallocated output assembly. |
-| `channels_last` | no | within tolerance (PSNR typically > 60 dB) | Everything in `lossless`, plus channels-last convolution weights, a single-pass channels-last RMSNorm+SiLU kernel that also absorbs the preceding `conv1` bias, and Conv2d for supported single-frame causal convolutions without history. |
+| `channels_last` | no | within tolerance (PSNR typically > 60 dB) | Everything in `lossless`, plus decoder convolution weights converted to channels-last memory format so cuDNN picks its channels-last kernels, and a single-pass channels-last RMSNorm+SiLU kernel that also absorbs the bias of the preceding `conv1`. |
 | `off` | no | bit-identical | Reference diffusers decoder. |
 
 ```bash
@@ -62,12 +61,6 @@ stages:
 - The `channels_last` level changes the order in which cuDNN accumulates
   convolutions, so outputs differ from the reference in the last bits. Use
   `lossless` when bitwise reproducibility against diffusers matters.
-- For a supported causal convolution with one input frame and no history,
-  the temporal input is `[0, 0, x]`. The `channels_last` level uses Conv2d with
-  the last temporal weight slice, preserving the convolution formula for
-  finite inputs and the existing cache update. Calls with history retain
-  Conv3d. The weight slice is taken on each call, so weight updates and device
-  moves do not leave a stale packed copy.
 - For VAEs kept in fp32, cuDNN's channels-last convolution algorithms run in
   TF32 under PyTorch's default `torch.backends.cudnn.allow_tf32 = True`, which
   dominates the difference to the reference (about 1e-3). Set
@@ -80,10 +73,10 @@ Cosmos3-Nano VAE, 1280x720 x 189 frames, bf16, one GB200 GPU, `bench_wan_vae_dec
 (best of 2 runs after warmup):
 
 | `--vae-fast-path` | Decode time | Speedup | Output vs. `off` |
-| --- | ------------- | --------- | --- |
-| `off` | 6.00 s | 1.00x | reference |
-| `lossless` | 3.26 s | 1.84x | bit-identical |
-| `channels_last` | 2.51 s | 2.39x | PSNR 62.6 dB, max abs diff 4.6e-2 |
+|---|-------------|---------|---|
+| `off` | 6.00 s      | 1.00x   | reference |
+| `lossless` | 3.26 s      | 1.84x   | bit-identical |
+| `channels_last` | 2.51 s      | 2.39x   | PSNR 62.6 dB, max abs diff 4.6e-2 |
 
 Peak decode memory was unchanged (about 13 GiB): it is set by the largest
 activations and the cuDNN workspace, not by the output assembly.
@@ -100,22 +93,6 @@ python benchmarks/diffusion/bench_wan_vae_decode.py --model nvidia/Cosmos3-Nano
     --size 1280x720 --frames 189 --fast-path off,lossless,channels_last --profile
 ```
 
-For a single-GPU comparison that isolates first-frame Conv2d within
-`channels_last`, use `--first-frame-ablation`. It alternates the two paths,
-disables cuDNN TF32 and autotuning, and prints JSON with every timing sample,
-summary statistics, peak allocated memory and output comparisons. Per-call
-weight packing is included in the measured decode time.
-
-```bash
-python benchmarks/diffusion/bench_wan_vae_decode.py \
-    --model Wan-AI/Wan2.1-T2V-1.3B-Diffusers --size 832x480 --frames 1 \
-    --dtype bf16 --first-frame-ablation --warmup 60 --iters 50 --seed 0
-```
-
-The default input is a seeded latent tensor. `--latents /path/to/latents.pt`
-can instead load a saved VAE input after latent mean/std scaling; the benchmark
-selects the temporal prefix requested by `--frames` and checks its shape.
-
 The same script benchmarks multi-GPU VAE decode when launched with `torchrun`;
 the decode is timed across all ranks and rank 0 reports:
 

docs/user_guide/diffusion_features.md

@@ -152,10 +152,9 @@ The following tables show which models support each feature:
 | **HiDream-O1-Image**     |     ❌     |     ✅      |           ❌           |       ❌        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **HunyuanImage3**        |     ❌     |     ✅      |           ❌           |       ❌        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ✅        |        ✅*        |
 | **Krea 2**               |     ❌     |     ✅      |           ❌           |       ❌        |         ❌         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
-| **JoyAI-Image-Edit**    |     ❌     |     ❌      |           ❌           |       ✅        |         ❌         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **LongCat-Image**        |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **LongCat-Image-Edit**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
-| **MammothModa2(T2I)**    |     ✅     |     ✅      |           ❌           |       ❌        |         ❌         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
+| **MammothModa2(T2I)**    |     ❌     |     ✅      |           ❌           |       ❌        |         ❌         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **Nextstep_1(T2I)**      |     ❓     |     ❓      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **OmniGen2**             |     ❌     |     ✅      |           ✅           |       ❌        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **Ovis-Image**           |     ❌     |     ✅      |           ❌           |       ✅        |         ❌         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
@@ -164,7 +163,6 @@ The following tables show which models support each feature:
 | **Qwen-Image-Edit**      |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Qwen-Image-Edit-2509** |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |        ✅ (decode)         |          ✅           |       ❌        |        ❌         |
 | **Qwen-Image-Layered**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
-| **Qwen-Image-2.1**       |     ❌     |     ❌      |           ❓           |       ✅        |         ✅         |          ❌          |   ❓    |             ❓             |      ✅ (decode)      |       ✅        |        ✅         |
 | **SenseNova-U1**         |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **SenseNova-U1.5**       |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ✅        |        ❌         |
 | **SenseNova-U1-A3B**     |     ❌     |     ❌      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
@@ -183,8 +181,7 @@ The following tables show which models support each feature:
 > 5. HunyuanImage3 supports step execution. Multi-request step execution requires `TORCH_SDPA`; see [Diffusion Execution Modes](diffusion/execution_modes.md#step-execution).
 > 6. BAGEL step execution supports image generation with `bagel.yaml`, `bagel_think.yaml`, and `bagel_single_stage.yaml`; two-stage Thinker execution and explicit single-stage text output remain on their existing complete-request paths. Image requests require `num_inference_steps >= 2`. BAGEL step execution cannot currently be combined with sequence parallelism or a diffusion cache backend; see [Diffusion Execution Modes](diffusion/execution_modes.md#step-execution).
 > 7. MammothModa2 runs its DiT stage on the diffusion runner (`StageExecutionType.DIFFUSION`); Cache-DiT is enabled through the standard diffusion-stage knobs on the stage entry of the deploy YAML (`cache_backend: cache_dit`, plus optional `cache_config` / `enable_cache_dit_summary`). The runner installs the backend at startup and the pipeline adopts it per request. Only the repeated main-layer stack is cached; requests with `text_guidance_scale = 1.0` bypass the cache hooks.
-> 8. Qwen-Image-2.1 (`QwenImage21Pipeline`) handles both text-to-image and image-conditioned generation (up to 4 condition images) and supports LoRA through `QwenImageLoraLoaderMixin`. Ulysses sequence parallelism is wired and ring attention is not supported, but multi-GPU padding e2e and a parity number are not in this PR's H200 results, so SP stays ❓. Cache acceleration (TeaCache / Cache-DiT) is unavailable; its built-in KV cache for the text/condition prefix is a separate feature from those cache backends. Online FP8 is documented in the quantization guide with H200 numbers.
-> 9. SenseNova-U1-A3B: `--cfg-parallel-size 2` and TP=2 both complete. CFG-P covers t2i 2-branch (`cond`+`uncond`), img2img 2-branch (`cond`+`img_cond` when `img_cfg_scale=1`), and img2img 3-branch (`cond`+`img_cond`+`uncond` when `img_cfg_scale` is neither 1 nor equal to `cfg_scale`; 3 branches on 2 ranks dispatch as `[[0, 2], [1]]`). Cache-DiT does not: the adapter is declared on `SenseNovaU1Model`, but discovery walks `_dit_modules` (`language_model` / `SenseNovaU1ForCausalLM`) and raises. Layerwise CPU offload is a no-op for the same reason — that class has no `_layerwise_offload_blocks_attrs`, so the 72.2 GiB expert weights stay resident.
+> 8. SenseNova-U1-A3B: `--cfg-parallel-size 2` and TP=2 both complete. CFG-P covers t2i 2-branch (`cond`+`uncond`), img2img 2-branch (`cond`+`img_cond` when `img_cfg_scale=1`), and img2img 3-branch (`cond`+`img_cond`+`uncond` when `img_cfg_scale` is neither 1 nor equal to `cfg_scale`; 3 branches on 2 ranks dispatch as `[[0, 2], [1]]`). Cache-DiT does not: the adapter is declared on `SenseNovaU1Model`, but discovery walks `_dit_modules` (`language_model` / `SenseNovaU1ForCausalLM`) and raises. Layerwise CPU offload is a no-op for the same reason — that class has no `_layerwise_offload_blocks_attrs`, so the 72.2 GiB expert weights stay resident.
 
 ### VideoGen
 
@@ -199,7 +196,6 @@ The following tables show which models support each feature:
 |  **SANA-Video-2B T2V I2V**   |     ❌     |     ❌      |   ✅ (`--usp`, frame-sharded)   |       ✅       |  ✅ (TP=2 only)   |         ❌         |   ❌   |            ❌             |          ❌          |       ❌       |        ❌        |
 | **Helios**                   |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ✅*        |
 | **HunyuanVideo-1.5 T2V I2V** |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
-| **Kandinsky 6 TI2VA**        | ✅* (uncalibrated) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ✅ (FP8) | ✅ (1 request) |
 | **Cosmos3**                  |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
 | **LongCat-Video-Avatar-1.5** |     ❌     |     ❌      |           ❌           |       ❌        |         ❌         |         ❌         |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **MiniMax-H3**               | ✅ (FL2VA) |     ✅      |           ✅           |       ❌        |       ✅ (DiT/TE)  |         ❌         |   ✅    |             ✅             |       ✅ (tile)       |      ✅ (DiT)      |        ❌         |

docs/user_guide/examples/offline_inference/image_to_image.md

@@ -3,7 +3,7 @@
 Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/image_to_image>.
 
 
-This example edits an input image with `Qwen/Qwen-Image-Edit` and `JoyAI-Image-Edit` using the `image_edit.py` CLI.
+This example edits an input image with `Qwen/Qwen-Image-Edit` using the `image_edit.py` CLI.
 
 ## Local CLI Usage
 
@@ -26,27 +26,6 @@ python image_edit.py \
   --cfg-scale 4.0
 ```
 
-### JoyAI-Image-Edit
-
-JoyAI-Image-Edit uses the same `image_edit.py` CLI and supports one input
-image per request. Requested dimensions are snapped to the nearest supported
-Joy bucket; for square outputs, use `1024x1024`.
-
-```bash
-python image_edit.py \
-  --model jdopensource/JoyAI-Image-Edit-Diffusers \
-  --image input.png \
-  --prompt "Change the background to a clean studio while preserving the subject." \
-  --height 1024 \
-  --width 1024 \
-  --num-inference-steps 50 \
-  --guidance-scale 4.0 \
-  --output output_joyai_edit.png
-```
-
-JoyAI defaults to guidance scale `4.0`; pass `--guidance-scale` explicitly when
-you need a different value.
-
 ### Multiple Image Editing (Qwen-Image-Edit-2509)
 
 For multiple image inputs, use `Qwen/Qwen-Image-Edit-2509` or  `Qwen/Qwen-Image-Edit-2511`:
@@ -64,7 +43,7 @@ python image_edit.py \
 
 Key arguments:
 
-- `--model`: model name or path. Use `Qwen/Qwen-Image-Edit-2509` or later for multiple image support. `jdopensource/JoyAI-Image-Edit-Diffusers` supports one input image.
+- `--model`: model name or path. Use `Qwen/Qwen-Image-Edit-2509` or later for multiple image support.
 - `--image`: path(s) to the source image(s) (PNG/JPG, converted to RGB). Can specify multiple images.
 - `--prompt` / `--negative-prompt`: text description (string).
 - `--cfg-scale`: true classifier-free guidance scale (default: 4.0). Classifier-free guidance is enabled by setting cfg_scale > 1 and providing a negative_prompt. Higher guidance scale encourages images closely linked to the text prompt, usually at the expense of lower image quality.

docs/user_guide/examples/offline_inference/text_to_speech.md

@@ -130,7 +130,7 @@ python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
 ### Notes
 
 - Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform.
-- H200 automatically selects `cosyvoice3_packed_streaming_optimized_standard.yaml` when the CUDA MPS control tool is available; other devices and environments use `cosyvoice3.yaml`. See the [serving defaults and tradeoffs](../online_serving/text_to_speech.md#cosyvoice3). Pass `--deploy-config cosyvoice3.yaml` for the previous profile or `--deploy-config <path>` to override. The example uses the resolved profile's sampling parameters.
+- Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config <path>` to override.
 - `async_chunk: true` is the default; pass `--no-async-chunk` to switch to the legacy synchronous path.
 
 ---

docs/user_guide/examples/online_serving/image_to_image.md

@@ -3,14 +3,10 @@
 Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/image_to_image>.
 
 
-This example demonstrates how to deploy image-to-image models for online image editing service using vLLM-Omni.
-
-Supported models include Qwen-Image-Edit, BAGEL, JoyAI-Image-Edit, and other image-to-image pipelines.
+This example demonstrates how to deploy Qwen-Image-Edit model for online image editing service using vLLM-Omni.
 
 For **multi-image** input editing, use **Qwen-Image-Edit-2509** (QwenImageEditPlusPipeline) and send multiple images in the user message content.
 
-For **JoyAI-Image-Edit**, use `jdopensource/JoyAI-Image-Edit-Diffusers` and send exactly one input image.
-
 ## Start Server
 
 ### Basic Start
@@ -28,12 +24,6 @@ vllm serve Qwen/Qwen-Image-Edit --omni --port 8092
 vllm serve Qwen/Qwen-Image-Edit-2509 --omni --port 8092
 ```
 
-### JoyAI-Image-Edit
-
-```bash
-vllm serve jdopensource/JoyAI-Image-Edit-Diffusers --omni --port 8092
-```
-
 ### Start with Parameters
 
 
@@ -86,11 +76,6 @@ curl -s http://localhost:8092/v1/chat/completions \
   | cut -d',' -f2 | base64 -d > output.png
 ```
 
-For JoyAI-Image-Edit, set classifier-free guidance with `guidance_scale`
-in `extra_body` (default: `4.0`). `true_cfg_scale` remains supported as an
-alias; if both are provided, their values must match. Values at or below `1.0`
-disable CFG.
-
 ### Method 2: Using OpenAI Python SDK
 
 ```python

docs/user_guide/examples/online_serving/text_to_speech.md

@@ -155,22 +155,16 @@ vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 --omni --port 8091 --trust-remot
 ./cosyvoice3/run_server.sh
 ```
 
-Streaming is on by default. On CUDA Hopper GPUs with at least 140 GiB of device memory (H200), serving automatically selects `cosyvoice3_packed_streaming_optimized_standard.yaml` when `nvidia-cuda-mps-control` is available on `PATH`: AR32 / codec32, standard sampling, packed Flow, cached ISTFT, HiFT decode graphs, and local CUDA MPS. No `--hf-overrides` is needed. Other devices and environments without the MPS control tool use `cosyvoice3.yaml` with RAS sampling and AR8 / codec8.
+Streaming is on by default via `async_chunk: true` in `vllm_omni/deploy/cosyvoice3.yaml`. Pass `--no-async-chunk` (or `NO_ASYNC_CHUNK=1 ./cosyvoice3/run_server.sh`) for the legacy synchronous path.
 
-The optimized profile is not a universal hardware configuration. Its packed kernels are Hopper-only; the talker's Model Runner V2 is unsupported on NPU/XPU, and CUDA MPS requires a local NVIDIA GPU and its control tool. Larger batches, 25-frame chunks, and graph allocations also require per-device validation on smaller or slower GPUs. Automatic selection retains the generic profile outside the validated device class; an explicit deploy configuration takes precedence.
-
-The H200 default targets streaming throughput at C32/C64. It changes sampling from RAS to standard (temperature 0.7, top-p 0.8, top-k 20, repetition penalty 1.21); output lengths and quality can differ. HiFT captures a recurring exact shape on its third use, which adds latency to that request, and its graph pool needs memory headroom beyond the engine reservations. This profile is not a claim of optimal first-audio latency at low concurrency. Pass `--deploy-config cosyvoice3.yaml` to retain the previous configuration, or `--deploy-config <path>` to select another profile.
-
-Pass `--no-async-chunk` (or `NO_ASYNC_CHUNK=1 ./cosyvoice3/run_server.sh`) for the legacy synchronous path; use `--deploy-config cosyvoice3.yaml` alongside it when selecting that path on H200.
-
-Cross-request Stage-1 flow batching is enabled in the H200 profile. For the generic configuration, enable it in the server environment when concurrent requests should share a flow-estimator call:
+Cross-request Stage-1 flow batching is opt-in. Enable it in the server environment when concurrent requests should share a flow-estimator call:
 
 ```bash
 export COSYVOICE3_BATCH_FLOW=1
 vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 --omni --port 8091 --trust-remote-code
 ```
 
-Batching preserves output lengths and streaming cache alignment, but the different GEMM shapes can produce small waveform differences compared with processing each request separately. Use `--deploy-config cosyvoice3.yaml` and leave `COSYVOICE3_BATCH_FLOW` unset (or set it to `0`) when request-independent numerical behavior is required. Set `COSYVOICE3_BATCH_FLOW_DEBUG=1` to log the observed group-size distribution and enable detailed Stage-1 profiler scopes; diagnostics are disabled by default to avoid per-step profiling overhead.
+Batching preserves output lengths and streaming cache alignment, but the different GEMM shapes can produce small waveform differences compared with processing each request separately. Leave `COSYVOICE3_BATCH_FLOW` unset (or set it to `0`) when request-independent numerical behavior is required. Set `COSYVOICE3_BATCH_FLOW_DEBUG=1` to log the observed group-size distribution and enable detailed Stage-1 profiler scopes; diagnostics are disabled by default to avoid per-step profiling overhead.
 
 ### CLI client
 
@@ -200,7 +194,7 @@ The client supports `--api-base`, `--model`, `--text`, `--ref-audio`, `--ref-tex
 ### Notes
 
 - Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform.
-- Deploy config auto-loads the device-specific CosyVoice3 default described above. Pass `--deploy-config <path>` to override.
+- Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config <path>` to override.
 - For offline inference and the end-to-end script, see the [offline CosyVoice3 section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#cosyvoice3).
 
 ---

docs/user_guide/quantization/fp8.md

@@ -1,4 +1,4 @@
-# FP8 Quantization
+# FP8 Quantization
 
 ## Overview
 
@@ -17,7 +17,7 @@ in deep DiT blocks.
 ## Hardware Support
 
 | Device | Support |
-| -------- | --------- |
+|--------|---------|
 | NVIDIA Blackwell GPU (SM 100+) | ✅ |
 | NVIDIA Ada/Hopper GPU (SM 89+) | ✅ |
 | NVIDIA Ampere GPU (SM 80+) | ✅ |
@@ -93,9 +93,8 @@ warmup_quack_fp8([(14040, 2048, 6144), (14040, 2048, 2048)])
 ### Diffusion Models
 
 | Model | HF models | Online | Pre-calibrated | Recommendation | `ignored_layers` | Text-Encoder quantization |
-| ------- | ----------- | :-------: | :------: | ---------------- | ------------------ | ------------------ |
+|-------|-----------|:-------:|:------:|----------------|------------------|------------------|
 | Qwen-Image | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes | Yes | Skip sensitive image-stream MLPs when quality regresses | `img_mlp` | |
-| Qwen-Image-2.1 | `Qwen/Qwen-Image-2.1` | Yes | Not validated | Skip image-stream MLPs for best fidelity; all-layer FP8 is loadable. On H200 (1024×1024, teapot prompt) DiT FP8-all is ~30.4 GB / 6.06 s vs BF16 ~36.9 GB / 7.34 s (~33 dB vs BF16). Text encoder FP8 covers the language model only — the vision tower and LM head stay BF16 | `img_mlp` | ✅︎ (language model only) |
 | Wan2.2 | Wan2.2 diffusion pipelines | Not validated | Not validated | Validate against BF16 before documenting as supported | TBD | |
 | LTX-2 | `Lightricks/LTX-2`, `rootonchair/LTX-2-19b-distilled` | Yes | Not validated | Transformer only; use dynamic phase LoRA for ordinary two-stage | None | |
 | LTX-2.3 | `diffusers/LTX-2.3-Diffusers`, `diffusers/LTX-2.3-Distilled-Diffusers` | Yes | Not validated | Transformer only; use dynamic phase LoRA for ordinary two-stage | None | |
@@ -111,20 +110,20 @@ warmup_quack_fp8([(14040, 2048, 6144), (14040, 2048, 2048)])
 
 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
 
-| Model      | Scope                        | Format                                   | Status                              |
-| ---------- | ---------------------------- | ---------------------------------------- | ----------------------------------- |
+| Model | Scope | Format | Status |
+|-------|-------|--------|--------|
 | Qwen3-Omni | Thinker language-model stage | [ModelOpt](modelopt.md) `quant_algo=FP8` | Tested for thinker memory reduction |
-| Qwen3-TTS  | TTS language-model stage     | Checkpoint config                        | Not validated                       |
+| Qwen3-TTS | TTS language-model stage | Checkpoint config | Not validated |
 
 Audio encoder, vision encoder, talker, and code2wav stay in BF16 unless a
 model-specific guide says otherwise.
 
 ### Multi-Stage Diffusion Model (BAGEL, GLM-Image)
 
-| Model     | Scope                                    | Status        | Notes                                           |
-| --------- | ---------------------------------------- | ------------- | ----------------------------------------------- |
-| BAGEL     | Stage-specific transformer or DiT module | Not validated | Route FP8 to the intended stage before enabling |
-| GLM-Image | Stage-specific transformer or DiT module | Not validated | Validate quality against BF16 baseline          |
+| Model | Scope | Status | Notes |
+|-------|-------|--------|-------|
+| BAGEL | Stage-specific transformer or DiT module | Not validated | Route FP8 to the intended stage before enabling |
+| GLM-Image | Stage-specific transformer or DiT module | Not validated | Validate quality against BF16 baseline |
 
 ## Configuration
 
@@ -163,7 +162,7 @@ For a pipeline that exposes both a transformer and a quantization-aware text
 encoder, the scope is:
 
 | Configuration | Transformer | Text encoder | Components without supported quantizable layers |
-| --------------- | ------------- | -------------- | ------------------------------------------------- |
+|---------------|-------------|--------------|-------------------------------------------------|
 | `quantization="fp8"` | FP8 | FP8 | checkpoint precision |
 | `{"transformer": {"method": "fp8"}}` | FP8 | checkpoint precision | checkpoint precision |
 | `{"text_encoder": {"method": "fp8"}}` | checkpoint precision | FP8 | checkpoint precision |
@@ -212,7 +211,7 @@ does not fit the workload.
 ## Parameters
 
 | Parameter | Type | Default | Description |
-| ----------- | ------ | --------- | ------------- |
+|-----------|------|---------|-------------|
 | `method` | str | - | Quantization method (`"fp8"`) |
 | `ignored_layers` | list[str] | `[]` | Layer name patterns to keep in BF16 |
 | `activation_scheme` | str | `"dynamic"` | `"dynamic"` selects online activation scaling, or `"static"` when scales are available |

examples/offline_inference/image_to_image/image_edit.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 """
 Example script for image editing with OmniGen2.
@@ -39,20 +39,6 @@ Usage (multiple images):
         --cfg-scale 4.0 \
         --guidance-scale 1.0
 
-Usage (JoyAI-Image-Edit, single image):
-    python image_edit.py \
-        --model jdopensource/JoyAI-Image-Edit-Diffusers \
-        --image input.png \
-        --prompt "Change the background to a clean studio while preserving the subject." \
-        --height 1024 \
-        --width 1024 \
-        --num-inference-steps 50 \
-        --guidance-scale 4.0 \
-        --output output_joyai_edit.png
-
-    Note: JoyAI-Image-Edit snaps requested dimensions to the nearest supported
-    Joy bucket; for square outputs, use 1024x1024.
-
 Usage (with cache-dit acceleration):
     python image_edit.py \
         --image input.png \
@@ -223,14 +209,14 @@ def parse_profiler_config(value: str) -> dict[str, Any]:
 
 
 def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Edit images with vLLM-Omni.")
+    parser = argparse.ArgumentParser(description="Edit an image with Qwen-Image-Edit.")
     parser.add_argument(
         "--model",
         default="Qwen/Qwen-Image-Edit",
         help=(
             "Diffusion model name or local path. "
-            "For multiple image inputs, use Qwen/Qwen-Image-Edit-2509, "
-            "Qwen/Qwen-Image-Edit-2511, or Qwen/Qwen-Image-2.1."
+            "For multiple image inputs, use Qwen/Qwen-Image-Edit-2509 or Qwen/Qwen-Image-Edit-2511"
+            "which supports QwenImageEditPlusPipeline."
         ),
     )
     parser.add_argument(
@@ -284,10 +270,9 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--cfg-scale",
         type=float,
-        default=None,
+        default=4.0,
         help=(
-            "True classifier-free guidance scale. When omitted, the pipeline uses its model-specific default. "
-            "Guidance scale as defined in Classifier-Free "
+            "True classifier-free guidance scale (default: 4.0). Guidance scale as defined in Classifier-Free "
             "Diffusion Guidance. Classifier-free guidance is enabled by setting cfg_scale > 1 and providing "
             "a negative_prompt. Higher guidance scale encourages images closely linked to the text prompt, "
             "usually at the expense of lower image quality."
@@ -296,9 +281,9 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--guidance-scale",
         type=float,
-        default=None,
+        default=1.0,
         help=(
-            "Guidance scale for guidance-distilled models. When omitted, the pipeline uses its model-specific default. "
+            "Guidance scale for guidance-distilled models (default: 1.0, disabled). "
             "Unlike classifier-free guidance (--cfg-scale), guidance-distilled models take the guidance scale "
             "directly as an input parameter. Enabled when guidance_scale > 1. Ignored when not using guidance-distilled models."
         ),
@@ -383,7 +368,7 @@ def parse_args() -> argparse.Namespace:
         "--color-format",
         type=str,
         default="RGB",
-        help="For Qwen-Image-Layered or Qwen-Image-2.1, set to RGBA to preserve transparency.",
+        help="For Qwen-Image-Layered, set to RGBA.",
     )
 
     # Cache-DiT specific parameters
@@ -538,12 +523,6 @@ def parse_args() -> argparse.Namespace:
             "silently masking with a possibly-wrong prompt."
         ),
     )
-    parser.add_argument(
-        "--init-timeout",
-        type=int,
-        default=600,
-        help="Overall pipeline initialization timeout in seconds.",
-    )
     return parser.parse_args()
 
 
@@ -614,7 +593,6 @@ def main():
         enable_cpu_offload=args.enable_cpu_offload,
         enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
         profiler_config=args.profiler_config,
-        init_timeout=args.init_timeout,
     )
     if args.enforce_eager is not None:
         omni_kwargs["enforce_eager"] = args.enforce_eager
@@ -669,7 +647,6 @@ def main():
 
     diffusion_params = OmniDiffusionSamplingParams(
         generator=generator,
-        seed=args.seed,
         true_cfg_scale=args.cfg_scale,
         guidance_scale=args.guidance_scale,
         guidance_scale_2=args.guidance_scale_2,

examples/offline_inference/image_to_image/image_to_image.md

@@ -1,6 +1,6 @@
 # Image-To-Image
 
-This example edits an input image with `Qwen/Qwen-Image-Edit` and `JoyAI-Image-Edit` using the `image_edit.py` CLI.
+This example edits an input image with `Qwen/Qwen-Image-Edit` using the `image_edit.py` CLI.
 
 ## Local CLI Usage
 
@@ -23,27 +23,6 @@ python image_edit.py \
   --cfg-scale 4.0
 ```
 
-### JoyAI-Image-Edit
-
-JoyAI-Image-Edit uses the same `image_edit.py` CLI and supports one input
-image per request. Requested dimensions are snapped to the nearest supported Joy
-bucket; for square outputs, use 1024x1024.
-
-```bash
-python image_edit.py \
-  --model jdopensource/JoyAI-Image-Edit-Diffusers \
-  --image input.png \
-  --prompt "Change the background to a clean studio while preserving the subject." \
-  --height 1024 \
-  --width 1024 \
-  --num-inference-steps 50 \
-  --guidance-scale 4.0 \
-  --output output_joyai_edit.png
-```
-
-JoyAI defaults to guidance scale `4.0`; pass `--guidance-scale` explicitly when
-you need a different value.
-
 ### Multiple Image Editing (Qwen-Image-Edit-2509)
 
 For multiple image inputs, use `Qwen/Qwen-Image-Edit-2509` or  `Qwen/Qwen-Image-Edit-2511`:
@@ -61,7 +40,7 @@ python image_edit.py \
 
 Key arguments:
 
-- `--model`: model name or path. Use `Qwen/Qwen-Image-Edit-2509` or later for multiple image support. `jdopensource/JoyAI-Image-Edit-Diffusers` supports one input image.
+- `--model`: model name or path. Use `Qwen/Qwen-Image-Edit-2509` or later for multiple image support.
 - `--image`: path(s) to the source image(s) (PNG/JPG, converted to RGB). Can specify multiple images.
 - `--prompt` / `--negative-prompt`: text description (string).
 - `--cfg-scale`: true classifier-free guidance scale (default: 4.0). Classifier-free guidance is enabled by setting cfg_scale > 1 and providing a negative_prompt. Higher guidance scale encourages images closely linked to the text prompt, usually at the expense of lower image quality.

examples/offline_inference/image_to_image/run_qwen_image_21.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
-
-set -euo pipefail
-
-python "$(dirname "${BASH_SOURCE[0]}")/image_edit.py" \
-    --model Qwen/Qwen-Image-2.1 \
-    --color-format RGBA \
-    --seed 42 \
-    --image qwen_bear.png \
-    --prompt "Let this mascot dance under the moon, surrounded by floating stars and poetic bubbles such as 'Be Kind'" \
-    --output qwen_image_21_edit.png \
-    --num-inference-steps 50 \
-    --cfg-scale 1.0 \
-    "$@"

examples/offline_inference/image_to_video/README.md

@@ -1,7 +1,7 @@
 # Image-To-Video
 
 This shared example generates videos from images with VACE, Wan2.2, LTX-2,
-HunyuanVideo-1.5, SANA-Video, Cosmos3, MAGI-2, Kandinsky 6, and other compatible pipelines.
+HunyuanVideo-1.5, SANA-Video, Cosmos3, MAGI-2, and other compatible pipelines.
 
 - `image_to_video.py`: command-line script for single video generation with advanced options.
 
@@ -30,7 +30,6 @@ This folder provides a unified CLI script for image-to-video generation using vL
 | `sand-ai/MAGI-2-preview` | 512 x 896 | 125 | 100 | Model-fixed | Native four-GPU TP/SP; resident SP4 default; DLO available |
 | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers` | 480 x 832 | 81 | 50 | 6.0 | Native `SanaImageToVideoPipeline`; Wan VAE |
 | `Efficient-Large-Model/SANA-Video_2B_720p_diffusers` | 704 x 1280 | 81 | 50 | 6.0 | Native `SanaImageToVideoPipeline`; LTX-2 Video VAE |
-| `kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers` | 480 x 864 | 125 | 50 | 5.0 | ~75 GiB reserved (H100, CPU offload); gated Hub repo |
 
 MAGI-2 native Preview setup, four-GPU topology and DLO choices, I2VA commands,
 and eight-GPU validation status are documented in the

examples/offline_inference/text_to_image/text_to_image.py

@@ -14,7 +14,7 @@ from diffusers.utils import numpy_to_pil
 
 from vllm_omni.diffusion.data import logger
 from vllm_omni.diffusion.utils.image_output import extract_images_from_outputs
-from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args, ar_grid_max_tokens
+from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.entrypoints.openai.stage_params import clone_sampling_params
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
@@ -72,24 +72,6 @@ def build_text_to_image_prompt(prompt: str, negative_prompt: str | None) -> dict
     return result
 
 
-def build_parallel_knob_kwargs(args: argparse.Namespace) -> dict[str, Any]:
-    """Forward explicitly-set parallel knobs so they can override a deploy YAML.
-
-    The argparse defaults are ``None``; only explicitly supplied values enter
-    the returned dict, letting a deploy YAML's parallel_config stay in effect
-    when the flags are omitted (and letting explicit values equal to the old
-    argparse defaults, e.g. ``--ulysses-degree 1``, still override the YAML).
-    """
-    knob_names = (
-        "ulysses_degree",
-        "ring_degree",
-        "ulysses_mode",
-        "cfg_parallel_size",
-        "vae_patch_parallel_size",
-    )
-    return {name: getattr(args, name) for name in knob_names if getattr(args, name) is not None}
-
-
 def _normalize_images_for_save(images: list[Any]) -> list[Any]:
     """Convert NumPy diffusion outputs to PIL images before saving."""
     normalized = []
@@ -101,15 +83,13 @@ def _normalize_images_for_save(images: list[Any]) -> list[Any]:
     return normalized
 
 
-def build_parser() -> argparse.ArgumentParser:
-    """Build the example's CLI parser (kept separate for testability)."""
+def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="Generate an image with supported diffusion models.")
     parser.add_argument(
         "--model",
         default="Qwen/Qwen-Image",
         help="Diffusion model name or local path. Supported models: "
-        "Qwen/Qwen-Image, Tongyi-MAI/Z-Image-Turbo, Qwen/Qwen-Image-2512, Qwen/Qwen-Image-2.1, "
-        "stepfun-ai/NextStep-1.1, "
+        "Qwen/Qwen-Image, Tongyi-MAI/Z-Image-Turbo, Qwen/Qwen-Image-2512, stepfun-ai/NextStep-1.1, "
         "black-forest-labs/FLUX.1-dev, black-forest-labs/FLUX.2-klein-9B, "
         "black-forest-labs/FLUX.2-dev, tencent/HunyuanImage-3.0-Instruct, "
         "HiDream-ai/HiDream-O1-Image, meituan-longcat/LongCat-Image, OvisAI/Ovis-Image, "
@@ -186,31 +166,28 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument(
         "--ulysses-degree",
         type=int,
-        default=None,
-        help="Number of GPUs used for ulysses sequence parallelism. "
-        "Explicitly passing 1 overrides a deploy YAML value.",
+        default=1,
+        help="Number of GPUs used for ulysses sequence parallelism.",
     )
     parser.add_argument(
         "--ulysses-mode",
         type=str,
-        default=None,
+        default="strict",
         choices=["strict", "advanced_uaa"],
-        help="Ulysses sequence-parallel mode: 'strict' (divisibility required) or 'advanced_uaa' (UAA). "
-        "Explicitly passing 'strict' overrides a deploy YAML value.",
+        help="Ulysses sequence-parallel mode: 'strict' (divisibility required) or 'advanced_uaa' (UAA).",
     )
     parser.add_argument(
         "--ring-degree",
         type=int,
-        default=None,
-        help="Number of GPUs used for ring sequence parallelism. Explicitly passing 1 overrides a deploy YAML value.",
+        default=1,
+        help="Number of GPUs used for ring sequence parallelism.",
     )
     parser.add_argument(
         "--cfg-parallel-size",
         type=int,
-        default=None,
+        default=1,
         choices=[1, 2],
-        help="Number of GPUs used for classifier free guidance parallel size. "
-        "Explicitly passing 1 overrides a deploy YAML value.",
+        help="Number of GPUs used for classifier free guidance parallel size.",
     )
     parser.add_argument(
         "--enforce-eager",
@@ -319,9 +296,8 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument(
         "--vae-patch-parallel-size",
         type=int,
-        default=None,
-        help="Number of ranks used for VAE patch/tile parallelism (decode/encode). "
-        "Explicitly passing 1 overrides a deploy YAML value.",
+        default=1,
+        help="Number of ranks used for VAE patch/tile parallelism (decode/encode).",
     )
     # NextStep-1.1 specific arguments
     parser.add_argument(
@@ -437,11 +413,7 @@ def build_parser() -> argparse.ArgumentParser:
         ),
     )
     current_omni_platform.pre_register_and_update(parser)
-    return parser
-
-
-def parse_args() -> argparse.Namespace:
-    return build_parser().parse_args()
+    return parser.parse_args()
 
 
 def _apply_ar_stage_inputs(
@@ -555,8 +527,14 @@ def main():
             "scm_steps_policy": "dynamic",  # SCM steps policy: "dynamic" or "static"
         }
     elif cache_backend == "tea_cache":
-        # Let TeaCache select the model-specific threshold and coefficients.
-        cache_config = {}
+        # TeaCache configuration
+        # All parameters marked with [tea_cache only] in DiffusionCacheConfig
+        cache_config = {
+            # TeaCache parameters [tea_cache only]
+            "rel_l1_thresh": 0.2,  # Threshold for accumulated relative L1 distance
+            # Note: coefficients will use model-specific defaults based on model_type
+            #       (e.g., QwenImagePipeline or FluxPipeline)
+        }
 
     profiler_enabled = args.profiler_config is not None
 
@@ -584,11 +562,21 @@ def main():
 
     omni_kwargs = {
         "model": args.model,
+        "enable_layerwise_offload": args.enable_layerwise_offload,
+        "vae_use_slicing": args.vae_use_slicing,
+        "vae_use_tiling": args.vae_use_tiling,
         "cache_backend": args.cache_backend,
         "cache_config": cache_config,
+        "enable_cache_dit_summary": args.enable_cache_dit_summary,
+        "ulysses_degree": args.ulysses_degree,
+        "ring_degree": args.ring_degree,
+        "ulysses_mode": args.ulysses_mode,
+        "cfg_parallel_size": args.cfg_parallel_size,
+        "vae_patch_parallel_size": args.vae_patch_parallel_size,
         "enable_expert_parallel": args.enable_expert_parallel,
         "enable_cpu_offload": args.enable_cpu_offload,
         "log_stats": args.log_stats,
+        "enable_diffusion_pipeline_profiler": args.enable_diffusion_pipeline_profiler,
         "profiler_config": args.profiler_config,
         "init_timeout": args.init_timeout,
         "stage_init_timeout": args.stage_init_timeout,
@@ -596,8 +584,6 @@ def main():
         **lora_args,
         **quant_kwargs,
     }
-    # Diffusion-only engine args: pass only when explicitly set, so pipelines
-    # without a DIFFUSION stage pass ownership validation.
     if args.tensor_parallel_size is not None:
         omni_kwargs["tensor_parallel_size"] = args.tensor_parallel_size
     if args.enforce_eager is not None:
@@ -617,18 +603,6 @@ def main():
     # gate is an engine-level config (offline analog of the server's --no-guardrails).
     if args.extra_body and "guardrails" in args.extra_body:
         omni_kwargs["model_config"] = {"guardrails": bool(args.extra_body["guardrails"])}
-    if args.enable_layerwise_offload:
-        omni_kwargs["enable_layerwise_offload"] = True
-    if args.vae_use_slicing:
-        omni_kwargs["vae_use_slicing"] = True
-    if args.vae_use_tiling:
-        omni_kwargs["vae_use_tiling"] = True
-    if args.enable_cache_dit_summary:
-        omni_kwargs["enable_cache_dit_summary"] = True
-    if args.enable_diffusion_pipeline_profiler:
-        omni_kwargs["enable_diffusion_pipeline_profiler"] = True
-    omni_kwargs.update(build_parallel_knob_kwargs(args))
-
     omni = Omni(**omni_kwargs)
     model_class_name = get_model_class_name(omni)
     declared_extra_body_params = get_extra_body_params(model_class_name)
@@ -647,17 +621,11 @@ def main():
     if ignored_layers:
         print(f"  Ignored layers: {ignored_layers}")
     tp_display = args.tensor_parallel_size if args.tensor_parallel_size is not None else "deploy/default"
-
-    def _knob_display(value: Any, default: Any) -> Any:
-        return value if value is not None else f"deploy/default ({default})"
-
     print(
         f"  Parallel configuration: tensor_parallel_size={tp_display}, "
-        f"ulysses_degree={_knob_display(args.ulysses_degree, 1)}, "
-        f"ulysses_mode={_knob_display(args.ulysses_mode, 'strict')}, "
-        f"ring_degree={_knob_display(args.ring_degree, 1)}, "
-        f"cfg_parallel_size={_knob_display(args.cfg_parallel_size, 1)}, "
-        f"vae_patch_parallel_size={_knob_display(args.vae_patch_parallel_size, 1)}, "
+        f"ulysses_degree={args.ulysses_degree}, ulysses_mode={args.ulysses_mode}, "
+        f"ring_degree={args.ring_degree}, cfg_parallel_size={args.cfg_parallel_size}, "
+        f"vae_patch_parallel_size={args.vae_patch_parallel_size}, "
         f"enable_expert_parallel={args.enable_expert_parallel}."
     )
     print(f"  CPU offload: {args.enable_cpu_offload}; CPU Layerwise Offload: {args.enable_layerwise_offload}")
@@ -768,9 +736,8 @@ def main():
             if idx == 0 and prompt_info.get("omni_task") == ["t2i"]:
                 ar_width = int(prompt_info.get("ar_width", [0])[0])
                 ar_height = int(prompt_info.get("ar_height", [0])[0])
-                ar_grid_budget = ar_grid_max_tokens(ar_width, ar_height)
-                if ar_grid_budget is not None:
-                    params.max_tokens = ar_grid_budget
+                if ar_width > 0 and ar_height > 0:
+                    params.max_tokens = ar_height * (ar_width + 1) + 1
 
     if not diffusion_replaced and len(sampling_params_list) == 1:
         sampling_params_list = [diffusion_params]

examples/offline_inference/text_to_speech/cosyvoice3/end2end.py

@@ -1,7 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 import argparse
-import copy
 import functools
 import json
 import os
@@ -11,11 +10,13 @@ from typing import Any
 
 import numpy as np
 import soundfile as sf
+from vllm import SamplingParams
 from vllm.multimodal.media.audio import load_audio
 
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
 from vllm_omni.model_executor.models.cosyvoice3.utils import extract_text_token
+from vllm_omni.transformers_utils.configs.cosyvoice3 import CosyVoice3Config
 
 # Upstream zero-shot reference clip
 ZERO_SHOT_PROMPT_URL = "https://raw.githubusercontent.com/FunAudioLLM/CosyVoice/main/asset/zero_shot_prompt.wav"
@@ -58,7 +59,8 @@ def run_e2e():
         "--deploy-config",
         type=str,
         default=None,
-        help="Override the deploy config path. If unset, selects the bundled CosyVoice3 profile for the device.",
+        help="Override the deploy config path. If unset, auto-loads "
+        "vllm_omni/deploy/cosyvoice3.yaml based on the HF model_type.",
     )
     parser.add_argument("--text", type=str, default="Hello, this is a test of the CosyVoice system capability.")
     parser.add_argument(
@@ -103,6 +105,8 @@ def run_e2e():
         profiler_config=args.profiler_config,
     )
 
+    sampling_cfg = {"top_p": 0.8, "top_k": 25, "eos_token_id": 6561 + 1}
+
     print("Model initialized. Preparing inputs...")
     ref_audio_path = args.ref_audio or _default_ref_audio()
     if not os.path.exists(ref_audio_path):
@@ -134,7 +138,7 @@ def run_e2e():
 
     print(f"Generating for prompt: {args.text}")
 
-    config = omni.engine.stage_vllm_configs[0].model_config.hf_config
+    config = CosyVoice3Config()
     tokenizer = get_qwen_tokenizer(
         token_path=args.tokenizer,
         skip_special_tokens=config.skip_special_tokens,
@@ -145,10 +149,29 @@ def run_e2e():
     min_len = int(base_len * config.min_token_text_ratio)
     max_len = int(base_len * config.max_token_text_ratio)
 
-    # Keep the deploy profile's sampling policy, penalties and required stops.
-    sampling_params_list = copy.deepcopy(omni.default_sampling_params_list)
-    sampling_params_list[0].max_tokens = max(1, min(2048, max_len))
-    sampling_params_list[0].min_tokens = min(max(1, min_len), sampling_params_list[0].max_tokens)
+    # Build SamplingParams for each stage (GPT, S2Mel, Vocoder)
+    gpt_sampling = SamplingParams(
+        temperature=1.0,
+        top_p=sampling_cfg["top_p"],
+        top_k=sampling_cfg["top_k"],
+        repetition_penalty=2.0,
+        min_tokens=min_len,
+        max_tokens=max_len,
+        stop_token_ids=[sampling_cfg["eos_token_id"]],
+        # allowed_token_ids=list(range(6561+3)),
+        detokenize=False,
+    )
+    # Not used
+    s2mel_sampling = SamplingParams(
+        temperature=1.0,
+        top_p=1.0,
+        top_k=-1,
+        repetition_penalty=2.0,
+        max_tokens=256,
+        detokenize=False,
+    )
+
+    sampling_params_list = [gpt_sampling, s2mel_sampling]
 
     profiler_enabled = args.profiler_config is not None
     if profiler_enabled:

examples/offline_inference/text_to_video/text_to_video.md

@@ -14,7 +14,6 @@ A unified script for text-to-video generation. Supports multiple models with mod
 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required |
 | `nvidia/Cosmos3-Nano` | 720x1280 | 189 | 35 | 6.0 | ~46 GiB (peak, 720p) |
 | `BestWishYsh/Helios-Base` / `Helios-Mid` / `Helios-Distilled` | 384x640 | 99 | 50 | 5.0 / 5.0 / 1.0 | — |
-| `kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers` | 480x864 | 125 | 50 | 5.0 | ~75 GiB reserved (H100, CPU offload, SDPA) |
 | `sand-ai/MAGI-2-preview` | 512x896 | 125 | 100 | Model-fixed | Native four-GPU TP/SP; resident SP4 default; DLO available |
 | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers` | 480x832 | 81 | 50 | 6.0 | BF16 DiT + FP32 Wan VAE wrapper |
 | `Efficient-Large-Model/SANA-Video_2B_720p_diffusers` | 704x1280 | 81 | 50 | 6.0 | BF16 DiT + LTX-2 Video VAE wrapper |
@@ -250,24 +249,6 @@ python text_to_video.py \
 > conditioning tensors that cannot be passed through the JSON `--extra-body`
 > flag; they are out of scope for this text-to-video example.
 
-### Kandinsky 6 (TI2VA)
-
-The Hub checkpoint is gated; export `HF_TOKEN` if download returns 401.
-`--enable-cpu-offload` is required on an 80 GB H100. Audio is on by default.
-
-```bash
-python text_to_video.py \
-  --model kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers \
-  --prompt "A golden retriever runs along a sunny beach, waves crashing, cinematic footage" \
-  --enable-cpu-offload \
-  --output kandinsky6_t2va.mp4
-```
-
-Video only: add `--extra-body '{"sample_audio": false}'`. Image-to-video uses
-the shared [`image_to_video.py`](../image_to_video/README.md) example with
-`--image`. Serving numbers and flags are in the
-[`Kandinsky 6 TI2VA recipe`](../../../recipes/Kandinsky/Kandinsky6-TI2VA.md).
-
 ## Key Arguments
 
 ### Common

examples/offline_inference/text_to_video/text_to_video.py

@@ -235,7 +235,7 @@ def parse_extra_body(value: str) -> dict[str, Any]:
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(
         description="Generate a video from a text prompt. "
-        "Supports Wan2.2, HunyuanVideo-1.5, Helios, LingBot-Video, MAGI-2, Kandinsky 6, "
+        "Supports Wan2.2, HunyuanVideo-1.5, Helios, LingBot-Video, MAGI-2, "
         "and other text-to-video models."
     )
     parser.add_argument(

examples/offline_inference/yue2/end2end.py

@@ -28,9 +28,7 @@ import torch
 from vllm_omni import Omni
 from vllm_omni.model_executor.models.yue2.yue2 import (
     ABC_SAMPLING,
-    CONTEXT,
     KEY_MAX_AUDIO_FRAMES,
-    KEY_MAX_HOLD_STEPS,
     KEY_MIN_TOKENS,
     KEY_PENALTY_WINDOW,
     KEY_PHASE,
@@ -43,12 +41,10 @@ from vllm_omni.model_executor.models.yue2.yue2 import (
     KEY_TOP_P,
     SEMANTIC_SAMPLING,
     STOP_TOKEN_IDS,
-    SYNTHESIS_HOLD_STEPS,
 )
 from vllm_omni.tokenizers.yue2_prompt import (
     abc_ids_from_generated,
     abc_prefix_ids,
-    semantic_frames,
     semantic_prefix_ids,
 )
 from vllm_omni.tokenizers.yue2_tokenizer import YuE2TextTokenizer
@@ -75,13 +71,7 @@ def sampling_params(engine, *, phase, seed, max_frames, prompt_ids):
         # conditioning prefix from the scheduled tokens alone.
         KEY_PREFIX_IDS: list(prompt_ids),
     }
-    if phase == "abc":
-        params.max_tokens = preset["max_tokens"]
-    else:
-        # The song is synthesized on a side stream while the request is held;
-        # leave max_tokens headroom for the hold tokens.
-        params.max_tokens = min(max_frames + 1 + SYNTHESIS_HOLD_STEPS, CONTEXT - len(prompt_ids))
-        params.extra_args[KEY_MAX_HOLD_STEPS] = max(0, params.max_tokens - max_frames - 1)
+    params.max_tokens = preset["max_tokens"] if phase == "abc" else max_frames + 1
     params.stop_token_ids = list(STOP_TOKEN_IDS)
     params.detokenize = False
     return params
@@ -154,9 +144,6 @@ def main() -> None:
         payload = {
             "prompt_token_ids": prompt_ids,
             "generated_token_ids": generated_ids,
-            # Codec frames only: without the end token and the hold tokens
-            # the request emits while its song is synthesized.
-            "semantic_frames": semantic_frames(generated_ids),
             "seed": args.seed,
             "cot": args.cot,
             "max_frames": args.max_frames,

examples/online_serving/image_to_image/README.md

@@ -2,12 +2,10 @@
 
 This example demonstrates how to deploy image-to-image models for online image editing service using vLLM-Omni.
 
-Supported models include Qwen-Image-Edit, BAGEL, JoyAI-Image-Edit, and other image-to-image pipelines.
+Supported models include Qwen-Image-Edit, BAGEL, and other image-to-image pipelines.
 
 For **multi-image** input editing, use **Qwen-Image-Edit-2509** (QwenImageEditPlusPipeline) and send multiple images in the user message content.
 
-For **JoyAI-Image-Edit**, use `jdopensource/JoyAI-Image-Edit-Diffusers` and send exactly one input image.
-
 ## Start Server
 
 ### Basic Start
@@ -31,12 +29,6 @@ vllm serve Qwen/Qwen-Image-Edit-2509 --omni --port 8092
 vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091
 ```
 
-### JoyAI-Image-Edit
-
-```bash
-vllm serve jdopensource/JoyAI-Image-Edit-Diffusers --omni --port 8092
-```
-
 ### Start with Parameters
 
 
@@ -85,11 +77,6 @@ EOF
 curl -s http://localhost:8092/v1/chat/completions   -H "Content-Type: application/json"   -d @request.json | jq -r '.choices[0].message.content[0].image_url.url' | cut -d',' -f2 | base64 -d > output.png
 ```
 
-For JoyAI-Image-Edit, set classifier-free guidance with `guidance_scale`
-in `extra_body` (default: `4.0`). `true_cfg_scale` remains supported as an
-alias; if both are provided, their values must match. Values at or below `1.0`
-disable CFG.
-
 ### Method 2: Using OpenAI Python SDK
 
 ```python

examples/online_serving/qwen3_omni/README.md

@@ -277,6 +277,61 @@ cd examples/online_serving/qwen3_omni
 python examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py --model Qwen/Qwen3-Omni-30B-A3B-Instruct --query-type use_image --port 8091 --host "localhost"
 ```
 
+#### Realtime WebSocket client (`openai_realtime_client.py`)
+
+[`openai_realtime_client.py`](./openai_realtime_client.py) connects to **`ws://<host>:<port>/v1/realtime`**, streams a local WAV as **PCM16 mono @ 16 kHz** in fixed-size chunks, and receives **`response.output_audio.*`**, **`response.output_text.*`**, and transcript events. By default it uses explicit `input_audio_buffer.commit`; pass **`--server-vad`** to let the server detect speech endpoints and commit the turn automatically. The client concatenates audio deltas and writes **`--output-wav`** (model output is typically **24 kHz**). Optional **`--delta-dump-dir`** saves each delta as `delta_000001.wav`, … for debugging.
+
+Streaming input works well for translation-style use cases; if the Thinker runs while input is still incomplete, consider limiting **`max_tokens`** in your session / server defaults to avoid over-generation.
+
+**Dependencies:**
+
+```bash
+pip install websockets
+```
+
+**From this directory** (`examples/online_serving/qwen3_omni`):
+
+```bash
+python openai_realtime_client.py \
+  --url ws://localhost:8091/v1/realtime \
+  --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
+  --input-wav /path/to/input_16k_mono.wav \
+  --output-wav realtime_output.wav \
+  --server-vad \
+  --delta-dump-dir ./rt_delta_wavs
+```
+
+**Arguments:**
+
+| Flag | Default | Description |
+| ------ | --------- | ------------- |
+| `--url` | `ws://localhost:8091/v1/realtime` | Full WebSocket URL including path |
+| `--model` | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | Must match the served model (sent in `session.update`) |
+| `--input-wav` | *(required)* | Input WAV: mono, 16-bit PCM, **16 kHz** |
+| `--output-wav` | `realtime_output.wav` | Output path for concatenated reply audio |
+| `--output-text` | *(optional)* | If set, write final transcription text to this path |
+| `--chunk-ms` | `200` | Size of each uploaded audio chunk (milliseconds of audio) |
+| `--send-delay-ms` | `0` | Delay between chunk sends (simulate realtime upload) |
+| `--delta-dump-dir` | *(optional)* | Directory to write per-`response.output_audio.delta` WAV files |
+| `--server-vad` | disabled | Enable server-side Silero VAD instead of sending explicit commits |
+| `--num-requests` | `1` | Number of sequential sessions (see `--concurrency`) |
+| `--concurrency` | `1` | Max concurrent WebSocket sessions when `--num-requests` > 1 |
+
+Server VAD uses the pinned Silero v6.2 ONNX artifact from `istupakov/silero-vad-onnx` and never downloads model files while processing a session. Make the artifact available in the Hugging Face cache before startup, or configure a local artifact in the deployment YAML:
+
+```yaml
+base_config: /path/to/vllm-omni/vllm_omni/deploy/qwen3_omni_duplex.yaml
+duplex_session:
+  server_vad_model_path: /models/silero_vad.onnx
+```
+
+For the engine-owned plugin, use an overlay based on
+`vllm_omni/deploy/qwen3_omni_duplex.yaml`, with `duplex_session.server_vad_model_path`
+pointing to your local artifact. See the [complete duplex setup](../realtime_web/README.md#qwen3-with-server-vad-automatic-turns).
+The default `qwen3_omni_moe.yaml` remains a turn deployment. Duplex mode serves
+`/v1/realtime?duplex=1` and `/v1/chat/completions`; the legacy STT handler requires
+a turn deployment.
+
 The Python client supports the following command-line arguments:
 
 - `--query-type` (or `-q`): Query type (default: `use_video`). Options: `text`, `use_audio`, `use_image`, `use_video`
@@ -303,13 +358,6 @@ python examples/online_serving/openai_chat_completion_client_for_multimodal_gene
 bash run_curl_multimodal_generation.sh use_image
 ```
 
-### Realtime WebSocket
-
-The [shared Realtime UI](../realtime_web/README.md#qwen3-explicit-turn-realtime)
-supports explicit turns through `/v1/realtime?duplex=0`. The
-[Server VAD setup](../realtime_web/README.md#qwen3-with-server-vad-automatic-turns)
-uses the duplex endpoint.
-
 ### FAQ
 
 ## Modality control
@@ -534,6 +582,26 @@ The gradio script supports the following arguments:
 
 Run these commands from the repository root with the vLLM-Omni environment activated.
 
+### Without VAD: manually submit each turn
+
+Start the backend:
+
+```bash
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
+```
+
+In another terminal, check backend readiness and start the UI:
+
+```bash
+curl --fail http://127.0.0.1:8091/health
+python -m examples.online_serving.qwen3_omni.realtime_web \
+    --backend ws://127.0.0.1:8091 --stt --port 7863
+```
+
+Open `http://localhost:7863`, start a session, speak, and press **Send turn**.
+`--stt` is the default. Each turn uses a separate connection without shared model
+history; the conversation shown in the browser is a local log.
+
 ### With VAD: automatically submit after silence
 
 Start the backend with a Server VAD deployment overlay and a compatible Silero
@@ -542,7 +610,7 @@ ONNX artifact, following the
 After its health check succeeds, start the UI:
 
 ```bash
-python -m examples.online_serving.realtime_web --profile qwen3-turn \
+python -m examples.online_serving.qwen3_omni.realtime_web \
     --backend ws://127.0.0.1:8091 --vad --port 7863
 ```
 
@@ -553,7 +621,9 @@ VAD mode keeps uploading audio during replies and supports speech interruption.
 The engine-owned Qwen duplex plugin maintains conversation history and tracks
 played replies through playback acknowledgements. Each committed utterance starts
 a new generation request; the model does not use native streaming-input KV decoding.
+STT mode pauses microphone upload while the reply is generated and played.
 In VAD mode, click **Camera** to upload sampled frames with your spoken question;
 see [camera setup and limits](../realtime_web/README.md#qwen-vad-camera-input).
-For explicit turns, use `--manual` with the shared UI as described in the
-[shared UI guide](../realtime_web/README.md#qwen3-explicit-turn-realtime).
+STT remains audio-only. To switch back to manual turns, use the default turn
+deployment, restart the UI with `--stt`, refresh the page, and reconnect. Stop the existing UI before reusing port 7863.
+See the [shared UI guide](../realtime_web/README.md) for HTTPS access and testing.

examples/online_serving/realtime_web/README.md

@@ -7,14 +7,14 @@ This implements [RFC #7222](https://github.com/vllm-project/vllm-omni/issues/722
 | Profile | Turn control | Camera | Playback acknowledgement |
 | --- | --- | --- | --- |
 | `minicpm-native` | Model-controlled listen/speak, continuous audio input | Frames accompany audio | Yes |
-| `qwen3-turn --manual` | Press **Send turn**; mic pauses while the reply plays | No | No |
+| `qwen3-turn --stt` (default) | User presses **Send turn** | No | No |
 | `qwen3-turn --vad` | Server detects trailing silence; speech interrupts replies | Sampled frames with each spoken turn | Yes |
 | `aura-ptt` | Hold **Hold to talk**; release commits. Hold also stops local playback | Sticky frames with speech; vision-follow every 2 frames only while unlocked (locked from PTT release until text-final / listen) | Yes |
 
 Qwen3 VAD uses an engine-owned duplex plugin: microphone upload continues
 while replies stream, and speech can interrupt generation and playback. The
 checkpoint still generates committed turns; this is not native streaming-input
-KV decoding.
+KV decoding. STT remains explicit-turn and audio-only.
 
 ## MiniCPM: existing command stays valid
 
@@ -62,36 +62,45 @@ just-committed speech turn on Stage0/1 (`max_num_seqs=1`). Audio still playing
 does not hold the next vision turn once text is done.
 MiniCPM / Qwen profiles do not set `pushToTalk`, so the PTT control stays hidden.
 
-## Qwen3: explicit-turn Realtime
+## Qwen3: explicit-turn STT adapter
 
-The shared UI sends OpenAI Realtime events to `/v1/realtime?duplex=0` and keeps
-conversation history on the same connection:
-
-Start the turn-based backend:
+Start the backend, then the UI in a second terminal:
 
 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
+
+python -m examples.online_serving.qwen3_omni.realtime_web \
+    --backend ws://127.0.0.1:8091 --stt --port 7863
 ```
 
-In another terminal, check readiness and start the UI:
+Open `http://localhost:7863`, start a session, speak, and press **Send turn**.
+Wait for the answer to finish playing before speaking again. The UI opens a
+fresh STT connection for each turn; the displayed conversation is a local log,
+and **previous turns are not provided as model history**. The STT endpoint
+accepts only model selection, so system-prompt controls are hidden in this mode.
 
-```bash
-curl --fail http://127.0.0.1:8091/health
-python -m examples.online_serving.realtime_web --profile qwen3-turn \
-    --backend ws://127.0.0.1:8091 --manual --port 7863
-```
+The adapter follows the shipped `qwen3_omni/openai_realtime_client.py` and
+`vllm_omni/entrypoints/openai/realtime_connection.py`:
 
-Speak, then press **Send turn**. The client commits the audio and requests a
-response; recording pauses while the reply plays.
+
+1. Explicit `duplex=0` selects the legacy STT handler on a turn deployment.
+   It does not enable STT on a duplex deployment.
+2. Send `{type: "session.update", model: ...}` and `commit(final=false)`.
+3. Stream mono PCM16 at 16 kHz; **Send turn** flushes buffered audio, then sends `commit(final=true)`.
+4. `response.output_audio.delta.audio` contains PCM; `response.output_audio.done`
+   terminates this adapter, including a response with no audio.
+5. The local implementation emits **model output** on `transcription.*`, so the
+   UI displays it as assistant text. This differs from the early RFC's suggested
+   user-transcript mapping. The VAD adapter also supports `response.output_text.*`
+   and audio-transcript events without duplicating the answer.
 
 ## Qwen3 with Server VAD (automatic turns)
 
 Run commands from the repository root with the vLLM-Omni environment activated.
-The plugin uses the pinned Silero v6.2 ONNX artifact from
-`istupakov/silero-vad-onnx` and does not download it while serving. Make it
-available in the Hugging Face cache or set `server_vad_model_path` to a local
-copy in the overlay below. The backend requires ONNX Runtime; setting `--vad`
-on the UI alone does not enable VAD.
+Prepare the pinned Silero v6.2 ONNX artifact as described in the
+[Qwen3 Server VAD instructions](../qwen3_omni/README.md#realtime-websocket-client-openai_realtime_clientpy).
+The backend requires ONNX Runtime and a compatible local artifact; setting
+`--vad` on the UI alone does not enable VAD on the backend.
 
 Create an overlay named `qwen3_vad.yaml`, replacing both paths with absolute paths
 on the backend host:
@@ -115,7 +124,7 @@ In a second terminal, wait until the health check succeeds, then start the UI:
 
 ```bash
 curl --fail http://127.0.0.1:8091/health
-python -m examples.online_serving.realtime_web --profile qwen3-turn \
+python -m examples.online_serving.qwen3_omni.realtime_web \
     --backend ws://127.0.0.1:8091 --vad --port 7863
 ```
 
@@ -139,17 +148,21 @@ The bundled duplex deploy allows four audio inputs and eight images per prompt.
 Use headphones to avoid speaker audio triggering VAD interruption. Reference
 voices and tool calls are not supported by this plugin.
 
-### Using the Qwen3 web UI
+### Switching between VAD and manual turns
 
-The shared UI supports both explicit turns and Server VAD. Use the manual UI
-command above with the turn-based deployment, or start the VAD deployment above
-and launch the UI with `--vad`.
+| Mode | Backend | UI flag | Submit a turn |
+| --- | --- | --- | --- |
+| Without VAD | Default Qwen turn deployment | `--stt` (default) | Press **Send turn** |
+| With VAD | Server VAD configuration and Silero artifact | `--vad` | Pause after speaking |
 
 Stop the existing UI process before starting another on the same port, then
-refresh the browser and reconnect. To enable VAD on a default backend, restart
-that backend with the overlay. Make `--backend` match the actual backend port;
-the UI and backend use separate ports. `Connection refused` means the target
-backend is unavailable: check its startup log and `/health` before connecting.
+refresh the browser and reconnect. To use STT, start a backend with the default
+turn deployment and restart the UI with `--stt`. The duplex backend exposes
+Realtime duplex and ChatCompletion; it does not expose the legacy STT handler.
+To enable VAD on a default backend, first restart that backend with the overlay.
+Make `--backend` match the actual backend port; the UI and backend use separate
+ports. `Connection refused` means the target backend is unavailable: check its
+startup log and `/health` before connecting.
 
 VAD connects with `duplex=1` and sends nested `session.audio.input` configuration
 with `create_response: true` and `interrupt_response: true`. It sends playback
@@ -197,7 +210,8 @@ native camera protocol, where Stage 0 interleaves frames at unit boundaries.
 
 This is a supported subset, not full OpenAI Realtime API compatibility.
 `duplex=1`, `overlap_policy`, `playback.ack`, and `session.close` remain vLLM-Omni
-extensions. This demo does not add WebRTC, semantic VAD, or tool calling.
+extensions. The legacy `--stt` mode is unchanged and is not the standard Realtime
+conversation API. This demo does not add WebRTC, semantic VAD, or tool calling.
 
 ## Shared host options
 
@@ -205,14 +219,11 @@ The shared entry point can select a profile explicitly:
 
 ```bash
 python -m examples.online_serving.realtime_web --profile qwen3-turn \
-    --backend ws://127.0.0.1:8091 --vad --port 7863
+    --backend ws://127.0.0.1:8091 --stt --port 7863
 ```
 
-This Qwen3 UI profile requires the server-VAD deployment above.
-
 - `--backend` / `--ws-backend`: backend WebSocket origin (not the full `/v1/realtime` URL).
 - `--model`: override the wrapper's model name.
-- `--manual` / `--vad`: Qwen turn control; manual turns are the default.
 - `--public-realtime-url`: optional browser-visible WebSocket URL; otherwise the
   static host proxies `/v1/realtime` on the same origin.
 - `--host`, `--port`: UI bind address and port (default port 7862).
@@ -234,7 +245,7 @@ pytest -m cpu tests/model_executor/models/qwen3_omni/test_duplex_plugin.py \
        tests/engine/test_duplex_orchestrator.py
 ```
 
-These CPU tests cover profile message serialization, VAD terminal/drain ordering,
+These CPU tests cover profile messages, two STT turns, VAD terminal/drain ordering,
 backpressure, native capture during playback, server config injection, and shared
 worklets. Backend regressions cover current audio inclusion in history-based
 prompts, delayed playback ACK ordering, and no extra generation during silence

examples/online_serving/realtime_web/app/static/profiles/qwen3_turn.js

@@ -2,24 +2,24 @@
   'use strict';
   const profiles = root.OmniRealtimeProfiles;
   profiles['qwen3-turn'] = (config) => {
-    const vad = config.turnMode === 'vad';
+    const vad = config.adapter === 'vad';
     let interruptOnSpeech = vad;
     let imageSeq = 0;
     let imageItems = [];
     // The engine refuses a ninth image (or 4 MiB of them) for the whole
     // session, so the camera track retires its own oldest frame instead.
     const MAX_IMAGE_ITEMS = 8;
-    const hint = 'This Qwen deployment needs session_mode: duplex with Server VAD enabled.';
+    const hint = 'This Qwen deploy needs session_mode: duplex and Server VAD support; use --stt or enable Server VAD.';
     return {
-      title: 'Qwen3-Omni Voice', eyebrow: vad ? 'Engine-owned duplex call' : 'Turn-based Realtime call',
+      title: 'Qwen3-Omni Voice', eyebrow: vad ? 'Engine-owned duplex call' : 'Turn-based realtime call',
       policy: vad ? 'Server VAD • interruptible responses' : 'Send turn • one response at a time',
       description: vad
         ? 'Pause to send your turn; speak again to interrupt. Microphone upload continues during playback. Requires the Qwen duplex plugin and Silero. Optional camera frames accompany each spoken turn.'
-        : 'Speak, then press Send turn. Microphone upload pauses while the model answers. Conversation history stays on the Realtime session. No barge-in or camera input.',
+        : 'Speak, then press Send turn. Microphone upload pauses while the model answers. Each turn uses a fresh connection; conversation history is not carried between turns. No barge-in or camera input.',
       waiting: 'Waiting for you', camera: vad, cameraMaxDimension: 448, playbackAck: vad, clientCommit: !vad,
-      waitForResponseDone: true, halfDuplex: !vad, deduplicateTranscript: true, closeSession: vad,
-      inputSampleRate: 24000,
-      readyEvent: 'session.updated', instructions: true, sendIntervalMs: 200,
+      waitForResponseDone: true, halfDuplex: !vad, deduplicateTranscript: true, closeSession: vad, reconnectEachTurn: !vad,
+      inputSampleRate: vad ? 24000 : 16000,
+      readyEvent: vad ? 'session.updated' : 'session.created', instructions: vad, sendIntervalMs: 200,
       presets: { assistant: 'You are a helpful assistant. Answer clearly and concisely.' },
       url(config, location) {
         const url = profiles.url(config, location);
@@ -33,17 +33,22 @@
       initialMessages(config, instructions) {
         imageItems = [];
         interruptOnSpeech = vad;
-        const session = { model: config.model, audio: { input: {
+        if (!vad) return [
+          { type: 'session.update', model: config.model },
+          { type: 'input_audio_buffer.commit', final: false },
+        ];
+        const session = { model: config.model, overlap_policy: 'barge_in_on_speech', audio: { input: {
           format: { type: 'audio/pcm', rate: 24000 },
-          turn_detection: vad ? { type: 'server_vad', threshold: 0.5, prefix_padding_ms: 300,
-            silence_duration_ms: 500, create_response: true, interrupt_response: true } : null,
+          turn_detection: { type: 'server_vad', threshold: 0.5, prefix_padding_ms: 300,
+            silence_duration_ms: 500, create_response: true, interrupt_response: true },
         } } };
-        if (vad) session.overlap_policy = 'barge_in_on_speech';
-        else session.type = 'realtime';
         if (instructions) session.instructions = instructions;
         return [{ type: 'session.update', session }];
       },
-      append: (audio) => ({ type: 'input_audio_buffer.append', audio }),
+      append(audio) {
+        return vad ? { type: 'input_audio_buffer.append', audio }
+          : { type: 'input_audio_buffer.append', audio, format: 'pcm16', sample_rate_hz: 16000 };
+      },
       // Qwen is a turn model: a frame is a picture added to the conversation,
       // not a track interleaved into the audio. That is exactly the OpenAI
       // Realtime image interface, so the camera sends conversation items
@@ -63,10 +68,7 @@
         } });
         return messages;
       },
-      commitMessages: () => vad ? [] : [
-        { type: 'input_audio_buffer.commit' },
-        { type: 'response.create' },
-      ],
+      commitMessages: () => vad ? [] : [{ type: 'input_audio_buffer.commit', final: true }],
       ack: (responseId, playedMs) => vad ? { type: 'playback.ack', response_id: responseId,
         item_id: `item_${responseId}`, played_ms: playedMs, committed_ms: playedMs } : null,
       mapEvent(event) {
@@ -85,28 +87,30 @@
           return { kind: 'interrupt' };
         }
         if (event.type === 'playback.acknowledged') return { kind: 'ack', committedMs: (event.event || event).committed_ms || 0 };
-        if (event.type === 'response.output_text.delta') {
+        if (event.type === 'response.output_text.delta' || event.type === 'transcription.delta') {
+          // On the shipped STT route transcription.* is model-generated text,
+          // not a separate ASR transcript of the user. See realtime_connection.py.
           return { kind: 'text', role: 'assistant', channel: 'text', text: event.delta || '' };
         }
-        if (event.type === 'response.output_text.done') {
+        if (event.type === 'response.output_text.done' || event.type === 'transcription.done') {
           return { kind: 'text-final', role: 'assistant', channel: 'text', text: event.text || '' };
         }
         if (vad && (event.type === 'output_audio_buffer.cleared' ||
             (event.type === 'response.done' && event.response?.status === 'cancelled'))) {
           return { kind: 'interrupt', responseId: event.response_id || event.response?.id || null };
         }
-        if (vad && event.type === 'input_audio_buffer.committed') return { kind: 'begin' };
-        if (!vad && event.type === 'input_audio_buffer.cleared') return { kind: 'ignore' };
+        if (event.type === 'input_audio_buffer.committed') return { kind: 'begin' };
         if (event.type === 'input_audio_buffer.cleared') return { kind: 'backpressure', message: 'Input cleared. Model is answering; please wait.' };
+        if (!vad && event.type === 'response.output_audio.done') return { kind: 'done' };
         const action = profiles.event(event);
         if (action.kind === 'error') {
           if (action.code === 'input_backpressure') return { ...action, kind: 'backpressure', message: 'Input buffer is full; please repeat dropped speech after the response.' };
-          if (action.code === 'server_vad_initialization_failed') action.message = 'Server VAD could not load Silero. Configure the server artifact and restart the UI.';
+          if (action.code === 'server_vad_initialization_failed') action.message = 'Server VAD could not load Silero. Configure the server artifact or restart this UI with --stt.';
           else if (vad && action.code === 'unsupported') action.message += ` ${hint}`;
         }
         return action;
       },
-      connectionHint: vad ? hint : 'Check the Qwen turn-based /v1/realtime deployment and model name.',
+      connectionHint: vad ? hint : 'Check the Qwen /v1/realtime STT endpoint and model name.',
     };
   };
 })(globalThis);

examples/online_serving/realtime_web/server.py

@@ -74,12 +74,12 @@ def build_app(
     public_realtime_url: str | None = None,
     ref_audio: str | None = None,
     profile: str = "minicpm-native",
-    turn_mode: str = "manual",
+    adapter: str = "stt",
 ) -> FastAPI:
     if profile not in {"minicpm-native", "qwen3-turn", "aura-ptt"}:
         raise ValueError(f"Unknown profile: {profile}")
-    if turn_mode not in {"manual", "vad"}:
-        raise ValueError(f"Unknown turn mode: {turn_mode}")
+    if adapter not in {"stt", "vad"}:
+        raise ValueError(f"Unknown adapter: {adapter}")
     if profile == "qwen3-turn" and ref_audio:
         raise ValueError("Qwen3 profile does not accept --ref-audio")
     if profile == "aura-ptt" and ref_audio:
@@ -116,7 +116,7 @@ def build_app(
             {
                 "model": model,
                 "profile": profile,
-                "turnMode": turn_mode,
+                "adapter": adapter,
                 "realtimePath": public_realtime_url or "v1/realtime",
                 "refAudio": ref_audio_uri,
                 "appVersion": app_version,
@@ -190,9 +190,9 @@ def main(default_profile: str = "minicpm-native") -> None:
     parser.add_argument("--profile", choices=("minicpm-native", "qwen3-turn", "aura-ptt"), default=default_profile)
     parser.add_argument("--ws-backend", "--backend", dest="ws_backend")
     mode = parser.add_mutually_exclusive_group()
-    mode.add_argument("--manual", dest="turn_mode", action="store_const", const="manual")
-    mode.add_argument("--vad", dest="turn_mode", action="store_const", const="vad")
-    parser.set_defaults(turn_mode="manual")
+    mode.add_argument("--stt", dest="adapter", action="store_const", const="stt")
+    mode.add_argument("--vad", dest="adapter", action="store_const", const="vad")
+    parser.set_defaults(adapter="stt")
     parser.add_argument(
         "--public-realtime-url",
         help="Browser-visible ws:// or wss:// Realtime URL; defaults to the same-origin proxy.",
@@ -225,7 +225,7 @@ def main(default_profile: str = "minicpm-native") -> None:
             public_realtime_url=args.public_realtime_url,
             ref_audio=args.ref_audio,
             profile=args.profile,
-            turn_mode=args.turn_mode,
+            adapter=args.adapter,
         ),
         host=args.host,
         port=args.port,

examples/online_serving/text_to_speech/cosyvoice3/run_server.sh

@@ -1,15 +1,11 @@
 #!/bin/bash
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
-
 # Launch vLLM-Omni server for CosyVoice3 TTS
 #
 # Usage:
 #   ./run_server.sh
 #   CUDA_VISIBLE_DEVICES=0 ./run_server.sh
 #
-# Streaming (async-chunk) is on by default. H200 selects the optimized profile;
-# other devices use vllm_omni/deploy/cosyvoice3.yaml.
+# Streaming (async-chunk) is on by default via vllm_omni/deploy/cosyvoice3.yaml.
 # Set NO_ASYNC_CHUNK=1 to use the legacy synchronous path.
 
 set -e

pyproject.toml

@@ -232,11 +232,6 @@ ignore = [
 # F-correctness rules still apply.
 "vllm_omni/model_executor/models/nemotron_voicechat/nemo_vendored/**" = ["E501", "E721", "E741", "F401", "F841", "I001", "N803", "N814", "UP006", "UP007", "UP008", "UP011", "UP032", "UP035", "W291", "W293"]
 "vllm_omni/experimental/fullduplex/joyvl/decision/prompts.py" = ["E501"]  # System/summarizer prompts are long prose lines
-# Kandinsky 6 keeps native/BigVGAN names (T/H/W, `l`) and NVIDIA vocoder
-# docstring length so the port stays diffable against the source modules.
-"vllm_omni/diffusion/models/kandinsky6/kandinsky6_transformer.py" = ["N803", "E741"]
-"vllm_omni/diffusion/models/kandinsky6/pipeline_kandinsky6.py" = ["N803"]
-"vllm_omni/diffusion/models/kandinsky6/modeling_kandinsky6_audio.py" = ["E501", "E741", "N803", "UP008"]
 
 [tool.ruff.lint.flake8-tidy-imports.banned-api]
 "librosa".msg = "The librosa module is banned, use vllm.multimodal helpers instead"
@@ -282,7 +277,6 @@ markers = [
     "parallel: Parallelism/distributed tests",
     "sp: Sequence parallelism tests",
     "example: Doc example code tests",
-    "nvidia_only: Tests that require NVIDIA CUDA and must not run on ROCm",
     # platform markers
     # Tag with [hardware-platform] for tests.helpers.mark.get_supported_platforms().
     # Add [gpu] on accelerator platforms that should also get pytest.mark.gpu.
@@ -370,8 +364,6 @@ extend-ignore-identifiers-re = [
     # (output feature dim), not "doubt".
     ".*dout.*",
     ".*DOUT.*",
-    # Kandinsky nabla threshold argument (`thr`), not the word "the".
-    "^thr$",
 ]
 extend-ignore-re = [
     "\\.\\.\\.n,d->\\.\\.\\.nd",
@@ -412,8 +404,3 @@ IY = "IY"
 # Speech DAC codec identifiers (convnet_type, transconvnet_type) and is not a
 # typo of "convent".
 convnet = "convnet"
-# Upstream Kandinsky system prompt. "promt" and "scren" are part of the
-# conditioning text and must not be corrected.
-thr = "thr"
-promt = "promt"
-scren = "scren"

recipes/JD/JoyAI-Image-Edit.md

@@ -1,221 +0,0 @@
-# JoyAI-Image-Edit
-
-> Text-guided single-image editing with the native JoyAI-Image-Edit diffusion pipeline
-
-## Summary
-
-- Vendor: JD / JoyImage Team
-- Model: `jdopensource/JoyAI-Image-Edit-Diffusers`
-- Task: Text-guided single-image editing
-- Mode: Offline inference and OpenAI-compatible online serving
-- Maintainer: Community
-
-## When to use this recipe
-
-Use this recipe when you want to run JoyAI-Image-Edit through the shared
-image-to-image examples with a known-good baseline configuration. JoyAI-Image-Edit
-is a large single-image editing model: it requires one reference image per
-request, snaps requested sizes to Joy's supported resolution buckets, and has a
-validated 1024x1024 baseline on an 80 GB NVIDIA GPU.
-
-## References
-
-- Upstream model:
-  [`jdopensource/JoyAI-Image-Edit-Diffusers`](https://huggingface.co/jdopensource/JoyAI-Image-Edit-Diffusers)
-- Upstream project:
-  [`jd-opensource/JoyAI-Image`](https://github.com/jd-opensource/JoyAI-Image)
-- Related offline example:
-  [`examples/offline_inference/image_to_image/image_edit.py`](../../examples/offline_inference/image_to_image/image_edit.py)
-- Related online example:
-  [`examples/online_serving/image_to_image/README.md`](../../examples/online_serving/image_to_image/README.md)
-- User guide:
-  [`docs/user_guide/examples/offline_inference/image_to_image.md`](../../docs/user_guide/examples/offline_inference/image_to_image.md),
-  [`docs/user_guide/examples/online_serving/image_to_image.md`](../../docs/user_guide/examples/online_serving/image_to_image.md)
-
-## Hardware Support
-
-This recipe documents the CUDA configuration used to validate the PR's JoyAI
-integration. Other GPU types should be treated as unvalidated until you rerun
-the smoke and parity checks below.
-
-## GPU
-
-### 1x NVIDIA H800 PCIe 80GB
-
-#### Environment
-
-- OS: Linux
-- Python: `3.12.3`
-- Driver / runtime: NVIDIA driver `580.82.07`, CUDA runtime `13.0`
-- PyTorch: `2.11.0+cu130`
-- Diffusers: `0.39.0.dev0` with `JoyImageEditPipeline` available
-- vLLM: `0.21.0`
-- vLLM-Omni: use the commit you are deploying from
-- GPU: NVIDIA H800 PCIe, 81559 MiB
-
-#### Online Serving
-
-Start the OpenAI-compatible server from the repository root:
-
-```bash
-CUDA_VISIBLE_DEVICES=0 \
-PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
-vllm serve jdopensource/JoyAI-Image-Edit-Diffusers \
-  --omni \
-  --port 8092 \
-  --init-timeout 1200
-```
-
-The server is ready when the log shows `Application startup complete.`
-
-If you need extra GPU headroom, use one of the memory-saving variants below.
-These variants are useful on tighter 80 GB deployments, but the latency and
-quality numbers in this recipe are from the baseline command above.
-
-```bash
-# Pipeline-level CPU offload for large components.
-CUDA_VISIBLE_DEVICES=0 \
-PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
-vllm serve jdopensource/JoyAI-Image-Edit-Diffusers \
-  --omni \
-  --port 8092 \
-  --init-timeout 1200 \
-  --enable-cpu-offload
-```
-
-```bash
-# DiT blockwise offload. This trades latency for lower peak GPU memory.
-CUDA_VISIBLE_DEVICES=0 \
-PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
-vllm serve jdopensource/JoyAI-Image-Edit-Diffusers \
-  --omni \
-  --port 8092 \
-  --init-timeout 1200 \
-  --enable-layerwise-offload
-```
-
-#### Online Verification
-
-Download the public JoyAI plate example:
-
-```bash
-wget -O /tmp/joy_plate.jpg \
-  https://raw.githubusercontent.com/jd-opensource/JoyAI-Image/main/test_images/test_1.jpg
-```
-
-Send one image and one prompt. JoyAI-Image-Edit supports exactly one input image
-per request.
-
-```bash
-IMG_B64=$(base64 -w0 /tmp/joy_plate.jpg)
-
-curl -s http://localhost:8092/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -d "$(jq -n --arg img "$IMG_B64" '{
-    messages: [{
-      role: "user",
-      content: [
-        {type: "text", text: "Turn the plate blue"},
-        {type: "image_url", image_url: {url: ("data:image/jpeg;base64," + $img)}}
-      ]
-    }],
-    extra_body: {
-      height: 1024,
-      width: 1024,
-      num_inference_steps: 30,
-      guidance_scale: 4.0,
-      seed: 0
-    }
-  }')" \
-  | jq -r '.choices[0].message.content[0].image_url.url' \
-  | cut -d',' -f2- \
-  | base64 -d > /tmp/joyai_image_edit_plate_online.png
-```
-
-Expected result: `/tmp/joyai_image_edit_plate_online.png` is a 1024x1024 PNG
-where the plate color changes to blue while the surrounding scene is mostly
-preserved. Use a client timeout of at least 600 seconds for first-run or cold
-server checks.
-
-#### Offline Command
-
-Run the same smoke case without starting a server:
-
-```bash
-CUDA_VISIBLE_DEVICES=0 \
-PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
-python examples/offline_inference/image_to_image/image_edit.py \
-  --model jdopensource/JoyAI-Image-Edit-Diffusers \
-  --image /tmp/joy_plate.jpg \
-  --prompt "Turn the plate blue" \
-  --negative-prompt "" \
-  --height 1024 \
-  --width 1024 \
-  --num-inference-steps 30 \
-  --guidance-scale 4.0 \
-  --seed 0 \
-  --enforce-eager \
-  --init-timeout 1200 \
-  --output /tmp/joyai_image_edit_plate_offline.png
-```
-
-For a local checkpoint mirror, replace `--model` with the local model directory.
-The PR validation used the same command shape with
-`/autodl-fs/data/joyai_models/JoyAI-Image-Edit-Diffusers`.
-
-#### Validated Results
-
-The table below is from the 1x H800 PCIe environment above. Resolution requests
-were snapped to the Joy 1024x1024 bucket, batch size was 1, and CFG was 4.0.
-`vLLM peak` is the observed peak GPU memory for the vLLM-Omni run.
-
-| Case | Steps | Seed | vLLM latency | Diffusers latency | vLLM peak | Diffusers peak | SSIM | PSNR |
-| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
-| `plate_seed123` | 30 | 123 | 50.87 s | 52.74 s | 57.52 GiB | 57.28 GiB | 0.9869 | 34.7051 |
-| `plate_seed0` | 30 | 0 | 51.02 s | 52.01 s | 57.52 GiB | 57.28 GiB | 0.9946 | 41.4351 |
-| `astronaut_seed0` | 40 | 0 | 67.90 s | 69.22 s | 57.52 GiB | 57.28 GiB | 0.9948 | 38.2498 |
-| `crane_removal_seed0` | 40 | 0 | 68.00 s | 69.27 s | 57.52 GiB | 57.28 GiB | 0.9975 | 50.0135 |
-| `object_move_seed0` | 40 | 0 | 68.06 s | 69.32 s | 57.52 GiB | 57.28 GiB | 0.9969 | 46.1138 |
-| `object_rotation_seed0` | 40 | 0 | 67.94 s | 69.28 s | 57.52 GiB | 57.28 GiB | 0.9898 | 35.5365 |
-| `camera_control_seed0` | 40 | 0 | 68.15 s | 69.51 s | 57.52 GiB | 57.28 GiB | 0.9952 | 41.7697 |
-
-#### Output Size Behavior
-
-JoyAI-Image-Edit does not use arbitrary image sizes directly. The pipeline maps
-the input or requested `height` / `width` to the nearest supported Joy bucket,
-then resizes and center-crops the reference image to that bucket before
-denoising.
-
-- Provide both `height` and `width`, or omit both.
-- For square outputs, request `height=1024` and `width=1024`.
-- A request such as `512x512` is also square and is snapped to the 1024x1024 Joy
-  bucket, so the final image size will not match the literal request.
-- Non-square requests are snapped to the nearest aspect-ratio bucket supported
-  by the model.
-
-#### Guidance Parameters
-
-Use `guidance_scale` for JoyAI classifier-free guidance, matching Diffusers.
-The pipeline defaults to `4.0` when the parameter is omitted. `true_cfg_scale`
-remains supported as an alias; if both parameters are supplied, their values
-must match. Values greater than `1.0` enable CFG; an omitted negative prompt is
-treated as an empty string. Values at or below `1.0` disable CFG.
-
-The offline example passes `--guidance-scale 4.0` explicitly for clarity; if it
-is omitted, the JoyAI pipeline uses the same `4.0` default.
-
-#### Known Limitations
-
-- Single image only: JoyAI-Image-Edit rejects requests with zero or multiple
-  input images.
-- Single prompt only: batched prompt requests are rejected.
-- HSDP is not supported yet. Starting with HSDP enabled raises a `ValueError`
-  before the generic HSDP setup reaches the transformer.
-- Tensor Parallelism is not supported yet. Keep `tensor_parallel_size=1`;
-  larger values raise a `ValueError` during pipeline initialization.
-- LoRA is not included for this JoyAI pipeline.
-- Cache-DiT and TeaCache are not included for this JoyAI pipeline. Do not treat
-  the generic image-to-image cache flags as validated JoyAI acceleration knobs.
-- Text encoder quantization is not included.
-- The validation above is CUDA-only; ROCm and NPU configurations are not
-  documented as supported by this recipe.

recipes/Kandinsky/Kandinsky6-TI2VA.md

@@ -1,247 +0,0 @@
-# Kandinsky 6 TI2VA — H100 (v1, single GPU, CPU offload)
-
-> Text/image-to-video-and-audio serving with Kandinsky 6 Pro
-
-## Summary
-
-- Vendor: Kandinsky Lab ([`kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers`](https://huggingface.co/kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers))
-- Model: Kandinsky 6 Pro (TI2VA — text/image-to-video-and-audio)
-- Task: Joint text/image-to-video-and-audio generation
-- Mode: Offline (`Omni(...)`) and online serving with the OpenAI-compatible
-  `/v1/videos` API
-- Hardware: 1x NVIDIA H100 80GB HBM3 with component-level CPU offload
-- Maintainer: Community
-
-## When to use this recipe
-
-Use this recipe to serve Kandinsky 6's TI2VA pipeline — a single DiT that
-jointly denoises video and audio latents, conditioned on a text prompt and,
-optionally, a reference image (image-to-video). Tensor parallel, CFG
-parallel, Ulysses, ring, pipeline parallel, HSDP, layerwise offload,
-MagCache, TeaCache, Cache-DiT, FP8, and tiled VAE patch parallel are wired.
-NaviCache and a registered NABLA backend are not. TeaCache coefficients are
-uncalibrated. See "Known limitations".
-
-## Supported model contract
-
-| Task | Entrypoint | Input | Output |
-|---|---|---|---|
-| Text-to-video-and-audio | `vllm serve kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers --omni` / `Omni(model="kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers")` | text prompt | synchronized MP4 (H.264 video + AAC 44.1 kHz mono audio) |
-| Image-to-video-and-audio | same | text prompt + reference image | synchronized MP4, first frame conditioned on the reference image |
-
-- Audio generation is on by default (`sample_audio=True`); pass
-  `extra_args.sample_audio=false` to generate video only.
-- Batch size is 1 request per generation call (upstream limitation of the
-  ported denoise loop; concurrent requests are still served by vLLM-Omni's
-  own request scheduler, just not batched together within one DiT forward
-  pass yet).
-- Serving defaults follow the Pro production geometry registered in
-  `vllm_omni/model_extras/kandinsky6.py`: **864x480, 125 frames @ 24 fps
-  (5.2 s), 50 steps, CFG 5.0**. Any `4k+1` frame count and 16-divisible
-  resolution can be requested explicitly via `height`/`width`/`num_frames`/
-  `num_inference_steps` — the model is *not* fixed-duration.
-- Audio is always 44.1 kHz; its length is derived from the video duration.
-- Negative prompt defaults to the pipeline's built-in K6 negative prompt
-  when omitted.
-
-## References
-
-- Hub checkpoint: [`kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers`](https://huggingface.co/kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers)
-- Joint video+audio native-model reference: [`recipes/MiniMaxAI/MiniMax-H3.md`](../MiniMaxAI/MiniMax-H3.md)
-- Offline example: [`examples/offline_inference/text_to_video/text_to_video.py`](../../examples/offline_inference/text_to_video/text_to_video.py)
-- Image-to-video example: [`examples/offline_inference/image_to_video/image_to_video.py`](../../examples/offline_inference/image_to_video/image_to_video.py)
-
-## Hardware
-
-- Accelerator model and per-device memory: NVIDIA H100 80GB HBM3
-  (driver 570.133.20)
-- Number of devices: 1 (multi-GPU tensor parallelism is implemented in the
-  transformer's linear layers but not yet validated end-to-end)
-- Device interconnect: N/A (single-device path)
-- Host memory: 1.4 TB installed; the bf16 DiT (~56 GB) plus the text
-  encoders are staged on the host under `--enable-cpu-offload`, so budget
-  at least ~90 GB of free host RAM.
-- Qualification scope: single-GPU T2VA at the Pro default geometry and at a
-  reduced smoke geometry, offline and via `vllm serve`. Numbers below were
-  measured once on the shared box described above; treat them as
-  indicative, not as a benchmark.
-
-## Software environment
-
-- OS: Ubuntu 22.04.5 LTS
-- Python: 3.12.13
-- Driver / runtime: CUDA 12.9 (torch 2.13.0+cu129)
-- vLLM version: 0.29.0
-- vLLM-Omni version or commit: this repository, `vllm_omni/diffusion/models/kandinsky6/`
-- diffusers 0.40.0, transformers 5.14.1
-- Attention: FlashAttention-3 on this box (`flash_attn_interface`). The Pro
-  timings below use it. Without FA3, `attention_engine: "auto"` falls back
-  to PyTorch SDPA; an earlier SDPA Pro run on the same GPU was 26.9 s/it
-  and 1377 s.
-
-## Weights
-
-`Kandinsky6TI2VAPipeline._load_components` loads the public Diffusers layout
-from [`kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers`](https://huggingface.co/kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers)
-(`model_index.json`, `transformer/`, `vae/`, `text_encoder/`, `tokenizer/`,
-`text_encoder_2/`, `tokenizer_2/`, `audio_vae/`, `scheduler/`). Pass that
-repo id (or a local snapshot of it) to `vllm serve` / `Omni(model=...)`.
-The Hub repo is gated; export `HF_TOKEN` if download returns 401.
-
-A raw Kandinsky 6 weight tree is not required. If you already have a local
-Diffusers-layout snapshot, point `--model` at that directory instead of the
-Hub id.
-
-Each weight folder is loaded on its own (`transformer/`, `vae/`, `text_encoder/`, `text_encoder_2/`, `audio_vae/`). The DiT is built with initialization skipped, then the framework loader fills it from `transformer/`.
-
-## Command
-
-```bash
-vllm serve kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers --omni \
-  --host 127.0.0.1 --port 8091 \
-  --num-gpus 1 --enable-cpu-offload
-```
-
-`--enable-cpu-offload` enables model-level offload with mutual exclusion
-between the transformer and the two text encoders (they never co-reside on
-the GPU). Without it the bf16 DiT (~56 GB) plus Qwen2.5-VL-7B (~16 GB) plus
-the VAEs and activations do not fit an 80 GB device at the Pro geometry.
-
-Offline (writes an MP4 with the audio track muxed in):
-
-```bash
-python examples/offline_inference/text_to_video/text_to_video.py \
-  --model kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers \
-  --prompt "A golden retriever runs along a sunny beach, waves crashing, cinematic footage" \
-  --seed 42 --enable-cpu-offload --output kandinsky6_t2va.mp4
-# Video only: add --extra-body '{"sample_audio": false}'
-```
-
-Image-to-video-and-audio uses the shared image-to-video example:
-
-```bash
-python examples/offline_inference/image_to_video/image_to_video.py \
-  --model kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers \
-  --image first_frame.png \
-  --prompt "A golden retriever runs along a sunny beach, waves crashing, cinematic footage" \
-  --seed 42 --enable-cpu-offload --output kandinsky6_i2va.mp4
-```
-
-The generic scripts pick up the Pro defaults from
-`vllm_omni/model_extras/kandinsky6.py`.
-
-## Verification
-
-`/v1/videos` takes multipart form fields (not a JSON body). Submit, poll,
-then download:
-
-```bash
-VID=$(curl -s -X POST http://127.0.0.1:8091/v1/videos \
-  -F prompt="A golden retriever runs along a sunny beach, waves crashing" \
-  -F size=864x480 -F num_frames=125 -F num_inference_steps=50 -F seed=42 \
-  | python3 -c "import sys,json;print(json.load(sys.stdin)['id'])")
-
-# Poll until "status": "completed"
-curl -s http://127.0.0.1:8091/v1/videos/$VID
-
-curl -s -o kandinsky6.mp4 http://127.0.0.1:8091/v1/videos/$VID/content
-# Expect an MP4 with an H.264 video stream (125 frames, 24 fps, 864x480)
-# and an AAC audio stream (44100 Hz, mono) of matching duration.
-```
-
-Smoke-size request for quick validation (~30 s end-to-end on one H100):
-`-F size=512x320 -F num_frames=25 -F num_inference_steps=10`.
-
-## Measurements
-
-Single H100 80GB, seed 42, prompt
-"A golden retriever runs along a sunny beach, waves crashing, cinematic footage",
-negative prompt left at the pipeline default, MagCache off, prompt expansion
-off. Pro geometry: 864x480, 125 frames, 50 steps, CFG 5.0, audio on.
-
-vLLM-Omni: `text_to_video.py --model kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers --enable-cpu-offload --model-class-name Kandinsky6TI2VAPipeline`,
-FlashAttention-3.
-
-| Wall / generation | Step time | Peak GPU | After load |
-|---|---|---|---|
-| 751.7 s generation (12.5 min); engine init 41 s | 14.4 s/it average | 78.39 GiB reserved (78.0 GiB nvidia-smi) | 17.74 GiB |
-
-An earlier SDPA vLLM-Omni Pro run on the same GPU was 26.9 s/it and 1377 s,
-peak reserved 74.97 GiB. A 512x320 / 25-frame / 10-step SDPA smoke was 1.4 s/it
-and 29.5 s at the same reserved peak.
-
-The MP4 is H.264 864x480, 125 frames at 24 fps (5.22 s), plus AAC 44.1 kHz
-mono (230400 samples): a golden retriever on a sunny beach with breaking waves.
-
-vLLM-Omni's VAE decode logged two failed CUDA allocations (7.15 GiB, then
-4.27 GiB) and still wrote a complete file.
-
-Audio is peak-normalized to full scale by default; pass
-`extra_args.audio_normalization="clip"` to keep the raw decoded amplitude
-instead.
-
-## Notes
-
-- Key flags: `extra_args.sample_audio` (bool, default true),
-  `extra_args.audio_normalization` (`"normalize"` default | `"clip"`),
-  `extra_args.visual_cond_scheme` (defaults to `pretrain` for text-only,
-  `tail_cond_first_frame` when an image is supplied).
-- The post-process payload is a flat dict (`video`, `audio` as float32 in
-  `[-1, 1]`, `audio_sample_rate=44100`, `fps`) so the serving muxer picks
-  up the correct sample rate.
-- Acceleration that is wired:
-  - Tensor parallel: the DiT loader narrows full checkpoint tensors onto
-    each rank's `ColumnParallelLinear` / `RowParallelLinear` shard.
-  - CFG parallel (`--cfg-parallel-size 2`): rank 0 runs the conditional
-    DiT, rank 1 the unconditional one, then both all-gather video and
-    audio velocities.
-  - HSDP: `_hsdp_shard_conditions` covers the indexed transformer blocks.
-    Do not combine with tensor parallel.
-  - Sequence parallel: visual tokens (and their RoPE) are sharded;
-    Ulysses all-to-all runs inside visual self-attention, Ring uses the
-    PyTorch ring kernel. Text stays replicated. Audio queries gather the
-    full visual sequence.
-  - Pipeline parallel: `visual_transformer_blocks` are split with
-    `PPMissingLayer`. Embeddings stay on the first stage, output heads on
-    the last. Activations and the final velocity are sent between stages.
-  - VAE patch parallel: Hunyuan decode splits the latent width when
-    `vae_patch_parallel_size > 1` and tiling is on.
-  - MagCache / TeaCache / Cache-DiT are registered. MagCache reuses the
-    Pro `mag_ratios`. TeaCache coefficients are **uncalibrated** (copied
-    from Qwen-Image) so skips are not quality-neutral. Cache-DiT targets
-    `visual_transformer_blocks` only and falls back to the same step cache
-    if the fused `(video, audio)` block return is rejected.
-  - FP8: online quantization runs in the DiT loader via
-    `process_weights_after_loading` after the sharded assign.
-  - `attention_engine: auto` uses FlashAttention-3 when
-    `flash_attn_interface` imports.
-- Known limitations:
-  - NABLA sparse attention is not registered in the framework attention
-    backend. NaviCache is not wired.
-  - Prompt expansion, NF4 Qwen, and the super-resolution cascade are not
-    part of this port.
-  - Distributed layerwise offload still uses one process unless another
-    parallel axis sets `world_size > 1` (for example tensor parallel).
-  - `_load_components` loads the Diffusers checkpoint directly rather than the
-    streamed-prefetch loader.
-
-## Supported features
-
-| Feature | Status | Guide |
-|---|---|---|
-| Text-to-video-and-audio | Supported (validated on H100) | — |
-| Image-to-video-and-audio | Supported (not yet validated on hardware) | — |
-| CPU offload (`--enable-cpu-offload`) | Supported (required on 80 GB) | — |
-| Layerwise offload (`--enable-layerwise-offload`) | Supported (about 36 GiB peak on the smoke geometry) | — |
-| Tensor parallelism | Wired (sharded DiT load) | — |
-| CFG parallelism | Wired (`--cfg-parallel-size 2`) | — |
-| Ulysses / Ring sequence parallel | Wired (visual tokens only) | — |
-| Pipeline parallelism | Wired (visual block split) | — |
-| HSDP | Wired (not with tensor parallel) | — |
-| VAE patch parallel | Wired (width-split decode) | — |
-| MagCache | Wired (Pro ratios, step reuse) | — |
-| TeaCache | Wired, coefficients uncalibrated | — |
-| Cache-DiT | Wired on `visual_transformer_blocks` | — |
-| FP8 | Wired (online quant; smoke peak about 46 GiB vs 75 GiB bf16) | [`FP8 quantization`](../../docs/user_guide/quantization/fp8.md) |
-| NaviCache | Not supported | — |
-| LoRA | Not supported | — |

recipes/MammothModa2/MammothModa2.md

@@ -18,24 +18,12 @@ The generic example formats the AR prompt, drives the AR → DiT stage pipeline,
 and forwards MammothModa2-specific generation parameters through the
 pipeline-declared `extra_body` contract.
 
-MammothModa2's DiT stage runs in the shared diffusion runtime. The default
-`mammoth_moda2.yaml` uses request-level batching with up to eight compatible
-requests (`max_num_seqs: 8`) and one image per request
-(`num_outputs_per_prompt: 1`). TeaCache and Cache-DiT are optional alternative
-cache backends in request mode. Enable TeaCache with `--cache-backend tea_cache`
-as shown below, or enable Cache-DiT using the commented settings in that YAML.
-Compilation, DiT quantization, parallelism, and offload are not enabled by
-these presets.
-
-To use step execution and continuous batching, select
-[`mammoth_moda2_step.yaml`](../../vllm_omni/deploy/mammoth_moda2_step.yaml)
-with `--deploy-config vllm_omni/deploy/mammoth_moda2_step.yaml` in the
-text-to-image commands below. This preset inherits the same stage placement
-and capacity, enables step execution for Stage 1, and disables request-batch
-admission waiting and diffusion cache acceleration. Compatible requests can
-join between denoising steps and finish independently, including requests
-with different inference-step counts. Step mode cannot be combined with
-TeaCache, Cache-DiT, or other diffusion cache backends.
+MammothModa2's DiT stage runs in the shared diffusion runtime in request mode.
+The first integration intentionally supports one request and one image per
+forward only (`max_num_seqs: 1`, `num_outputs_per_prompt: 1`). Request-level
+batching, step execution, continuous batching, cache acceleration,
+compilation, quantization, parallelism, and offload are not enabled by this
+recipe.
 
 Image size, seed, guidance, and denoising steps use the standard diffusion
 request fields. `cfg_range` remains a MammothModa2-specific `extra_body`
@@ -188,30 +176,7 @@ sets the relative step range `[start, end]` over which CFG is applied (default
 `[0.0, 1.0]`). For compatibility, `text_guidance_scale` and
 `num_inference_steps` remain accepted `extra_body` aliases and, when non-null,
 take precedence over the standard request fields. Model extras are filtered
-against the declared `extra_body_params`.
-
-TeaCache can be enabled for the DiT stage with the same user-facing sampling
-parameters:
-
-```bash
-python examples/offline_inference/text_to_image/text_to_image.py \
-  --model ./MammothModa2-Preview \
-  --deploy-config vllm_omni/deploy/mammoth_moda2.yaml \
-  --prompt "A stylish woman riding a motorcycle in NYC, movie poster style" \
-  --height 1024 \
-  --width 1024 \
-  --guidance-scale 4.0 \
-  --num-inference-steps 50 \
-  --cache-backend tea_cache \
-  --extra-body '{"cfg_range": [0.0, 1.0]}' \
-  --output mammoth_t2i_teacache.png
-```
-
-The bundled TeaCache coefficients were fitted from MammothModa2 full-compute
-traces. MammothModa2 uses the model-specific default `rel_l1_thresh=0.075`,
-selected for the evaluated 1024x1024, 50-step configuration.
-
-The model-specific keys are declared in
+against the declared `extra_body_params` (see
 [`vllm_omni/model_extras/mammothmodal2_preview.py`](../../vllm_omni/model_extras/mammothmodal2_preview.py)),
 so unknown MammothModa2 extras may be dropped.
 
@@ -322,44 +287,6 @@ python3 examples/offline_inference/text_to_image/text_to_image.py \
 
 The first request took 85.224 seconds. The AR stage generated 4,161 visual tokens in 72.996 seconds, and the DiT stage took 12.163 seconds. AR weight loading used 21.4 GiB and took 8.250 seconds. DiT weight loading used 5.49 GiB and took 1.824 seconds. The largest one second whole device memory sample was 106.57 GiB, including the AR KV cache reserved by the 0.5 memory setting.
 
-### 1x A800 80GB, MammothModa2 Preview (serving)
-
-#### Environment
-
-- OS: Linux, x86_64
-- CPU: 18 cores
-- GPU: one NVIDIA A800-SXM4-80GB
-- Deploy config: `vllm_omni/deploy/mammoth_moda2.yaml` (default split: AR `gpu_memory_utilization` 0.5, DiT 0.3)
-- vLLM-Omni version or commit: use the commit you are deploying from
-
-#### Serving Commands
-
-```bash
-vllm-omni serve bytedance-research/MammothModa2-Preview --omni \
-    --port 8091 \
-    --trust-remote-code
-
-python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --model bytedance-research/MammothModa2-Preview \
-    --endpoint /v1/images/generations \
-    --host 127.0.0.1 --port 8091 \
-    --height 1024 --width 1024 --num-inference-steps 50 \
-    --extra-body '{"text_guidance_scale": 9.0, "cfg_range": [0.0, 1.0]}' \
-    --num-prompts 8 --seed 142 --warmup-requests 0 \
-    --output-file mm2_1024_s50_c1.json
-```
-
-#### Verification
-
-On 1024x1024 / 50 steps, single-concurrency end-to-end mean latency is ~96 s
-(P99 ~101 s) per image and concurrency-4 mean is ~151 s (~0.024 img/s);
-steady-state combined GPU memory is ~50.4 GiB (AR ~39.2 GiB, DiT ~11.1 GiB).
-The AR stage dominates (~77 s of a ~96 s request) because it decodes a fixed
-4,161-token visual grid per image, so latency is insensitive to prompt length
-and scales only partially with DiT step count. See the
-[serving performance dashboard](../../benchmarks/diffusion/performance_dashboard/mammoth_moda2_serving_performance.md)
-for the full sweep, peak-memory, and component-attribution data.
-
 The output was a valid 1024 by 1024 RGB PNG.
 
 ### Startup benchmark

recipes/MiniMaxAI/MiniMax-H3.md

@@ -988,42 +988,6 @@ settings, and both videos with any comparison; the command alone is not quality
 evidence. A matching seed does not imply matching noise over differently sized
 full and windowed latents.
 
-## Request-scoped sampler
-
-H3 supports `euler` (the default) and `res_multistep`. Select the sampler per
-request; it applies to both video and audio, using their separate sigma schedules.
-For the HTTP examples above, set `num_inference_steps=20` and add
-`"sampler":"res_multistep"` to the existing `extra_params` object. For example,
-the T2VA fields become:
-
-```bash
--F 'num_inference_steps=20' \
--F 'extra_params={"task":"t2va","duration":8.7,"audio_flow_shift":3.0,"sampler":"res_multistep"}'
-```
-
-For the offline Python API, use `extra_args`:
-
-```python
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-
-sampling_params = OmniDiffusionSamplingParams(
-    num_inference_steps=20,
-    seed=42,
-    extra_args={"task": "t2va", "sampler": "res_multistep"},
-)
-```
-
-Both samplers use one DiT evaluation per step in request mode and step execution.
-`res_multistep` retains the previous denoised prediction independently for each
-request and each stream; its first and final updates use Euler. Omitting `sampler`
-or selecting `euler` preserves the existing Euler update. Reducing the step count
-reduces model evaluations, but quality depends on the prompt and configuration;
-20 RES steps are a candidate to compare against an Euler-50 reference.
-
-Requests using a fixed distilled `base_schedule` from a checkpoint or adapter
-must keep `sampler="euler"`. RES sampling with those schedules has not been
-validated and is rejected with `ValueError`.
-
 ## Request-scoped quality
 
 Add one of these fields to any HTTP request above. No Cache-DiT startup option
@@ -1444,7 +1408,6 @@ four-step adapter keeps its original sampling and fixed-top-k behavior.
 | `duration` | Workload-specific | Decimal seconds in `extra_params`; converted to H3-compatible frame count |
 | `fps` | `24` | H3 output FPS is fixed |
 | `num_inference_steps` | `50` | Matches the reference accuracy workloads |
-| `extra_params.sampler` | `euler` | Per-request `euler` or `res_multistep` for both streams; fixed distilled schedules require Euler |
 | `flow_shift` | `12` | Video sigma shift |
 | `audio_flow_shift` | `3` | Audio sigma shift, passed in `extra_params` |
 | `seed` | Task-specific | Use a fixed value for reproducibility |

recipes/NVIDIA/PersonaPlex.md

@@ -84,10 +84,8 @@ duplex (`is_enabled()` is unconditionally true). Turn-based models
 
 ## Hardware Support
 
-Verified on GPU (Hopper-class). The shipped deploy config runs the Stage 0
-talker with CUDA graphs and replays the streaming Mimi encode (Stage 0) and
-decode (Stage 1) frame steps from model-local CUDA graphs, bitwise equal to
-eager. Other CUDA GPUs with enough memory are expected to work.
+Verified on GPU (Hopper-class). The serving path is plain PyTorch eager plus the
+native modules, so other CUDA GPUs with enough memory are expected to work.
 
 ## GPU
 
@@ -139,11 +137,9 @@ python tests/e2e/online_serving/personaplex_realtime_duplex.py \
 
 #### Notes
 
-- Realtime budget is 80 ms/frame. The shipped config admits two sessions; on a
-  141 GB card, 64 concurrent 30 s sessions stayed realtime after raising
-  `duplex_session.max_sessions` and both stages' `max_num_seqs` (see the
-  sizing notes in `vllm_omni/deploy/personaplex.yaml`). Longer sessions cost
-  more per step as their context fills.
+- Realtime budget is 80 ms/frame. Verified eager per-tick latency at four
+  concurrent sessions is ~70-74 ms on this hardware class, i.e. all four
+  conversations stay realtime; single-session has comfortable headroom.
 - Decoding is greedy only (temperature/top-k knobs are intentionally not
   exposed): greedy is what the parity gates pin against the reference
   implementation, and sampling amplifies sub-bit numeric drift into divergence.

recipes/OpenMOSS/MOSS-TTS.md

@@ -35,10 +35,6 @@ the requested `input` speech. Without a nonblank `ref_text`, Local uses
 audio-reference generation. Reference transcripts must match the reference
 audio; they are not style instructions.
 
-For the CUDA MRV2 runner and platform fallbacks, see the
-[Local 1.5 deployment profile](../../docs/configuration/stage_configs.md#moss-tts-local-15-with-model-runner-v2).
-Local 1.5 defaults to MRV2 on CUDA and retains V1 on other platforms.
-
 ## References
 
 - Offline inference example: [`examples/offline_inference/text_to_speech/moss_tts/`](../../examples/offline_inference/text_to_speech/moss_tts/)
@@ -156,340 +152,3 @@ curl -X POST http://localhost:8091/v1/audio/speech \
 - No `ref_audio` required or accepted for MOSS-SoundEffect.
 - Input field maps to the `ambient_sound` parameter in the upstream processor.
 - Rate: ~12.5 tokens per second; longer descriptions produce longer audio.
-
-## Local 1.5 MRV2 and slot attention
-
-`MOSS-TTS-Local-Transformer-v1.5` defaults to the native MRV2 pipeline on
-CUDA, with the original Local projection and sampler. When
-`nvidia-cuda-mps-control` is on `PATH`, it starts private full-quota MPS.
-GPUs with at least 140 GiB total memory use the C128 system profile with
-prefix caching, Triton backbone attention, bounded codec first-chunk decode,
-native Torch sampler compilation and dense codec graph buckets through 128. Its Talker KV budget is 32 GiB; smaller GPUs or a failed memory
-query use C64 with utilization-based memory budgets. When the MPS executable
-is unavailable, the automatic default is C64 MRV2 without MPS. NPU, XPU,
-ROCm and MUSA retain V1. Explicit deploy configs override automatic selection.
-
-```bash
-CUDA_VISIBLE_DEVICES=0 OMP_NUM_THREADS=1 \
-vllm serve OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 --omni \
-  --stage-init-timeout 1200 --init-timeout 1500
-```
-
-Use `--deploy-config vllm_omni/deploy/moss_tts_local_v1.yaml` to select the
-previous V1 profile explicitly. C128's batch-prefill and direct-token switches
-remain confined to the throughput profile; they are not enabled in C64.
-
-The same `moss_tts_local_mrv2_optimized.yaml` system profile can be selected
-explicitly. It retains the original Local depth projection and sampling algorithm.
-The codec admits fast first chunks below 32 active streams; crowded streams use
-the regular decoder with a dispatch target of 16 and a maximum wait of 6 ms.
-This preserves the low-load first-audio path while coalescing high-load work.
-`local_compile_audio_sampler: true` in the Talker HF overrides compiles the native
-Torch sampler; explicit request generators use the original helper. Set the
-override to `false` in a deployment file to disable that compilation. The setting
-is specific to this C128 system profile.
-Both stages share one GPU, with a 32 GiB Talker KV budget; this profile requires H200-class memory and
-`nvidia-cuda-mps-control` on `PATH`.
-
-```bash
-CUDA_VISIBLE_DEVICES=0 OMP_NUM_THREADS=1 \
-vllm serve OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 --omni \
-  --deploy-config vllm_omni/deploy/moss_tts_local_mrv2_optimized.yaml \
-  --stage-init-timeout 1200 --init-timeout 1500
-```
-
-Evaluate WER, speaker similarity and streaming latency for your workload;
-throughput gains do not establish quality equivalence. Do not apply partial
-MPS SM quotas: BF16 GEMM outputs were observed incomplete in that configuration
-on the validation environment. MPS uses an owned control socket or an
-explicitly supplied operator socket; only the owned daemon is stopped at shutdown.
-
-The explicit C64 MRV2 and capped-C128 throughput profiles remain available.
-Both profiles below preserve 1-frame initial and 15-frame steady codec chunks
-and the model's sampling defaults.
-
-```bash
-vllm serve OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 --omni \
-  --deploy-config vllm_omni/deploy/moss_tts_local_mrv2.yaml
-```
-
-This C64 profile bounds Talker prefill to 512 tokens and retains codec-owned
-CUDA graphs without compiling the codec with Inductor.
-
-For sustained high concurrency on a large-memory CUDA GPU, use:
-
-```bash
-CUDA_VISIBLE_DEVICES=0 OMP_NUM_THREADS=4 \
-vllm serve OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 --omni \
-  --deploy-config vllm_omni/deploy/moss_tts_local_mrv2_high_concurrency.yaml \
-  --stage-init-timeout 1200 --init-timeout 1500
-```
-
-The high-concurrency profile places both stages on one GPU, sets each stage's
-capacity to 128 and fixes the Talker KV budget at 32 GiB. The Talker uses
-mixed FULL CUDA graphs with both its token budget and capture limit set to
-512, so new prompt chunks remain within graph coverage. It also enables
-`hf_overrides.mrv2_gpu_slot_state`: Local hidden states, audio codes and
-continuation control stay in GPU request slots, gathered through MRV2's
-batch mapping. Prefill conditioning and explicit per-request sampling seeds
-retain their original behavior. Published audio codes own their storage, so
-slot reuse cannot overwrite an in-flight output. Set the override to `false`
-to compare with the generic Omni model state. The profile selects
-`codec_attention_backend: triton_slot`, Inductor mode 3 with combo kernels
-disabled, and codec CUDA graph buckets through 128. The configuration was
-validated on one H200. The earlier capacity-256 variant used about 110 GiB of
-sampled peak GPU memory, including loading and warmup. Reduce the stage capacities,
-graph buckets and coalescing target together when adapting it to smaller GPUs. Cold codec
-compilation can take several minutes; subsequent starts can reuse the AOT
-cache. Graph capture is still performed at startup.
-
-The high-concurrency profile also coalesces ready codec streams with
-`connectors.shm.extra.generation_min_batch_size: 32` and
-`generation_max_wait_ms: 12`. The target controls when to stop waiting; it
-neither caps execution at 32 nor guarantees every codec group has 32 rows.
-First and final chunks share the bounded window at high concurrency. When
-fewer than 32 requests remain, dispatch is immediate. Pending outputs are
-retired before waiting, with the original deadline preserved, so waiting does
-not prevent the previous batch from releasing its in-flight state. Cancellation
-and input notifications keep their ordinary scheduler bookkeeping. This option
-requires a stateful native MRV2 generation stage with TP1/PP1. Set the wait to
-`0` to disable coalescing; other profiles keep the immediate-dispatch default.
-
-The high-concurrency profile enables two additional Local model-state
-optimizations under the CUDA stage-0 `hf_overrides`, alongside
-`mrv2_gpu_slot_state: true`:
-
-- `mrv2_batch_prefill: true` reuses the batch's text embeddings and combines
-  CPU reference-code slices and their destination positions into one pinned
-  upload. Host staging is reused only after the upload event completes;
-  GPU-resident references keep the original path.
-- `mrv2_direct_tokens: true` returns the already determined text token after
-  the Local audio/stop decision. It avoids constructing and sampling the
-  full text vocabulary. Audio-code and binary-stop sampling are unchanged.
-  Requests requiring distribution metadata or token constraints, and
-  unsupported execution modes, retain the normal sampler.
-
-Other profiles leave both flags disabled. Set both flags explicitly when
-comparing their combined effect. Deployment
-inheritance replaces a stage's `hf_overrides` mapping, so an overlay must also
-retain `mrv2_gpu_slot_state: true`. Measure first-packet latency as well as
-throughput: faster prompt preparation can change competition between the
-Talker and codec sharing the GPU.
-
-For the single-H200 capacity-128 throughput baseline, keep both switches on.
-In complete Seed-TTS EN1088 runs at client concurrency 128, four combined-switch
-rounds pooled 344.11 audio-s/s versus 335.78 for batch prefill alone; direct
-tokens alone had no reliable gain. The combined path had 685 ms mean first
-audio versus 679 ms across six control rounds, with lower mean completion time.
-A fixed-seed 128-row audio sample showed mean Whisper WER of 3.68% versus
-3.78% for the control. This sample does not establish speaker similarity or
-exclude small quality changes. At concurrency 256, the combined path had
-raised mean first-audio latency by about 0.33 s, so other profiles leave the
-switches off until their latency and quality are checked.
-
-The optional connector setting `generation_coalescing_policy: idle_wait`
-waits for an inbox notification when no stream is runnable, all receivers
-are parked and registered, and no output needs retirement. The first ready
-arrival starts a fresh coalescing window, preserving the batch budget.
-Control messages, including cancellations, can wait up to two windows
-instead of one (24 ms at the 12 ms setting). The default policy remains
-`fixed`; neither policy changes the global orchestration default.
-
-The codec backends differ in state access:
-
-- `sdpa` uses PyTorch attention after gathering and updating the ring cache.
-- `triton` replaces the attention calculation, retaining the ring-cache
-  gather/copy and explicit mask construction.
-- `triton_slot` writes surviving K/V directly into request slots, attends
-  directly to the ring, and advances the active slot offsets in three
-  ordered kernels. Graph-padding rows do not advance persistent state.
-
-The slot path preserves the existing chunk-complete ring semantics, including
-retaining the final cache-capacity tokens when a chunk exceeds the ring.
-It does not introduce another cache owner or change request-slot lifetime.
-Zero-length padding rows skip the attention computation and emit zeros.
-
-The high-concurrency profile includes intermediate codec batch buckets 6, 12
-and 24. To compare with power-of-two buckets, change the stage-1
-`cudagraph_capture_sizes` to `[1, 2, 4, 8, 16, 32, 64, 128]`.
-Keep the maximum bucket equal to the state capacity to retain terminal-tail
-coalescing. Smaller buckets reduce padding work but require more graphs;
-measure complete serving runs before selecting them for a deployment.
-
-For a codec output-path comparison, set
-`connectors.shm.extra.codec_gpu_stream_output: 0` to select the synchronous
-codec output path; `1` selects GPU output snapshots on a private codec
-stream. Both configurations use MRV2 and keep streaming audio responses.
-The option changes input ownership, metadata staging and snapshot handling
-as well as transfer scheduling, so its timing difference is not just D2H
-copy time.
-
-Those chunk-complete semantics truncate the causal window: every ring holds
-exactly `context` entries and a whole chunk is written before attention runs,
-so token `i` of a `T`-token chunk sees `capacity - T + i + 1` keys. With the
-tokenizer-v2 decoder's per-layer contexts (10/10/8/4/2/1 s, i.e. 125/250/400/
-400/400/400 tokens) a 15-frame chunk is 480 tokens at the last transformer, so
-its first 80 tokens attend to nothing and no chunk sees the previous one there.
-Padding a terminal tail to 15 frames therefore decodes it differently from an
-exact-length decode; the difference is deterministic (identical in fp32 and
-fp64), not bf16 noise. The connector option `codec_ring_headroom: 1` sizes each
-ring as `context + max_chunk_frames * tokens_per_frame` (including ramp shapes); chunked streaming
-then reproduces whole-sequence decoding bit-for-bit in fp64 and padded tails
-equal exact tails. It increases state/graph memory and capture time; historical C256 experiments
-observed about a 2% throughput cost. It is off by default and its perceptual
-quality benefit has not been established. Terminal tails are padded
-into the regular 15-frame graph bucket in either mode.
-The slot kernel is specific to the CUDA tokenizer-v2 decoder; other paths
-retain their existing attention implementation. Event-driven orchestration
-remains independently selectable and its default is unchanged.
-
-### Optional progressive chunks
-
-Set `codec_chunk_ramp` under `connectors.shm.extra` to insert smaller chunks
-before steady decoding, for example:
-
-```yaml
-connectors:
-  shm:
-    extra:
-      codec_chunk_ramp: [1, 4, 15]
-```
-
-The ramp's first entry overrides `initial_codec_chunk_frames`. After the last
-entry, chunks use `codec_chunk_frames`. Each request advances independently;
-the final partial chunk is flushed and an empty terminal carries no codec
-tokens. The codec captures every ramp length, sizes the maximum execution
-step for the largest entry, and uses the ramp's first entry for its dedicated
-first-chunk graph. Additional shapes increase startup and graph memory.
-
-Ramps are opt-in. They can reduce early playback underrun while increasing
-codec calls, first-audio latency or total generation time. The measured C128
-throughput presets retain 1→15; their published figures do not establish an
-additional benefit from enabling ramps. The processor and graph-shape tests
-run without GPU or model weights:
-
-```bash
-CUDA_VISIBLE_DEVICES= PYTHONPATH=. python -m pytest -q \
-  tests/model_executor/stage_input_processors/test_moss_tts_async_chunk.py \
-  -m 'core_model and cpu' --run-level core_model
-```
-
-### Low-latency and reference-encoding options
-
-`moss_tts_local_mrv2_low_latency.yaml` keeps both stage capacities at 128 and
-uses prefill 2048, GPU slot state, batch prefill, direct tokens and eager MTP.
-Its codec first-chunk fast path uses dedicated graphs and a stream handoff
-before the regular decoder reuses the request slot. A gate limits contention
-with regular codec work. Slot waits are bounded at 30 seconds. A failed or
-stalled decode raises an error rather than replaying a partially advanced
-slot; a failed output enqueue retains owned PCM for regular delivery.
-Closing the decoder rejects new jobs. Its regular-batch limit of 16 is internal dispatch
-policy; it does not change the client concurrency or stage capacities.
-
-Reference encoding runs in the API layer, independently of MRV2. Enable
-reference graphs explicitly with `VLLM_OMNI_MOSS_REF_GRAPHS=1`; the default
-eager encoder is retained when the option is absent. The graph path uses the
-loaded tokenizer's encoder/quantizer, length buckets and optional compilation
-(`VLLM_OMNI_MOSS_REF_COMPILE=0` disables compilation). It also uses windowed
-attention; `VLLM_OMNI_MOSS_REF_ATTN=sdpa` retains the original attention.
-Compilation and attention changes need not produce bit-identical codes.
-
-For multiple API processes, `VLLM_OMNI_MOSS_REF_CODES_SHARED_DIR` enables shared
-reference-code storage and, by default, a single encoder host with four
-workers. Use a dedicated directory per service, checkpoint and encoding
-configuration. Workers have their own graph resources; all graph captures
-complete before serving begins. `VLLM_OMNI_MOSS_REF_SHARED_ENCODER=0` retains
-separate encoders while sharing codes. More workers/graphs consume memory;
-they do not imply more GPUs. Example after selecting an available GPU and
-configuring any operator-managed MPS daemon:
-
-```bash
-VLLM_WORKER_MULTIPROC_METHOD=spawn \
-VLLM_OMNI_EVENT_DRIVEN_ORCH=1 \
-VLLM_OMNI_CONNECTOR_RECV_POLL_MS=1 \
-VLLM_OMNI_MOSS_REF_GRAPHS=1 \
-VLLM_OMNI_MOSS_REF_ENCODER_WORKERS=4 \
-VLLM_OMNI_MOSS_REF_BATCH_WINDOW_MS=0 \
-VLLM_OMNI_MOSS_REF_INFLIGHT=4 \
-VLLM_OMNI_MOSS_REF_HOST_WINDOW_MS=2 \
-VLLM_OMNI_MOSS_REF_CODES_SHARED_DIR=/dev/shm/moss-local-service \
-vllm serve OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 --omni \
-  --api-server-count 4 \
-  --deploy-config vllm_omni/deploy/moss_tts_local_mrv2_low_latency.yaml \
-  --stage-init-timeout 1200 --init-timeout 1500
-```
-
-This example is a configurable serving profile, not the complete launch
-configuration of a historical benchmark. In particular, the low-latency YAML
-uses synchronous stage-1 scheduling; the retained source46 experiment used
-asynchronous stage-1 scheduling, FP8 backbone, MPS and additional encoder
-environment settings. Do not infer a throughput result from the YAML alone.
-
-### Cold versus hot reference measurements
-
-A cold reference misses the encoded-reference cache: the request pays for
-reference parsing/preparation and GPU encoding before speech generation.
-A hot reference reuses reference codes but still synthesizes new speech;
-generated audio is not cached. Pre-registering a reference moves encoding
-into registration and can make subsequent synthesis hot, but registration
-latency is still part of the first-use cost.
-
-Model startup/compilation is separate from reference coldness. EN1088 contains
-repeated references, so its first pass is not uniformly cold. For an all-unique
-cold pass, use its 666 first-occurrence references, keep order, warm execution
-with disjoint references, then repeat the same 666 requests. A second pass
-through independent API-local caches is not guaranteed hot; verify cache
-coverage before comparing it with a shared-cache result. Report requests/s
-alongside audio-s/s and retain failed, empty, long and near-silent outputs.
-
-### Reproduce the serving benchmark
-
-Use the complete Seed-TTS English test set, including reference audio and text.
-Set `SEED_TTS_DATA` to the directory containing `en/meta.lst` and its 1088
-entries. Run the native benchmark after the server is ready:
-
-```bash
-export VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=48000
-export VLLM_OMNI_BENCH_AUDIO_CHANNELS=2
-export SEED_TTS_WER_EVAL=0
-
-for concurrency in 128; do
-  for phase in warm r1 r2; do
-    vllm bench serve --omni \
-      --model OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 \
-      --backend openai-audio-speech --endpoint /v1/audio/speech \
-      --dataset-name seed-tts --dataset-path "$SEED_TTS_DATA" \
-      --seed-tts-locale en --disable-shuffle --num-prompts 1088 \
-      --num-warmups 0 --ready-check-timeout-sec 0 \
-      --output-len 256 --max-concurrency "$concurrency" \
-      --request-rate inf --seed 42 \
-      --extra-body '{"task_type":"Base","max_new_tokens":256}' \
-      --save-result --save-detailed --result-dir results/moss-local \
-      --result-filename "c${concurrency}-${phase}.json"
-  done
-done
-```
-
-Discard the complete `warm` pass and combine measured runs as total generated
-audio seconds divided by total benchmark duration. Require 1088 successes,
-zero failures and 1088 nonempty-audio metric samples in each run. The output
-cap matches the benchmark protocol; success alone does not establish speech
-quality or that every sentence ended before the cap. Dataset seed 42 does
-not fix an independent sampling seed for every request.
-
-For an attention-only comparison, copy the high-concurrency YAML beside the
-original to preserve relative `base_config` resolution. Change its
-codec `compilation_config.mode` to `0` while retaining `cudagraph_mode: FULL`,
-and compare `codec_attention_backend: triton` against `triton_slot`. Keep all
-other settings, warmup and client concurrency identical. Restart the server
-between configurations and use separate result directories. Comparing the
-compiled slot profile to uncompiled Triton includes both changes.
-
-Kernel and MHA regression tests require CUDA, compatible vLLM/Triton packages,
-and no model weights:
-
-```bash
-python -m pytest -q tests/model_executor/models/moss_tts/test_slot_attention.py \
-  tests/model_executor/models/moss_tts/test_streaming_attention.py \
-  -m 'core_model and cuda' --run-level=core_model
-```

recipes/Qwen/Qwen-Image-2.1.md

@@ -1,418 +0,0 @@
-# Qwen-Image-2.1
-
-> Text-to-image and image-conditioned generation with Qwen-Image 2.1
-
-## Summary
-
-- Vendor: Qwen
-- Model: `Qwen/Qwen-Image-2.1`
-- Task: Text-to-image generation and image-conditioned generation (editing)
-- Mode: Offline inference and online serving with optional step-wise execution
-- Maintainer: Community
-
-> **Model id:** `Qwen/Qwen-Image-2.1` follows the reference diffusers
-> documentation and may change before the official release — the final
-> checkpoint name is subject to the official release.
-
-## When to use this recipe
-
-Use this recipe as a starting point for running `Qwen/Qwen-Image-2.1`. A single
-pipeline class, `QwenImage21Pipeline`, handles both pure text-to-image requests
-(no condition image) and image-conditioned requests with up to 4 condition
-images: prompt and condition images are encoded together by a Qwen3-VL text
-encoder, so editing-style requests go through the same serving path as
-text-to-image.
-
-## References
-
-- Related offline examples:
-  [`examples/offline_inference/text_to_image/text_to_image.py`](../../examples/offline_inference/text_to_image/text_to_image.py),
-  [`examples/offline_inference/image_to_image/image_edit.py`](../../examples/offline_inference/image_to_image/image_edit.py)
-- Supported models table:
-  [`docs/models/supported_models.md`](../../docs/models/supported_models.md)
-- Feature compatibility matrix:
-  [`docs/user_guide/diffusion_features.md`](../../docs/user_guide/diffusion_features.md)
-- Sibling recipes: [`Qwen-Image.md`](./Qwen-Image.md),
-  [`Qwen-Image-Edit.md`](./Qwen-Image-Edit.md)
-
-## Offline Inference
-
-### Text-to-image
-
-```bash
-python examples/offline_inference/text_to_image/text_to_image.py \
-  --model Qwen/Qwen-Image-2.1 \
-  --prompt "A ceramic teapot on a wooden table" \
-  --output qwen_image_21_t2i.png \
-  --num-inference-steps 50 \
-  --cfg-scale 1.0
-```
-
-### Image-conditioned generation (editing)
-
-```bash
-python examples/offline_inference/image_to_image/image_edit.py \
-  --model Qwen/Qwen-Image-2.1 \
-  --color-format RGBA \
-  --seed 42 \
-  --image qwen_bear.png \
-  --prompt "Let this mascot dance under the moon, surrounded by floating stars" \
-  --output qwen_image_21_edit.png \
-  --num-inference-steps 50 \
-  --cfg-scale 1.0
-```
-
-or via the bundled launcher:
-
-```bash
-bash examples/offline_inference/image_to_image/run_qwen_image_21.sh
-```
-
-Up to 4 condition images can be passed via `--image`:
-
-```bash
-python examples/offline_inference/image_to_image/image_edit.py \
-  --model Qwen/Qwen-Image-2.1 \
-  --color-format RGBA \
-  --seed 42 \
-  --image input1.png input2.png \
-  --prompt "Combine these images into a single scene" \
-  --output qwen_image_21_multi.png \
-  --num-inference-steps 50 \
-  --cfg-scale 1.0
-```
-
-Use `--color-format RGBA` to preserve transparency in condition images. The
-shared example otherwise loads inputs as RGB. Add `--width` and `--height`
-(multiples of 32) to set the output size; when omitted, Qwen-Image 2.1 derives
-it from the last condition image's aspect ratio at approximately 1024×1024.
-Use `--num-outputs-per-prompt` to generate multiple images and `--vae-use-tiling`
-to reduce VAE memory usage. The launcher accepts these options as well.
-
-## Online Serving
-
-Start the server:
-
-```bash
-vllm serve Qwen/Qwen-Image-2.1 --omni --port 8091
-```
-
-To enable the step-wise runtime:
-
-```bash
-vllm serve Qwen/Qwen-Image-2.1 --omni \
-  --port 8091 \
-  --step-execution \
-  --max-num-seqs 8
-```
-
-Step execution batches compatible requests at the same KV-cache phase. The
-pipeline tags every request with a `batch_compatibility_key` (conditioning
-mode plus condition-image layout), so a newly arriving request that would
-otherwise join a batch already past its first denoising step is isolated by
-the scheduler and deferred until the in-flight batch drains — it no longer
-fails the whole batch. `--max-num-seqs` controls batch capacity.
-
-### CUDA Graph decode
-
-Qwen-Image-2.1 automatically uses CUDA Graph for supported fixed-shape denoising
-decode steps when `enable_cuda_graph_decode=True` (the default) and
-`enforce_eager=False`. Prefill remains eager. This works with both request
-execution and `--step-execution`.
-
-To disable graph capture while keeping the rest of the compilation setup:
-
-```bash
-vllm serve Qwen/Qwen-Image-2.1 --omni --no-enable-cuda-graph-decode
-```
-
-The same control is available as `Omni(..., enable_cuda_graph_decode=False)`
-and as `enable_cuda_graph_decode: false` on a stage in the deployment YAML.
-`--enforce-eager` / `enforce_eager: true` remains the broader switch: it
-disables both graph capture and automatic `torch.compile`. The two
-optimizations stack: the DiT blocks are regionally `torch.compile`'d and
-graph capture records the compiled (fused) kernels, while inductor's own
-cudagraphs stay disabled so the two graph layers never nest. Measured on a
-single H200 (1024×1024, 50 steps, seed 42, BF16, `true_cfg_scale=1.0`,
-warmup 1 + median of 3): compile+graph **5.93 s**, compile-only **6.88 s**,
-graph-only **9.52 s**, eager **8.28 s** end-to-end per image; combo and
-compile-only vs eager are **38.10 dB** PSNR (compile fusion drift), and
-graph-only vs eager is bit-identical.
-
-The autoregressive engine's `compilation_config.cudagraph_mode` does not
-control this diffusion path; use `enable_cuda_graph_decode` (or
-`enforce_eager` for everything) to force eager decode execution.
-
-TP/SP/ring parallelism, HSDP, cache hooks, quantized KV caches,
-dynamic LoRA, padded text masks, and a second in-flight request whose cache
-aliases an already-owned graph key fall back to eager decode.
-Model-level CPU offload does not disable decode graphs while
-`enable_cuda_graph_decode` stays on (the default). DiT weights stay on fixed
-device storage so captured pointers remain valid, and that storage is not
-released while the text encoder runs. Pass `--no-enable-cuda-graph-decode`
-(or `enable_cuda_graph_decode: false`) for the eager offload path, which
-moves DiT weights off the device and frees that VRAM. Peak memory for the
-default graph-plus-offload combination was not re-measured in this PR.
-Graphs keep separate entries for different image layouts and copy
-request-owned prefix K/V into static buffers before replay; each graph key
-has a single live owner at a time, so concurrent same-shape requests never
-overwrite each other's prefix. Those buffers require additional memory; graph
-entries are bounded by the transformer's cache limit.
-
-### Verification
-
-For a direct API smoke test:
-
-```bash
-curl http://localhost:8091/v1/images/generations \
-  -H "Content-Type: application/json" \
-  -d '{
-    "model": "Qwen/Qwen-Image-2.1",
-    "prompt": "A ceramic teapot on a wooden table",
-    "size": "1024x1024",
-    "num_inference_steps": 50,
-    "true_cfg_scale": 1.0,
-    "seed": 42
-  }'
-```
-
-For image-conditioned generation, pass condition images through the
-OpenAI-compatible multimodal endpoint (`/v1/chat/completions` with
-`image_url` content parts, or `/v1/images/edits`), same as
-[`Qwen-Image-Edit.md`](./Qwen-Image-Edit.md).
-
-## Parameters
-
-| Parameter | Default | Notes |
-| --- | --- | --- |
-| `true_cfg_scale` | 1.0 | Disables the negative-prompt branch by default. Set above 1 with a `negative_prompt` to enable true CFG. Qwen-Image 2.1 supports true CFG only — there is no `guidance_scale` parameter. |
-| `num_inference_steps` | 50 | Number of denoising steps. |
-| `height` / `width` | 1024×1024 | Must be multiples of 32. For image-conditioned requests, leaving them unset derives the output size from the last condition image's aspect ratio at ~1024×1024. |
-| `negative_prompt` | None | Ignored at the default CFG scale of 1.0; used only when `true_cfg_scale > 1`. |
-| condition images | — | Up to 4 input images per request; more than 4 raises an error. |
-| `seed` | — | Fix for reproducible outputs. |
-
-The default CFG scale of 1.0 runs only the positive-prompt branch at each
-denoising step. To enable negative-prompt guidance, explicitly set
-`--cfg-scale` (offline) or `true_cfg_scale` (API) above 1 and provide a
-negative prompt. This changes the generated output as well as compute cost.
-
-### Prefix KV cache
-
-The transformer caches the text/condition-image prefix across denoising steps
-(a prefill/decode split keyed by CFG branch). The cache is enabled
-automatically when the checkpoint declares `causal_condition: true` and is
-re-created per generation, so no state leaks between requests. It changes the
-compute path — cached prefix vs. full recompute at every step — so exact
-numerical reproduction against a no-cache reference (e.g. the diffusers
-pipeline) requires matching the cache setting on both sides. The prefix KV
-cache is orthogonal to weight quantization and works unchanged under FP8.
-
-### FP8 quantization
-
-Online FP8 quantization of the transformer is supported; the fused
-`to_qkv` projections load correctly, and the prefix KV cache path is
-unaffected. Only block-internal linears are eligible — the boundary
-projections (`img_in`/`txt_in`/`modulation`/`proj_out`/`time_text_embed`) stay
-in BF16 by construction:
-
-```bash
-python examples/offline_inference/text_to_image/text_to_image.py \
-  --model Qwen/Qwen-Image-2.1 \
-  --prompt "A ceramic teapot on a wooden table" \
-  --output qwen_image_21_fp8.png \
-  --num-inference-steps 50 --cfg-scale 1.0 \
-  --quantization fp8 --ignored-layers "img_mlp"
-```
-
-`ignored_layers` entries are matched as name patterns for this model (e.g.
-`img_mlp` matches every block's `img_mlp.proj`/`img_mlp.gate_layer`/
-`img_mlp.out`); exact full prefixes like
-`transformer.transformer_blocks.3.attn.to_qkv` also work.
-
-A per-module sensitivity sweep (fixed prompt embeds / latent / timestep, 5
-timesteps across the trajectory, relative L2 of the noise prediction vs BF16)
-ranks the groups: `img_mlp` is the most sensitive (3.8% max error on its own,
-driven by `img_mlp.proj` 2.8% and `img_mlp.out` 2.5%), then `attn.to_out`
-(2.7%), then `attn.to_qkv` (1.6%); errors are largest at the late (low-sigma)
-steps. End-to-end on H200, single GPU, same prompt as Quality Comparison
-(`"A ceramic teapot on a wooden table"`, 1024×1024, 50 steps, seed 42,
-eager):
-
-| Config | Quantized block linears | PSNR vs BF16 | Steady-state time/image | Peak memory |
-| --- | --- | --- | --- | --- |
-| BF16 | 0 / 160 | — | **7.34 s** | **36.9 GB** |
-| FP8 all layers | 160 / 160 | **33.0 dB** | **6.06 s** | **30.4 GB** |
-| FP8, `ignored_layers=["img_mlp"]` | 64 / 160 | **34.3 dB** | **6.98 s** | **34.9 GB** |
-
-Keeping `img_mlp` in BF16 is still the higher-fidelity DiT setting vs
-all-layer FP8. On H200, all-layer FP8 is both leaner and faster than BF16
-at this shape.
-
-## Quantization
-
-Online FP8 is supported per component via `quantization_config`:
-
-```python
-from vllm_omni import Omni
-
-omni = Omni(
-    model="Qwen/Qwen-Image-2.1",
-    quantization_config={
-        # DiT: skip the sensitive image-stream MLPs if quality regresses.
-        "transformer": {"method": "fp8", "ignored_layers": ["img_mlp"]},
-        # Qwen3-VL text encoder: only the language-model linear layers are
-        # quantized; the vision tower and the (unused) LM head stay BF16.
-        "text_encoder": {"method": "fp8"},
-    },
-)
-```
-
-Either component can be quantized on its own. Measured on H200 at 1024×1024
-(seed 42, 50 steps, same teapot prompt, eager): text-encoder FP8 lowers peak
-GPU memory from **36.9 GB** (BF16) to **30.4 GB** (−6.5 GB); T2I PSNR vs BF16
-**32.1 dB**. DiT FP8-all is **30.4 GB** / **6.06 s**. Combined DiT+text-encoder
-FP8 was not re-measured on H200. The edit path routes condition images through
-the BF16 vision tower, so it is unaffected by text-encoder FP8. See
-[`docs/user_guide/quantization/fp8.md`](../../docs/user_guide/quantization/fp8.md)
-for the scope rules.
-
-### FP8 prefix KV storage
-
-The cached prefix K/V can be stored in FP8 E4M3 (with per-token-per-head fp32
-scales), halving the prefix-cache memory — relevant for long prompts (up to
-8192 text tokens) and up to 4 condition images (~4096 latent tokens each).
-Prefill quantizes once; decode dequantizes per step, so attention compute is
-unchanged. Enable it through the model-specific `extras` config (e.g. in the
-deploy YAML's stage `extras` or as an `Omni(...)` engine kwarg):
-
-```yaml
-extras:
-  prefix_kv_cache_dtype: "fp8"
-```
-
-Accepted values: `"fp8"` / `"fp8_e4m3"` (K and V quantized), `"fp8_v"` (V
-only — K stays in the native dtype), and `None` / `"auto"` (default, native
-dtype — behavior unchanged). This is independent of
-`diffusion_kv_cache_dtype`, which quantizes attention Q/K/V *compute* per
-forward pass on supported backends.
-
-Measured on one NVIDIA H200, T2I 1024x1024 (seed 42, 50 steps, true CFG 4.0,
-eager, PSNR vs. the bf16 baseline on the same GPU): **`"fp8"` 34.2 dB**,
-**`"fp8_v"` 44.0 dB**. The error is dominated by
-K quantization: post-RoPE keys are the precision-sensitive half of the
-cache, so `"fp8_v"` buys back about 9.8 dB at 75% (instead of 50%) of the original
-cache size. The residual error is inherent e4m3 precision accumulated
-coherently over the denoising trajectory; finer scale granularity
-(per-tensor/per-head/per-token were compared) or Hadamard-rotated V did not
-improve it. The saving scales with prefix length. These sizes are calculated at the
-limit (8192 text tokens + 4 condition images, ~24.6k prefix tokens), not
-measured on the 1024 T2I run above: ~12.9 GB per CFG branch in bf16, ~6.6 GB
-in `"fp8"` and ~9.7 GB in `"fp8_v"`. That is where this option matters. Treat it as an opt-in for
-memory-bound long-prompt / multi-image workloads, not a free lunch;
-`"fp8_v"` is the better default trade-off when quality matters.
-
-Note: a quantized prefix cache is not CUDA-graph capturable — requests using
-this option stay on the eager decode path (a warning is logged once).
-
-### Why not the scheduler-managed paged KV (`DiffusionKVCacheMode.PAGED_SCHEDULER`)?
-
-Evaluated and deliberately not adopted (see the `qwen21-p0-kv` commit message
-for the full analysis): the paged scheduler targets *cross-request* KV
-management — a statically sized page pool, block sharing/dedup, and eviction —
-whereas this prefix cache lives exactly as long as one generation and is
-sized once per request. Migrating would require routing decode attention
-through the worker paged adapter (writing fresh target K/V into pages every
-step), per-request page tables, and re-validating the block-causal
-piecewise-span masking and Ulysses/CFG constraints, for no memory benefit over
-the per-request dense tensors. The FP8 storage above captures the actual
-memory win at ~1% of the integration cost.
-
-## Quality Comparison (head vs diffusers)
-
-The reference is the diffusers Qwen-Image 2.1 pipeline
-([huggingface/diffusers#14804](https://github.com/huggingface/diffusers/pull/14804)).
-Both sides must pin the same checkpoint, inputs, seed, precision (BF16), and
-hardware.
-
-vLLM-Omni (this recipe):
-
-```bash
-python examples/offline_inference/text_to_image/text_to_image.py \
-  --model Qwen/Qwen-Image-2.1 \
-  --prompt "A ceramic teapot on a wooden table" \
-  --seed 42 \
-  --output qwen_image_21_vllm_omni.png \
-  --num-inference-steps 50 \
-  --cfg-scale 1.0
-```
-
-diffusers reference:
-
-```python
-import torch
-from diffusers import QwenImage21Pipeline  # diffusers PR #14804
-
-pipe = QwenImage21Pipeline.from_pretrained(
-    "Qwen/Qwen-Image-2.1", torch_dtype=torch.bfloat16
-).to("cuda")
-image = pipe(
-    prompt="A ceramic teapot on a wooden table",
-    num_inference_steps=50,
-    true_cfg_scale=1.0,
-    generator=torch.Generator(device="cuda").manual_seed(42),
-    height=1024,
-    width=1024,
-).images[0]
-image.save("qwen_image_21_diffusers.png")
-```
-
-| Metric | vLLM-Omni | diffusers (#14804) | Status |
-| --- | --- | --- | --- |
-| Output image (1024×1024, 50 steps, seed 42, BF16, H200) | generated (eager + default CUDA-graph) | generated | measured |
-| PSNR / LPIPS vs reference | eager vs Diffusers **47.17 dB**; **default compile+graph** vs Diffusers **38.02 dB**; **default compile+graph** vs eager **38.10 dB** | — | measured (H200) |
-| Latency (time/image, same GPU, H200) | **default compile+graph** **6.04 s** (120.8 ms/step); eager **7.31 s** (146.2 ms/step) | **7.74 s** (154.9 ms/step) | measured |
-
-The prefix KV cache changes the compute path (cached prefix vs. full recompute
-at every step), so exact numerical parity is not expected; match the cache
-setting on both sides or compare with PSNR/LPIPS rather than exact pixels.
-Eager stays numerically close to Diffusers (~47 dB). The **default** serving
-path stacks regional `torch.compile` with CUDA-graph decode; on H200 that
-combo lands at **~38.1 dB** vs eager (~1.21× on the Quality Comparison timed
-run). Graph-only vs eager is bit-identical; the ~38 dB drift is the compile
-fusion, not CUDA-graph replay. See "CUDA Graph decode" for the four-mode
-H200 breakdown (warmup 1 + median of 3).
-
-Parity / feature measurements (same prompt/seed/steps/CFG, BF16, eager decode
-unless noted; 4× NVIDIA H200):
-
-- [x] TP=2/4 output parity vs single-card — measured (1024×1024): TP1 vs TP2
-  **39.32 dB**, TP1 vs TP4 **38.37 dB**, TP2 vs TP4 **45.77 dB**. Latency:
-  TP1 **7.30 s**, TP2 **4.89 s**, TP4 **5.12 s** per image.
-- [x] VAE tiling vs non-tiled decode — measured: 1024×1024 nontile vs
-  `--vae-use-tiling` **43.55 dB** (peak mem ~37.7 GB → ~32.7 GB); 2048×2048
-  nontile vs tiled **44.14 dB** (peak mem ~57.9 GB → ~33.3 GB; gen ~36.8 s vs
-  ~37.1 s).
-- [x] CUDA-graph four-mode vs eager — measured on H200 (1024×1024, 50 steps,
-  seed 42, BF16, warmup 1 + median of 3): compile+graph **5.93 s**,
-  compile-only **6.88 s**, graph-only **9.52 s**, eager **8.28 s**. Combo /
-  compile-only vs eager **38.10 dB**; graph-only vs eager bit-identical.
-  Quality Comparison table uses a separate single-run A/B vs Diffusers
-  (compile+graph **6.04 s**, eager **7.31 s**).
-
-## Known Limitations
-
-- Cache acceleration backends (`cache_dit`, `tea_cache`) are not supported;
-  `QwenImage21Pipeline` is registered in `_NO_CACHE_ACCELERATION`.
-- Ulysses sequence parallelism is wired; ring attention is not a supported path.
-  A short H200 smoke (1024×1024, 8 steps, seed 42, true CFG 1.0, eager) finished:
-  Ulysses SP=2 vs SP=1 is **46.5 dB**. That size's target length is divisible by 4,
-  so SP padding was not exercised, and this is not the 50-step parity next to the
-  TP tables. The features matrix therefore stays ❓. Ring SP=2 also ran, at
-  **29.9 dB** vs SP=1, which is why it stays unsupported.
-- Only true CFG is exposed; there is no `guidance_scale` knob.
-- VAE tiling (`--vae-use-tiling`, also implied by `--vae-patch-parallel-size > 1`)
-  decodes in 512px tiles with 384px stride (32 latent pixels per tile at 16x
-  spatial compression). On CUDA OOM the tile size is automatically halved
-  (keeping 25% overlap) and decoding retried, down to 128px tiles. Tile sizes
-  are not yet exposed as engine arguments.

recipes/Qwen/Qwen3-Omni.md

@@ -73,20 +73,8 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
     "1": {"max_num_seqs": 4},
     "2": {"max_num_seqs": 4}
   }'
-
-# Replay the thinker's image encoder from CUDA graphs (images only)
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
-  --stage-overrides '{"0": {"compilation_config": {"cudagraph_mm_encoder": true}}}'
 ```
 
-With the encoder graphs on, the thinker captures one graph per token budget
-(64 to 256 image tokens and up to 4 images per replay by default). A step's
-images replay together when one captured graph holds them all; otherwise they
-run through the eager encoder in one call. Set `encoder_cudagraph_token_budgets`
-and `encoder_cudagraph_max_vision_items_per_batch` in the same
-`compilation_config` to change them; each captured budget adds graph memory on
-the thinker's GPU.
-
 ### Stage-based launch (one stage per process)
 
 Use three terminals (one per stage). Start with the default commands below, then

recipes/README.md

@@ -64,12 +64,8 @@ recipes/
 | [`MiniMaxAI/MiniMax-H3-Spark-GB10.md`](./MiniMaxAI/MiniMax-H3-Spark-GB10.md) | MiniMax H3 T2VA and Ref2VA on DGX Spark | 1x or 2x DGX Spark (GB10) |
 | [`MiniMaxAI/MiniMax-H3-MUSA.md`](./MiniMaxAI/MiniMax-H3-MUSA.md) | MiniMax H3 T2VA, FL2VA, and Ref2VA on MUSA | 4x MTT S5000 (TP4 Ref2VA profile) |
 | [`krea/Krea-2.md`](./krea/Krea-2.md) | Text-to-image (Turbo + Raw), offline + online, with LoRA | 1x H100 80GB / 1x Intel Arc Pro B70 32GB |
-| [`Kandinsky/Kandinsky6-TI2VA.md`](./Kandinsky/Kandinsky6-TI2VA.md) | Text/image-to-video-and-audio, offline and `/v1/videos` | 1x H100 80GB with CPU offload; 2x H100 for parallel modes |
 | [`LTX/LTX-2.md`](./LTX/LTX-2.md) | LTX-2/LTX-2.3 text-to-video and image-to-video with synchronized audio | H200 141GB / 96GB-class GPU / 1x Intel Arc Pro B70 32GB |
 | [`LTX/LTX-2.5.md`](./LTX/LTX-2.5.md) | LTX-2.5-Diffusers: Full/SFT one-stage and distilled two-stage T2V/I2V with synchronized audio | NVIDIA B300; cuDNN-qualified / 1x Intel Arc Pro B70 32GB |
-| [`JD/JoyAI-Image-Edit.md`](./JD/JoyAI-Image-Edit.md) | Text-guided single-image editing | 1x H800 PCIe 80GB |
-| [`m-a-p/SheetSage2-H200.md`](./m-a-p/SheetSage2-H200.md) | Official audio-to-score preprocessing and YuE2 request export | 1x H200 141GB |
-| [`m-a-p/YuE2-3B.md`](./m-a-p/YuE2-3B.md) | Offline and online text-to-music with optional ABC score | 1x RTX 4090 24GB |
 | [`MammothModa2/MammothModa2.md`](./MammothModa2/MammothModa2.md) | Preview and Dev text-to-image (AR → DiT); Dev text/image understanding | Preview: 1x L40S 48GB / 1x ≥40GB GPU; Dev: 1x NVIDIA GPU with sufficient cache headroom |
 | [`NVIDIA/SANA-Video-2B.md`](./NVIDIA/SANA-Video-2B.md) | Native and Diffusers-adapter 480p/720p T2V+I2V | 1x RTX 5090 32GB |
 | [`meituan-longcat/LongCat-Video-Avatar-1.5.md`](./meituan-longcat/LongCat-Video-Avatar-1.5.md) | Audio-driven avatar video generation (AT2V / AI2V, single- and multi-speaker, AVC continuation) | 1x H100 80GB |
@@ -88,7 +84,6 @@ recipes/
 | [`NVIDIA/NemotronLabs-VoiceChat.md`](./NVIDIA/NemotronLabs-VoiceChat.md) | Offline speech-to-speech voice chat (11B, frame-locked 12.5 Hz, 3-stage thinker/talker/code2wav) | 1x H100 80GB |
 | [`NVIDIA/SANA-WM.md`](./NVIDIA/SANA-WM.md) | First-frame image-to-video serving with camera control | 1x 24GB+ CUDA GPU; ~40GB disk |
 | [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt mixed FP8/NVFP4; offline text-to-image on XPU | 1x A100 80GB / 2x B200 / 1x Intel Arc Pro B70 32GB |
-| [`Qwen/Qwen-Image-2.1.md`](./Qwen/Qwen-Image-2.1.md) | Text-to-image and image-conditioned generation (editing) with prefix KV cache and CUDA graph decode | 1x GPU with CPU offload / 1x H200 |
 | [`Qwen/Qwen-Image-2512.md`](./Qwen/Qwen-Image-2512.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt FP8 / mixed FP8/NVFP4 | 1x A800 80GB / 2x B200 |
 | [`Qwen/Qwen-Image-Edit.md`](./Qwen/Qwen-Image-Edit.md) | Text-guided single-image editing | 1x or 2x H200 141GB |
 | [`Qwen/Qwen3-Omni.md`](./Qwen/Qwen3-Omni.md) | Online serving for multimodal chat; offline end-to-end text-to-speech on XPU | 1x A100 80GB / 5x Intel Arc Pro B70 32GB |