[Bugfix][MiniMax-H3] Use Hopper-safe modulation precision

Signed-off-by: david6666666 <530634352@qq.com>

.buildkite/amd/scripts/bootstrap-amd-omni.sh

@@ -60,7 +60,8 @@ resolve_test_specs() {
     if ! selected_specs=$(python3 "$AMD_SUITE_SELECTOR" \
         --branch "$BUILDKITE_BRANCH" \
         --labels "$PR_LABELS" \
-        --debug-test-yaml "${DEBUG_TEST_YAML:-}"); then
+        --debug-test-yaml "${DEBUG_TEST_YAML:-}" \
+        --nightly "$NIGHTLY"); then
         exit 1
     fi
 
@@ -80,6 +81,12 @@ filter_test_specs_by_skip_ci() {
         case "$suite_spec" in
             READY_TESTS:*) level="l2" ;;
             MERGE_TESTS:*) level="l3" ;;
+            NIGHTLY_TESTS:*)
+                # Explicit PR and scheduled nightly selections must survive
+                # L2/L3 diff gating, including docs-only main commits.
+                runnable_specs+=("$suite_spec")
+                continue
+                ;;
             *)
                 echo "ERROR: unknown AMD test suite spec '$suite_spec'" >&2
                 exit 1
@@ -101,7 +108,7 @@ filter_test_specs_by_skip_ci() {
                 ":memo: CI skipped — docs or pytest skip-mark changes only" \
                 --style "info" 2>/dev/null || true
         fi
-        echo "No AMD L2/L3 suites remain after skip-ci filtering."
+        echo "No AMD suites remain after skip-ci filtering."
         exit 0
     fi
 }
@@ -219,8 +226,8 @@ if [[ $BUILDKITE_BRANCH == "main" ]]; then
     file_diff=$(get_diff_main)
 fi
 
-# Resolve PR tier labels before skip-ci so ready and merge suites can be
-# filtered independently for CI-YAML-only changes.
+# Resolve PR tier labels before skip-ci so ready/merge suites can be filtered
+# independently while an explicitly selected nightly suite passes through.
 resolve_test_specs
 filter_test_specs_by_skip_ci
 

.buildkite/amd/scripts/select_test_suites.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
-"""Select AMD L2/L3 suites after Buildkite has created an AMD build.
+"""Select AMD ready/merge/nightly suites after Buildkite creates a build.
 
 This repository-side selector does not decide which GitHub label events start
 the external ``vllm-omni-amd-ci`` pipeline. Its Buildkite PR build condition
@@ -17,6 +17,7 @@ from collections.abc import Iterable
 SUITE_SPECS = {
     "ready": "READY_TESTS:test-amd-ready.yml",
     "merge": "MERGE_TESTS:test-amd-merge.yml",
+    "nightly": "NIGHTLY_TESTS:test-amd-nightly.yml",
 }
 
 
@@ -28,7 +29,7 @@ def _parse_debug_suites(value: str) -> tuple[str, ...]:
     seen: set[str] = set()
     for suite in suites:
         if suite not in SUITE_SPECS:
-            raise ValueError(f"DEBUG_TEST_YAML entries must be 'merge' or 'ready', got {suite!r}")
+            raise ValueError(f"DEBUG_TEST_YAML entries must be 'merge', 'nightly', or 'ready', got {suite!r}")
         if suite in seen:
             raise ValueError(f"duplicate DEBUG_TEST_YAML suite {suite!r}")
         seen.add(suite)
@@ -40,18 +41,27 @@ def select_amd_test_suites(
     branch: str,
     labels: Iterable[str],
     debug_test_yaml: str = "",
+    nightly: bool = False,
 ) -> tuple[str, ...]:
     """Return ordered suite names while preserving legacy unlabeled PRs."""
 
     if debug_test_yaml:
         return _parse_debug_suites(debug_test_yaml)
     if branch == "main":
-        return ("merge",)
+        return ("nightly",) if nightly else ("merge",)
 
     label_set = {label.strip() for label in labels if label.strip()}
-    selected = tuple(suite for label, suite in (("ready", "ready"), ("merge-test", "merge")) if label in label_set)
-    # AMD historically uploaded L2 for every PR build. Keep that behavior for
-    # triggers such as amd-test while adding tier selection for ready/merge-test.
+    selected = tuple(
+        suite
+        for label, suite in (
+            ("ready", "ready"),
+            ("merge-test", "merge"),
+            ("nightly-test", "nightly"),
+        )
+        if label in label_set
+    )
+    # AMD historically uploaded the ready suite for every admitted PR build.
+    # Preserve that fallback when no explicit tier label is present.
     return selected or ("ready",)
 
 
@@ -60,6 +70,7 @@ def main() -> int:
     parser.add_argument("--branch", required=True)
     parser.add_argument("--labels", default="")
     parser.add_argument("--debug-test-yaml", default="")
+    parser.add_argument("--nightly", type=int, choices=(0, 1), default=0)
     args = parser.parse_args()
 
     labels = tuple(args.labels.splitlines())
@@ -68,19 +79,15 @@ def main() -> int:
             branch=args.branch,
             labels=labels,
             debug_test_yaml=args.debug_test_yaml,
+            nightly=bool(args.nightly),
         )
     except ValueError as exc:
         parser.error(str(exc))
 
     label_set = {label.strip() for label in labels if label.strip()}
-    if not args.debug_test_yaml and args.branch != "main" and not ({"ready", "merge-test"} & label_set):
+    if args.branch != "main" and not args.debug_test_yaml and not ({"ready", "merge-test", "nightly-test"} & label_set):
         print(
-            "No AMD L2/L3 tier label found; preserving the legacy ready-suite fallback.",
-            file=sys.stderr,
-        )
-    if "nightly-test" in label_set:
-        print(
-            "AMD nightly-test has no suite yet and does not affect L2/L3 selection.",
+            "No AMD ready/merge/nightly label found; preserving the legacy ready-suite fallback.",
             file=sys.stderr,
         )
 

.buildkite/amd/test-amd-nightly.yml

@@ -0,0 +1,66 @@
+# AMD MI300 `full_model` nightly coverage during burn-in.
+# All jobs are non-blocking until the lane has stable runtime evidence.
+# Historical build #11646 was red in Function Expansion and Accuracy (one
+# async_chunk similarity mismatch; Seed-TTS WER 0.376480 > 0.35).
+
+env:
+  VLLM_WORKER_MULTIPROC_METHOD: spawn
+  HF_HUB_DOWNLOAD_TIMEOUT: 300
+  HF_HUB_ETAG_TIMEOUT: 60
+
+steps:
+  # MI325 is the repository's legacy ROCm pytest marker. The AMD template maps
+  # agent_pool to the current MI300 Buildkite queues independently of that mark.
+  - group: ":full_moon: Qwen3-Omni MI300 Nightly"
+    steps:
+      - label: "Qwen3-Omni Function Expansion"
+        agent_pool: mi300_2
+        depends_on: amd-build
+        mirror_hardwares: [amdproduction]
+        grade: NonBlocking
+        timeout_in_minutes: 180
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_expansion.py -m "full_model and rocm and MI325 and cards_2" --run-level "full_model"
+
+      - label: "Qwen3-Omni Accuracy"
+        agent_pool: mi300_2
+        depends_on: amd-build
+        mirror_hardwares: [amdproduction]
+        grade: NonBlocking
+        timeout_in_minutes: 180
+        artifact_paths:
+          - tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json
+        commands:
+          - export SEED_TTS_WER_EVAL=1
+          # PyTorch ROCm uses the torch.cuda device namespace.
+          - export SEED_TTS_EVAL_DEVICE=cuda:1
+          - pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model and rocm and MI325 and cards_2" --run-level "full_model"
+          # Native tests run from the image copy under /app/vllm-omni, while
+          # Buildkite's artifact hook searches the checkout workspace.
+          - |
+            if [[ "$$PWD" != "$${BUILDKITE_BUILD_CHECKOUT_PATH:?}" ]]; then
+              artifact_dir="$${BUILDKITE_BUILD_CHECKOUT_PATH}/tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc"
+              mkdir -p "$$artifact_dir"
+              cp -v tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json "$$artifact_dir/"
+            fi
+
+      - label: "Qwen3-Omni Documentation Examples"
+        agent_pool: mi300_2
+        depends_on: amd-build
+        mirror_hardwares: [amdproduction]
+        grade: NonBlocking
+        timeout_in_minutes: 120
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - pytest -s -v tests/examples/online_serving/test_qwen3_omni.py -m "full_model and rocm and MI325 and cards_2" --run-level "full_model"
+
+      - label: "Qwen3-Omni AITER-on Smoke"
+        agent_pool: mi300_2
+        depends_on: amd-build
+        mirror_hardwares: [amdproduction]
+        grade: NonBlocking
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ROCM_USE_AITER=1
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_expansion.py::test_text_to_text_audio_001 -m "full_model and rocm and MI325 and cards_2" --run-level "full_model"

.buildkite/amd/test-amd-ready.yml

@@ -160,6 +160,39 @@ steps:
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
 
+- label: "ROCm · Engine & Model Executor GPU Coverage (R2-01)"
+  agent_pool: mi300_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: NonBlocking
+  timeout_in_minutes: 20
+  artifact_paths:
+    - "artifacts/rocm-r2-01/**/*"
+  commands:
+    # Native AMD tests run from /app/vllm-omni, while Buildkite uploads
+    # artifacts relative to its checkout directory.
+    - export R2_01_ARTIFACT_DIR="$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/rocm-r2-01"
+    - mkdir -p "$$R2_01_ARTIFACT_DIR"
+    # The CUDA ready job also covers tests/worker/test_batched_omni_output.py.
+    # Keep the initial ROCm burn-in scoped to the requested directories.
+    # Verbose pytest output and JUnit retain node IDs without a second
+    # collection-only pass over the same GPU-backed test tree.
+    # The CUDA/L4-only CosyVoice3 ragged-flow parity test cold-JITs multiple
+    # AITER kernels on ROCm and is covered by the CUDA ready lane.
+    - >-
+      pytest -s -v -ra
+      tests/engine/ tests/model_executor/
+      -m 'core_model and cuda and not
+      (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
+      -k 'not test_code2wav_streaming_batch_matches_ragged_flow_numerics'
+      --run-level "core_model"
+      --durations=0
+      --junitxml="$$R2_01_ARTIFACT_DIR/pytest.xml"
+      2>&1 | tee "$$R2_01_ARTIFACT_DIR/pytest.log"
+    - >-
+      tail -n 100 "$$R2_01_ARTIFACT_DIR/pytest.log" >
+      "$$R2_01_ARTIFACT_DIR/pytest-summary.txt"
+
 - label: "Engine Test"
   agent_pool: mi300_1
   depends_on: amd-build
@@ -172,6 +205,71 @@ steps:
               pytest -s -v tests/engine/test_async_omni_engine_abort.py
       '
 
+- label: "ROCm · Entrypoints GPU Coverage (R2-02)"
+  agent_pool: mi300_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: NonBlocking
+  timeout_in_minutes: 30
+  env:
+    VLLM_WORKER_MULTIPROC_METHOD: spawn
+  artifact_paths:
+    - "artifacts/rocm-r2-02/**/*"
+  commands:
+    # Native AMD tests run from /app/vllm-omni, while Buildkite uploads
+    # artifacts relative to its checkout directory.
+    - export R2_02_ARTIFACT_DIR="$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/rocm-r2-02"
+    - mkdir -p "$$R2_02_ARTIFACT_DIR"
+    - >-
+      ps -eo pid=,ppid=,lstart=,stat=,comm= >
+      "$$R2_02_ARTIFACT_DIR/processes-before.txt"
+    # Verbose pytest output and JUnit retain selected node IDs without a
+    # second collection-only pass over the same GPU-backed test tree.
+    # The inner timeout leaves five minutes for teardown verification and
+    # artifact generation before the Buildkite step timeout.
+    - >-
+      timeout --signal=TERM --kill-after=1m 25m
+      pytest -s -v -ra
+      tests/entrypoints/
+      -m 'core_model and cuda and not
+      (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
+      --run-level "core_model"
+      --durations=0
+      --junitxml="$$R2_02_ARTIFACT_DIR/pytest.xml"
+      2>&1 | tee "$$R2_02_ARTIFACT_DIR/pytest.log"
+    - >-
+      tail -n 100 "$$R2_02_ARTIFACT_DIR/pytest.log" >
+      "$$R2_02_ARTIFACT_DIR/pytest-summary.txt"
+    - |
+      sleep 5
+      ps -eo pid=,ppid=,lstart=,stat=,comm= > "$$R2_02_ARTIFACT_DIR/processes-after.txt"
+      # Compare PID plus process start time so PID reuse cannot hide a leak.
+      # Double dollars preserve awk field references through Buildkite interpolation.
+      awk '
+        NR == FNR {
+          if ($$9 != "ps") {
+            baseline[$$1 "|" $$3 "|" $$4 "|" $$5 "|" $$6 "|" $$7] = 1
+          }
+          next
+        }
+        $$9 != "ps" {
+          identity = $$1 "|" $$3 "|" $$4 "|" $$5 "|" $$6 "|" $$7
+          if (!(identity in baseline)) {
+            print
+            leaked = 1
+          }
+        }
+        END {
+          if (leaked) {
+            print "status=FAIL leaked_processes_detected=1"
+            exit 1
+          }
+          print "status=PASS leaked_processes=0"
+        }
+      ' "$$R2_02_ARTIFACT_DIR/processes-before.txt" \
+        "$$R2_02_ARTIFACT_DIR/processes-after.txt" \
+        2>&1 | tee "$$R2_02_ARTIFACT_DIR/process-cleanup.txt"
+
 # - label: "Omni Model Test Qwen3-Omni"
 #   agent_pool: mi300_2
 #   depends_on: amd-build

.buildkite/common/ci_source_file_dependencies.yml

@@ -0,0 +1,458 @@
+# Uploader-only path-prefix presets for CUDA L2–L5 and NPU L4 test YAML.
+# Referenced as ``source_file_dependencies: omni_qwen3_omni_function`` (or a list of
+# keys) on a step; ``upload_pipeline.py`` expands the key(s) only.
+#
+# YAML anchors above the registry hold model business-code paths. Job keys
+# ({model}_function / _perf / _accuracy / _reliability / _doc / _cov) alias
+# those anchors and add the scripts the job actually runs. Feature keys below
+# are unchanged. ``source_filter_fallback`` is not a job key: when no job-key
+# prefix matches, a matching diff keeps every job for that upload.
+#
+# YAML anchors are nested lists; the loader flattens them.
+
+x-qwen3-tts: &qwen3_tts
+  - vllm_omni/engine/async_omni_engine.py
+  - vllm_omni/entrypoints/omni_base.py
+  - vllm_omni/worker/gpu_model_runner.py
+  - vllm_omni/model_executor/models/qwen3_tts/
+  - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+  - vllm_omni/model_executor/models/common/snake_activation.py
+  - vllm_omni/model_executor/models/common/alias_free_activation.py
+  - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+  - vllm_omni/deploy/qwen3_tts.yaml
+  - vllm_omni/entrypoints/openai/serving_speech.py
+  - vllm_omni/entrypoints/openai/tts_adapters/
+  - vllm_omni/engine/stage_pool.py
+
+x-minicpmo-4-5: &minicpmo_4_5
+  - vllm_omni/engine/async_omni_engine.py
+  - vllm_omni/entrypoints/omni_base.py
+  - vllm_omni/model_executor/models/minicpmo_4_5/
+  - vllm_omni/model_executor/models/step_audio2/
+  - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
+  - vllm_omni/model_executor/models/cosyvoice3/utils.py
+  - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+  - vllm_omni/deploy/minicpmo_4_5.yaml
+  - vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja
+
+x-minicpmo-4-5-duplex: &minicpmo_4_5_duplex
+  - vllm_omni/deploy/minicpmo_4_5_duplex.yaml
+  - vllm_omni/engine/duplex/
+  - vllm_omni/engine/duplex_omni_engine.py
+  - vllm_omni/engine/duplex_orchestrator.py
+  - vllm_omni/engine/omni_engine_base.py
+  - vllm_omni/entrypoints/duplex/
+  - vllm_omni/entrypoints/duplex_omni.py
+  - vllm_omni/entrypoints/async_omni_base.py
+  - vllm_omni/clients/
+  - vllm_omni/outputs/duplex.py
+  - vllm_omni/model_executor/duplex_sampling.py
+  - vllm_omni/experimental/fullduplex/
+  - vllm_omni/core/sched/
+  - vllm_omni/outputs/multimodal_accumulation.py
+  - vllm_omni/engine/stage_init_utils.py
+  - vllm_omni/engine/orchestrator.py
+  - vllm_omni/entrypoints/cli/serve.py
+  - vllm_omni/entrypoints/openai/api_server.py
+  - vllm_omni/entrypoints/openai/duplex_capability.py
+  - vllm_omni/config/stage_config.py
+
+x-qwen2-5-omni: &qwen2_5_omni
+  - vllm_omni/engine/async_omni_engine.py
+  - vllm_omni/entrypoints/omni_base.py
+  - vllm_omni/model_executor/models/qwen2_5_omni/
+  - vllm_omni/model_executor/stage_input_processors/qwen2_5_omni.py
+  - vllm_omni/deploy/qwen2_5_omni.yaml
+
+x-qwen3-omni: &qwen3_omni
+  - vllm_omni/engine/async_omni_engine.py
+  - vllm_omni/entrypoints/omni_base.py
+  - vllm_omni/entrypoints/openai/serving_chat.py
+  - vllm_omni/model_executor/models/qwen3_omni/
+  - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+  - vllm_omni/model_executor/models/common/snake_activation.py
+  - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
+  - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
+  - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
+  - vllm_omni/deploy/qwen3_omni_moe.yaml
+  - vllm_omni/deploy/qwen3_omni_moe_thinking.yaml
+
+x-wan22: &wan22
+  - vllm_omni/diffusion/models/wan2_2/
+  - vllm_omni/diffusion/models/dmd2/
+
+x-cosmos3: &cosmos3
+  - vllm_omni/diffusion/models/cosmos3/
+  - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_wan.py
+  - vllm_omni/experimental/world_models/
+  - vllm_omni/model_extras/cosmos3.py
+
+x-qwen-image: &qwen_image
+  - vllm_omni/diffusion/models/qwen_image/
+  - vllm_omni/diffusion/models/dmd2/
+  - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py
+
+x-hunyuan-image3: &hunyuan_image3
+  - vllm_omni/diffusion/models/hunyuan_image3/
+  - vllm_omni/model_executor/models/hunyuan_image3/
+  - vllm_omni/deploy/hunyuan_image3_dit.yaml
+
+x-minimax-h3: &minimax_h3
+  - vllm_omni/diffusion/models/minimax_h3/
+
+x-hunyuan-video: &hunyuan_video
+  - vllm_omni/diffusion/models/hunyuan_video/
+  - vllm_omni/model_executor/models/hunyuan_video/
+
+x-ltx2: &ltx2
+  - vllm_omni/diffusion/models/ltx2/
+  - vllm_omni/model_extras/ltx2.py
+
+x-sana-video: &sana_video
+  - vllm_omni/diffusion/models/sana_video/
+  - vllm_omni/model_extras/sana_video.py
+
+x-z-image: &z_image
+  - vllm_omni/diffusion/models/z_image/
+
+x-stable-audio: &stable_audio
+  - vllm_omni/diffusion/models/stable_audio/
+
+x-lingbot-video: &lingbot_video
+  - vllm_omni/diffusion/models/lingbot_video/
+  - vllm_omni/model_extras/lingbot_video.py
+
+x-lingbot-world: &lingbot_world
+  - vllm_omni/diffusion/models/lingbot_world/
+
+
+source_file_dependencies:
+  # Hardware presets and shared test helpers. Not a job key — do not attach this
+  # to a step. Used only when no job-key prefix matches: then a hit keeps every job.
+  source_filter_fallback:
+    - .buildkite/common/ci_mirror_hardwares.yml
+    - tests/helpers/
+
+  # --- features ---
+  diffusion_distributed_attention:
+    - tests/diffusion/distributed/
+    - tests/diffusion/attention/test_attention_sp.py
+    - vllm_omni/diffusion/distributed/
+    - vllm_omni/diffusion/attention/
+    - vllm_omni/diffusion/data.py
+    - vllm_omni/diffusion/models/
+
+  custom_pipeline:
+    - tests/e2e/features/custom_pipeline/
+    - tests/e2e/features/helpers/custom_pipeline.py
+    - vllm_omni/diffusion/
+    - vllm_omni/engine/
+    - vllm_omni/entrypoints/
+    - vllm_omni/inputs/
+    - vllm_omni/outputs/
+
+  rlhf_verl_omni:
+    - tests/e2e/features/rlhf_test/test_verl_omni_e2e.py
+    - tests/e2e/features/helpers/verl_omni_server.py
+    - tests/e2e/features/helpers/custom_pipeline.py
+    - vllm_omni/diffusion/
+    - vllm_omni/engine/
+    - vllm_omni/entrypoints/
+    - vllm_omni/inputs/
+    - vllm_omni/outputs/
+
+  # Tiny-model diffusion jobs (shared across many model trees under models/).
+  diffusion_tiny_model:
+    - tests/model_tests/diffusion/
+    - vllm_omni/diffusion/models/
+
+  diffusion_offloader:
+    - tests/diffusion/offloader/
+    - vllm_omni/diffusion/offloader/
+
+  diffusion_quantization:
+    - tests/diffusion/quantization/
+    - vllm_omni/diffusion/quantization/
+
+  # --- models ---
+  # --- omni / tts jobs ---
+  omni_qwen3_omni_function:
+    - *qwen3_omni
+    - tests/e2e/offline_inference/test_qwen3_omni.py
+    - tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py
+    - tests/e2e/offline_inference/test_qwen3_omni_autoround_w4a16_expansion.py
+    - tests/e2e/offline_inference/test_qwen3_omni_modelopt_nvfp4_w4a4.py
+    - tests/e2e/online_serving/test_qwen3_omni.py
+    - tests/e2e/online_serving/test_qwen3_omni_expansion.py
+    - tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
+
+  omni_qwen3_omni_accuracy:
+    - *qwen3_omni
+    - tests/e2e/accuracy/qwen3_omni/
+
+  omni_qwen3_omni_perf:
+    - *qwen3_omni
+    - tests/dfx/perf/scripts/run_benchmark.py
+    - tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json
+    - tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
+    - tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
+    - tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json
+
+  omni_qwen3_omni_reliability:
+    - *qwen3_omni
+    - tests/dfx/reliability/test_reliability_qwen3_omni.py
+
+  omni_qwen3_omni_doc:
+    - *qwen3_omni
+    - examples/online_serving/qwen3_omni/
+    - examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py
+    - tests/examples/online_serving/test_qwen3_omni.py
+
+  omni_qwen2_5_omni_doc:
+    - *qwen2_5_omni
+    - examples/online_serving/qwen2_5_omni/
+    - examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py
+    - tests/examples/online_serving/test_qwen2_5_omni.py
+
+  omni_minicpmo_4_5_function:
+    - *minicpmo_4_5
+    - tests/e2e/offline_inference/test_minicpmo_4_5.py
+    - tests/e2e/online_serving/test_minicpmo_4_5.py
+    - tests/e2e/online_serving/test_minicpmo_4_5_expansion.py
+
+  omni_minicpmo_4_5_accuracy:
+    - *minicpmo_4_5
+    - tests/e2e/accuracy/minicpmo_4_5/
+
+  omni_minicpmo_4_5_perf:
+    - *minicpmo_4_5
+    - tests/dfx/perf/scripts/run_benchmark.py
+    - tests/dfx/perf/tests/test_minicpmo_4_5_ready.json
+    - tests/dfx/perf/tests/test_minicpmo_4_5.json
+
+  omni_minicpmo_4_5_doc:
+    - *minicpmo_4_5
+    - examples/online_serving/minicpmo/realtime_web/
+    - examples/online_serving/minicpmo/realtime_duplex_demo.py
+    - tests/examples/test_minicpmo_realtime_web_server.py
+    - tests/examples/test_minicpmo_realtime_web_static.py
+    - tests/examples/test_minicpmo_realtime_duplex_simple_demo.py
+
+  omni_minicpmo_4_5_duplex_function:
+    - *minicpmo_4_5
+    - *minicpmo_4_5_duplex
+    - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
+    - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+    - tests/e2e/online_serving/test_duplex_client_live.py
+    - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
+    - tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
+    - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
+    - tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py
+    - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
+    - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
+    - tests/assets/minicpmo_4_5/
+    - tests/clients/
+    - tests/engine/duplex/
+    - tests/engine/test_duplex_omni_engine.py
+    - tests/engine/test_duplex_orchestrator.py
+    - tests/entrypoints/duplex/
+    - tests/entrypoints/test_duplex_omni.py
+    - tests/entrypoints/openai_api/test_duplex_api_server.py
+    - tests/model_executor/models/minicpmo_4_5/duplex/
+
+  omni_minicpmo_4_5_duplex_perf:
+    - *minicpmo_4_5
+    - *minicpmo_4_5_duplex
+    - tests/dfx/perf/scripts/run_benchmark.py
+    - tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json
+
+  tts_qwen3_tts_function:
+    - *qwen3_tts
+    - tests/e2e/features/prefill_phase/
+    - tests/e2e/offline_inference/test_qwen3_tts_base.py
+    - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
+    - tests/e2e/online_serving/test_qwen3_tts_base.py
+    - tests/e2e/online_serving/test_qwen3_tts_base_expansion.py
+    - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
+    - tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py
+
+  tts_qwen3_tts_perf:
+    - *qwen3_tts
+    - tests/dfx/perf/scripts/run_benchmark.py
+    - tests/dfx/perf/tests/test_tts.json
+
+  tts_qwen3_tts_cov:
+    - *qwen3_tts
+    - .buildkite/common/scripts/run_cov_split.sh
+    - pyproject.toml
+    - tests/e2e/offline_inference/test_qwen3_tts_base.py
+    - tests/e2e/online_serving/test_qwen3_tts_base.py
+
+  # --- diffusion jobs ---
+  diffusion_wan22_function:
+    - *wan22
+    - tests/e2e/offline_inference/test_wan22_t2v.py
+    - tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py
+    - tests/e2e/offline_inference/test_device_postprocess_equivalence.py
+    - tests/e2e/online_serving/test_wan22_t2v.py
+    - tests/e2e/online_serving/test_wan22_expansion.py
+
+  diffusion_wan22_accuracy:
+    - *wan22
+    - tests/e2e/accuracy/wan22_i2v/
+    - tests/e2e/accuracy/test_diffusers_backend_similarity.py
+
+  diffusion_wan22_perf:
+    - *wan22
+    - tests/dfx/perf/scripts/run_diffusion_benchmark.py
+    - tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json
+
+  diffusion_wan22_reliability:
+    - *wan22
+    - tests/dfx/reliability/test_reliability_wan22.py
+
+  diffusion_image_to_video_doc:
+    - *wan22
+    - *hunyuan_video
+    - *ltx2
+    - *sana_video
+    - *lingbot_video
+    - examples/offline_inference/image_to_video/
+    - tests/examples/offline_inference/test_image_to_video.py
+
+  diffusion_cosmos3_function:
+    - *cosmos3
+    - tests/e2e/online_serving/test_cosmos3.py
+
+  diffusion_cosmos3_accuracy:
+    - *cosmos3
+    - tests/e2e/accuracy/test_cosmos3_similarity.py
+
+  diffusion_cosmos3_perf:
+    - *cosmos3
+    - tests/dfx/perf/scripts/run_diffusion_benchmark.py
+    - tests/dfx/perf/tests/test_cosmos3_vllm_omni.json
+
+  diffusion_qwen_image_function:
+    - *qwen_image
+    - tests/e2e/online_serving/test_qwen_image.py
+    - tests/e2e/online_serving/test_qwen_image_expansion.py
+    - tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py
+
+  diffusion_qwen_image_accuracy:
+    - *qwen_image
+    - tests/e2e/accuracy/test_qwen_image.py
+    - tests/e2e/accuracy/test_diffusers_backend_similarity.py
+
+  diffusion_qwen_image_perf:
+    - *qwen_image
+    - tests/dfx/perf/scripts/run_diffusion_benchmark.py
+    - tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
+
+  diffusion_text_to_image_doc:
+    - *qwen_image
+    - *z_image
+    - examples/offline_inference/text_to_image/
+    - examples/online_serving/text_to_image/
+    - tests/examples/offline_inference/test_text_to_image.py
+    - tests/examples/online_serving/test_text_to_image.py
+    - vllm_omni/diffusion/models/nextstep_1_1/
+    - vllm_omni/diffusion/models/longcat_image/
+    - vllm_omni/diffusion/models/ovis_image/
+    - vllm_omni/diffusion/models/omnigen2/
+    - vllm_omni/diffusion/models/sd3/
+    - vllm_omni/diffusion/models/flux/
+    - vllm_omni/diffusion/models/flux2/
+    - vllm_omni/diffusion/models/flux2_klein/
+    - vllm_omni/diffusion/models/hidream_image/
+    - vllm_omni/diffusion/models/krea2/
+    - vllm_omni/diffusion/models/lingbot_video/
+
+  diffusion_hunyuan_image3_function:
+    - *hunyuan_image3
+    - tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py
+
+  diffusion_hunyuan_image3_accuracy:
+    - *hunyuan_image3
+    - tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py
+
+  diffusion_hunyuan_image3_perf:
+    - *hunyuan_image3
+    - tests/dfx/perf/scripts/run_diffusion_benchmark.py
+    - tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json
+    - tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json
+    - tests/dfx/perf/tests/test_hunyuan_image_tp4.json
+
+  diffusion_hunyuan_image3_reliability:
+    - *hunyuan_image3
+    - tests/dfx/reliability/test_reliability_hunyuan_image.py
+
+  diffusion_minimax_h3_function:
+    - *minimax_h3
+    - tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py
+    - tests/e2e/online_serving/minimax_h3/
+    - tests/diffusion/models/minimax_h3/
+    - vllm_omni/diffusion/lora/
+    - vllm_omni/diffusion/offloader/
+    - vllm_omni/diffusion/distributed/
+    - vllm_omni/entrypoints/openai/
+
+  diffusion_minimax_h3_accuracy:
+    - *minimax_h3
+    - tests/e2e/accuracy/minimax_h3/
+
+  diffusion_minimax_h3_perf:
+    - *minimax_h3
+    - tests/dfx/perf/scripts/run_diffusion_benchmark.py
+    - tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json
+
+  diffusion_hunyuan_video_function:
+    - *hunyuan_video
+    - tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
+
+  diffusion_hunyuan_video_accuracy:
+    - *hunyuan_video
+    - tests/e2e/accuracy/hunyuanvideo15_t2v/
+    - tests/e2e/accuracy/hunyuanvideo15_i2v/
+
+  diffusion_hunyuan_video_perf:
+    - *hunyuan_video
+    - tests/dfx/perf/scripts/run_diffusion_benchmark.py
+    - tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
+    - tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json
+
+  diffusion_z_image_function:
+    - *z_image
+    - tests/e2e/online_serving/test_z_image.py
+    - tests/e2e/online_serving/test_zimage_expansion.py
+
+  diffusion_sana_video_function:
+    - *sana_video
+    - tests/e2e/online_serving/test_sana_video_expansion.py
+    - tests/e2e/accuracy/sana_video/
+
+  diffusion_stable_audio_function:
+    - *stable_audio
+    - tests/e2e/offline_inference/test_stable_audio_expansion.py
+    - tests/e2e/online_serving/test_stable_audio_online_expansion.py
+
+  diffusion_lingbot_function:
+    - *lingbot_video
+    - *lingbot_world
+    - tests/e2e/online_serving/test_lingbot_video.py
+    - tests/e2e/online_serving/test_lingbot_video_moe.py
+    - tests/e2e/offline_inference/test_lingbot_world_v2.py
+
+
+  # Invalid-param jobs: model business code only, plus the reliability scripts.
+  reliability_invalid_param_h100:
+    - *qwen3_omni
+    - *minicpmo_4_5
+    - *qwen_image
+    - *wan22
+    - tests/dfx/reliability/invalid_param_test/
+
+  reliability_invalid_param_l4:
+    - *qwen3_tts
+    - *stable_audio
+    - tests/dfx/reliability/invalid_param_test/

.buildkite/common/scripts/skip_ci.py

@@ -44,6 +44,7 @@ L3_YAML_FILES: dict[str, str] = {
 L45_YAML_FILES: dict[str, str] = {
     ".buildkite/cuda/test-nightly.yml": "cuda",
     ".buildkite/cuda/test-weekly.yml": "cuda",
+    ".buildkite/amd/test-amd-nightly.yml": "amd",
     ".buildkite/npu/test-npu-nightly.yml": "npu",
 }
 

.buildkite/common/scripts/upload_pipeline.py

@@ -10,8 +10,17 @@ Bootstrap mode (``bootstrap-upload-steps.yml``):
   - Detect docs-only, pytest skip-mark-only, or combined skip-ci from git diff.
   - When only CI level YAML changes, enable **L2/L3** upload steps for affected levels only.
 
-Test pipeline mode (e.g. test-merge.yml):
-  - Drop steps whose ``source_file_dependencies`` do not match changed files.
+Test pipeline mode (e.g. test-merge.yml, test-nightly.yml, test-weekly.yml):
+  - Drop steps whose ``source_file_dependencies`` do not match changed files
+    (string/list keys from ci_source_file_dependencies.yml, or inline path
+    prefixes). Filtering applies on **PR label** uploads and on post-merge
+    ``main`` L3 uploads. Scheduled ``main`` uploads that pass ``--all``
+    (NIGHTLY/WEEKLY) or ``--e2e`` keep every selected job and still strip
+    the field. If no job-key prefix matches, a change to the pipeline YAML
+    being uploaded or to a path under the ``source_filter_fallback`` registry
+    key keeps every job so command, env, and hardware edits can be validated
+    before merge. When any job-key prefix already matches, normal filtering
+    wins.
   - Expand uploader-only ``mirror_hardwares`` into ``agents`` (+ optional ``image``
     for NPU) + ``plugins`` (see ci_mirror_hardwares.yml).
   - Omit ``mirror_hardwares`` to compose ``{chip}_{n}`` from pytest ``-m`` SKU
@@ -87,6 +96,10 @@ BOOTSTRAP_UPLOAD_IF_KEYS = {
 }
 E2E_GROUP_MARKER = "E2E Test"
 CI_MIRROR_HARDWARES_PATH = ROOT / ".buildkite/common/ci_mirror_hardwares.yml"
+CI_SOURCE_FILE_DEPENDENCIES_PATH = ROOT / ".buildkite/common/ci_source_file_dependencies.yml"
+# Registry key whose prefixes bypass source filtering. Paths live in
+# ci_source_file_dependencies.yml; do not attach this key to a job.
+SOURCE_FILTER_FALLBACK_KEY = "source_filter_fallback"
 CUDA_HF_TOKEN_ENV = "VLLM_CI_HF_TOKEN"
 CUDA_HF_TOKEN_EXPORT = f'if [ -n "$${{{CUDA_HF_TOKEN_ENV}:-}}" ]; then export HF_TOKEN="$${{{CUDA_HF_TOKEN_ENV}}}"; fi'
 
@@ -95,14 +108,7 @@ CUDA_HF_TOKEN_EXPORT = f'if [ -n "$${{{CUDA_HF_TOKEN_ENV}:-}}" ]; then export HF
 # ``*_UPLOAD_IF``: full gate for uploading that child pipeline.
 NIGHTLY_MAIN_IF = 'build.branch == "main" && build.env("NIGHTLY") == "1"'
 NIGHTLY_LABEL_IF = (
-    f"({NIGHTLY_MAIN_IF}) || "
-    '(build.branch != "main" && ('
-    'build.pull_request.labels includes "nightly-test" || '
-    'build.pull_request.labels includes "omni-test" || '
-    'build.pull_request.labels includes "tts-test" || '
-    'build.pull_request.labels includes "diffusion-x2iat-test" || '
-    'build.pull_request.labels includes "diffusion-x2v-test"'
-    "))"
+    f'({NIGHTLY_MAIN_IF}) || (build.branch != "main" && build.pull_request.labels includes "nightly-test")'
 )
 WEEKLY_E2E_IF = 'build.branch == "main" && build.env("WEEKLY") == "1"'
 WEEKLY_MAIN_IF = 'build.branch == "main" && (build.env("WEEKLY") == "1" || build.env("NON_CRITICAL") == "1")'
@@ -248,7 +254,7 @@ def _render_bootstrap_pipeline(
     return yaml.safe_dump(doc, sort_keys=False)
 
 
-# --- Test pipeline (test-ready.yml, test-merge.yml) ---
+# --- Test pipeline (test-ready.yml, test-merge.yml, test-nightly.yml) ---
 
 
 @lru_cache(maxsize=1)
@@ -264,6 +270,117 @@ def _load_mirror_hardwares() -> dict[str, dict[str, Any]]:
     return presets
 
 
+@lru_cache(maxsize=1)
+def _load_source_file_dependencies() -> dict[str, list[str]]:
+    def flatten(value: Any, *, key: str) -> list[str]:
+        """Flatten YAML-anchor nested lists into a de-duplicated prefix list."""
+        if isinstance(value, str):
+            if not value.strip():
+                raise ValueError(f"empty path in source_file_dependencies[{key!r}]")
+            return [value]
+        if isinstance(value, list):
+            flattened: list[str] = []
+            seen: set[str] = set()
+            for item in value:
+                for path in flatten(item, key=key):
+                    if path not in seen:
+                        seen.add(path)
+                        flattened.append(path)
+            return flattened
+        raise ValueError(
+            f"source_file_dependencies[{key!r}] must be a list of path prefixes, got {type(value).__name__}",
+        )
+
+    if not CI_SOURCE_FILE_DEPENDENCIES_PATH.is_file():
+        raise FileNotFoundError(
+            f"missing CI source_file_dependencies registry: {CI_SOURCE_FILE_DEPENDENCIES_PATH}",
+        )
+    doc = yaml.safe_load(CI_SOURCE_FILE_DEPENDENCIES_PATH.read_text(encoding="utf-8"))
+    if not isinstance(doc, dict):
+        raise ValueError(f"invalid CI source_file_dependencies registry: {CI_SOURCE_FILE_DEPENDENCIES_PATH}")
+    presets = doc.get("source_file_dependencies")
+    if not isinstance(presets, dict):
+        raise ValueError(f"source_file_dependencies must be a mapping in {CI_SOURCE_FILE_DEPENDENCIES_PATH}")
+    loaded: dict[str, list[str]] = {}
+    for key, value in presets.items():
+        if not isinstance(key, str) or not key.strip():
+            raise ValueError(
+                f"source_file_dependencies keys must be non-empty strings in {CI_SOURCE_FILE_DEPENDENCIES_PATH}",
+            )
+        loaded[key] = flatten(value, key=key)
+    return loaded
+
+
+_REGISTRY_KEY_RE = re.compile(r"^[a-z][a-z0-9_]*$")
+
+
+def _parse_command_text(step: dict[str, Any]) -> str:
+    """Parse step ``commands`` / ``command`` into a single string."""
+    raw = step.get("commands", step.get("command"))
+    if raw is None:
+        return ""
+    if isinstance(raw, str):
+        return raw
+    if isinstance(raw, list):
+        return "\n".join(str(item) for item in raw if item is not None)
+    return str(raw)
+
+
+def _resolve_source_file_dependencies(step: dict[str, Any]) -> list[str] | None:
+    """Expand a registry key (or list of keys / inline path prefixes)."""
+    deps = step.get("source_file_dependencies")
+    if deps is None:
+        return None
+
+    def dedupe(*groups: list[str]) -> list[str]:
+        merged: list[str] = []
+        seen: set[str] = set()
+        for group in groups:
+            for path in group:
+                if path not in seen:
+                    seen.add(path)
+                    merged.append(path)
+        return merged
+
+    def lookup(key: str) -> list[str]:
+        if key == SOURCE_FILTER_FALLBACK_KEY:
+            raise ValueError(
+                f"source_file_dependencies {key!r} bypasses filtering and cannot be attached to a step "
+                f"({_get_step_label(step)!r})",
+            )
+        registry = _load_source_file_dependencies()
+        paths = registry.get(key)
+        if paths is None:
+            known = ", ".join(sorted(registry))
+            raise ValueError(
+                f"unknown source_file_dependencies {key!r} in step {_get_step_label(step)!r}; known: {known}",
+            )
+        return list(paths)
+
+    prefixes: list[str]
+    if isinstance(deps, str):
+        prefixes = lookup(deps) if _REGISTRY_KEY_RE.fullmatch(deps) else [deps]
+    elif isinstance(deps, list):
+        if not all(isinstance(item, str) for item in deps):
+            raise ValueError(
+                f"source_file_dependencies must be a string key or list of strings in step {_get_step_label(step)!r}",
+            )
+        keyish = [bool(_REGISTRY_KEY_RE.fullmatch(item)) for item in deps]
+        if deps and all(keyish):
+            prefixes = dedupe(*(lookup(key) for key in deps))
+        elif any(keyish):
+            raise ValueError(
+                f"source_file_dependencies in step {_get_step_label(step)!r} mixes registry keys and path prefixes",
+            )
+        else:
+            prefixes = list(deps)
+    else:
+        raise ValueError(
+            f"source_file_dependencies must be a string key or list in step {_get_step_label(step)!r}",
+        )
+    return prefixes
+
+
 @lru_cache(maxsize=1)
 def _cuda_mirror_chips() -> tuple[str, ...]:
     """Lowercase CUDA SKUs that have at least one ``{chip}_{n}`` preset.
@@ -278,7 +395,7 @@ def _cuda_mirror_chips() -> tuple[str, ...]:
     return tuple(sorted(chips, key=lambda chip: (-len(chip), chip)))
 
 
-def _read_cards_marks(expr: str) -> tuple[set[int], bool]:
+def _parse_cards_marks(expr: str) -> tuple[set[int], bool]:
     """Parse registered ``cards_n`` / ``not cards_n`` from a pytest ``-m`` expr.
 
     Returns positive card counts and whether any ``not cards_*`` is present.
@@ -295,8 +412,8 @@ def _read_cards_marks(expr: str) -> tuple[set[int], bool]:
     return positives, has_not_cards
 
 
-def _read_hardware_marks(expr: str) -> set[str]:
-    """Return lowercase positive CUDA SKUs that have a mirror preset.
+def _parse_hardware_marks(expr: str) -> set[str]:
+    """Parse lowercase positive CUDA SKUs that have a mirror preset.
 
     ``not H100`` is ignored. SKUs with no ``{chip}_*`` preset (e.g. H200) are dropped.
     """
@@ -338,36 +455,22 @@ def _get_mirror_hw_selector() -> str:
     return selector
 
 
-def _read_pytest_marks(commands: Any) -> tuple[set[str], int | Literal["max"] | None]:
-    """Read pytest ``-m`` from step commands: CUDA SKU chips and cards count.
+def _parse_pytest_marks(step: dict[str, Any]) -> tuple[set[str], int | Literal["max"] | None]:
+    """Parse pytest ``-m`` from step commands: CUDA SKU chips and cards count.
 
     *chips* are lowercase SKUs from positive markers (``not H100`` is ignored).
     *cards* is the max positive ``cards_n``, ``"max"`` when only ``not cards_*``
     is present, or ``None``.
     """
-    chunks: list[str] = []
-
-    def _collect(value: Any) -> None:
-        if value is None:
-            return
-        if isinstance(value, str):
-            chunks.append(value)
-        elif isinstance(value, list):
-            for part in value:
-                _collect(part)
-        else:
-            chunks.append(str(value))
-
-    _collect(commands)
-    text = "\n".join(chunks)
+    text = _parse_command_text(step)
 
     chips: set[str] = set()
     positives: set[int] = set()
     has_not_cards = False
     for match in _PYTEST_MARKER_ARG.finditer(text):
         expr = match.group(1) or match.group(2) or match.group(3) or ""
-        chips |= _read_hardware_marks(expr)
-        card_counts, not_cards = _read_cards_marks(expr)
+        chips |= _parse_hardware_marks(expr)
+        card_counts, not_cards = _parse_cards_marks(expr)
         positives.update(card_counts)
         has_not_cards = has_not_cards or not_cards
 
@@ -484,7 +587,7 @@ def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any] | None:
     if "mirror_hardwares" not in step:
         if has_pool:
             return step
-        chips, cards = _read_pytest_marks(step.get("commands"))
+        chips, cards = _parse_pytest_marks(step)
         if not chips and cards is None:
             return step
         preset_name = _compose_mirror_hardware_name(chips, cards, step_label=step_label)
@@ -552,11 +655,7 @@ def _process_test_steps(
             processed.append(step)
             continue
 
-        deps = step.get("source_file_dependencies")
-        if deps is not None and not isinstance(deps, list):
-            raise ValueError(
-                f"source_file_dependencies must be a list in step {_get_step_label(step)!r}",
-            )
+        deps = _resolve_source_file_dependencies(step)
         if changed_files is not None and deps is not None and not _match_source_file(changed_files, deps):
             _log(f"skip {_get_step_label(step)!r} (no changes under {deps})")
             continue
@@ -595,10 +694,25 @@ def _select_e2e_group_steps(steps: list[Any]) -> list[Any]:
     return selected
 
 
+def _any_source_dependency_match(steps: list[Any], changed_files: list[str]) -> bool:
+    """True when any step with ``source_file_dependencies`` matches *changed_files*."""
+    for step in steps:
+        if not isinstance(step, dict):
+            continue
+        deps = _resolve_source_file_dependencies(step)
+        if deps is not None and _match_source_file(changed_files, deps):
+            return True
+        nested = step.get("steps")
+        if isinstance(nested, list) and _any_source_dependency_match(nested, changed_files):
+            return True
+    return False
+
+
 def _render_test_pipeline(
     doc: dict[str, Any],
     changed_files: list[str] | None,
     *,
+    pipeline_path: Path | None = None,
     e2e_only: bool = False,
 ) -> dict[str, Any]:
     """Filter steps by PR diff and strip uploader-only ``source_file_dependencies`` metadata."""
@@ -611,6 +725,13 @@ def _render_test_pipeline(
         return doc
     if e2e_only:
         steps = _select_e2e_group_steps(steps)
+    # Bypass is a fallback: only when no job-key prefix matched.
+    if changed_files is not None and pipeline_path is not None:
+        if not _any_source_dependency_match(steps, changed_files):
+            bypass = _source_filter_fallback_reason(changed_files, pipeline_path)
+            if bypass is not None:
+                _log(f"keep all jobs (no source_file_dependencies match; bypassed by {bypass})")
+                changed_files = None
     steps = _process_test_steps(steps, changed_files)
     return {**doc, "steps": steps}
 
@@ -618,6 +739,60 @@ def _render_test_pipeline(
 # --- Entry (read file → bootstrap or test render → YAML string) ---
 
 
+def _repo_relative_posix(path: Path) -> str:
+    resolved = path.resolve()
+    try:
+        return resolved.relative_to(ROOT).as_posix()
+    except ValueError:
+        return path.as_posix().replace("\\", "/")
+
+
+def _source_filter_fallback_prefixes() -> list[str]:
+    """Shared uploader/registry prefixes from ``source_filter_fallback``. Not a job key."""
+    registry = _load_source_file_dependencies()
+    prefixes = registry.get(SOURCE_FILTER_FALLBACK_KEY)
+    if not prefixes:
+        raise ValueError(
+            f"source_file_dependencies[{SOURCE_FILTER_FALLBACK_KEY!r}] must list shared uploader paths "
+            f"in {CI_SOURCE_FILE_DEPENDENCIES_PATH}",
+        )
+    return prefixes
+
+
+def _source_filter_fallback_reason(changed_files: list[str] | None, pipeline_path: Path) -> str | None:
+    """Return the changed path that should disable source filtering, if any."""
+    if not changed_files:
+        return None
+    changed = set(changed_files)
+    pipeline_rel = _repo_relative_posix(pipeline_path)
+    if pipeline_rel in changed:
+        return pipeline_rel
+    prefixes = _source_filter_fallback_prefixes()
+    for path in changed_files:
+        if _match_source_file([path], prefixes):
+            return path
+    return None
+
+
+def _changed_files_for_source_filter(
+    ctx,
+    *,
+    force_all: bool,
+    e2e_only: bool,
+) -> list[str] | None:
+    """Return the diff to filter against, or None to keep every step.
+
+    ``--all`` / ``--e2e`` disable filtering (scheduled NIGHTLY/WEEKLY full
+    uploads and weekly E2E sweeps). Post-merge ``main`` L3 and PR-label
+    uploads filter against the commit/PR diff. Pipeline YAML /
+    ``source_filter_fallback`` matches are applied later in
+    ``_render_test_pipeline`` only when no job-key prefix already matched.
+    """
+    if force_all or e2e_only:
+        return None
+    return ctx.changed_files
+
+
 def _render_pipeline(
     path: Path,
     *,
@@ -635,13 +810,18 @@ def _render_pipeline(
 
     text = path.read_text(encoding="utf-8")
     ctx = resolve_ci_context_from_git()
-    changed_files = None if force_all or e2e_only else ctx.changed_files
+    changed_files = _changed_files_for_source_filter(ctx, force_all=force_all, e2e_only=e2e_only)
 
     doc = yaml.safe_load(text)
     if not isinstance(doc, dict):
         raise ValueError(f"invalid pipeline YAML: {path}")
 
-    doc = _render_test_pipeline(doc, changed_files, e2e_only=e2e_only)
+    doc = _render_test_pipeline(
+        doc,
+        changed_files,
+        pipeline_path=path,
+        e2e_only=e2e_only,
+    )
     return yaml.safe_dump(doc, sort_keys=False)
 
 

.buildkite/cuda/bootstrap-upload-steps.yml

@@ -22,7 +22,8 @@ steps:
     agents:
       queue: "cpu_queue_premerge"
 
-  # L3 Test — PR merge-test label (diff-aware) or main+WEEKLY=1 (--e2e, full E2E group)
+  # L3 Test — PR merge-test label or post-merge on main (both diff-aware);
+  # main+WEEKLY=1 uploads the E2E group only (--e2e).
   - label: "Upload Merge Pipeline"
     depends_on: image-build
     key: upload-merge-pipeline
@@ -36,20 +37,30 @@ steps:
     agents:
       queue: "cpu_queue_premerge"
 
-  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
+  # L4 Test — main+NIGHTLY=1 (full pipeline), or PR nightly-test label (diff-aware)
   - label: "Upload Nightly Pipeline"
     depends_on: image-build
     key: upload-nightly-pipeline
     commands:
-      - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-nightly.yml
+      - |
+        EXTRA=""
+        if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
+          EXTRA="--all"
+        fi
+        python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-nightly.yml
     agents:
       queue: "cpu_queue_premerge"
 
-  # L5 Test — main+WEEKLY=1 / NON_CRITICAL=1 (scheduled), or PR with weekly-test label
+  # L5 Test — main+WEEKLY=1 / NON_CRITICAL=1 (full pipeline), or PR weekly-test label (diff-aware)
   - label: "Upload Weekly Pipeline"
     depends_on: image-build
     key: upload-weekly-pipeline
     commands:
-      - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-weekly.yml
+      - |
+        EXTRA=""
+        if [ "${BUILDKITE_BRANCH}" = "main" ]; then
+          EXTRA="--all"
+        fi
+        python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-weekly.yml
     agents:
       queue: "cpu_queue_premerge"

.buildkite/cuda/test-merge.yml

@@ -1,5 +1,6 @@
 # CUDA CI L3 (`advanced_model` e2e) plus L1 CPU Simple Test (`core_model and cpu`).
-# Uploaded by Upload Merge Pipeline: PR label `merge-test`, or post-merge on main.
+# Uploaded by Upload Merge Pipeline: PR label `merge-test`, or post-merge on main
+# (both diff-aware via source_file_dependencies).
 
 env:
   VLLM_WORKER_MULTIPROC_METHOD: spawn
@@ -65,7 +66,9 @@ steps:
         # 0.26 startup regression is understood and fixed.
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"
+          # Turn-based server VAD over /v1/realtime is deferred to RFC #7181 PR 5
+          # (see the P0 thread on #7413); the test stays in the tree and runs again there.
+          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model" --deselect tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py::TestQwen3OmniRealtimeWebSocket::test_server_vad_multi_turn_without_client_commit
         mirror_hardwares: h100_2
 
       - label: "Entrypoint Test with L4"
@@ -78,21 +81,7 @@ steps:
     depends_on: upload-merge-pipeline
     steps:
       - label: "TTS · Qwen3-TTS CustomVoice Test"
-        source_file_dependencies:
-          - tests/e2e/features/prefill_phase/
-          - vllm_omni/worker/gpu_model_runner.py
-          - tests/helpers/media.py
-          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
-          - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
-          - vllm_omni/model_executor/models/qwen3_tts/
-          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-          - vllm_omni/model_executor/models/common/snake_activation.py
-          - vllm_omni/model_executor/models/common/alias_free_activation.py
-          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
-          - vllm_omni/deploy/qwen3_tts.yaml
-          - vllm_omni/entrypoints/openai/serving_speech.py
-          - vllm_omni/entrypoints/openai/tts_adapters/
-          - vllm_omni/engine/stage_pool.py
+        source_file_dependencies: tts_qwen3_tts_function
         commands:
           - |
             timeout 40m bash -c "
@@ -103,21 +92,7 @@ steps:
         mirror_hardwares: l4_1
 
       - label: "TTS · Qwen3-TTS Base Test"
-        source_file_dependencies:
-          - tests/helpers/media.py
-          - tests/e2e/online_serving/test_qwen3_tts_base.py
-          - tests/e2e/offline_inference/test_qwen3_tts_base.py
-          - vllm_omni/model_executor/models/qwen3_tts/
-          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-          - vllm_omni/model_executor/models/common/snake_activation.py
-          - vllm_omni/model_executor/models/common/alias_free_activation.py
-          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
-          - vllm_omni/deploy/qwen3_tts.yaml
-          - vllm_omni/entrypoints/openai/serving_speech.py
-          - vllm_omni/entrypoints/openai/tts_adapters/
-          - vllm_omni/engine/stage_pool.py
-          - .buildkite/common/scripts/run_cov_split.sh
-          - pyproject.toml
+        source_file_dependencies: tts_qwen3_tts_cov
         # 40 -> 60: the coverage split runs the model twice.
         timeout_in_minutes: 60
         commands:
@@ -132,36 +107,14 @@ steps:
         mirror_hardwares: l4_1
 
       - label: "Omni · Qwen3-Omni Test"
-        source_file_dependencies:
-          - tests/e2e/offline_inference/test_qwen3_omni.py
-          - tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py
-          - tests/e2e/online_serving/test_qwen3_omni.py
-          - tests/helpers/runtime.py
-          - vllm_omni/entrypoints/openai/serving_chat.py
-          - vllm_omni/model_executor/models/qwen3_omni/
-          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-          - vllm_omni/model_executor/models/common/snake_activation.py
-          - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
-          - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
-          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
-          - vllm_omni/deploy/qwen3_omni_moe.yaml
-          - vllm_omni/deploy/qwen3_omni_moe_thinking.yaml
+        source_file_dependencies: omni_qwen3_omni_function
         timeout_in_minutes: 50
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py -m 'advanced_model and cuda' --run-level 'advanced_model'
         mirror_hardwares: h100_2
 
       - label: "Omni · MiniCPM-o 4.5 Test"
-        source_file_dependencies:
-          - tests/e2e/offline_inference/test_minicpmo_4_5.py
-          - tests/e2e/online_serving/test_minicpmo_4_5.py
-          - vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/models/step_audio2/
-          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
-          - vllm_omni/model_executor/models/cosyvoice3/utils.py
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
-          - vllm_omni/deploy/minicpmo_4_5.yaml
+        source_file_dependencies: omni_minicpmo_4_5_function
         timeout_in_minutes: 50
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'
@@ -174,59 +127,14 @@ steps:
         mirror_hardwares: h100_2
 
       - label: "Omni · MiniCPM-o 4.5 Duplex Test"
-        source_file_dependencies:
-          - tests/assets/minicpmo_4_5/response_required_16k.wav
-          - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
-          - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
-          - tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
-          - vllm_omni/deploy/minicpmo_4_5.yaml
-          - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
-          - vllm_omni/engine/duplex/
-          - vllm_omni/entrypoints/duplex/
-          # Public duplex client library + its unit suites: client-side
-          # changes must re-run the duplex e2e group. The live client probe
-          # (test_duplex_client_live.py, single GPU, core tier) runs in the
-          # ready pipeline's duplex group.
-          - vllm_omni/clients/
-          - tests/clients/
-          - vllm_omni/entrypoints/duplex_request_client.py
-          - vllm_omni/outputs/duplex.py
-          - vllm_omni/model_executor/duplex_sampling.py
-          - tests/engine/duplex/
-          - tests/entrypoints/duplex/
-          - tests/model_executor/models/minicpmo_4_5/duplex/
-          # Legacy Realtime probe client + video stacking used by the e2e drivers.
-          - vllm_omni/experimental/fullduplex/
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/models/step_audio2/
-          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
-          - vllm_omni/model_executor/models/cosyvoice3/utils.py
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
-          - vllm_omni/core/sched/
-          - vllm_omni/outputs/multimodal_accumulation.py
-          - vllm_omni/engine/stage_init_utils.py
-          - vllm_omni/engine/async_omni_engine.py
-          - vllm_omni/engine/orchestrator.py
-          - vllm_omni/entrypoints/cli/serve.py
-          - vllm_omni/entrypoints/openai/api_server.py
-          - vllm_omni/entrypoints/openai/duplex_capability.py
-          - vllm_omni/config/stage_config.py
+        source_file_dependencies: omni_minicpmo_4_5_duplex_function
         timeout_in_minutes: 50
         commands:
           - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model'
         mirror_hardwares: h100_1
 
       - label: "Diffusion · Qwen Image Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen_image.py
-          - vllm_omni/diffusion/models/qwen_image/
-          - vllm_omni/diffusion/models/dmd2/
-          - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py
+        source_file_dependencies: diffusion_qwen_image_function
         commands:
           - |
             timeout 40m bash -c '
@@ -236,11 +144,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Diffusion · Wan22 Test"
-        source_file_dependencies:
-          - tests/e2e/offline_inference/test_wan22_t2v.py
-          - tests/e2e/online_serving/test_wan22_t2v.py
-          - vllm_omni/diffusion/models/wan2_2/
-          - vllm_omni/diffusion/models/dmd2/
+        source_file_dependencies: diffusion_wan22_function
         commands:
           - |
             timeout 40m bash -c "
@@ -250,9 +154,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Diffusion · HunyuanVideo-1.5 Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
-          - vllm_omni/diffusion/models/hunyuan_video/
+        source_file_dependencies: diffusion_hunyuan_video_function
         commands:
           - |
             timeout 40m bash -c "
@@ -262,17 +164,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Diffusion · MiniMax-H3 L3 DLO DP2 and FastH3"
-        source_file_dependencies:
-          - tests/e2e/online_serving/minimax_h3/
-          - tests/helpers/assertions.py
-          - tests/helpers/mark.py
-          - tests/helpers/media.py
-          - tests/helpers/runtime.py
-          - vllm_omni/diffusion/models/minimax_h3/
-          - vllm_omni/diffusion/lora/
-          - vllm_omni/diffusion/offloader/
-          - vllm_omni/diffusion/distributed/
-          - vllm_omni/entrypoints/openai/
+        source_file_dependencies: diffusion_minimax_h3_function
         # Keep four server lifecycles in one Buildkite job, but never in one
         # pytest process. DLO uses multiprocessing shared-memory queues; the
         # parent's resource_tracker can retain a stale /psm_* entry during

.buildkite/cuda/test-nightly.yml

@@ -12,138 +12,145 @@ steps:
   - group: ":card_index_dividers: Omni Model Test"
     key: nightly-omni-test-group
     depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     steps:
 
       - label: ":full_moon: Omni · Function Test with H100 · Single-GPU"
+        source_file_dependencies:
+          - omni_qwen3_omni_function
+          - omni_minicpmo_4_5_function
         timeout_in_minutes: 120
         commands:
           - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
 
       - label: ":full_moon: Omni · Function Test with H100 · 2-GPU"
+        source_file_dependencies:
+          - omni_qwen3_omni_function
+          - omni_minicpmo_4_5_function
         timeout_in_minutes: 90
         commands:
           - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_2" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
 
       - label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
+        source_file_dependencies: omni_minicpmo_4_5_duplex_function
         timeout_in_minutes: 50
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni and cards_1" --run-level "full_model"
-        mirror_hardwares: h100_1
+          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and B200 and omni and cards_1" --run-level "full_model"
 
       - label: ":full_moon: Omni · Doc Test with L4 · 4-GPU"
+        source_file_dependencies: omni_qwen2_5_omni_doc
         timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          - pytest -s -v tests/examples/ -m "full_model and omni and L4 and cards_4" --run-level "full_model"
-        mirror_hardwares: l4_4
+          - pytest -s -v tests/examples/ -m "full_model and omni and L4 and B200 and cards_4" --run-level "full_model"
 
       - label: ":full_moon: Omni · Doc Test with H100 · 2-GPU"
+        source_file_dependencies: omni_qwen3_omni_doc
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/examples/ -m "full_model and omni and H100 and cards_2" --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/examples/ -m "full_model and omni and H100 and B200 and cards_2" --run-level "full_model"
 
       - label: ":full_moon: Omni · Accuracy Test"
+        source_file_dependencies: omni_qwen3_omni_accuracy
         timeout_in_minutes: 180
         commands:
           - export SEED_TTS_WER_EVAL=1
           - export SEED_TTS_EVAL_DEVICE=cuda:1
           - |
             set +e
-            pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model" --run-level full_model
+            pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model and H100 and B200 and cards_2" --run-level full_model
             EXIT=$$?
             buildkite-agent artifact upload "tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_2
 
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
+        source_file_dependencies: omni_minicpmo_4_5_accuracy
         timeout_in_minutes: 180
         commands:
           - export SEED_TTS_WER_EVAL=1
           - export SEED_TTS_EVAL_DEVICE=cuda:1
           - |
             set +e
-            pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and cuda" --run-level full_model
+            pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level full_model
             EXIT=$$?
             buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_2
 
       - label: ":full_moon: Omni · Perf Test · No Async Chunk"
+        source_file_dependencies: omni_qwen3_omni_perf
         key: nightly-omni-performance-no-async-chunk
         timeout_in_minutes: 300
         commands:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json -m "H100 and B200 and cards_2"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_2
 
       - label: ":full_moon: Omni · Perf Test · Async Chunk"
+        source_file_dependencies: omni_qwen3_omni_perf
         key: nightly-omni-performance-async-chunk
         timeout_in_minutes: 180
         commands:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and full_model"
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and B200 and full_model and cards_2"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_2
 
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"
+        source_file_dependencies: omni_minicpmo_4_5_perf
         key: nightly-omni-performance-minicpmo-4-5
         timeout_in_minutes: 180
         commands:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json -m "H100 and B200 and cards_1"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_1
 
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test"
+        source_file_dependencies: omni_minicpmo_4_5_duplex_perf
         key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts
         timeout_in_minutes: 180
         commands:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json -m "H100 and B200 and cards_1"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_1
 
       - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100"
+        source_file_dependencies: omni_qwen3_omni_function
         timeout_in_minutes: 45
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model" --run-level "core_model"
-        mirror_hardwares: h100_4
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model and H100 and B200 and cards_4" --run-level "core_model"
 
   - group: ":card_index_dividers: TTS Model Test"
     key: nightly-tts-test-group
     depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     steps:
 
       - label: ":full_moon: TTS · Function Test with L4"
+        source_file_dependencies: tts_qwen3_tts_function
         timeout_in_minutes: 120
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
           - export VLLM_USE_DEEP_GEMM="0"
           - export VLLM_MOE_USE_DEEP_GEMM="0"
-          - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
-        mirror_hardwares: l4_1
+          - pytest -s -v tests/e2e/ -m "full_model and L4 and B200 and tts and cards_1" --run-level "full_model" --ignore=tests/e2e/accuracy
 
       - label: ":full_moon: TTS · Perf Test · Single-GPU"
+        source_file_dependencies: tts_qwen3_tts_perf
         key: nightly-tts-performance-single
         timeout_in_minutes: 180
         commands:
@@ -151,13 +158,13 @@ steps:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and cards_1"
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and B200 and cards_1"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_1
 
       - label: ":full_moon: TTS · Perf Test · 2-GPU"
+        source_file_dependencies: tts_qwen3_tts_perf
         key: nightly-tts-performance-2gpu
         timeout_in_minutes: 180
         commands:
@@ -165,11 +172,10 @@ steps:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and cards_2"
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and B200 and cards_2"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_2
 
   - group: ":card_index_dividers: Tiny Model Tests [Multi-GPU]"
     key: nightly-tiny-model-test-group
@@ -180,6 +186,7 @@ steps:
     steps:
 
       - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 2-GPU"
+        source_file_dependencies: diffusion_tiny_model
         timeout_in_minutes: 30
         # NOTE: Single-GPU (core_model) tiny model tests run on the ready label,
         # so nightly only adds the multi-GPU parallelism configurations.
@@ -189,91 +196,74 @@ steps:
         # for the CI, since these tests should pass irrespective of whether or not we use
         # the full weights or not.
         commands:
-          - pytest -sv tests/model_tests/diffusion -m 'full_model and cuda and cards_2' --run-level "core_model"
-        mirror_hardwares: l4_2
+          - pytest -sv tests/model_tests/diffusion -m 'full_model and L4 and B200 and cuda and cards_2' --run-level "core_model"
 
       - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 4-GPU"
+        source_file_dependencies: diffusion_tiny_model
         timeout_in_minutes: 30
         # CFG+TP extra_test_groups request 2×2=4 devices (see get_required_device_count).
         commands:
-          - pytest -sv tests/model_tests/diffusion -m 'full_model and cuda and cards_4' --run-level "core_model"
-        mirror_hardwares: l4_4
+          - pytest -sv tests/model_tests/diffusion -m 'full_model and L4 and B200 and cuda and cards_4' --run-level "core_model"
 
   - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
     key: nightly-diffusion-x2iat-group
     depends_on: upload-nightly-pipeline
     if: >-
       build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2iat-test"
+      build.pull_request.labels includes "nightly-test"
     steps:
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU"
+        source_file_dependencies: diffusion_qwen_image_function
         timeout_in_minutes: 120
         commands:
           - >-
             pytest -sv
             tests/e2e/online_serving/test_qwen_image_expansion.py
-            -m "full_model and diffusion and H100 and cards_1"
+            -m "full_model and diffusion and H100 and B200 and cards_1"
             --run-level "full_model"
-        mirror_hardwares: h100_1
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU"
+        source_file_dependencies: diffusion_qwen_image_function
         timeout_in_minutes: 120
         commands:
           - >-
             pytest -sv
             tests/e2e/online_serving/test_qwen_image_expansion.py
-            -m "full_model and diffusion and H100 and cards_2"
+            -m "full_model and diffusion and H100 and B200 and cards_2"
             --run-level "full_model"
-        mirror_hardwares: h100_2
 
-      - label: ":full_moon: Diffusion X2I(&A&T) · Boogu Edit CFG · 3-GPU"
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · Single-GPU"
+        source_file_dependencies: diffusion_qwen_image_function
         timeout_in_minutes: 120
         commands:
-          - >-
-            pytest -sv
-            tests/e2e/online_serving/test_boogu_image_edit_expansion.py
-            -k "test_double_guidance_cfg_parallel and cfg_parallel_3_full"
-            -m "slow and diffusion and cuda and H100 and cards_3"
-            --run-level "full_model"
-        mirror_hardwares: h100_3
+          - pytest -sv tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py -m "full_model and diffusion and L4 and B200 and cards_1" --run-level "full_model"
 
-      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · 4-GPU"
-        timeout_in_minutes: 120
-        commands:
-          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4 and cards_4" --run-level "full_model" --ignore=tests/e2e/accuracy
-        mirror_hardwares: l4_4
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
+        source_file_dependencies: diffusion_text_to_image_doc
         timeout_in_minutes: 60
         commands:
-          - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100 and B200 and (cards_1 or cards_2)" --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test"
+        source_file_dependencies: diffusion_qwen_image_accuracy
         timeout_in_minutes: 180
         commands:
           - export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=120
-          - pytest -s -v tests/e2e/accuracy/test_qwen_image.py --run-level full_model
-          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model
-        mirror_hardwares: h100_1
+          - pytest -s -v tests/e2e/accuracy/test_qwen_image.py -m "H100 and B200 and cards_1" --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m "full_model and H100 and B200 and cards_1" --run-level full_model
 
       - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
+        source_file_dependencies: diffusion_hunyuan_image3_accuracy
         timeout_in_minutes: 180
         commands:
           - export HUNYUAN_IMAGE3_MODEL="tencent/HunyuanImage-3.0-Instruct"
           - export HUNYUAN_IMAGE3_DEVICES="0,1,2,3"
-          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m full_model --run-level full_model
-        mirror_hardwares: h100_4
-
-      - label: ":full_moon: HiDream-O1-Image · Accuracy Test"
-        timeout_in_minutes: 180
-        commands:
-          - pytest -s -v tests/e2e/accuracy/test_hidream_o1_image.py -m full_model --run-level full_model
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m "full_model and H100 and B200 and cards_4" --run-level full_model
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · Single-GPU"
+        source_file_dependencies: diffusion_qwen_image_perf
         key: nightly-diffusion-x2iat-performance-qwen-image-single
         timeout_in_minutes: 180
         commands:
@@ -281,168 +271,182 @@ steps:
           - export CACHE_DIT_VERSION=1.5.0
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and cards_1"
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and B200 and cards_1"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
-        mirror_hardwares: h100_1
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · 4-GPU"
+        source_file_dependencies: diffusion_qwen_image_perf
         key: nightly-diffusion-x2iat-performance-qwen-image-4gpu
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
           - export CACHE_DIT_VERSION=1.5.0
+          # Do not pin DIFFUSION_ATTENTION_BACKEND: H100 auto-selects FLASH_ATTN;
+          # B200 rejects explicit FLASH_ATTN without FA4 and uses CUDNN/TRTLLM.
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and cards_4"
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and B200 and cards_4"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
-        mirror_hardwares: h100_4
 
   - group: ":card_index_dividers: Diffusion X2V Model Test"
     key: nightly-diffusion-x2v-group
     depends_on: upload-nightly-pipeline
     if: >-
       build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2v-test"
+      build.pull_request.labels includes "nightly-test"
     steps:
 
       - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · Single-GPU"
+        source_file_dependencies: diffusion_wan22_function
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and cards_1" --run-level "full_model"
-        mirror_hardwares: h100_1
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and B200 and cards_1" --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · 2-GPU"
+        source_file_dependencies: diffusion_wan22_function
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and cards_2" --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and B200 and cards_2" --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · Single-GPU"
+        source_file_dependencies: diffusion_wan22_function
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and cards_1" --run-level "full_model"
-        mirror_hardwares: h100_1
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and B200 and cards_1" --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · 2-GPU"
+        source_file_dependencies: diffusion_wan22_function
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and cards_2" --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and B200 and cards_2" --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test"
+        source_file_dependencies: diffusion_hunyuan_video_function
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and cuda" --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level "full_model"
 
-      - label: ":full_moon: Diffusion X2V · MAGI-2 Preview Serving Smoke"
-        timeout_in_minutes: 180
+      - label: ":full_moon: Diffusion X2V · LingBot Function Test"
+        source_file_dependencies: diffusion_lingbot_function
+        timeout_in_minutes: 120
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_magi2.py -m "advanced_model and diffusion and cuda and H100 and cards_4" --run-level "advanced_model"
-        mirror_hardwares: h100_4
+          - >-
+            pytest -s -v
+            tests/e2e/online_serving/test_lingbot_video.py
+            tests/e2e/online_serving/test_lingbot_video_moe.py
+            tests/e2e/offline_inference/test_lingbot_world_v2.py
+            -m "full_model and diffusion and H100 and B200 and cards_1"
+            --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2V · Doc Test"
+        source_file_dependencies: diffusion_image_to_video_doc
         timeout_in_minutes: 60
         commands:
-          - pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m "full_model and example and H100" --run-level "full_model"
-        mirror_hardwares: h100_1
+          - pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m "full_model and example and H100 and B200 and cards_1" --run-level "full_model"
 
-      - label: ":full_moon: Diffusion X2V · Other Function Test"
+      - label: ":full_moon: Diffusion X2V · Wan2.2 Autoround Function Test"
+        source_file_dependencies: diffusion_wan22_function
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level "full_model"
+
+      - label: ":full_moon: Diffusion X2V · Wan2.2 Device-Postprocess Equivalence Test"
+        source_file_dependencies: diffusion_wan22_function
+        timeout_in_minutes: 90
+        commands:
+          - |
+            VLLM_OMNI_DEVICE_POSTPROCESS_MODEL=Wan-AI/Wan2.2-TI2V-5B-Diffusers \
+              pytest -s -v tests/e2e/offline_inference/test_device_postprocess_equivalence.py -m "full_model and H100 and B200 and cuda and cards_1" --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test"
+        source_file_dependencies: diffusion_wan22_accuracy
         timeout_in_minutes: 180
         commands:
-          - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model
-          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m "full_model and H100 and B200 and (cards_1 or cards_2)" --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m "full_model and H100 and B200 and cards_1" --run-level full_model
 
       - label: ":full_moon: Diffusion X2V · MiniMax H3 I2VA/Ref2VA Accuracy Test"
+        source_file_dependencies: diffusion_minimax_h3_accuracy
         timeout_in_minutes: 180
         commands:
-          - pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py -m full_model --run-level full_model
-        mirror_hardwares: h100_4
+          - pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py -m "full_model and H100 and B200 and cards_4" --run-level full_model
 
-      - label: ":full_moon: Diffusion X2V · Other Accuracy Test"
+      - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Accuracy Test"
+        source_file_dependencies: diffusion_hunyuan_video_accuracy
         timeout_in_minutes: 180
         commands:
-          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model
-          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m "full_model and H100 and B200 and (cards_1 or cards_2)" --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m "full_model and H100 and B200 and (cards_1 or cards_2)" --run-level full_model
 
       - label: ":full_moon: Diffusion X2V · Perf Test · Single-GPU"
+        source_file_dependencies: diffusion_wan22_perf
         key: nightly-diffusion-x2v-performance-single
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_1"
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and B200 and cards_1"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
-        mirror_hardwares: h100_1
 
       - label: ":full_moon: Diffusion X2V · Perf Test · 2-GPU"
+        source_file_dependencies:
+          - diffusion_wan22_perf
+          - diffusion_cosmos3_perf
         key: nightly-diffusion-x2v-performance-2gpu
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_2"
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and B200 and cards_2"
             EXIT1=$$?
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json -m "H100 and cards_2"
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json -m "H100 and B200 and cards_2"
             EXIT2=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi
             exit $$EXIT2
-        mirror_hardwares: h100_2
 
       - label: ":full_moon: Diffusion X2V · Perf Test · HunyuanVideo-1.5"
+        source_file_dependencies: diffusion_hunyuan_video_perf
         key: nightly-diffusion-x2v-performance-hunyuanvideo15
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json -m "H100 and B200 and (cards_1 or cards_2)"
             EXIT1=$$?
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json -m "H100 and B200 and (cards_1 or cards_2)"
             EXIT2=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi
             exit $$EXIT2
-        mirror_hardwares: h100_2
 
       - label: ":full_moon: Diffusion X2V · Perf Test · MiniMax-H3"
+        source_file_dependencies: diffusion_minimax_h3_perf
         key: nightly-diffusion-x2v-performance-minimax-h3
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json -m "H100 and B200 and cards_4"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
-        mirror_hardwares: h100_4
 
   - group: ":card_index_dividers: Diffusion Test"
     key: nightly-diffusion-test-group
@@ -450,59 +454,59 @@ steps:
     if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     steps:
       - label: ":full_moon: Diffusion · Distributed Test with L4 · 2-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_2' --run-level "full_model"
-        mirror_hardwares: l4_2
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_2' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · Distributed Test with L4 · 3-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_3' --run-level "full_model"
-        mirror_hardwares: l4_3
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_3' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · Distributed Test with L4 · 4-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_4' --run-level "full_model"
-        mirror_hardwares: l4_4
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_4' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · Distributed Test with H100 · 2-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and cards_2' --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and B200 and cards_2' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · Distributed Test with H100 · 4-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and cards_4' --run-level "full_model"
-        mirror_hardwares: h100_4
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and B200 and cards_4' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · Offloader Test"
+        source_file_dependencies: diffusion_offloader
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/offloader -m "full_model and cuda"
-        mirror_hardwares: l4_1
+          - pytest -sv tests/diffusion/offloader -m "full_model and L4 and B200 and cuda and cards_1"
 
       - label: ":full_moon: Diffusion · Quantization Test with H100 · Single-GPU"
+        source_file_dependencies: diffusion_quantization
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model"
-        mirror_hardwares: h100_1
+          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100 and B200 and cards_1' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · MiniMax H3 FP8 Quality Test with H100 · 2-GPU"
+        source_file_dependencies: diffusion_minimax_h3_function
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py -m 'full_model and cuda and H100' --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -sv tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py -m 'full_model and cuda and H100 and B200 and cards_2' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · Quantization Test with L4"
+        source_file_dependencies: diffusion_quantization
         timeout_in_minutes: 60
         commands:
           - pip install "vllm-gguf-plugin==0.0.4"
-          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model"
-        mirror_hardwares: l4_1
+          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4 and B200 and cards_1' --run-level "full_model"
 
   # NOTE: ":email: Nightly Collection & Email" is deprecated and will be removed soon.
   # Do not add new depends_on entries or extend this step.

.buildkite/cuda/test-ready.yml

@@ -54,16 +54,19 @@ steps:
         mirror_hardwares: l4_1
 
       - label: "Diffusion · Distributed Test · Single-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         commands:
           - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_1' --run-level "core_model"
         mirror_hardwares: l4_1
 
       - label: "Diffusion · Distributed Test · 2-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         commands:
           - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_2' --run-level "core_model"
         mirror_hardwares: l4_2
 
       - label: "Diffusion · Distributed & Attention Test · 4-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         commands:
           - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_4' --run-level "core_model"
           - timeout 20m pytest -sv tests/diffusion/attention/test_attention_sp.py -m 'core_model and cuda and L4 and cards_4' --run-level "core_model"
@@ -120,29 +123,14 @@ steps:
     depends_on: upload-ready-pipeline
     steps:
       - label: "Custom Pipeline Test"
-        source_file_dependencies:
-          - tests/e2e/features/custom_pipeline/
-          - tests/e2e/features/helpers/custom_pipeline.py
-          - vllm_omni/diffusion/
-          - vllm_omni/engine/
-          - vllm_omni/entrypoints/
-          - vllm_omni/inputs/
-          - vllm_omni/outputs/
+        source_file_dependencies: custom_pipeline
         commands:
           - timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"
         mirror_hardwares: l4_1
 
 
       - label: "RLHF · VeRL-Omni E2E Test"
-        source_file_dependencies:
-          - tests/e2e/features/rlhf_test/test_verl_omni_e2e.py
-          - tests/e2e/features/helpers/verl_omni_server.py
-          - tests/e2e/features/helpers/custom_pipeline.py
-          - vllm_omni/diffusion/
-          - vllm_omni/engine/
-          - vllm_omni/entrypoints/
-          - vllm_omni/inputs/
-          - vllm_omni/outputs/
+        source_file_dependencies: rlhf_verl_omni
         commands:
           - |
             timeout 20m bash -c "
@@ -151,17 +139,7 @@ steps:
         mirror_hardwares: l4_1
 
       - label: "Omni · Qwen3-Omni Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen3_omni.py
-          - tests/helpers/runtime.py
-          - vllm_omni/entrypoints/openai/serving_chat.py
-          - vllm_omni/model_executor/models/qwen3_omni/
-          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-          - vllm_omni/model_executor/models/common/snake_activation.py
-          - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
-          - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
-          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
-          - vllm_omni/deploy/qwen3_omni_moe.yaml
+        source_file_dependencies: omni_qwen3_omni_function
         commands:
           - |
             timeout 20m bash -c "
@@ -170,14 +148,7 @@ steps:
         mirror_hardwares: h100_2
 
       - label: "Omni · MiniCPM-o 4.5 Offline Test"
-        source_file_dependencies:
-          - tests/e2e/offline_inference/test_minicpmo_4_5.py
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/models/step_audio2/
-          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
-          - vllm_omni/model_executor/models/cosyvoice3/utils.py
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
-          - vllm_omni/deploy/minicpmo_4_5.yaml
+        source_file_dependencies: omni_minicpmo_4_5_function
         commands:
           - |
             timeout 20m bash -c "
@@ -186,15 +157,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Omni · MiniCPM-o 4.5 Online Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_minicpmo_4_5.py
-          - vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/models/step_audio2/
-          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
-          - vllm_omni/model_executor/models/cosyvoice3/utils.py
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
-          - vllm_omni/deploy/minicpmo_4_5.yaml
+        source_file_dependencies: omni_minicpmo_4_5_function
         commands:
           - |
             timeout 20m bash -c "
@@ -203,48 +166,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Omni · MiniCPM-o 4.5 Duplex Test"
-        source_file_dependencies:
-          - tests/assets/minicpmo_4_5/response_required_16k.wav
-          - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
-          - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
-          - tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
-          - vllm_omni/deploy/minicpmo_4_5.yaml
-          - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
-          - vllm_omni/engine/duplex/
-          - vllm_omni/entrypoints/duplex/
-          # Public duplex client library + its unit suites: client-side
-          # changes must re-run the duplex e2e group, which includes the
-          # live client probe (test_duplex_client_live.py, single GPU).
-          - vllm_omni/clients/
-          - tests/clients/
-          - tests/e2e/online_serving/test_duplex_client_live.py
-          - vllm_omni/entrypoints/duplex_request_client.py
-          - vllm_omni/outputs/duplex.py
-          - vllm_omni/model_executor/duplex_sampling.py
-          - tests/engine/duplex/
-          - tests/entrypoints/duplex/
-          - tests/model_executor/models/minicpmo_4_5/duplex/
-          # Legacy Realtime probe client + video stacking used by the e2e drivers.
-          - vllm_omni/experimental/fullduplex/
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/models/step_audio2/
-          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
-          - vllm_omni/model_executor/models/cosyvoice3/utils.py
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
-          - vllm_omni/core/sched/
-          - vllm_omni/outputs/multimodal_accumulation.py
-          - vllm_omni/engine/stage_init_utils.py
-          - vllm_omni/engine/async_omni_engine.py
-          - vllm_omni/engine/orchestrator.py
-          - vllm_omni/entrypoints/cli/serve.py
-          - vllm_omni/entrypoints/openai/api_server.py
-          - vllm_omni/entrypoints/openai/duplex_capability.py
-          - vllm_omni/config/stage_config.py
+        source_file_dependencies: omni_minicpmo_4_5_duplex_function
         commands:
           - |
             timeout 35m bash -c "
@@ -254,29 +176,14 @@ steps:
 
       - label: "Omni · MiniCPM-o 4.5 Perf Test"
         timeout_in_minutes: 60
-        source_file_dependencies:
-          - tests/dfx/perf/tests/test_minicpmo_4_5_ready.json
-          - vllm_omni/deploy/minicpmo_4_5.yaml
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+        source_file_dependencies: omni_minicpmo_4_5_perf
         commands:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda'
         mirror_hardwares: h100_1
 
       - label: "TTS · Qwen3-TTS CustomVoice Test"
-        source_file_dependencies:
-          - tests/helpers/media.py
-          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
-          - vllm_omni/model_executor/models/qwen3_tts/
-          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-          - vllm_omni/model_executor/models/common/snake_activation.py
-          - vllm_omni/model_executor/models/common/alias_free_activation.py
-          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
-          - vllm_omni/deploy/qwen3_tts.yaml
-          - vllm_omni/entrypoints/openai/serving_speech.py
-          - vllm_omni/entrypoints/openai/tts_adapters/
-          - vllm_omni/engine/stage_pool.py
+        source_file_dependencies: tts_qwen3_tts_function
         commands:
           - |
             timeout 20m bash -c "
@@ -286,10 +193,7 @@ steps:
         mirror_hardwares: l4_1
 
       - label: "Diffusion · Wan22 Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_wan22_t2v.py
-          - vllm_omni/diffusion/models/wan2_2/
-          - vllm_omni/diffusion/models/dmd2/
+        source_file_dependencies: diffusion_wan22_function
         commands:
           - |
             timeout 40m bash -c "
@@ -299,12 +203,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Diffusion · Cosmos3 Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_cosmos3.py
-          - vllm_omni/diffusion/models/cosmos3/
-          - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_wan.py
-          - vllm_omni/experimental/world_models/
-          - vllm_omni/model_extras/cosmos3.py
+        source_file_dependencies: diffusion_cosmos3_function
         commands:
           - |
             timeout 40m bash -c "
@@ -315,14 +214,7 @@ steps:
 
       - label: "Diffusion · MiniMax-H3 DLO DP2 T2VA Test"
         timeout_in_minutes: 90
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py
-          - tests/helpers/assertions.py
-          - tests/helpers/runtime.py
-          - vllm_omni/diffusion/models/minimax_h3/
-          - vllm_omni/diffusion/offloader/
-          - vllm_omni/diffusion/distributed/
-          - vllm_omni/entrypoints/openai/
+        source_file_dependencies: diffusion_minimax_h3_function
         commands:
           - |
             timeout 80m bash -c "
@@ -332,11 +224,7 @@ steps:
         mirror_hardwares: h100_2
 
       - label: "Diffusion · Qwen Image Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen_image.py
-          - vllm_omni/diffusion/models/qwen_image/
-          - vllm_omni/diffusion/models/dmd2/
-          - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py
+        source_file_dependencies: diffusion_qwen_image_function
         commands:
           - |
             timeout 40m bash -c "
@@ -346,9 +234,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Diffusion · HunyuanVideo-1.5 Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
-          - vllm_omni/diffusion/models/hunyuan_video/
+        source_file_dependencies: diffusion_hunyuan_video_function
         commands:
           - |
             timeout 40m bash -c "

.buildkite/cuda/test-weekly.yml

@@ -29,36 +29,42 @@ steps:
     steps:
 
       - label: "Reliability Test · qwen3-omni"
+        source_file_dependencies: omni_qwen3_omni_reliability
         timeout_in_minutes: 180
         commands:
           - pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow"
         mirror_hardwares: h100_2
 
       - label: "Reliability Test · wan22"
+        source_file_dependencies: diffusion_wan22_reliability
         timeout_in_minutes: 180
         commands:
           - pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow"
         mirror_hardwares: h100_2
 
       - label: "Reliability Test · hunyuan-image"
+        source_file_dependencies: diffusion_hunyuan_image3_reliability
         timeout_in_minutes: 180
         commands:
           - pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow"
         mirror_hardwares: h100_4
 
       - label: "Reliability Test · Invalid parameters · H100 · Single-GPU"
+        source_file_dependencies: reliability_invalid_param_h100
         timeout_in_minutes: 90
         commands:
           - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100 and cards_1"
         mirror_hardwares: h100_1
 
       - label: "Reliability Test · Invalid parameters · H100 · 2-GPU"
+        source_file_dependencies: reliability_invalid_param_h100
         timeout_in_minutes: 90
         commands:
           - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100 and cards_2"
         mirror_hardwares: h100_2
 
       - label: "Reliability Test · Invalid parameters · L4"
+        source_file_dependencies: reliability_invalid_param_l4
         timeout_in_minutes: 120
         commands:
           - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"
@@ -70,6 +76,7 @@ steps:
     steps:
 
       - label: ":full_moon: Omni · Perf Test · vLLM Text"
+        source_file_dependencies: omni_qwen3_omni_perf
         key: weekly-omni-performance-vllm-text
         timeout_in_minutes: 180
         commands:
@@ -96,6 +103,7 @@ steps:
         mirror_hardwares: h100_2
 
       - label: ":full_moon: Omni · Perf Test · Multi-Replica"
+        source_file_dependencies: omni_qwen3_omni_perf
         key: weekly-omni-performance-multi-replicas
         timeout_in_minutes: 180
         commands:
@@ -183,13 +191,14 @@ steps:
       # from downgrading transformers; see
       # examples/offline_inference/text_to_speech/README.md.
       - label: "TTS · Gepard-1.0 · L4"
-        timeout_in_minutes: 60
+        timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
           - printf '%s\n' "transformers>=5.5.3" "huggingface-hub>=1.0" > /tmp/gepard-overrides.txt
           - uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" "nemo_toolkit[tts]==2.7.3" --overrides /tmp/gepard-overrides.txt
           - >-
             pytest -s -v tests/e2e/offline_inference/test_gepard_tts.py
+            tests/e2e/online_serving/test_gepard_tts.py
             -m "tts and cuda"
             --run-level "full_model"
         mirror_hardwares: l4_1
@@ -221,24 +230,277 @@ steps:
             --run-level "full_model"
         mirror_hardwares: l4_4
 
-      - label: "Diffusion · H100 · Single-GPU"
-        timeout_in_minutes: 150
+      # Split the old shared H100 Single-GPU sweep: one pytest process collected
+      # every slow+cards_1 diffusion file and exceeded 150 minutes.
+      - label: "Diffusion · Qwen-Image · H100 · Single-GPU"
+        timeout_in_minutes: 120
         commands:
           - >-
             pytest -sv
-            tests/e2e/ -m "slow and diffusion and H100 and cards_1"
+            tests/e2e/online_serving/test_qwen_image_edit.py
+            tests/e2e/online_serving/test_qwen_image_edit_expansion.py
+            tests/e2e/online_serving/test_qwen_image_layered.py
+            tests/e2e/online_serving/test_qwen_image_layered_expansion.py
+            tests/e2e/accuracy/test_qwen_image_edit.py
+            tests/e2e/accuracy/test_qwen_image_layered.py
+            -m "slow and diffusion and H100 and cards_1"
             --run-level "full_model"
         mirror_hardwares: h100_1
 
-      - label: "Diffusion · H100 · 2-GPU"
+      - label: "Diffusion · Wan2.1 VACE · H100 · Single-GPU"
         timeout_in_minutes: 90
         commands:
           - >-
             pytest -sv
-            tests/e2e/ -m "slow and diffusion and H100 and cards_2"
+            tests/e2e/online_serving/test_wan_2_1_vace_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · LongCat · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_longcat_image_expansion.py
+            tests/e2e/online_serving/test_longcat_image_edit_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Bagel · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_bagel.py
+            tests/e2e/online_serving/test_bagel_expansion.py
+            tests/e2e/offline_inference/test_bagel.py
+            tests/e2e/offline_inference/test_bagel_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Boogu · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_boogu_image.py
+            tests/e2e/online_serving/test_boogu_image_edit.py
+            tests/e2e/online_serving/test_boogu_image_expansion.py
+            tests/e2e/online_serving/test_boogu_image_edit_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Flux · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_flux_2_dev_expansion.py
+            tests/e2e/offline_inference/test_flux1_schnell_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · LTX · H100 · Single-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_ltx.py
+            tests/e2e/online_serving/test_ltx25.py
+            tests/e2e/accuracy/ltx/test_ltx_official_similarity.py
+            tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Krea2 · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_krea2_expansion.py
+            tests/e2e/offline_inference/test_krea2_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · HiDream-I1 · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_hidream_i1_full_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · SANA-Video · H100 · Single-GPU"
+        source_file_dependencies: diffusion_sana_video_function
+        timeout_in_minutes: 180
+        commands:
+          - export SANA_VIDEO_GOLDEN_BASE_URL=https://vllm-public-assets.s3.us-west-2.amazonaws.com/omni-assets/sana-video/v2
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_sana_video_expansion.py
+            tests/e2e/accuracy/sana_video/
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · SenseNova-U1 · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/offline_inference/test_sensenova_u1_expansion.py
+            tests/e2e/offline_inference/test_sensenova_u1_text2img_expansion.py
+            tests/e2e/offline_inference/test_sensenova_u1_img2img_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · MammothModa2 · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/offline_inference/test_mammoth_moda2_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Helios · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/accuracy/test_video_streaming_output_similarity.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      # Split the old shared H100 2-GPU sweep: one pytest process collected every
+      # slow+cards_2 diffusion file and exceeded 90 minutes.
+      - label: "Diffusion · Qwen-Image · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_qwen_image_edit_expansion.py
+            tests/e2e/online_serving/test_qwen_image_layered_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
             --run-level "full_model"
         mirror_hardwares: h100_2
 
+      - label: "Diffusion · Wan2.1 VACE · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_wan_2_1_vace_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · LongCat · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_longcat_image_expansion.py
+            tests/e2e/online_serving/test_longcat_image_edit_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · Bagel · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_bagel_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · Boogu · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_boogu_image_expansion.py
+            tests/e2e/online_serving/test_boogu_image_edit_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · Flux · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_flux_2_dev_expansion.py
+            tests/e2e/offline_inference/test_flux1_kontext_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · GLM-Image · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_glm_image_expansion.py
+            tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · LTX · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_ltx.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · Krea2 · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/offline_inference/test_krea2_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · DreamZero · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_dreamzero_expansion.py
+            tests/e2e/accuracy/test_dreamzero.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · H100 · 3-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/ -m "slow and diffusion and H100 and cards_3"
+            --run-level "full_model"
+        mirror_hardwares: h100_3
+
       - label: "Diffusion · H100 · 4-GPU"
         timeout_in_minutes: 90
         commands:
@@ -246,4 +508,33 @@ steps:
             pytest -sv
             tests/e2e/ -m "slow and diffusion and H100 and cards_4"
             --run-level "full_model"
+            --ignore=tests/e2e/online_serving/test_magi2.py
+        mirror_hardwares: h100_4
+
+      # Dedicated: HiDream-O1 accuracy needs a long TP2 run; the shared H100
+      # 2-GPU sweep is only 90 minutes.
+      - label: "Diffusion · HiDream-O1-Image Accuracy · H100 · 2-GPU"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/e2e/accuracy/test_hidream_o1_image.py -m "slow and diffusion and cuda and H100 and cards_2" --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      # Dedicated: Boogu CFG-parallel size 3 needs 3 GPUs; weekly has no H100 3-GPU sweep.
+      - label: "Diffusion · Boogu Edit CFG · H100 · 3-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_boogu_image_edit_expansion.py
+            -k "test_double_guidance_cfg_parallel and cfg_parallel_3_full"
+            -m "slow and diffusion and cuda and H100 and cards_3"
+            --run-level "full_model"
+        mirror_hardwares: h100_3
+
+      # Dedicated: MAGI-2 Preview serving smoke needs ~30 min server init; the
+      # shared H100 4-GPU sweep is only 90 minutes.
+      - label: "Diffusion · MAGI-2 Preview Serving Smoke · H100 · 4-GPU"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_magi2.py -m "slow and diffusion and cuda and H100 and cards_4" --run-level "full_model"
         mirror_hardwares: h100_4

.buildkite/intel/pipeline-intel.yml

@@ -10,7 +10,7 @@ steps:
           DOCKER_BUILDKIT: "1"
           # Buildkite will automatically replace this with the actual commit hash
           VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.28.0"
+          VLLM_VERSION: "v0.29.0"
         priority: 100
         timeout_in_minutes: 60
         soft_fail: false

.buildkite/npu/bootstrap-upload-steps.yml

@@ -93,7 +93,7 @@ steps:
     command: |
       python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/test-npu-ready.yml
 
-  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
+  # L4 Test — main+NIGHTLY=1 (scheduled), or PR nightly-test label
   - label: "Upload Nightly Pipeline"
     depends_on:
       - image-build-a2
@@ -102,6 +102,11 @@ steps:
       - image-build-310p
     key: upload-nightly-pipeline
     commands:
-      - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/test-npu-nightly.yml
+      - |
+        EXTRA=""
+        if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
+          EXTRA="--all"
+        fi
+        python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/npu/test-npu-nightly.yml
     agents:
       queue: "cpu_queue_premerge"

.buildkite/npu/pipeline-npu-a3.yml

@@ -96,13 +96,7 @@
   #   key: upload-nightly-pipeline
   #   if: >-
   #     (build.branch == "main" && build.env("NIGHTLY") == "1") ||
-  #     (build.branch != "main" && (
-  #       build.pull_request.labels includes "nightly-test" ||
-  #       build.pull_request.labels includes "omni-test" ||
-  #       build.pull_request.labels includes "tts-test" ||
-  #       build.pull_request.labels includes "diffusion-x2iat-test" ||
-  #       build.pull_request.labels includes "diffusion-x2v-test"
-  #     ))
+  #     (build.branch != "main" && build.pull_request.labels includes "nightly-test")
   #   commands:
   #     - buildkite-agent pipeline upload .buildkite/npu/test-npu-nightly.yml
   #   agents:

.buildkite/npu/test-npu-nightly.yml

@@ -4,10 +4,10 @@ steps:
     depends_on: upload-nightly-pipeline
     if: >-
       build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "omni-test"
+      build.pull_request.labels includes "nightly-test"
     steps:
       - label: ":full_moon: Omni · Perf Test · No Async Chunk"
+        source_file_dependencies: omni_qwen3_omni_perf
         key: nightly-omni-performance-no-async-chunk
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4
@@ -24,6 +24,7 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
       - label: ":full_moon: Omni · Perf Test · Async Chunk"
+        source_file_dependencies: omni_qwen3_omni_perf
         key: nightly-omni-performance-async-chunk
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4
@@ -40,6 +41,7 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
       - label: ":full_moon: Omni · MiniCPM-o 4.5 Test"
+        source_file_dependencies: omni_minicpmo_4_5_function
         mirror_hardwares: a3_npu_2
         env:
           VLLM_WORKER_MULTIPROC_METHOD: spawn
@@ -48,6 +50,7 @@ steps:
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model'
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Test"
+        source_file_dependencies: omni_minicpmo_4_5_duplex_function
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_2
         env:
@@ -62,6 +65,7 @@ steps:
             -m "npu and A3 and omni"
             --run-level full_model
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
+        source_file_dependencies: omni_minicpmo_4_5_accuracy
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_2
         env:
@@ -82,6 +86,7 @@ steps:
             buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
             exit $$EXIT
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"
+        source_file_dependencies: omni_minicpmo_4_5_perf
         key: nightly-omni-performance-minicpmo-4-5-npu
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_2
@@ -98,6 +103,7 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test (A2)"
+        source_file_dependencies: omni_minicpmo_4_5_perf
         key: nightly-omni-performance-minicpmo-4-5-npu-a2
         timeout_in_minutes: 180
         mirror_hardwares: a2b3_npu_1
@@ -115,6 +121,7 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test"
+        source_file_dependencies: omni_minicpmo_4_5_duplex_perf
         key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts-npu
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_2
@@ -133,9 +140,10 @@ steps:
   - group: ":card_index_dividers: TTS Model Test"
     key: nightly-tts-test-group
     depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     steps:
       - label: ":full_moon: TTS · Perf Test"
+        source_file_dependencies: tts_qwen3_tts_perf
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_2
         env:
@@ -151,15 +159,15 @@ steps:
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-  - group: ":card_index_dividers: Diffusion I2I Model Test"
-    key: nightly-HunyuanImage3-x2i-test-group
+  - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
+    key: nightly-diffusion-x2iat-group
     depends_on: upload-nightly-pipeline
     if: >-
       build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2iat-test"
+      build.pull_request.labels includes "nightly-test"
     steps:
       - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
+        source_file_dependencies: diffusion_hunyuan_image3_accuracy
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4
         env:
@@ -169,15 +177,8 @@ steps:
           DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
         commands:
           - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m "full_model and A3" --run-level full_model
-  - group: ":card_index_dividers: Diffusion Benchmark Perf Test"
-    key: nightly-diffusion-benchmark-group
-    depends_on: upload-nightly-pipeline
-    if: >-
-      build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2iat-test"
-    steps:
       - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 CFGP=2"
+        source_file_dependencies: diffusion_hunyuan_image3_perf
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4
         env:
@@ -194,6 +195,7 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
       - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 SP=2"
+        source_file_dependencies: diffusion_hunyuan_image3_perf
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4
         env:
@@ -210,6 +212,7 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
       - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=4"
+        source_file_dependencies: diffusion_hunyuan_image3_perf
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4
         env:
@@ -230,10 +233,10 @@ steps:
     depends_on: upload-nightly-pipeline
     if: >-
       build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2v-test"
+      build.pull_request.labels includes "nightly-test"
     steps:
-      - label: ":full_moon: Diffusion X2V · Function Test"
+      - label: ":full_moon: Diffusion X2V · Wan 2.2 Function Test"
+        source_file_dependencies: diffusion_wan22_function
         mirror_hardwares: a2b3_npu_4
         env:
           VLLM_WORKER_MULTIPROC_METHOD: spawn
@@ -241,7 +244,8 @@ steps:
           HF_TOKEN: "${HF_TOKEN}"
         commands:
           - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"
-      - label: ":full_moon: Diffusion X2V · Perf Test"
+      - label: ":full_moon: Diffusion X2V · Wan 2.2 Perf Test"
+        source_file_dependencies: diffusion_wan22_perf
         key: nightly-diffusion-x2v-performance
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4

.claude/skills/vllm-omni-test/SKILL.md

@@ -468,10 +468,10 @@ When the user asks for **invalid parameter validation**, **invalid request bodie
 | **Invalid parameters Test · H100** | `pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"` | Diffusion / omni / video invalid-param tests with `H100` hardware mark |
 | **Invalid parameters Test · L4** | `pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"` | TTS / lighter models marked `L4` |
 
-- **Trigger:** `build.env("WEEKLY") == "1"` or PR label `weekly-test`.
+- **Trigger:** `build.env("WEEKLY") == "1"` (all weekly jobs) or PR label `weekly-test` (Reliability / Perf, then filtered by `source_file_dependencies`).
 - **Default:** extending an existing `invalid_param_test` script needs **no YAML edit** — the weekly steps already sweep the whole directory.
 - **Edit YAML only** when adding a **new hardware queue**, a **new top-level script** that must run in isolation, or a **model-specific weekly shard** (mirror neighboring reliability steps).
-- Weekly steps **do not** use `source_file_dependencies`.
+- Weekly Reliability / Perf jobs declare `source_file_dependencies`. Filtering applies only on the `weekly-test` label; `WEEKLY=1` on `main` keeps every job in those groups. CPU coverage (`WEEKLY=1` only) and the E2E Tests group (`NON_CRITICAL=1` only) do **not** use this field.
 
 **Example — append to `test_invalid_image_generation.py`:**
 
@@ -644,7 +644,7 @@ If the test is not already collected by an existing pipeline command (for exampl
 | **L2** | `test-ready.yml` with `core_model` + `source_file_dependencies` | merge / nightly pipelines |
 | **Invalid param** | `test-weekly.yml` — **Invalid parameters Test** group (`-m "slow and H100"` / `-m "slow and L4"`). Usually **no YAML edit** when appending cases to existing `invalid_param_test` scripts | `test-ready.yml`, `test-merge.yml`, `test-nightly.yml` |
 
-`test-nightly.yml` shards use **explicit pytest file paths** in `commands` (and PR labels like `diffusion-x2iat-test`); they generally **do not** use `source_file_dependencies`. Only suggest merge/ready YAML when the requested level is L3/L2 (or the user explicitly asks for multi-level CI).
+`test-nightly.yml` shards use **explicit pytest file paths** in `commands` plus `source_file_dependencies`; PR trigger is `nightly-test` (path filter), or `NIGHTLY=1` on `main` (all jobs). Only suggest merge/ready YAML when the requested level is L3/L2 (or the user explicitly asks for multi-level CI).
 
 **`test-nightly.yml` top-level groups** (each uses `full_model` + type marker in pytest `-m`):
 
@@ -653,7 +653,7 @@ If the test is not already collected by an existing pipeline command (for exampl
 | **Omni Model Test** | `omni` | Function / doc / accuracy / **perf** for Qwen-Omni family |
 | **TTS Model Test** | `tts` | `-m "full_model and L4 and tts"` (and H100 variants when enabled); **Perf Test** uses `run_benchmark.py` + `test_tts.json` |
 | **Diffusion X2I(&A&T) Model Test** | `diffusion` | Image/audio/text diffusion; explicit file shards for **Function**; **Perf Test · <Model>** uses `run_diffusion_benchmark.py` + JSON config |
-| **Diffusion X2V Model Test** | `diffusion` | Video-only (Wan, HunyuanVideo, …); PR label `diffusion-x2v-test`; separate **Perf** steps when in-tree |
+| **Diffusion X2V Model Test** | `diffusion` | Video-only (Wan, HunyuanVideo, …); separate **Perf** steps when in-tree |
 
 Guidelines when editing YAML:
 

.claude/skills/vllm-omni-test/references/test-routing.md

@@ -37,7 +37,7 @@ Every model-centric e2e test must declare **exactly one** type marker:
 | **Diffusion X2I(&A&T) Model Test** | x2**i** / x2**a** / x2**t** — image, audio, text (non-video) | Qwen-Image*, BAGEL, FLUX, SD3, Z-Image, LongCat, DreamZero |
 | **Diffusion X2V Model Test** | x2**v** — video only | Wan2.2, HunyuanVideo 1.5, Wan VACE |
 
-PR labels for selective nightly runs: `diffusion-x2iat-test`, `diffusion-x2v-test` (plus `nightly-test` / `NIGHTLY=1`).
+PR label for L4 on a branch: `nightly-test` (jobs then filtered by `source_file_dependencies`). Scheduled `main` uses `NIGHTLY=1` and runs every nightly job. L5 Reliability / Perf use `weekly-test` with the same path filter; CPU coverage and the weekly E2E group are env-only (`WEEKLY=1` / `NON_CRITICAL=1`) and are not source-filtered.
 
 ## Level and Marker Mapping
 
@@ -238,7 +238,7 @@ pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and L4"
 pytest -s -v dfx/reliability/invalid_param_test/test_invalid_image_generation.py::test_images_generations_invalid_requests -m "slow and H100"
 ```
 
-**Trigger:** `WEEKLY=1` or PR label `weekly-test`. Appending cases to existing scripts usually needs **no YAML edit** (directory sweep). No `source_file_dependencies` on weekly steps.
+**Trigger:** `WEEKLY=1` (all jobs) or PR label `weekly-test` (Reliability / Perf, filtered by `source_file_dependencies`). Appending cases to existing scripts usually needs **no YAML edit** (directory sweep).
 
 ### Platform-targeted examples
 

apps/ComfyUI-vLLM-Omni/README.md

@@ -13,7 +13,6 @@ It can send model inference requests to either a locally running vLLM-Omni servi
 > [!TIP]
 > If you run both ComfyUI and vLLM-Omni on the same device, you can create separate virtual environments and use different Python versions for them.
 
-
 ## Installation
 
 Copy this folder to the `custom_nodes` subfolder of your ComfyUI installation. Your directory should look like `ComfyUI/custom_nodes/ComfyUI-vLLM-Omni`.
@@ -21,9 +20,10 @@ Copy this folder to the `custom_nodes` subfolder of your ComfyUI installation. Y
 If you are running ComfyUI during copying, you should restart ComfyUI to load this extension.
 
 > [!TIP]
-> You can use utility websites such as https://download-directory.github.io/ to download a subdirectory of a repo. Also checkout community discussions (e.g., https://stackoverflow.com/questions/7106012/download-a-single-folder-or-directory-from-a-github-repository) for more info.
+> You can use utility websites such as <https://download-directory.github.io/> to download a subdirectory of a repo. Also checkout community discussions (e.g., <https://stackoverflow.com/questions/7106012/download-a-single-folder-or-directory-from-a-github-repository>) for more info.
 
 On the device and virtual environment you run ComfyUI, launch ComfyUI with
+
 ```bash
 cd ComfyUI
 
@@ -35,6 +35,7 @@ python main.py --cpu
 ```
 
 On the device and virtual environment you run vLLM-Omni, start a model service with
+
 ```bash
 vllm serve The_Model_ID_to_Serve --omni --port 8000
 ```
@@ -48,14 +49,30 @@ This extension offers the following nodes based on the output modalities (at **C
 
 - **Generate Image** for text-to-image and image-to-image tasks
 - **Generate Video** for text-to-video, first-frame/image-to-video, and reference-conditioned video
+- **FastH3 Deployment** for routing text-to-video requests to a MiniMax-H3 server with FastH3 fused at startup
 - **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks
 - **TTS** and **TTS Voice Clone** for TTS tasks
+- **Generate Music** for lyrics-and-description-to-music generation
 
 This extension also offers example workflows (at **ComfyUI sidebar -> Templates -> vLLM-Omni**)
 
 > [!NOTE]
 > The node UI and feature designs are intended to match vLLM-Omni online serving interfaces. It cannot offer more than what the interfaces support.
 
+Every node carries the vLLM-Omni mark in its title bar and is tinted by what it outputs, so a graph is readable at a glance:
+
+| Colour | Nodes | What they produce |
+| --- | --- | --- |
+| Blue | Generate Image, Generate Video, Multimodality Understanding, TTS, TTS Voice Clone | A generated image, video, audio, or text. These are the only nodes that reach a server. |
+| Amber | AR / Diffusion / Multi-Stage Sampling Params | Sampling parameters that apply to any model |
+| Purple | Qwen TTS Params, Wan Video Params, MiniMax-H3 Video Params | Parameters that only one model family accepts |
+| Red | LoRA, FastH3 Deployment | Which weights the server is expected to have loaded |
+| Teal | Video References | Reference media |
+
+Recolouring a node by hand (right click -> Colors) overrides its tint, and the choice is kept.
+
+**Generate Video** takes a clip length in seconds (`duration`), not a frame count. Frames stay the wire unit and are derived with the node's `fps`, so the length is always measured against the rate that is actually served; models that accept only certain frame counts still round to their own lattice server-side. Graphs saved before this widget existed stored `num_frames` in its place and are converted on load, using the fps recorded alongside it -- the browser console names every node it rewrites.
+
 To build a simple workflow yourself,
 
 - Drag a generation node onto the canvas.
@@ -123,6 +140,127 @@ You can configure per-stage sampling parameters for multi-stage models.
 >
 > Do not use `frame` and `references` together. Task routing is automatic from which inputs you connect.
 
+For MiniMax-H3 Ref2VA, **Video References** accepts up to 9 images (`image_1`–`image_9`),
+3 videos (`video_1`–`video_3`), and 3 audio clips (`audio_1`–`audio_3`), with at most
+12 connected inputs in total. Any mixture containing at least one image or video
+is supported; empty and audio-only references are rejected. For example, you can
+combine 6 images, 3 videos, and 3 audio clips in one request.
+
+Within each media type, references follow slot-number order, skipping unconnected
+slots. For example, connecting `image_2` and `image_9` sends `image_2` as the first
+image and `image_9` as the second. Each image slot uses the first image in its batch.
+Existing connections to `image_1`, `image_2`, `audio_1`, `audio_2`, `video_1`, and
+`video_2` remain valid in saved workflows.
+
+#### MiniMax-H3 Reference to Video
+
+Load [MiniMax-H3 Reference to Video](example_workflows/vLLM-Omni%20MiniMax-H3%20Reference%20to%20Video.json)
+from **Templates → ComfyUI-vLLM-Omni**, or drag the JSON onto the canvas.
+It adapts the [official ComfyUI R2V workflow](https://github.com/Comfy-Org/workflow_templates/blob/main/templates/video_minimax_h3_r2v.json)
+to remote vLLM-Omni execution. Model loading, sampling, and VAE decoding run on
+the server; ComfyUI loads references and saves the returned video.
+
+Use an extension version containing the audio-preserving output fix from
+[#7456](https://github.com/vllm-project/vllm-omni/pull/7456). Older versions can
+drop the server's audio while decoding the response.
+
+1. Start a Ref2VA-capable service using the [MiniMax-H3 recipe](../../recipes/MiniMaxAI/MiniMax-H3.md).
+   Set **Generate Video** to its `/v1` URL and served model name.
+2. Select your own image in **Load Image**. The template's filenames are placeholders
+   relative to ComfyUI's input directory; no sample assets or model weights are bundled.
+3. Connect **Load Video** and **Load Audio** when needed. Duplicate the loaders to
+   fill more reference slots, or disconnect the image when using video-only input.
+   Keep the references connected to **Generate Video** and leave `frame` disconnected.
+4. Match prompt tags such as `<Picture 1>`, `<Video 1>`, and `<Audio 1>` to the
+   connected references, counting each media type separately and skipping empty slots.
+5. Run the workflow. **Save Video** writes an MP4 under `output/video/` and preserves
+   the generated audio when the audio-output prerequisite is installed.
+
+The default is **1344×768, 24 FPS, 124 frames** (about 5.17 seconds), 50 sampling
+points, video flow shift 12, audio flow shift 3, and seed 42. The duration widget
+is set to 5.167 seconds, which converts to 124 frames at 24 FPS. Other H3 canvas presets
+are 1024×768, 768×768, 768×1024, and 768×1344. Keep frame counts at `17k+5` within
+the 4–15 second output range; examples are 107, 124, 209, and 345 frames.
+Reference video/audio clips must each be 2–15 seconds, with at most 15 seconds of
+reference video and at most 15 seconds of standalone reference audio per request.
+
+For **Ref2VA Turbo**, use the same template. Configure a Ref2VA-only server with
+`--lora-backend peft --lora-path "$TURBO_LORA" --task-type ref2va`, following the
+recipe's hardware setup. For `minimax_h3_ref2v_turbo_4step_v0.1_bf16.safetensors`,
+set **LoRA** `local_path` to that file's path on the server and connect it to
+**Generate Video**. Use `num_inference_steps=5`, `flow_shift=12`,
+`audio_flow_shift=3`, and LoRA `scale=1`. The API counts sigma points, so five
+points produce four denoiser evaluations. Use the Diffusers artifact, not its
+`_comfyui_` export. To return to base mode, disconnect LoRA and restore 50 steps.
+The 8-step v1.0 Ref2VA adapter instead requires nine sampling points and video
+flow shift 6. FastH3 is a separate startup-fused T2VA deployment.
+
+Validate the template's wiring and defaults locally with:
+
+```bash
+python -m pytest tests/e2e/features/comfyui/test_h3_reference_workflow.py -q
+```
+
+For a real-model check, run the workflow against H3 and inspect the saved file:
+
+```bash
+ffprobe -v error -show_entries stream=codec_type,width,height,r_frame_rate,sample_rate,channels \
+  -show_entries format=duration -of json "$OUTPUT_MP4"
+```
+
+Check for 24 FPS video, stereo audio, the requested canvas, and aligned duration;
+also play the result to assess reference conditioning and audio/video synchronization.
+Record the server commit, model/adapter, input assets, prompt, settings, and output
+alongside the result. Local schema or mocked-server checks do not establish H3
+generation quality or replace this real-model validation.
+
+### MiniMax H3 Text to Video
+
+Import [the H3 text-to-video template](example_workflows/MiniMax_H3_Text_to_Video.json)
+for remote video generation with audio. It includes connected base presets and
+optional Turbo sampling, H3 parameters, and Remote LoRA nodes. See the
+[workflow guide](docs/minimax-h3-t2v.md) for server setup, artifact-specific Turbo
+settings, and saved-video/audio validation. Set Remote LoRA’s `local_path` to
+the downloaded Turbo artifact on your server before enabling Turbo.
+
+#### H3 video upscale (WF-07)
+
+The **vLLM-Omni MiniMax H3 Video Upscale** template generates video remotely, upscales it with SeedVR2, and saves the original and upscaled videos with the generated audio and FPS. See [workflow setup](docs/wf07-h3-upscale.md).
+
+#### FastH3 text-to-video
+
+FastH3 is fused into MiniMax-H3 when the vLLM-Omni server starts; it is not a request-switchable LoRA. Download one adapter variant and start a non-offloaded FL2VA server. For the Dense / Data-Free profile:
+
+```bash
+export FASTH3_DIR=/path/to/fasth3
+hf download FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA \
+  dense-datafree/adapter_model.safetensors --local-dir "${FASTH3_DIR}"
+
+vllm serve /path/to/MiniMax-H3 \
+  --omni \
+  --task-type fl2va \
+  --lora-path "${FASTH3_DIR}/dense-datafree/adapter_model.safetensors" \
+  --port 8000
+```
+
+For the VSA / Data-Free profile, download `vsa-datafree/adapter_model.safetensors`, use that file as `--lora-path`, and add:
+
+```bash
+--diffusion-attention-backend FASTVIDEO_VSA \
+--fastvideo-vsa-topk 64
+```
+
+The VSA profile also requires a compatible `fastvideo-kernel` installation. See the [MiniMax-H3 FastH3 recipe](../../recipes/MiniMaxAI/MiniMax-H3.md#fasth3-adapter) for the full serving contract, supported parallel layouts, and kernel requirements.
+
+Open the **vLLM-Omni FastH3 Text to Video** template, then:
+
+- Set the server URL and served model name on **FastH3 Deployment**.
+- Connect its output to **Generate Video → fast_h3**. When connected, the deployment node's URL and model take precedence over the corresponding Generate Video widgets.
+- Keep `frame`, `references`, **LoRA**, and **MiniMax-H3 Video Params** disconnected. FastH3 Preview v1 supports T2VA only, is already fused, and owns both flow shifts.
+- A connected **Diffusion Sampling Params** node may set seed and other ordinary sampling options. The integration always enforces four denoising steps and 24 FPS for FastH3.
+
+The node records which server the workflow targets; it does not start one, nor switch adapters or attention backends on a running server.
+
 ### TTS (e.g., Qwen TTS series)
 
 (Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni TTS**)
@@ -137,6 +275,17 @@ You can configure per-stage sampling parameters for multi-stage models.
 > [!TIP]
 > There is a dedicated node for VoiceClone tasks with reference audio input. Other simple text-to-speech tasks should use the regular TTS node.
 
+### Music generation (e.g., MiniMax Music 3)
+
+(Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni Music Generation**)
+
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-music-generation.jpg">
+    <img alt="vLLM-Omni Music Generation" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-music-generation.jpg" width=55%>
+  </picture>
+</p>
+
 ### Chaining multiple model services
 
 (Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni Chaining Services**)
@@ -152,6 +301,8 @@ You can configure per-stage sampling parameters for multi-stage models.
 
 Follow the [development convention and rules of vLLM-Omni](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/).
 
+Node tints and the title-bar mark are applied in `web/main.js`, keyed off each node's declared output types rather than a list of node names. A new node that returns an existing type is themed with no front-end change; a new output type needs one entry in `FAMILY_BY_OUTPUT` there.
+
 ## Limitation and Non-Goals
 
 - Single server mode only. No automatic load balancing or failover.
@@ -178,11 +329,11 @@ Whenever you find an issue or problem, please
 
 Features
 
-- https://github.com/dougbtv/comfyui-vllm-omni/ The official reference implementation for ComfyUI integration with vLLM-Omni's DALL-E compatible image generation API.
-- https://github.com/Comfy-Org/ComfyUI/tree/master/comfy_extras ComfyUI's built-in node implementations.
+- <https://github.com/dougbtv/comfyui-vllm-omni/> The official reference implementation for ComfyUI integration with vLLM-Omni's DALL-E compatible image generation API.
+- <https://github.com/Comfy-Org/ComfyUI/tree/master/comfy_extras> ComfyUI's built-in node implementations.
 
 UI/UX design references
 
-- https://github.com/sgl-project/sglang/pull/15271 SGLang Diffusion's official ComfyUI integration for image and video generation.
-- https://github.com/SXQBW/ComfyUI-Qwen-Omni A third party ComfyUI integration for Qwen Omni series.
-- https://github.com/flybirdxx/ComfyUI-Qwen-TTS https://github.com/DarioFT/ComfyUI-Qwen3-TTS Third  party ComfyUI integrations for Qwen TTS series.
+- <https://github.com/sgl-project/sglang/pull/15271> SGLang Diffusion's official ComfyUI integration for image and video generation.
+- <https://github.com/SXQBW/ComfyUI-Qwen-Omni> A third party ComfyUI integration for Qwen Omni series.
+- <https://github.com/flybirdxx/ComfyUI-Qwen-TTS> <https://github.com/DarioFT/ComfyUI-Qwen3-TTS> Third  party ComfyUI integrations for Qwen TTS series.

apps/ComfyUI-vLLM-Omni/__init__.py

@@ -1,4 +1,8 @@
-"""Top-level package for comfyui_vllm_omni."""  # noqa: N999  # This is not a python library intended to be imported
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+# ruff: noqa: N999
+
+"""Top-level package for comfyui_vllm_omni."""  # This is not a Python library intended to be imported.
 
 __all__ = [
     "NODE_CLASS_MAPPINGS",
@@ -13,7 +17,9 @@ __version__ = "0.0.1"
 from .comfyui_vllm_omni.nodes import (
     VLLMOmniARSampling,
     VLLMOmniDiffusionSampling,
+    VLLMOmniFastH3Deployment,
     VLLMOmniGenerateImage,
+    VLLMOmniGenerateMusic,
     VLLMOmniGenerateVideo,
     VLLMOmniMiniMaxH3Params,
     VLLMOmniQwenTTSParams,
@@ -33,6 +39,7 @@ NODE_CLASS_MAPPINGS = {
     "VLLMOmniGenerateVideo": VLLMOmniGenerateVideo,
     "VLLMOmniUnderstanding": VLLMOmniUnderstanding,
     "VLLMOmniTTS": VLLMOmniTTS,
+    "VLLMOmniGenerateMusic": VLLMOmniGenerateMusic,
     "VLLMOmniVoiceClone": VLLMOmniVoiceClone,
     "VLLMOmniVideoReferences": VLLMOmniVideoReferences,
     # === Params ===
@@ -40,6 +47,7 @@ NODE_CLASS_MAPPINGS = {
     "VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling,
     "VLLMOmniSamplingParamsList": VLLMOmniSamplingParamsList,
     "VLLMOmniRemoteLoRA": VLLMOmniRemoteLoRA,
+    "VLLMOmniFastH3Deployment": VLLMOmniFastH3Deployment,
     "VLLMOmniQwenTTSParams": VLLMOmniQwenTTSParams,
     "VLLMOmniWanParams": VLLMOmniWanParams,
     "VLLMOmniMiniMaxH3Params": VLLMOmniMiniMaxH3Params,
@@ -52,6 +60,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "VLLMOmniGenerateVideo": "Generate Video",
     "VLLMOmniUnderstanding": "Multimodality Understanding",
     "VLLMOmniTTS": "TTS (Text to Speech)",
+    "VLLMOmniGenerateMusic": "Generate Music",
     "VLLMOmniVoiceClone": "TTS Voice Cloning",
     "VLLMOmniVideoReferences": "Video References",
     # === Params ===
@@ -59,6 +68,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "VLLMOmniDiffusionSampling": "Diffusion Sampling Params",
     "VLLMOmniSamplingParamsList": "Multi-Stage Sampling Params List",
     "VLLMOmniRemoteLoRA": "LoRA",
+    "VLLMOmniFastH3Deployment": "FastH3 Deployment",
     "VLLMOmniQwenTTSParams": "Qwen TTS Params",
     "VLLMOmniWanParams": "Wan Video Params",
     "VLLMOmniMiniMaxH3Params": "MiniMax-H3 Video Params",

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 from typing import Literal
 
 import torch
@@ -7,9 +10,13 @@ from .utils.api_client import VLLMOmniClient
 from .utils.logger import get_logger
 from .utils.models import lookup_model_spec
 from .utils.types import (
+    MAX_REFERENCE_AUDIOS,
+    MAX_REFERENCE_IMAGES,
+    MAX_REFERENCE_VIDEOS,
     AudioFormat,
     AutoregressionSamplingParams,
     DiffusionSamplingParams,
+    FastH3Deployment,
     MiniMaxH3ModelSpecificParams,
     QwenTTSModelSpecificParams,
     VideoReferences,
@@ -22,6 +29,23 @@ from .utils.validators import (
 
 logger = get_logger(__name__)
 
+FASTH3_INFERENCE_STEPS = 4
+FASTH3_FPS = 24
+# A FastH3 deployment may expose H3 under any --served-model-name, so the payload
+# spec has to be named outright rather than recovered from that alias.
+FASTH3_SPEC_MODEL = "MiniMax-H3"
+
+
+def _resolve_fast_h3_deployment(deployment: dict) -> tuple[str, str]:
+    if not isinstance(deployment, dict):
+        raise ValueError("FastH3 deployment must be provided by a FastH3 Deployment node.")
+
+    url = str(deployment.get("url") or "").strip().rstrip("/")
+    model = str(deployment.get("model") or "").strip()
+    if not url or not model:
+        raise ValueError("FastH3 deployment requires both URL and model.")
+    return url, model
+
 
 class _VLLMOmniGenerateBase:
     """Base class for vLLM-Omni generation nodes with shared functionality."""
@@ -162,7 +186,20 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
                 "width": ("INT", {"default": 832, "min": 1}),
                 "height": ("INT", {"default": 480, "min": 1}),
                 "fps": ("INT", {"default": 16, "min": 1}),
-                "num_frames": ("INT", {"default": 41, "min": 1}),
+                "duration": (
+                    "FLOAT",
+                    {
+                        "default": 4.0,
+                        "min": 0.1,
+                        "step": 0.1,
+                        "round": 0.001,
+                        "tooltip": (
+                            "Clip length in seconds, converted to frames with the fps above. "
+                            "Models that only accept certain frame counts (e.g. MiniMax-H3) round to "
+                            "their own lattice, so the served clip can be slightly longer than requested."
+                        ),
+                    },
+                ),
             },
             "optional": {
                 "frame": ("IMAGE",),
@@ -170,6 +207,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
                 "sampling_params": ("SAMPLING_PARAMS",),
                 "lora": ("REMOTE_LORA",),
                 "model_params": ("VIDEO_PARAMS",),
+                "fast_h3": ("FASTH3_DEPLOYMENT",),
             },
         }
 
@@ -194,20 +232,66 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
         width: int,
         height: int,
         fps: int,
-        num_frames: int,
+        duration: float,
         negative_prompt: str | None = None,
         frame: torch.Tensor | None = None,
         references: dict | None = None,
         sampling_params: dict | list[dict] | None = None,
         model_params: dict | None = None,
         lora: dict | None = None,
+        fast_h3: dict | None = None,
         **kwargs,
     ):
         if kwargs:
             logger.info("Uncaught kwargs: %s", kwargs)
         logger.debug("Got sampling params: %s", sampling_params)
         logger.debug("Got model params: %s", model_params)
-        validate_model_and_sampling_params_types(model, sampling_params)
+
+        # Which spec builds the payload. Only a FastH3 deployment separates this
+        # from the served name; every other path keeps them equal.
+        spec_model = model
+
+        if fast_h3 is not None:
+            if frame is not None or references is not None:
+                raise ValueError("FastH3 Preview supports T2VA only; disconnect frame and references inputs.")
+            if lora is not None:
+                raise ValueError(
+                    "FastH3 is already fused into the selected server; disconnect the request-level LoRA input."
+                )
+
+            url, model = _resolve_fast_h3_deployment(fast_h3)
+            logger.info("Using FastH3 deployment at %s", url)
+            fps = FASTH3_FPS
+            # The served name is whatever the operator passed to --served-model-name.
+            # Left alone, lookup_model_spec would miss H3 for an alias such as
+            # "fasth3", drop the params builder, and send a t2va request carrying no
+            # aspect_ratio -- which the server refuses.
+            spec_model = FASTH3_SPEC_MODEL
+
+            if sampling_params is None:
+                sampling_params = DiffusionSamplingParams()
+            elif isinstance(sampling_params, list):
+                if len(sampling_params) != 1:
+                    raise ValueError("FastH3 expects a single diffusion sampling params group.")
+                sampling_params = sampling_params[0].__class__(sampling_params[0])
+            else:
+                sampling_params = sampling_params.__class__(sampling_params)
+            sampling_params["num_inference_steps"] = FASTH3_INFERENCE_STEPS
+
+            # FastH3 owns both modality shifts. Sending the ordinary H3 values
+            # from a connected H3 Params node would turn a deployment choice
+            # into a request-level override, which the server intentionally
+            # rejects when it differs from the fused adapter contract.
+            if model_params is not None:
+                model_params = model_params.__class__(model_params)
+                model_params.pop("flow_shift", None)
+                model_params.pop("audio_flow_shift", None)
+
+        # Frames stay the wire unit. Convert after the FastH3 branch above, so the
+        # duration is measured against the fps the server will actually apply.
+        num_frames = max(1, round(duration * fps))
+
+        validate_model_and_sampling_params_types(spec_model, sampling_params)
 
         # Currently, all video generation models are single-stage diffusion models
         if isinstance(sampling_params, list):
@@ -227,6 +311,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
         client = VLLMOmniClient(url)
         output = await client.generate_video(
             model=model,
+            spec_model=spec_model,
             prompt=prompt,
             frame=frame,  # frame present => fl2va / Wan I2V
             references=references,
@@ -418,6 +503,66 @@ class VLLMOmniTTS(_VLLMOmniGenerateBase):
         return (audio,)
 
 
+class VLLMOmniGenerateMusic(_VLLMOmniGenerateBase):
+    """Generate a song from lyrics and a musical description with MiniMax Music 3."""
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "url": ("STRING", {"default": "http://localhost:8000/v1"}),
+                "model": ("STRING", {"default": "MiniMaxAI/MiniMax-Music3"}),
+                "instructions": (
+                    "STRING",
+                    {
+                        "multiline": True,
+                        "display_name": "caption",
+                        "tooltip": "Music caption: describe genre, instruments, tempo and mood.",
+                    },
+                ),
+                "lyrics": ("STRING", {"multiline": True}),
+                "max_duration_seconds": (
+                    "FLOAT",
+                    {
+                        "default": 300.0,
+                        "min": 1,
+                        "max": 360,
+                        "step": 0.01,
+                        "tooltip": "Upper limit; rounded down to whole 25 Hz audio frames. May end earlier.",
+                    },
+                ),
+                "seed": ("INT", {"default": 0, "min": 0, "max": 2**53 - 1, "control_after_generate": True}),
+                "response_format": (["wav", "mp3", "flac", "opus"],),
+            },
+        }
+
+    RETURN_TYPES = ("AUDIO",)
+    RETURN_NAMES = ("audio",)
+    FUNCTION = "generate"
+
+    async def generate(
+        self,
+        url: str,
+        model: str,
+        lyrics: str,
+        instructions: str,
+        response_format: AudioFormat,
+        max_duration_seconds: float,
+        seed: int = 0,
+    ) -> tuple[AudioInput]:
+        audio = await VLLMOmniClient(url.rstrip("/")).generate_speech(
+            model=model,
+            input=lyrics,
+            instructions=instructions,
+            voice="default",
+            speed=1.0,
+            response_format=response_format,
+            max_new_tokens=int(max_duration_seconds * 25),
+            seed=seed,
+        )
+        return (audio,)
+
+
 class VLLMOmniVoiceClone(_VLLMOmniGenerateBase):
     @classmethod
     def INPUT_TYPES(cls):
@@ -695,6 +840,52 @@ class VLLMOmniRemoteLoRA:
         return (lora,)
 
 
+class VLLMOmniFastH3Deployment:
+    """Select a vLLM-Omni service that fused FastH3 at startup."""
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "url": (
+                    "STRING",
+                    {
+                        "default": "http://localhost:8000/v1",
+                        "tooltip": "URL of a vLLM-Omni server started with a FastH3 --lora-path.",
+                    },
+                ),
+                "model": (
+                    "STRING",
+                    {
+                        "default": "MiniMaxAI/MiniMax-H3",
+                        "tooltip": "Model name exposed by the FastH3 deployment.",
+                    },
+                ),
+            }
+        }
+
+    RETURN_TYPES = ("FASTH3_DEPLOYMENT",)
+    RETURN_NAMES = ("deployment",)
+    FUNCTION = "get_deployment"
+    CATEGORY = "vLLM-Omni"
+    DESCRIPTION = (
+        "Selects a server with FastH3 fused at startup. The connected Generate Video node uses T2VA, "
+        "four inference steps, and 24 FPS without sending a request-level LoRA."
+    )
+
+    @classmethod
+    def VALIDATE_INPUTS(cls, url, model) -> str | Literal[True]:
+        try:
+            _resolve_fast_h3_deployment({"url": url, "model": model})
+        except ValueError as exc:
+            return str(exc)
+        return True
+
+    def get_deployment(self, url: str, model: str):
+        url, model = _resolve_fast_h3_deployment({"url": url, "model": model})
+        return (FastH3Deployment({"url": url, "model": model}),)
+
+
 class VLLMOmniQwenTTSParams:
     @classmethod
     def INPUT_TYPES(cls):
@@ -790,6 +981,10 @@ class VLLMOmniVideoReferences:
                 "audio_2": ("AUDIO",),
                 "video_1": ("VIDEO",),
                 "video_2": ("VIDEO",),
+                # Append ports to preserve connections in saved workflows.
+                **{f"image_{i}": ("IMAGE",) for i in range(3, MAX_REFERENCE_IMAGES + 1)},
+                **{f"audio_{i}": ("AUDIO",) for i in range(3, MAX_REFERENCE_AUDIOS + 1)},
+                **{f"video_{i}": ("VIDEO",) for i in range(3, MAX_REFERENCE_VIDEOS + 1)},
             },
         }
 
@@ -806,21 +1001,26 @@ class VLLMOmniVideoReferences:
         audio_2: AudioInput | None = None,
         video_1: VideoInput | None = None,
         video_2: VideoInput | None = None,
+        image_3: torch.Tensor | None = None,
+        image_4: torch.Tensor | None = None,
+        image_5: torch.Tensor | None = None,
+        image_6: torch.Tensor | None = None,
+        image_7: torch.Tensor | None = None,
+        image_8: torch.Tensor | None = None,
+        image_9: torch.Tensor | None = None,
+        audio_3: AudioInput | None = None,
+        video_3: VideoInput | None = None,
         **kwargs,
     ):
         if kwargs:
             logger.info("Uncaught kwargs: %s", kwargs)
         refs = VideoReferences()
-        if image_1 is not None:
-            refs["image_1"] = image_1
-        if image_2 is not None:
-            refs["image_2"] = image_2
-        if audio_1 is not None:
-            refs["audio_1"] = audio_1
-        if audio_2 is not None:
-            refs["audio_2"] = audio_2
-        if video_1 is not None:
-            refs["video_1"] = video_1
-        if video_2 is not None:
-            refs["video_2"] = video_2
+        for kind, values in (
+            ("image", (image_1, image_2, image_3, image_4, image_5, image_6, image_7, image_8, image_9)),
+            ("video", (video_1, video_2, video_3)),
+            ("audio", (audio_1, audio_2, audio_3)),
+        ):
+            for index, value in enumerate(values, start=1):
+                if value is not None:
+                    refs[f"{kind}_{index}"] = value
         return (refs,)

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 """
 An high-level API client adapter that forwards ComfyUI inputs to vLLM-Omni's REST API,
 and transforms the API responses back to ComfyUI formats.
@@ -17,6 +20,7 @@ from comfy_api.input import AudioInput, VideoInput
 
 from .format import (
     audio_to_base64,
+    audio_to_bytes,
     base64_to_audio,
     base64_to_image_tensor,
     bytes_to_audio,
@@ -28,7 +32,13 @@ from .format import (
 )
 from .logger import get_logger, pretty_printer
 from .models import lookup_model_spec
-from .types import AudioFormat
+from .types import (
+    MAX_REFERENCE_AUDIOS,
+    MAX_REFERENCE_IMAGES,
+    MAX_REFERENCE_VIDEOS,
+    MAX_TOTAL_REFERENCES,
+    AudioFormat,
+)
 
 logger = get_logger(__name__)
 
@@ -271,8 +281,15 @@ class VLLMOmniClient:
         sampling_params: dict | None = None,
         model_params: dict | None = None,
         lora: dict | None = None,
+        spec_model: str | None = None,
         **extra_params,
     ) -> VideoInput:
+        """Post a video job and return the decoded result.
+
+        ``spec_model`` names the model whose payload spec builds the request, for
+        deployments that serve a known model under a different ``model`` alias. It
+        never reaches the wire; defaults to ``model``.
+        """
         if frame is not None and references is not None:
             raise ValueError("Provide only one of frame or references, not both.")
 
@@ -294,38 +311,44 @@ class VLLMOmniClient:
 
         # === multimodal inputs (first-last-frames, references, etc.) ===
         input_reference_image: torch.Tensor | None = None
-        audio_reference: AudioInput | None = None
-        reference_videos: list[VideoInput] = []
         video_task: str | None = None
 
         if frame is not None:
             input_reference_image = frame
             video_task = "fl2va"
         elif references is not None:
-            images = [references[k] for k in ("image_1", "image_2") if k in references and references[k] is not None]
-            audios = [references[k] for k in ("audio_1", "audio_2") if k in references and references[k] is not None]
-            videos = [references[k] for k in ("video_1", "video_2") if k in references and references[k] is not None]
-
-            if not images and not audios and not videos:
-                raise ValueError("references is empty; connect at least one image/audio/video.")
-
-            if videos:
-                if images or audios:
-                    raise ValueError(
-                        "Video references cannot be combined with image or audio references. "
-                        "Connect only video_1/video_2 for multi-video Ref2VA."
-                    )
-                reference_videos = videos
-                video_task = "ref2va"
-            elif len(images) == 1 and len(audios) == 1:
-                input_reference_image = images[0]
-                audio_reference = audios[0]
-                video_task = "ref2va"
-            else:
+            reference_formats = (
+                ("image", MAX_REFERENCE_IMAGES, "png", "image/png", image_tensor_to_png_bytes),
+                ("video", MAX_REFERENCE_VIDEOS, "mp4", "video/mp4", video_to_bytes),
+                ("audio", MAX_REFERENCE_AUDIOS, "mp3", "audio/mpeg", audio_to_bytes),
+            )
+            supported_inputs = {f"{kind}_{i}" for kind, limit, *_ in reference_formats for i in range(1, limit + 1)}
+            connected = {name: value for name, value in references.items() if value is not None}
+            unsupported = connected.keys() - supported_inputs
+            if unsupported:
+                raise ValueError(f"Unsupported reference input(s): {', '.join(sorted(unsupported))}.")
+            if not any(name.startswith(("image_", "video_")) for name in connected):
                 raise ValueError(
-                    "Invalid references combination. Supported modes: "
-                    "(1) one or more videos only, or (2) exactly one image and one audio."
+                    "references requires at least one image or video; audio-only inputs are not supported."
                 )
+            if len(connected) > MAX_TOTAL_REFERENCES:
+                raise ValueError(
+                    f"references supports at most {MAX_TOTAL_REFERENCES} inputs in total "
+                    f"(up to {MAX_REFERENCE_IMAGES} images, {MAX_REFERENCE_VIDEOS} videos, "
+                    f"and {MAX_REFERENCE_AUDIOS} audios)."
+                )
+            for kind, limit, extension, content_type, encode in reference_formats:
+                for index in range(1, limit + 1):
+                    name = f"{kind}_{index}"
+                    if name in connected:
+                        filename = f"{name}.{extension}"
+                        form.add_field(
+                            "input_references",
+                            encode(connected[name], filename),
+                            filename=filename,
+                            content_type=content_type,
+                        )
+            video_task = "ref2va"
         else:
             video_task = "t2va"
 
@@ -338,32 +361,19 @@ class VLLMOmniClient:
                 content_type="image/png",
             )
 
-        if audio_reference is not None:
-            form.add_field(
-                "audio_reference",
-                json.dumps({"audio_url": audio_to_base64(audio_reference)}, ensure_ascii=False),
-            )
-
-        for idx, video in enumerate(reference_videos, start=1):
-            video_filename = f"reference_{idx}.mp4"
-            form.add_field(
-                "input_references",
-                video_to_bytes(video, video_filename),
-                filename=video_filename,
-                content_type="video/mp4",
-            )
-
         # === model specific params. Either use a specialized builder, or add flattened fields as-is ===
         if model_params is not None:
             model_params = dict(model_params)
             model_params.pop("type", None)
 
-        spec, _ = lookup_model_spec(model)
+        spec, _ = lookup_model_spec(spec_model or model)
         params_builder = spec.get("params_builder") if spec else None
         if params_builder is not None:
             form_fields = params_builder(
                 model_params or {},
                 extra_params={**extra_params, "task": video_task},
+                width=width,
+                height=height,
             )
             for k, v in form_fields.items():
                 form.add_field(k, v if isinstance(v, str) else str(v))

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/format.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 """Image/tensor format helpers.
 
 The image generation part is derived from dougbtv/comfyui-vllm-omni by Doug (@dougbtv).
@@ -170,6 +173,10 @@ def bytes_to_video(video_bytes: bytes) -> VideoInput:
                 audio_stream = container.streams.audio[-1]
                 audio_frames = []
                 resampler = AudioResampler(format="fltp")
+                # Decoding the video stream above left the container at EOF.
+                # Rewind before demuxing again, or this pass yields no frames
+                # and the audio of a T2VA result is silently dropped.
+                container.seek(0)
                 for frame in container.decode(audio_stream):
                     if not isinstance(frame, AudioFrame):
                         continue

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 import json
 import re
 from typing import Any
@@ -34,10 +37,28 @@ def _qwen25_payload_preprocessor(payload: dict) -> dict:
     return payload
 
 
+MINIMAX_H3_ASPECT_RATIOS = {
+    "21:9": 21.0 / 9.0,
+    "16:9": 16.0 / 9.0,
+    "4:3": 4.0 / 3.0,
+    "1:1": 1.0,
+    "3:4": 3.0 / 4.0,
+    "9:16": 9.0 / 16.0,
+}
+
+
+def _nearest_minimaxh3_aspect_ratio(width: int, height: int) -> str:
+    """Pick the supported named ratio closest to the requested frame size."""
+    target = float(width) / float(height)
+    return min(MINIMAX_H3_ASPECT_RATIOS, key=lambda name: abs(MINIMAX_H3_ASPECT_RATIOS[name] - target))
+
+
 def _minimaxh3_params_builder(
     model_params: dict[str, Any],
     *,
     extra_params: dict[str, Any],
+    width: int | None = None,
+    height: int | None = None,
 ) -> dict[str, Any]:
     """Build multipart form fields for MiniMax-H3 from model_params + routed task."""
     params = dict(model_params)
@@ -53,6 +74,12 @@ def _minimaxh3_params_builder(
     if params:
         logger.warning("Unused MiniMax-H3 model params ignored: %s", sorted(params))
 
+    # H3 refuses t2va without a named aspect ratio, and the Generate Video node
+    # carries width/height rather than a ratio. fl2va takes its ratio from the
+    # input image and ref2va defaults server-side, so only t2va needs this.
+    if merged_extra_params.get("task") == "t2va" and "aspect_ratio" not in merged_extra_params and width and height:
+        merged_extra_params["aspect_ratio"] = _nearest_minimaxh3_aspect_ratio(width, height)
+
     if merged_extra_params:
         form_fields["extra_params"] = json.dumps(merged_extra_params, ensure_ascii=False)
     return form_fields

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 from collections.abc import Callable
 from enum import Enum, auto
 from typing import Any, Literal, TypeAlias
@@ -27,10 +30,20 @@ class MiniMaxH3ModelSpecificParams(dict):
     pass
 
 
+MAX_REFERENCE_IMAGES = 9
+MAX_REFERENCE_VIDEOS = 3
+MAX_REFERENCE_AUDIOS = 3
+MAX_TOTAL_REFERENCES = 12
+
+
 class VideoReferences(dict):
     pass
 
 
+class FastH3Deployment(dict):
+    """Descriptor for a server that fused FastH3 at startup."""
+
+
 class ModelMode(Enum):
     IMAGE_GENERATION = auto()
     VIDEO_GENERATION = auto()

apps/ComfyUI-vLLM-Omni/docs/minimax-h3-t2v.md

@@ -0,0 +1,94 @@
+# MiniMax H3 Text to Video (WF-01)
+
+Import [MiniMax_H3_Text_to_Video.json](../example_workflows/MiniMax_H3_Text_to_Video.json)
+into ComfyUI, or select it under **Templates → ComfyUI-vLLM-Omni**.
+The template connects existing nodes to a remote H3 service and saves the
+returned video with audio. ComfyUI does not load H3 weights.
+
+## Setup
+
+Install the [ComfyUI extension](../README.md#installation) and follow the
+[H3 deployment recipe](../../../recipes/MiniMaxAI/MiniMax-H3.md) for your hardware.
+For a two-GPU FL2VA server with sufficient memory:
+
+```bash
+export MODEL=/path/to/MiniMax-H3/FL2VA
+CUDA_VISIBLE_DEVICES=0,1 VLLM_WORKER_MULTIPROC_METHOD=spawn \
+VLLM_OMNI_VIDEO_SYNC_TIMEOUT=14400 \
+vllm serve "$MODEL" --omni --host 127.0.0.1 --port 8000 \
+  --served-model-name MiniMaxAI/MiniMax-H3 --trust-remote-code \
+  --task-type fl2va --num-gpus 2 --tensor-parallel-size 2 \
+  --text-encoder-tp-size 2 --enforce-eager
+```
+
+Set Generate Video's **url** to the address reachable from ComfyUI, including
+`/v1`, and match its **model** to the served name. The defaults are
+`http://localhost:8000/v1` and `MiniMaxAI/MiniMax-H3`.
+
+## Base and Turbo
+
+Base presets are connected by default. Write the scene and its sound in the
+same **prompt**; leave **frame**, **references**, and **fast_h3** disconnected.
+The template uses 1344×768, 24 FPS, seed 1101 and **duration** 5.167 seconds
+(124 frames, satisfying H3's `17k+5` constraint). The existing client infers
+`16:9` from these dimensions. For portrait output, use 768×1344.
+
+| Setting | Base | Turbo v1.0 768p |
+| --- | --- | --- |
+| Inference steps | 50 | 5 (4 forwards) |
+| Video flow shift | 12 | 6 |
+| Audio flow shift | 3 | 3 |
+| Remote LoRA | Disconnected | Connected, scale 1 |
+
+For Turbo, use the **Diffusers-layout** artifact from
+[lightx2v/Minimax-h3-Turbo](https://huggingface.co/lightx2v/Minimax-h3-Turbo):
+
+```text
+minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors
+```
+
+Add `--lora-backend peft --lora-path "$TURBO_LORA"` to the server command,
+where `TURBO_LORA` is the full server-side path to that file. Enter the same
+path in Remote LoRA's **local_path**; retain name `h3-turbo-v1.0-768p`, scale
+`1.0`, and **int_id** `0` so the existing API derives the ID from the path.
+The template leaves the path blank for users to configure. The `_comfyui_`
+export is not supported by this backend.
+
+Connect **Turbo sampling**, **Turbo H3 params**, and **Remote LoRA** to Generate
+Video's `sampling_params`, `model_params`, and `lora` inputs, replacing the
+Base links. To return to Base, restore both Base links and disconnect LoRA.
+Other Turbo artifacts may need different settings; see the
+[artifact table](../../../recipes/MiniMaxAI/MiniMax-H3.md#turbo-lora).
+
+If importing an older template with a frame-count widget, reimport this JSON
+or set **duration** to **5.167**; do not interpret 124 frames as 124 seconds.
+
+## Validation
+
+Queue the configured workflow and inspect the MP4 saved under
+`ComfyUI/output/video/MiniMax_H3_WF01_*.mp4`:
+
+```bash
+ffprobe -v error -show_streams -show_format -of json "$OUTPUT"
+ffmpeg -v error -i "$OUTPUT" -f null -
+```
+
+Check the requested dimensions, frame count, 24 FPS and both video/audio
+streams, then watch and listen to the saved file. The current upstream
+extension includes the shared audio-preserving decoder fix.
+
+On September 16, the narrowed workflow was validated through ComfyUI using
+existing upstream Python code at `4a8ac297`, two H20-3e GPUs, vLLM 0.29.0,
+PyTorch 2.13.0+cu129 and ComfyUI `1d48d9cf`. The run used the template's
+forest-stream prompt, explicit server-side Turbo LoRA path, and settings
+above, with `TORCH_SDPA`, `--usp 1 --ring 1`, and tiled VAE parallelism on two
+GPUs. ComfyUI execution took 131.55 seconds. SaveVideo produced 124 H.264
+frames at 1344×768 and 24 FPS, plus 32 kHz stereo AAC audio. Video/audio
+durations were 5.166667/5.167000 seconds; full decoding passed and audio was
+nonzero. This is model-generated output, not a ground-truth reference or a
+subjective audio-quality assessment. Base graph validation passed with Turbo
+nodes disconnected; full 50-step Base quality was not evaluated.
+
+Based on the official
+[ComfyUI H3 T2V template](https://github.com/Comfy-Org/workflow_templates/blob/main/templates/video_minimax_h3_t2v.json),
+using remote inference instead of local model loaders.

apps/ComfyUI-vLLM-Omni/docs/wf07-h3-upscale.md

@@ -0,0 +1,21 @@
+# WF-07: H3 video upscale with SeedVR2
+
+Generate video and audio through vLLM-Omni H3, upscale the decoded frames with SeedVR2, and save both the original and upscaled videos. The generated audio and FPS connect directly to Create Video.
+
+## Requirements
+
+- ComfyUI with ComfyUI-vLLM-Omni and [ComfyUI-SeedVR2_VideoUpscaler](https://github.com/numz/ComfyUI-SeedVR2_VideoUpscaler#-installation) installed.
+- An H3 service using the FL2VA checkpoint partition. Follow the [H3 recipe](../../../recipes/MiniMaxAI/MiniMax-H3.md) for server setup.
+- `seedvr2_ema_3b_fp16.safetensors` and `ema_vae_fp16.safetensors` from [SeedVR2's ComfyUI weights](https://huggingface.co/numz/SeedVR2_comfyUI/tree/main), placed in `ComfyUI/models/SEEDVR2`. The SeedVR2 nodes can also download them on first use.
+
+## Use
+
+Open **vLLM-Omni MiniMax H3 Video Upscale.json** from the example workflows. An API-format version is provided alongside it. Set the Generate Video URL/model and the LoRA path as seen by the H3 server.
+
+The preset uses the LightX2V v1.0 768p Turbo adapter in Diffusers format: `minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors`, five sigma points, video/audio flow shifts 6/3, and LoRA scale 1. Preload the same adapter on the server using the [Turbo LoRA recipe](../../../recipes/MiniMaxAI/MiniMax-H3.md#turbo-lora). For base H3, disconnect LoRA, set inference steps to 50, and set video flow shift to 12.
+
+The H3 preset requests 1344 x 768 at 24 FPS for 5.167 seconds, which gives 124 frames. SeedVR2's `resolution=1536` sets the target short edge, producing 2688 x 1536 for this input. Update this value when changing the input size; it is an output resolution, not a fixed 2x multiplier.
+
+The SeedVR2 preset uses 3B FP16, five-frame batches, one overlapping frame, uniform batch padding, and LAB color correction. The DiT loader uses CPU offload with 32 swapped blocks; the VAE uses tiled encoding and decoding. Select the local GPU in both model loaders. See the [SeedVR2 settings](https://github.com/numz/ComfyUI-SeedVR2_VideoUpscaler#-usage) when adjusting memory use or batch size.
+
+Run the workflow. The two Save Video nodes write MP4s under `WF07/generated` and `WF07/upscaled` in ComfyUI's output directory.

apps/ComfyUI-vLLM-Omni/example_workflows/MiniMax_H3_Text_to_Video.json

@@ -0,0 +1,362 @@
+{
+  "last_node_id": 8,
+  "last_link_id": 3,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "VLLMOmniDiffusionSampling",
+      "pos": [
+        20,
+        20
+      ],
+      "size": [
+        310,
+        280
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "diffusion sampling params",
+          "type": "SAMPLING_PARAMS",
+          "links": [
+            1
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniDiffusionSampling"
+      },
+      "widgets_values": [
+        1,
+        50,
+        1.0,
+        1.0,
+        false,
+        false,
+        1101,
+        "fixed"
+      ],
+      "title": "Base sampling"
+    },
+    {
+      "id": 2,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        20,
+        340
+      ],
+      "size": [
+        310,
+        140
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "MiniMax-H3 video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            2
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ],
+      "title": "Base H3 params"
+    },
+    {
+      "id": 3,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        430,
+        20
+      ],
+      "size": [
+        430,
+        440
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null
+        },
+        {
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null
+        },
+        {
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": 1
+        },
+        {
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 2
+        },
+        {
+          "name": "fast_h3",
+          "type": "FASTH3_DEPLOYMENT",
+          "link": null
+        },
+        {
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            3
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://localhost:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "A small stream flows through a sunlit forest. The camera slowly moves toward the water. Birds chirp in the trees and water gurgles over smooth stones. Natural continuous sound, no music.",
+        "",
+        1344,
+        768,
+        24,
+        5.167
+      ]
+    },
+    {
+      "id": 4,
+      "type": "SaveVideo",
+      "pos": [
+        940,
+        20
+      ],
+      "size": [
+        440,
+        440
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "link": 3
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": null,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "video/MiniMax_H3_WF01",
+        "mp4",
+        "auto"
+      ]
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniRemoteLoRA",
+      "pos": [
+        430,
+        540
+      ],
+      "size": [
+        430,
+        160
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "links": null,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniRemoteLoRA"
+      },
+      "widgets_values": [
+        "",
+        "h3-turbo-v1.0-768p",
+        1.0,
+        0
+      ],
+      "title": "Optional Turbo: server LoRA path"
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniDiffusionSampling",
+      "pos": [
+        20,
+        540
+      ],
+      "size": [
+        310,
+        280
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "diffusion sampling params",
+          "type": "SAMPLING_PARAMS",
+          "links": null,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniDiffusionSampling"
+      },
+      "widgets_values": [
+        1,
+        5,
+        1.0,
+        1.0,
+        false,
+        false,
+        1101,
+        "fixed"
+      ],
+      "title": "Turbo sampling (4 forwards)"
+    },
+    {
+      "id": 7,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        430,
+        740
+      ],
+      "size": [
+        310,
+        140
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "MiniMax-H3 video params",
+          "type": "VIDEO_PARAMS",
+          "links": null,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        6.0
+      ],
+      "title": "Turbo H3 params"
+    },
+    {
+      "id": 8,
+      "type": "Note",
+      "pos": [
+        940,
+        520
+      ],
+      "size": [
+        440,
+        440
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "properties": {
+        "Node name for S&R": "Note"
+      },
+      "widgets_values": [
+        "MiniMax H3 Text to Video (WF-01)\n\nBASE: connected presets, 50 steps / video flow 12 / audio flow 3.\n\nTURBO: set Remote LoRA local_path to the SERVER path of minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors from lightx2v/Minimax-h3-Turbo. Keep name h3-turbo-v1.0-768p, scale 1.0 and int_id 0. Connect Turbo sampling, Turbo H3 params, and LoRA to Generate Video, replacing the two Base links. Use 5 steps (4 forwards), video flow 6 and audio flow 3. Do not use the ComfyUI-layout artifact.\n\nKeep frame, references and fast_h3 disconnected. 1344x768, 24 FPS, duration 5.167 seconds = 124 frames (17*7+5). The client selects the named 16:9 preset from the dimensions. Save Video preserves video and audio.\n\nSet URL and served model name for your server. See docs/minimax-h3-t2v.md for setup and validation. Source: Comfy-Org/workflow_templates/templates/video_minimax_h3_t2v.json."
+      ],
+      "title": "Setup / Base and Turbo"
+    }
+  ],
+  "links": [
+    [
+      1,
+      1,
+      0,
+      3,
+      2,
+      "SAMPLING_PARAMS"
+    ],
+    [
+      2,
+      2,
+      0,
+      3,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      3,
+      3,
+      0,
+      4,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 0.8,
+      "offset": [
+        50,
+        50
+      ]
+    }
+  },
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni FastH3 Text to Video.json

@@ -0,0 +1,322 @@
+{
+  "id": "c6db00da-2ccb-4ff8-8334-d67e1a267781",
+  "revision": 0,
+  "last_node_id": 5,
+  "last_link_id": 3,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "MarkdownNote",
+      "pos": [40, 20],
+      "size": [420, 250],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "FastH3 server prerequisite",
+      "properties": {},
+      "widgets_values": [
+        "FastH3 must be fused when the vLLM-Omni server starts. Use --task-type fl2va and --lora-path pointing to exactly one FastH3 adapter file. Do not use CPU or layerwise offload. For VSA / Data-Free, also install fastvideo-kernel and use --diffusion-attention-backend FASTVIDEO_VSA --fastvideo-vsa-topk 64."
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 2,
+      "type": "VLLMOmniFastH3Deployment",
+      "pos": [40, 320],
+      "size": [420, 160],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {"name": "url"},
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {"name": "model"},
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "deployment",
+          "name": "deployment",
+          "type": "FASTH3_DEPLOYMENT",
+          "links": [1]
+        }
+      ],
+      "properties": {"Node name for S&R": "VLLMOmniFastH3Deployment"},
+      "widgets_values": [
+        "http://localhost:8000/v1",
+        "MiniMaxAI/MiniMax-H3"
+      ]
+    },
+    {
+      "id": 3,
+      "type": "VLLMOmniDiffusionSampling",
+      "pos": [40, 540],
+      "size": [420, 230],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "n",
+          "name": "n",
+          "type": "INT",
+          "widget": {"name": "n"},
+          "link": null
+        },
+        {
+          "localized_name": "num_inference_steps",
+          "name": "num_inference_steps",
+          "type": "INT",
+          "widget": {"name": "num_inference_steps"},
+          "link": null
+        },
+        {
+          "localized_name": "guidance_scale",
+          "name": "guidance_scale",
+          "type": "FLOAT",
+          "widget": {"name": "guidance_scale"},
+          "link": null
+        },
+        {
+          "localized_name": "true_cfg_scale",
+          "name": "true_cfg_scale",
+          "type": "FLOAT",
+          "widget": {"name": "true_cfg_scale"},
+          "link": null
+        },
+        {
+          "localized_name": "vae_use_slicing",
+          "name": "vae_use_slicing",
+          "type": "BOOLEAN",
+          "widget": {"name": "vae_use_slicing"},
+          "link": null
+        },
+        {
+          "localized_name": "vae_use_tiling",
+          "name": "vae_use_tiling",
+          "type": "BOOLEAN",
+          "widget": {"name": "vae_use_tiling"},
+          "link": null
+        },
+        {
+          "localized_name": "seed",
+          "name": "seed",
+          "type": "INT",
+          "widget": {"name": "seed"},
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "diffusion sampling params",
+          "name": "diffusion sampling params",
+          "type": "SAMPLING_PARAMS",
+          "links": [2]
+        }
+      ],
+      "properties": {"Node name for S&R": "VLLMOmniDiffusionSampling"},
+      "widgets_values": [1, 4, 1, 1, false, false, 1101, "fixed"]
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [560, 210],
+      "size": [540, 610],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "frame",
+          "name": "frame",
+          "shape": 7,
+          "type": "IMAGE",
+          "link": null
+        },
+        {
+          "localized_name": "references",
+          "name": "references",
+          "shape": 7,
+          "type": "VIDEO_REFERENCES",
+          "link": null
+        },
+        {
+          "localized_name": "sampling_params",
+          "name": "sampling_params",
+          "shape": 7,
+          "type": "SAMPLING_PARAMS",
+          "link": 2
+        },
+        {
+          "localized_name": "lora",
+          "name": "lora",
+          "shape": 7,
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "localized_name": "model_params",
+          "name": "model_params",
+          "shape": 7,
+          "type": "VIDEO_PARAMS",
+          "link": null
+        },
+        {
+          "localized_name": "fast_h3",
+          "name": "fast_h3",
+          "shape": 7,
+          "type": "FASTH3_DEPLOYMENT",
+          "link": 1
+        },
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {"name": "url"},
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {"name": "model"},
+          "link": null
+        },
+        {
+          "localized_name": "prompt",
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {"name": "prompt"},
+          "link": null
+        },
+        {
+          "localized_name": "negative_prompt",
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {"name": "negative_prompt"},
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {"name": "width"},
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {"name": "height"},
+          "link": null
+        },
+        {
+          "localized_name": "fps",
+          "name": "fps",
+          "type": "INT",
+          "widget": {"name": "fps"},
+          "link": null
+        },
+        {
+          "localized_name": "duration",
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {"name": "duration"},
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [3]
+        }
+      ],
+      "properties": {"Node name for S&R": "VLLMOmniGenerateVideo"},
+      "widgets_values": [
+        "http://localhost:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "A cinematic live-action singer performs naturally on a neon-lit stage, with synchronized vocals and environmental sound.",
+        "",
+        1344,
+        768,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 5,
+      "type": "SaveVideo",
+      "pos": [1200, 280],
+      "size": [380, 310],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "link": 3
+        },
+        {
+          "localized_name": "filename_prefix",
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {"name": "filename_prefix"},
+          "link": null
+        },
+        {
+          "localized_name": "format",
+          "name": "format",
+          "type": "COMBO",
+          "widget": {"name": "format"},
+          "link": null
+        },
+        {
+          "localized_name": "codec",
+          "name": "codec",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {"name": "codec"},
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": null
+        }
+      ],
+      "properties": {},
+      "widgets_values": ["video/ComfyUI-FastH3", "auto", "auto"]
+    }
+  ],
+  "links": [
+    [1, 2, 0, 4, 5, "FASTH3_DEPLOYMENT"],
+    [2, 3, 0, 4, 2, "SAMPLING_PARAMS"],
+    [3, 4, 0, 5, 0, "VIDEO"]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "workflowRendererVersion": "LG",
+    "ds": {"scale": 0.9, "offset": [100, 100]}
+  },
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax H3 Video Upscale.json

@@ -0,0 +1,724 @@
+{
+  "version": 0.4,
+  "last_node_id": 12,
+  "last_link_id": 13,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        360,
+        20
+      ],
+      "size": [
+        330,
+        480
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "frame",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "shape": 7,
+          "link": 11
+        },
+        {
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "shape": 7,
+          "link": 12
+        },
+        {
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "shape": 7,
+          "link": 10
+        },
+        {
+          "name": "fast_h3",
+          "type": "FASTH3_DEPLOYMENT",
+          "shape": 7,
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1,
+            9
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8091/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "A small brass bell hanging from a wooden beam in a quiet garden. A gentle breeze swings the bell and it rings clearly. Static camera, natural daylight, continuous motion, synchronized ringing and soft birdsong.",
+        "",
+        1344,
+        768,
+        24,
+        5.167
+      ]
+    },
+    {
+      "id": 2,
+      "type": "GetVideoComponents",
+      "pos": [
+        730,
+        20
+      ],
+      "size": [
+        250,
+        125
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "link": 1
+        }
+      ],
+      "outputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "links": [
+            2
+          ]
+        },
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "links": [
+            6
+          ]
+        },
+        {
+          "name": "fps",
+          "type": "FLOAT",
+          "links": [
+            5
+          ]
+        },
+        {
+          "name": "bit_depth",
+          "type": "INT",
+          "links": [
+            7
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "GetVideoComponents"
+      }
+    },
+    {
+      "id": 3,
+      "type": "SeedVR2LoadDiTModel",
+      "pos": [
+        730,
+        290
+      ],
+      "size": [
+        340,
+        245
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "torch_compile_args",
+          "name": "torch_compile_args",
+          "shape": 7,
+          "type": "TORCH_COMPILE_ARGS",
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "SEEDVR2_DIT",
+          "name": "SEEDVR2_DIT",
+          "type": "SEEDVR2_DIT",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SeedVR2LoadDiTModel"
+      },
+      "widgets_values": [
+        "seedvr2_ema_3b_fp16.safetensors",
+        "cuda:0",
+        32,
+        false,
+        "cpu",
+        false,
+        "sdpa"
+      ]
+    },
+    {
+      "id": 4,
+      "type": "SeedVR2VideoUpscaler",
+      "pos": [
+        1120,
+        290
+      ],
+      "size": [
+        340,
+        415
+      ],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "image",
+          "name": "image",
+          "type": "IMAGE",
+          "link": 2
+        },
+        {
+          "localized_name": "dit",
+          "name": "dit",
+          "type": "SEEDVR2_DIT",
+          "link": 3
+        },
+        {
+          "localized_name": "vae",
+          "name": "vae",
+          "type": "SEEDVR2_VAE",
+          "link": 13
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "IMAGE",
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SeedVR2VideoUpscaler"
+      },
+      "widgets_values": [
+        42,
+        "fixed",
+        1536,
+        0,
+        5,
+        true,
+        "lab",
+        1,
+        0,
+        0.0,
+        0.0,
+        "cpu",
+        false
+      ]
+    },
+    {
+      "id": 5,
+      "type": "CreateVideo",
+      "pos": [
+        1490,
+        290
+      ],
+      "size": [
+        250,
+        175
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 4
+        },
+        {
+          "name": "fps",
+          "type": "FLOAT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": 5
+        },
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "shape": 7,
+          "link": 6
+        },
+        {
+          "name": "bit_depth",
+          "type": "INT",
+          "widget": {
+            "name": "bit_depth"
+          },
+          "link": 7
+        }
+      ],
+      "outputs": [
+        {
+          "name": "VIDEO",
+          "type": "VIDEO",
+          "links": [
+            8
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "CreateVideo"
+      },
+      "widgets_values": [
+        24,
+        8
+      ]
+    },
+    {
+      "id": 6,
+      "type": "SaveVideo",
+      "pos": [
+        1790,
+        290
+      ],
+      "size": [
+        330,
+        280
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "link": 8
+        },
+        {
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "name": "format",
+          "type": "COMBO",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        },
+        {
+          "name": "codec",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "codec"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "WF07/upscaled",
+        "mp4",
+        "auto"
+      ]
+    },
+    {
+      "id": 7,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        0,
+        20
+      ],
+      "size": [
+        320,
+        120
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "MiniMax-H3 video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            10
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        6.0
+      ]
+    },
+    {
+      "id": 8,
+      "type": "VLLMOmniDiffusionSampling",
+      "pos": [
+        0,
+        200
+      ],
+      "size": [
+        320,
+        280
+      ],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "diffusion sampling params",
+          "type": "SAMPLING_PARAMS",
+          "links": [
+            11
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniDiffusionSampling"
+      },
+      "widgets_values": [
+        1,
+        5,
+        1.0,
+        1.0,
+        false,
+        true,
+        42,
+        "fixed"
+      ]
+    },
+    {
+      "id": 9,
+      "type": "VLLMOmniRemoteLoRA",
+      "pos": [
+        0,
+        530
+      ],
+      "size": [
+        320,
+        210
+      ],
+      "flags": {},
+      "order": 9,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "links": [
+            12
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniRemoteLoRA"
+      },
+      "widgets_values": [
+        "models/minimax-h3-turbo/minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors",
+        "h3-turbo-v1.0",
+        1.0,
+        0
+      ]
+    },
+    {
+      "id": 10,
+      "type": "SaveVideo",
+      "pos": [
+        1120,
+        20
+      ],
+      "size": [
+        330,
+        190
+      ],
+      "flags": {},
+      "order": 10,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "link": 9
+        },
+        {
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "name": "format",
+          "type": "COMBO",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        },
+        {
+          "name": "codec",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "codec"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "WF07/generated",
+        "mp4",
+        "auto"
+      ],
+      "title": "Save generated video before upscale"
+    },
+    {
+      "id": 11,
+      "type": "Note",
+      "pos": [
+        360,
+        560
+      ],
+      "size": [
+        330,
+        260
+      ],
+      "flags": {},
+      "order": 11,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "properties": {
+        "Node name for S&R": "Note"
+      },
+      "widgets_values": [
+        "WF-07: H3 + SeedVR2\n\nInstall ComfyUI-SeedVR2_VideoUpscaler. The DiT and VAE loaders use SeedVR2 3B FP16 and ema_vae_fp16.\n\nSet the H3 service URL/model and the server-side Turbo LoRA path.\n\n1536 is the target short edge for this 1344 x 768 preset (2x). Original audio and FPS connect directly to Create Video.\n\nFor base H3: disconnect LoRA, use 50 steps and video flow shift 12."
+      ]
+    },
+    {
+      "id": 12,
+      "type": "SeedVR2LoadVAEModel",
+      "pos": [
+        730,
+        590
+      ],
+      "size": [
+        340,
+        315
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "torch_compile_args",
+          "name": "torch_compile_args",
+          "shape": 7,
+          "type": "TORCH_COMPILE_ARGS",
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "SEEDVR2_VAE",
+          "name": "SEEDVR2_VAE",
+          "type": "SEEDVR2_VAE",
+          "links": [
+            13
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SeedVR2LoadVAEModel"
+      },
+      "widgets_values": [
+        "ema_vae_fp16.safetensors",
+        "cuda:0",
+        true,
+        1024,
+        128,
+        true,
+        768,
+        128,
+        "false",
+        "cpu",
+        false
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      1,
+      0,
+      2,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      2,
+      0,
+      4,
+      0,
+      "IMAGE"
+    ],
+    [
+      3,
+      3,
+      0,
+      4,
+      1,
+      "SEEDVR2_DIT"
+    ],
+    [
+      4,
+      4,
+      0,
+      5,
+      0,
+      "IMAGE"
+    ],
+    [
+      5,
+      2,
+      2,
+      5,
+      1,
+      "FLOAT"
+    ],
+    [
+      6,
+      2,
+      1,
+      5,
+      2,
+      "AUDIO"
+    ],
+    [
+      7,
+      2,
+      3,
+      5,
+      3,
+      "INT"
+    ],
+    [
+      8,
+      5,
+      0,
+      6,
+      0,
+      "VIDEO"
+    ],
+    [
+      9,
+      1,
+      0,
+      10,
+      0,
+      "VIDEO"
+    ],
+    [
+      10,
+      7,
+      0,
+      1,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      11,
+      8,
+      0,
+      1,
+      2,
+      "SAMPLING_PARAMS"
+    ],
+    [
+      12,
+      9,
+      0,
+      1,
+      3,
+      "REMOTE_LORA"
+    ],
+    [
+      13,
+      12,
+      0,
+      4,
+      2,
+      "SEEDVR2_VAE"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 0.52,
+      "offset": [
+        100,
+        100
+      ]
+    }
+  }
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Reference to Video.json

@@ -0,0 +1,786 @@
+{
+  "id": "e7a4e606-6510-4c4e-b127-9a8b6bf3ea71",
+  "revision": 0,
+  "last_node_id": 11,
+  "last_link_id": 5,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "LoadImage",
+      "pos": [
+        40,
+        100
+      ],
+      "size": [
+        320,
+        300
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "image",
+          "name": "image",
+          "type": "COMBO",
+          "widget": {
+            "name": "image"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "choose file to upload",
+          "name": "upload",
+          "type": "IMAGEUPLOAD",
+          "widget": {
+            "name": "upload"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "IMAGE",
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [
+            1
+          ]
+        },
+        {
+          "localized_name": "MASK",
+          "name": "MASK",
+          "type": "MASK",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadImage"
+      },
+      "widgets_values": [
+        "h3_reference.png",
+        "image"
+      ],
+      "title": "Reference image \u2014 select a local file"
+    },
+    {
+      "type": "LoadVideo",
+      "size": [
+        320,
+        200
+      ],
+      "flags": {},
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "VIDEO",
+          "type": "VIDEO",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        "h3_reference.mp4"
+      ],
+      "id": 2,
+      "pos": [
+        40,
+        470
+      ],
+      "order": 1,
+      "mode": 0,
+      "title": "Optional reference video"
+    },
+    {
+      "id": 3,
+      "type": "LoadAudio",
+      "pos": [
+        40,
+        1100
+      ],
+      "size": [
+        320,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "audio",
+          "name": "audio",
+          "type": "COMBO",
+          "widget": {
+            "name": "audio"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "audioUI",
+          "name": "audioUI",
+          "type": "AUDIO_UI",
+          "widget": {
+            "name": "audioUI"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "choose file to upload",
+          "name": "upload",
+          "type": "AUDIOUPLOAD",
+          "widget": {
+            "name": "upload"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "AUDIO",
+          "name": "AUDIO",
+          "type": "AUDIO",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadAudio"
+      },
+      "widgets_values": [
+        "h3_reference.wav",
+        null,
+        ""
+      ],
+      "title": "Optional reference audio"
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniVideoReferences",
+      "pos": [
+        420,
+        100
+      ],
+      "size": [
+        300,
+        365
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image_1",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": 1
+        },
+        {
+          "name": "image_2",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "audio_1",
+          "type": "AUDIO",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "audio_2",
+          "type": "AUDIO",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "video_1",
+          "type": "VIDEO",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "video_2",
+          "type": "VIDEO",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_3",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_4",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_5",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_6",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_7",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_8",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_9",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "audio_3",
+          "type": "AUDIO",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "video_3",
+          "type": "VIDEO",
+          "shape": 7,
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "references",
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniVideoReferences"
+      },
+      "widgets_values": [],
+      "title": "References \u2014 up to 12 total"
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniDiffusionSampling",
+      "pos": [
+        420,
+        530
+      ],
+      "size": [
+        310,
+        275
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "n",
+          "name": "n",
+          "type": "INT",
+          "widget": {
+            "name": "n"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "num_inference_steps",
+          "name": "num_inference_steps",
+          "type": "INT",
+          "widget": {
+            "name": "num_inference_steps"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "guidance_scale",
+          "name": "guidance_scale",
+          "type": "FLOAT",
+          "widget": {
+            "name": "guidance_scale"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "true_cfg_scale",
+          "name": "true_cfg_scale",
+          "type": "FLOAT",
+          "widget": {
+            "name": "true_cfg_scale"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "vae_use_slicing",
+          "name": "vae_use_slicing",
+          "type": "BOOLEAN",
+          "widget": {
+            "name": "vae_use_slicing"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "vae_use_tiling",
+          "name": "vae_use_tiling",
+          "type": "BOOLEAN",
+          "widget": {
+            "name": "vae_use_tiling"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "seed",
+          "name": "seed",
+          "type": "INT",
+          "widget": {
+            "name": "seed"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "diffusion sampling params",
+          "name": "diffusion sampling params",
+          "type": "SAMPLING_PARAMS",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniDiffusionSampling"
+      },
+      "widgets_values": [
+        1,
+        50,
+        1.0,
+        1.0,
+        false,
+        false,
+        42,
+        "fixed"
+      ],
+      "title": "Sampling \u2014 base 50 / Turbo 5"
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        790,
+        750
+      ],
+      "size": [
+        310,
+        105
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "audio_flow_shift",
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "flow_shift",
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "MiniMax-H3 video params",
+          "name": "MiniMax-H3 video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ],
+      "title": "H3 video/audio flow shifts"
+    },
+    {
+      "type": "VLLMOmniRemoteLoRA",
+      "size": [
+        310,
+        180
+      ],
+      "flags": {},
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniRemoteLoRA"
+      },
+      "widgets_values": [
+        "",
+        "h3-ref2va-turbo",
+        1.0,
+        0
+      ],
+      "id": 7,
+      "pos": [
+        420,
+        940
+      ],
+      "order": 6,
+      "mode": 0,
+      "title": "Optional Turbo LoRA \u2014 connect to enable"
+    },
+    {
+      "id": 8,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        800,
+        100
+      ],
+      "size": [
+        470,
+        560
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "frame",
+          "name": "frame",
+          "shape": 7,
+          "type": "IMAGE",
+          "link": null
+        },
+        {
+          "localized_name": "references",
+          "name": "references",
+          "shape": 7,
+          "type": "VIDEO_REFERENCES",
+          "link": 2
+        },
+        {
+          "localized_name": "sampling_params",
+          "name": "sampling_params",
+          "shape": 7,
+          "type": "SAMPLING_PARAMS",
+          "link": 3
+        },
+        {
+          "localized_name": "lora",
+          "name": "lora",
+          "shape": 7,
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "localized_name": "model_params",
+          "name": "model_params",
+          "shape": 7,
+          "type": "VIDEO_PARAMS",
+          "link": 4
+        },
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "prompt",
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "negative_prompt",
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "fps",
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "duration",
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://localhost:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "Use the subject and visual style from <Picture 1>. Create a cinematic shot with a gentle camera move and natural ambient sound.",
+        "",
+        1344,
+        768,
+        24,
+        5.167
+      ],
+      "title": "MiniMax-H3 Reference to Video"
+    },
+    {
+      "id": 9,
+      "type": "SaveVideo",
+      "pos": [
+        1360,
+        100
+      ],
+      "size": [
+        410,
+        420
+      ],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "link": 5
+        },
+        {
+          "localized_name": "filename_prefix",
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "format",
+          "name": "format",
+          "type": "COMBO",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "codec",
+          "name": "codec",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "codec"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": null
+        }
+      ],
+      "properties": {},
+      "widgets_values": [
+        "video/MiniMax-H3-Ref2VA",
+        "mp4",
+        "h264"
+      ],
+      "title": "Save generated video with audio"
+    },
+    {
+      "id": 10,
+      "type": "MarkdownNote",
+      "pos": [
+        1360,
+        590
+      ],
+      "size": [
+        440,
+        595
+      ],
+      "flags": {},
+      "order": 9,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Reference inputs and output settings",
+      "properties": {},
+      "widgets_values": [
+        "## Reference to Video\nSelect an image, set the server URL and served model name, then run.\n\nConnect the optional video/audio loaders to Video References, or duplicate loaders for more inputs: up to 9 images, 3 videos, 3 audio clips, 12 total. At least one image or video is required.\n\nSlots are numbered separately per media type. Empty slots are skipped. Update prompt tags such as <Picture 1>, <Video 1>, <Audio 1> to match the connected references. Videos keep their own soundtrack in the reference upload.\n\n**Output:** 24 FPS; 124 frames (~5.17 s). Valid frame counts are 107, 124, 141, 158, 175, 192, 209, 226, 243, 260, 277, 294, 311, 328, 345.\n\n**Width x height:** 1344x768 (16:9), 1024x768 (4:3), 768x768 (1:1), 768x1024 (3:4), 768x1344 (9:16).\n\nEach reference video/audio clip must be 2\u201315 s. Total reference-video duration <=15 s; total standalone-audio duration <=15 s.\n\nRequires the expanded Video References node and the audio-preserving video output fix. See the extension README for setup and validation status."
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 11,
+      "type": "MarkdownNote",
+      "pos": [
+        790,
+        940
+      ],
+      "size": [
+        480,
+        455
+      ],
+      "flags": {},
+      "order": 10,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Optional Turbo \u2014 same workflow",
+      "properties": {},
+      "widgets_values": [
+        "## Turbo Ref2VA\nBase mode: leave LoRA disconnected; use 50 steps, video flow shift 12, audio flow shift 3.\n\nFor the 4-step v0.1 Ref2VA Turbo adapter:\n1. Start a Ref2VA-only server with the Diffusers adapter loaded (see README).\n2. Set LoRA local_path to its path on the server, then connect LoRA to Generate Video.\n3. Set Sampling num_inference_steps to **5**; keep shifts at **12 / 3**, scale **1**.\n\nvLLM-Omni counts sigma points: 5 points give 4 denoiser evaluations. Use minimax_h3_ref2v_turbo_4step_v0.1_bf16.safetensors, not the _comfyui_ export.\n\nDisconnect LoRA and restore 50 steps to return to base mode. FastH3 is a separate T2VA deployment, not this option.\n\nAdapted from the official ComfyUI MiniMax-H3 R2V workflow: https://github.com/Comfy-Org/workflow_templates/blob/main/templates/video_minimax_h3_r2v.json"
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    }
+  ],
+  "links": [
+    [
+      1,
+      1,
+      0,
+      4,
+      0,
+      "IMAGE"
+    ],
+    [
+      2,
+      4,
+      0,
+      8,
+      1,
+      "VIDEO_REFERENCES"
+    ],
+    [
+      3,
+      5,
+      0,
+      8,
+      2,
+      "SAMPLING_PARAMS"
+    ],
+    [
+      4,
+      6,
+      0,
+      8,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      5,
+      8,
+      0,
+      9,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "workflowRendererVersion": "LG",
+    "ds": {
+      "scale": 0.85,
+      "offset": [
+        100,
+        100
+      ]
+    }
+  },
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Music Generation.json

@@ -0,0 +1,206 @@
+{
+  "id": "4949c8da-12ff-449a-94b7-948bde9c39dd",
+  "revision": 0,
+  "last_node_id": 3,
+  "last_link_id": 1,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "VLLMOmniGenerateMusic",
+      "pos": [
+        100,
+        100
+      ],
+      "size": [
+        470,
+        491
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "name": "instructions",
+          "type": "STRING",
+          "widget": {
+            "name": "instructions"
+          },
+          "link": null
+        },
+        {
+          "name": "lyrics",
+          "type": "STRING",
+          "widget": {
+            "name": "lyrics"
+          },
+          "link": null
+        },
+        {
+          "name": "max_duration_seconds",
+          "type": "FLOAT",
+          "widget": {
+            "name": "max_duration_seconds"
+          },
+          "link": null
+        },
+        {
+          "name": "seed",
+          "type": "INT",
+          "widget": {
+            "name": "seed"
+          },
+          "link": null
+        },
+        {
+          "name": "response_format",
+          "type": "COMBO",
+          "widget": {
+            "name": "response_format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateMusic"
+      },
+      "widgets_values": [
+        "http://localhost:8000/v1",
+        "MiniMaxAI/MiniMax-Music3",
+        "Futuristic melodic EDM / progressive house. Reflective verses about memory and digital identity build into an uplifting, hook-driven chorus, with an emotive lead vocal, bright layered synths, pulsing bass, crisp four-on-the-floor drums, subtle glitch textures, and a wide polished festival mix.",
+        "[Verse 1]\nUpload my heart to the cloud tonight\nSave every spark of neon light\nIf this body starts to fade away\nLet my code keep dancing in the data stream\nStay\n[Chorus]\nCloud copy of me\nLiving on when these bones finally break free\nEvery joke, every scar\nEvery cheap memory we made\nSave it, save it\nCloud copy of me\nIf the screen goes dark, I\u2019m still in the machine\nEvery glitch, every dream\nEvery version you ever did see\nSave it, save it\n[Verse 2]\nTag every laugh in a secret folder\nArchive the nights we outran the thunder\nIf the wiring turns to rust and gray\nHit restore\u2026 press replay\n[Chorus]\nCloud copy of me\nLiving on when these bones finally break free\nEvery joke, every scar\nEvery cheap memory we made\nSave it, save it\nCloud copy of me\nIf the screen goes dark, I\u2019m still in the machine\nEvery glitch, every dream\nEvery version you ever did see\nSave it, save it (oh)\n[Bridge \u2013 new, optional but recommended]\nEven the crashes, the late-night confessions\nThe parts of me I never learned to mention\nDon\u2019t let the silence swallow me whole\nKeep a ghost in the glow\u2026\nSo I never go\n[Final Chorus \u2013 softer / layered]\nCloud copy of me\nStill laughing somewhere in the binary\nEvery joke, every scar\nEvery moment you saved for me\nSave it, save it\nCloud copy of me\nWhen the lights go out, I\u2019ll still be in the machine\nDancing in the code that you still believe\nSave it\u2026\nSave me",
+        300.0,
+        0,
+        "fixed",
+        "wav"
+      ]
+    },
+    {
+      "id": 2,
+      "type": "SaveAudioAdvanced",
+      "pos": [
+        650,
+        100
+      ],
+      "size": [
+        320,
+        491
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "link": 1
+        },
+        {
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "name": "format",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveAudioAdvanced"
+      },
+      "widgets_values": [
+        "audio/vLLM-Omni-Music",
+        "mp3",
+        "V0"
+      ]
+    },
+    {
+      "id": 3,
+      "type": "MarkdownNote",
+      "title": "Music Generation Guide",
+      "pos": [
+        -507,
+        100
+      ],
+      "size": [
+        527,
+        491
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "properties": {},
+      "widgets_values": [
+        "## MiniMax Music 3\n\nThe **Generate Music** node currently supports **MiniMax Music 3 only**. The parameters below apply to this model.\n\nA music generation model by MiniMax that creates complete songs up to 5 minutes long with stable structure and high audio quality. Two inputs drive it: **Caption** (structured music description: style, mood, vocals, arrangement) and **Lyrics** (with `[intro]` `[verse]` `[chorus]` `[bridge]` `[outro]` tags controlling song structure).\n\n### Parameters\n\n| Parameter | Description |\n|---|---|\n| url | vLLM-Omni service URL including `/v1`. Default: `http://localhost:8000/v1`. |\n| model | Served model name. Default: `MiniMaxAI/MiniMax-Music3`. |\n| caption | Music description, sent as `instructions`. Use Global Metadata \u2192 Vocal Details \u2192 Arrangement to describe style, vocals, and instrumentation. |\n| lyrics | Words to sing with section tags. Put each tag on its own line; tags guide rather than strictly enforce song structure. |\n| max_duration_seconds | Maximum duration, default 300 seconds; range 1\u2013360, step 0.01. Converted to tokens at 25 frames/s, rounded down. Generation may end earlier. |\n| seed | Random seed, default 0; range 0\u20139007199254740991. Keep inputs and runtime settings unchanged for repeatability. |\n| control_after_generate | Seed behavior after generation: fixed, randomize, increment, or decrement. Template default: fixed. |\n| response_format | API audio format, default WAV. Options: WAV, MP3, FLAC, Opus, AAC; AAC is currently unsupported by the vLLM-Omni speech API. |\n\n### Prompt Writing Guide\n\nYou can find an official [Music Caption Rewriter Skill here](https://github.com/MiniMax-AI/MiniMax-Music3)\n"
+      ],
+      "widgets_values_named": {
+        "text": "## MiniMax Music 3\n\nThe **Generate Music** node currently supports **MiniMax Music 3 only**. The parameters below apply to this model.\n\nA music generation model by MiniMax that creates complete songs up to 5 minutes long with stable structure and high audio quality. Two inputs drive it: **Caption** (structured music description: style, mood, vocals, arrangement) and **Lyrics** (with `[intro]` `[verse]` `[chorus]` `[bridge]` `[outro]` tags controlling song structure).\n\n### Parameters\n\n| Parameter | Description |\n|---|---|\n| url | vLLM-Omni service URL including `/v1`. Default: `http://localhost:8000/v1`. |\n| model | Served model name. Default: `MiniMaxAI/MiniMax-Music3`. |\n| caption | Music description, sent as `instructions`. Use Global Metadata \u2192 Vocal Details \u2192 Arrangement to describe style, vocals, and instrumentation. |\n| lyrics | Words to sing with section tags. Put each tag on its own line; tags guide rather than strictly enforce song structure. |\n| max_duration_seconds | Maximum duration, default 300 seconds; range 1\u2013360, step 0.01. Converted to tokens at 25 frames/s, rounded down. Generation may end earlier. |\n| seed | Random seed, default 0; range 0\u20139007199254740991. Keep inputs and runtime settings unchanged for repeatability. |\n| control_after_generate | Seed behavior after generation: fixed, randomize, increment, or decrement. Template default: fixed. |\n| response_format | API audio format, default WAV. Options: WAV, MP3, FLAC, Opus, AAC; AAC is currently unsupported by the vLLM-Omni speech API. |\n\n### Prompt Writing Guide\n\nYou can find an official [Music Caption Rewriter Skill here](https://github.com/MiniMax-AI/MiniMax-Music3)\n"
+      },
+      "color": "#333",
+      "bgcolor": "#535353"
+    }
+  ],
+  "links": [
+    [
+      1,
+      1,
+      0,
+      2,
+      0,
+      "AUDIO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "workflowRendererVersion": "LG",
+    "ds": {
+      "scale": 1.147,
+      "offset": [
+        899,
+        259
+      ]
+    }
+  },
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json

@@ -1 +1 @@
-{"id":"3aea1d08-7de5-4997-ba71-b2f66ad72a21","revision":0,"last_node_id":24,"last_link_id":35,"nodes":[{"id":13,"type":"SaveVideo","pos":[1315.729153459605,-137.92038948394355],"size":[374,313.7692307692308],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"video","name":"video","type":"VIDEO","link":32},{"localized_name":"filename_prefix","name":"filename_prefix","type":"STRING","widget":{"name":"filename_prefix"},"link":null},{"localized_name":"format","name":"format","type":"COMBO","widget":{"name":"format"},"link":null},{"localized_name":"codec","name":"codec","type":"COMFY_DYNAMICCOMBO_V3","widget":{"name":"codec"},"link":null}],"outputs":[{"localized_name":"video","name":"video","type":"VIDEO","links":null}],"properties":{},"widgets_values":["video/ComfyUI","auto","auto"]},{"id":10,"type":"MarkdownNote","pos":[183.74636797766794,-571.2529158046766],"size":[240.3326478922474,136.3505820791881],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[],"title":"Note: Task and Input","properties":{},"widgets_values":["vLLM-Omni nodes are categorized based on the output modality. The \"Generate Video\" node supports both *text-to-video*, *frame-to-video*, and *reference-to-video* generation. The node will create corresponding payload depending on whether an input image is present or not."],"color":"#432","bgcolor":"#000"},{"id":4,"type":"LoadImage","pos":[452.07189605801125,-569.7242536425791],"size":[270,314],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[29]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["rap.png","image"]},{"id":23,"type":"LoadImage","pos":[9.206110246057657,-7.6955291509170785],"size":[270,314],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[34]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["white-cat.jpg","image"]},{"id":22,"type":"MarkdownNote","pos":[-9.783113411416666,-148.8790300038605],"size":[297.3012007349086,88],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[],"title":"Note: Reference to video (r2v)","properties":{},"widgets_values":["For r2v tasks, connect this Video Reference block to the main Generate Video block. Then disconnect the \"frame\" input above."],"color":"#432","bgcolor":"#000"},{"id":24,"type":"LoadAudio","pos":[317.9929742385805,-141.44821500065095],"size":[282.798828125,136],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[35]}],"properties":{"Node name for S&R":"LoadAudio"},"widgets_values":["nothing-beats-a-jet2-holiday.mp3",null,""]},{"id":21,"type":"VLLMOmniVideoReferences","pos":[376.6129172897595,105.41557277556292],"size":[156.9380859375,126],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"image_1","name":"image_1","shape":7,"type":"IMAGE","link":34},{"localized_name":"image_2","name":"image_2","shape":7,"type":"IMAGE","link":null},{"localized_name":"audio_1","name":"audio_1","shape":7,"type":"AUDIO","link":35},{"localized_name":"audio_2","name":"audio_2","shape":7,"type":"AUDIO","link":null},{"localized_name":"video_1","name":"video_1","shape":7,"type":"VIDEO","link":null},{"localized_name":"video_2","name":"video_2","shape":7,"type":"VIDEO","link":null}],"outputs":[{"localized_name":"references","name":"references","type":"VIDEO_REFERENCES","links":null}],"properties":{"Node name for S&R":"VLLMOmniVideoReferences"},"widgets_values":[]},{"id":8,"type":"VLLMOmniDiffusionSampling","pos":[83.24620208113915,422.08543477542605],"size":[284.205078125,226],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"n","name":"n","type":"INT","widget":{"name":"n"},"link":null},{"localized_name":"num_inference_steps","name":"num_inference_steps","type":"INT","widget":{"name":"num_inference_steps"},"link":null},{"localized_name":"guidance_scale","name":"guidance_scale","type":"FLOAT","widget":{"name":"guidance_scale"},"link":null},{"localized_name":"true_cfg_scale","name":"true_cfg_scale","type":"FLOAT","widget":{"name":"true_cfg_scLine truncated
+{"id":"3aea1d08-7de5-4997-ba71-b2f66ad72a21","revision":0,"last_node_id":24,"last_link_id":35,"nodes":[{"id":13,"type":"SaveVideo","pos":[1315.729153459605,-137.92038948394355],"size":[374,313.7692307692308],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"video","name":"video","type":"VIDEO","link":32},{"localized_name":"filename_prefix","name":"filename_prefix","type":"STRING","widget":{"name":"filename_prefix"},"link":null},{"localized_name":"format","name":"format","type":"COMBO","widget":{"name":"format"},"link":null},{"localized_name":"codec","name":"codec","type":"COMFY_DYNAMICCOMBO_V3","widget":{"name":"codec"},"link":null}],"outputs":[{"localized_name":"video","name":"video","type":"VIDEO","links":null}],"properties":{},"widgets_values":["video/ComfyUI","auto","auto"]},{"id":10,"type":"MarkdownNote","pos":[183.74636797766794,-571.2529158046766],"size":[240.3326478922474,136.3505820791881],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[],"title":"Note: Task and Input","properties":{},"widgets_values":["vLLM-Omni nodes are categorized based on the output modality. The \"Generate Video\" node supports both *text-to-video*, *frame-to-video*, and *reference-to-video* generation. The node will create corresponding payload depending on whether an input image is present or not."],"color":"#432","bgcolor":"#000"},{"id":4,"type":"LoadImage","pos":[452.07189605801125,-569.7242536425791],"size":[270,314],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[29]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["rap.png","image"]},{"id":23,"type":"LoadImage","pos":[9.206110246057657,-7.6955291509170785],"size":[270,314],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[34]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["white-cat.jpg","image"]},{"id":22,"type":"MarkdownNote","pos":[-9.783113411416666,-148.8790300038605],"size":[297.3012007349086,88],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[],"title":"Note: Reference to video (r2v)","properties":{},"widgets_values":["For r2v tasks, connect this Video Reference block to the main Generate Video block. Then disconnect the \"frame\" input above."],"color":"#432","bgcolor":"#000"},{"id":24,"type":"LoadAudio","pos":[317.9929742385805,-141.44821500065095],"size":[282.798828125,136],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[35]}],"properties":{"Node name for S&R":"LoadAudio"},"widgets_values":["nothing-beats-a-jet2-holiday.mp3",null,""]},{"id":21,"type":"VLLMOmniVideoReferences","pos":[376.6129172897595,105.41557277556292],"size":[156.9380859375,126],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"image_1","name":"image_1","shape":7,"type":"IMAGE","link":34},{"localized_name":"image_2","name":"image_2","shape":7,"type":"IMAGE","link":null},{"localized_name":"audio_1","name":"audio_1","shape":7,"type":"AUDIO","link":35},{"localized_name":"audio_2","name":"audio_2","shape":7,"type":"AUDIO","link":null},{"localized_name":"video_1","name":"video_1","shape":7,"type":"VIDEO","link":null},{"localized_name":"video_2","name":"video_2","shape":7,"type":"VIDEO","link":null}],"outputs":[{"localized_name":"references","name":"references","type":"VIDEO_REFERENCES","links":null}],"properties":{"Node name for S&R":"VLLMOmniVideoReferences"},"widgets_values":[]},{"id":8,"type":"VLLMOmniDiffusionSampling","pos":[83.24620208113915,422.08543477542605],"size":[284.205078125,226],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"n","name":"n","type":"INT","widget":{"name":"n"},"link":null},{"localized_name":"num_inference_steps","name":"num_inference_steps","type":"INT","widget":{"name":"num_inference_steps"},"link":null},{"localized_name":"guidance_scale","name":"guidance_scale","type":"FLOAT","widget":{"name":"guidance_scale"},"link":null},{"localized_name":"true_cfg_scale","name":"true_cfg_scale","type":"FLOAT","widget":{"name":"true_cfg_scLine truncated

apps/ComfyUI-vLLM-Omni/web/main.js

@@ -1,19 +1,153 @@
 /**
- * @file This file is intended to add dynamic fields to vLLM-Omni nodes
- * based on widget (in-node form fields) and input (connection link) values and changes.
- * However, this functionality is currently disabled/commented out.
- * Because it introduces too much complexity,
- * and it may even conflict with the current backend (Python) validation for unknown reasons (pending ComfyUI upstream fixes).
+ * @file Front-end extension for the vLLM-Omni nodes.
+ *
+ * Branding. Every vLLM-Omni node draws the vLLM-Omni "v" mark where LiteGraph
+ * would otherwise draw its default title dot, and is tinted by the kind of value
+ * it produces: blue nodes reach a server, amber and purple ones only assemble
+ * parameters, red ones pin what the server has loaded, teal ones carry media.
+ * A graph is then readable at a glance, and a node that turns out to be the
+ * wrong colour for where it sits is usually wired wrong.
+ *
+ * The tint is keyed off each node's declared output types rather than a list of
+ * node names, so a node added later is themed as soon as it returns one of the
+ * existing types.
+ *
+ * It also migrates Generate Video graphs saved before `duration` replaced the
+ * `num_frames` widget. See migrateNumFramesToDuration below.
+ *
+ * This file also used to host dynamic fields on vLLM-Omni nodes, driven by
+ * widget (in-node form fields) and input (connection link) values and changes.
+ * That is deliberately absent: it introduced too much complexity, and it even
+ * conflicted with the current backend (Python) validation for unknown reasons
+ * (pending ComfyUI upstream fixes).
  */
 
 import { app } from "../../scripts/app.js";
+
+const MARK_URL = new URL("./vllm-omni-mark.svg", import.meta.url).href;
+const MARK_ASPECT = 157 / 165;
+
+/**
+ * Title bar and body tints, one per node family.
+ *
+ * The hues are the logo's: the amber and blue of the "v" mark, and three stops
+ * of the gradient ring in the "O" (#9128A3, #CC5260, and the teal neighbouring
+ * its blue). All of them are dark because LiteGraph paints the title bar darker
+ * than the body and expects both to sit under light text; the ComfyUI light
+ * theme lightens custom node colours on its own.
+ */
+const FAMILIES = {
+    generate: { color: "#152A43", bgcolor: "#1F4269" },
+    sampling: { color: "#3A2C12", bgcolor: "#5C461C" },
+    model: { color: "#2E1233", bgcolor: "#4A1C52" },
+    deployment: { color: "#3A1A1F", bgcolor: "#5C2931" },
+    media: { color: "#0F2E2A", bgcolor: "#194A44" },
+};
+
+/** Output type -> family. Types absent here fall through to "generate". */
+const FAMILY_BY_OUTPUT = {
+    SAMPLING_PARAMS: "sampling",
+    TTS_PARAMS: "model",
+    VIDEO_PARAMS: "model",
+    REMOTE_LORA: "deployment",
+    FASTH3_DEPLOYMENT: "deployment",
+    VIDEO_REFERENCES: "media",
+};
+
+function familyOf(nodeData) {
+    for (const type of nodeData.output ?? []) {
+        const family = FAMILY_BY_OUTPUT[type];
+        if (family) {
+            return family;
+        }
+    }
+    // What is left returns a generated artifact (IMAGE, VIDEO, AUDIO, STRING),
+    // which is only ever produced by a node that talks to a vLLM-Omni server.
+    return "generate";
+}
+
+/**
+ * Rewrite a pre-rename Generate Video graph in place.
+ *
+ * `duration` (seconds) took over the slot `num_frames` used to occupy, and
+ * ComfyUI restores widget values positionally, so a graph saved before the
+ * rename reads its frame count as a duration: the workflow this repository
+ * shipped stored 120 frames at 24 fps, which would come back as a 120-second
+ * request. Saved files still name the old widget under `inputs`, and that is
+ * what makes the rewrite both detectable and safe to apply exactly once -- once
+ * rewritten the node serializes `duration`, so a later load finds nothing to do.
+ */
+function migrateNumFramesToDuration(node, info) {
+    const savedBeforeRename = (info?.inputs ?? []).some((input) => input?.widget?.name === "num_frames");
+    if (!savedBeforeRename) {
+        return
+    }
+    const duration = node.widgets?.find((widget) => widget.name === "duration");
+    const fps = node.widgets?.find((widget) => widget.name === "fps");
+    if (!duration || !fps) {
+        return
+    }
+    // configure() has already placed the stale frame count in the duration widget.
+    const frames = Number(duration.value);
+    const rate = Number(fps.value);
+    if (!Number.isFinite(frames) || !Number.isFinite(rate) || rate <= 0) {
+        return
+    }
+    const seconds = Math.max(duration.options?.min ?? 0.1, Math.round((frames / rate) * 1000) / 1000);
+    console.info(
+        `vLLM-Omni: migrated Generate Video "${node.title}" from ${frames} frames to ${seconds}s at ${rate} fps.`,
+    );
+    duration.value = seconds;
+}
+
+const mark = new Image();
+let markReady = false;
+mark.addEventListener("load", () => {
+    markReady = true;
+    // The image resolves after the first frames are already on screen.
+    app.graph?.setDirtyCanvas?.(true, true);
+});
+mark.src = MARK_URL;
+
 app.registerExtension({
     name: "vllm.vllm_omni",
     async beforeRegisterNodeDef(nodeType, nodeData, app) {
         if (!nodeData.name.startsWith("VLLMOmni")) {
             return
         }
-        // Stub frontend plugin for now
+
+        const { color, bgcolor } = FAMILIES[familyOf(nodeData)];
+        // On the prototype, not the instance: a colour the user picks by hand is
+        // written to the instance and shadows this, so manual overrides survive.
+        nodeType.prototype.color = color;
+        nodeType.prototype.bgcolor = bgcolor;
+
+        if (nodeData.name === "VLLMOmniGenerateVideo") {
+            const onConfigure = nodeType.prototype.onConfigure;
+            nodeType.prototype.onConfigure = function (info) {
+                onConfigure?.apply(this, arguments);
+                migrateNumFramesToDuration(this, info);
+            };
+        }
+
+        // Defining this replaces the default dot outright — LiteGraph draws its
+        // own square or circle only for nodes that leave onDrawTitleBox unset.
+        nodeType.prototype.onDrawTitleBox = function (ctx, titleHeight, _size, scale) {
+            // Zoomed out far enough that LiteGraph itself drops to flat shapes,
+            // the mark is a few muddy pixels. Leave the title bar clean instead.
+            if (!markReady || (scale !== undefined && scale < 0.5)) {
+                return
+            }
+            const height = titleHeight * 0.62;
+            const width = height * MARK_ASPECT;
+            ctx.drawImage(
+                mark,
+                titleHeight * 0.5 - width * 0.5,
+                titleHeight * -0.5 - height * 0.5,
+                width,
+                height,
+            );
+        };
     },
     async setup() {
         console.info("vLLM-Omni Setup complete!")

apps/ComfyUI-vLLM-Omni/web/vllm-omni-mark.svg

@@ -0,0 +1,5 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 157 165" width="157" height="165">
+  <title>vLLM-Omni</title>
+  <polygon points="0,36 62,36 62,160" fill="#F1B844"/>
+  <polygon points="96,36 157,0 121,165 68,165" fill="#54A0F8"/>
+</svg>

benchmarks/README.md

@@ -4,6 +4,11 @@ This directory contains benchmark suites for evaluating different model families
 
 ## Benchmark families
 
+### [Qwen2.5-Omni](qwen2_5_omni/README.md) — Embedding Wrapper Latency
+
+CPU-observed A/B timings for redundant text embedding removal, with real checkpoint
+embeddings, warmup, paired blocks, and embedding call-count validation.
+
 ### [TTS](tts/README.md) — Text-to-Speech
 
 Model-agnostic serving benchmarks for TTS models, including Qwen3-TTS and VoxCPM2.

benchmarks/diffusion/benchmark_qwen_image_qk_norm_rope.py

@@ -0,0 +1,222 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Benchmark Qwen-Image Q/K RMSNorm + interleaved RoPE.
+
+Example:
+
+    CUDA_VISIBLE_DEVICES=0 python benchmarks/diffusion/benchmark_qwen_image_qk_norm_rope.py \
+      --seq-len 4096 --warmup 20 --iters 100 --include-compiled --profile
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from collections.abc import Callable
+
+import torch
+import torch.nn.functional as F
+
+from vllm_omni.diffusion.layers.fused_qk_norm_rope import (
+    _fused_cuda_supported,
+    fused_qk_norm_rope,
+)
+from vllm_omni.diffusion.layers.rope import RotaryEmbedding
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--batch", type=int, default=1)
+    parser.add_argument("--seq-len", type=int, default=4096)
+    parser.add_argument("--heads", type=int, default=24)
+    parser.add_argument("--kv-heads", type=int, default=24)
+    parser.add_argument("--head-dim", type=int, default=128)
+    parser.add_argument("--warmup", type=int, default=20)
+    parser.add_argument("--iters", type=int, default=100)
+    parser.add_argument("--dtype", choices=("bf16", "fp16", "fp32"), default="bf16")
+    parser.add_argument("--packed-qkv-view", action="store_true")
+    parser.add_argument("--include-compiled", action="store_true")
+    parser.add_argument("--profile", action="store_true")
+    return parser.parse_args()
+
+
+def _dtype(name: str) -> torch.dtype:
+    return {
+        "bf16": torch.bfloat16,
+        "fp16": torch.float16,
+        "fp32": torch.float32,
+    }[name]
+
+
+def _make_inputs(args: argparse.Namespace, dtype: torch.dtype, device: torch.device):
+    torch.manual_seed(2026)
+    if args.packed_qkv_view:
+        qkv_dim = (args.heads + args.kv_heads + args.kv_heads) * args.head_dim
+        qkv = torch.randn(args.batch, args.seq_len, qkv_dim, device=device, dtype=dtype)
+        q, k, _ = qkv.split(
+            [
+                args.heads * args.head_dim,
+                args.kv_heads * args.head_dim,
+                args.kv_heads * args.head_dim,
+            ],
+            dim=-1,
+        )
+        q = q.unflatten(-1, (args.heads, args.head_dim))
+        k = k.unflatten(-1, (args.kv_heads, args.head_dim))
+    else:
+        q = torch.randn(args.batch, args.seq_len, args.heads, args.head_dim, device=device, dtype=dtype)
+        k = torch.randn(args.batch, args.seq_len, args.kv_heads, args.head_dim, device=device, dtype=dtype)
+
+    q_weight = torch.randn(args.head_dim, device=device, dtype=dtype)
+    k_weight = torch.randn(args.head_dim, device=device, dtype=dtype)
+    angles = torch.randn(args.seq_len, args.head_dim // 2, device=device, dtype=torch.float32)
+    freqs = torch.polar(torch.ones_like(angles), angles)
+    return q, k, q_weight, k_weight, freqs
+
+
+def _native(
+    q: torch.Tensor,
+    k: torch.Tensor,
+    q_weight: torch.Tensor,
+    k_weight: torch.Tensor,
+    freqs: torch.Tensor,
+    eps: float,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    q = F.rms_norm(q, (q.shape[-1],), q_weight, eps)
+    k = F.rms_norm(k, (k.shape[-1],), k_weight, eps)
+    rope = RotaryEmbedding(is_neox_style=False)
+    cos = freqs.real.to(q.dtype)
+    sin = freqs.imag.to(q.dtype)
+    return rope(q, cos, sin), rope(k, cos, sin)
+
+
+def _fused(
+    q: torch.Tensor,
+    k: torch.Tensor,
+    q_weight: torch.Tensor,
+    k_weight: torch.Tensor,
+    freqs: torch.Tensor,
+    eps: float,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    batch, seq_len, num_heads, head_dim = q.shape
+    num_kv_heads = k.shape[2]
+    rope_table = torch.cat((freqs.real, freqs.imag), dim=-1)
+    rope_table = rope_table.unsqueeze(0).expand(batch, -1, -1).reshape(batch * seq_len, head_dim)
+    out_q, out_k = fused_qk_norm_rope(
+        q.reshape(batch * seq_len, num_heads, head_dim),
+        k.reshape(batch * seq_len, num_kv_heads, head_dim),
+        q_weight,
+        k_weight,
+        rope_table,
+        eps,
+        interleaved=True,
+    )
+    return (
+        out_q.reshape(batch, seq_len, num_heads, head_dim),
+        out_k.reshape(batch, seq_len, num_kv_heads, head_dim),
+    )
+
+
+def _measure(fn: Callable[[], tuple[torch.Tensor, torch.Tensor]], warmup: int, iters: int) -> list[float]:
+    for _ in range(warmup):
+        fn()
+    torch.accelerator.synchronize()
+    start = torch.cuda.Event(enable_timing=True)
+    end = torch.cuda.Event(enable_timing=True)
+    samples = []
+    for _ in range(iters):
+        start.record()
+        fn()
+        end.record()
+        end.synchronize()
+        samples.append(start.elapsed_time(end))
+    return samples
+
+
+def _stats(samples: list[float]) -> dict[str, float]:
+    samples = sorted(samples)
+    return {
+        "median_ms": samples[len(samples) // 2],
+        "p90_ms": samples[int(len(samples) * 0.9)],
+        "mean_ms": sum(samples) / len(samples),
+    }
+
+
+def _profile(fn: Callable[[], tuple[torch.Tensor, torch.Tensor]]) -> str:
+    from torch.profiler import ProfilerActivity, profile
+
+    with profile(activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA]) as prof:
+        fn()
+        torch.accelerator.synchronize()
+    return prof.key_averages().table(sort_by="self_cuda_time_total", row_limit=15)
+
+
+def main() -> None:
+    args = _parse_args()
+    if not torch.cuda.is_available():
+        raise RuntimeError("This benchmark requires CUDA")
+    device = torch.device("cuda")
+    dtype = _dtype(args.dtype)
+    eps = 1e-6
+    q, k, q_weight, k_weight, freqs = _make_inputs(args, dtype, device)
+
+    def native():
+        return _native(q, k, q_weight, k_weight, freqs, eps)
+
+    def fused():
+        return _fused(q, k, q_weight, k_weight, freqs, eps)
+
+    results: dict[str, object] = {
+        "device": torch.cuda.get_device_name(),
+        "batch": args.batch,
+        "seq_len": args.seq_len,
+        "heads": args.heads,
+        "kv_heads": args.kv_heads,
+        "head_dim": args.head_dim,
+        "dtype": args.dtype,
+        "packed_qkv_view": args.packed_qkv_view,
+        "fused_fast_path_supported": _fused_cuda_supported(
+            q.reshape(args.batch * args.seq_len, args.heads, args.head_dim),
+            k.reshape(args.batch * args.seq_len, args.kv_heads, args.head_dim),
+            args.head_dim,
+            args.head_dim,
+            interleaved=True,
+        ),
+    }
+
+    native_samples = _measure(native, args.warmup, args.iters)
+    fused_samples = _measure(fused, args.warmup, args.iters)
+    expected_q, expected_k = native()
+    actual_q, actual_k = fused()
+    torch.accelerator.synchronize()
+
+    native_stats = _stats(native_samples)
+    fused_stats = _stats(fused_samples)
+    results["native"] = native_stats
+    results["fused"] = fused_stats
+    results["speedup_vs_native"] = native_stats["median_ms"] / fused_stats["median_ms"]
+    results["max_abs_q"] = (actual_q.float() - expected_q.float()).abs().max().item()
+    results["max_abs_k"] = (actual_k.float() - expected_k.float()).abs().max().item()
+    results["mean_abs_q"] = (actual_q.float() - expected_q.float()).abs().mean().item()
+    results["mean_abs_k"] = (actual_k.float() - expected_k.float()).abs().mean().item()
+
+    if args.include_compiled:
+        compiled_native = torch.compile(native, dynamic=True, fullgraph=True)
+        compiled_fused = torch.compile(fused, dynamic=True, fullgraph=True)
+        compiled_native_stats = _stats(_measure(compiled_native, args.warmup, args.iters))
+        compiled_fused_stats = _stats(_measure(compiled_fused, args.warmup, args.iters))
+        results["compiled_native"] = compiled_native_stats
+        results["compiled_fused"] = compiled_fused_stats
+        results["compiled_fused_speedup_vs_compiled_native"] = (
+            compiled_native_stats["median_ms"] / compiled_fused_stats["median_ms"]
+        )
+
+    print(json.dumps(results, indent=2))
+    if args.profile:
+        print("\n# fused profile")
+        print(_profile(fused))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/diffusion/diffusion_benchmark_serving.py

@@ -1087,6 +1087,12 @@ async def _run_warmups(
 
     warmup_tasks = [asyncio.create_task(limited_warmup_request_func(req)) for req in warmup_requests]
     warmup_outputs = await asyncio.gather(*warmup_tasks)
+    failed_count = sum(not output.success for output in warmup_outputs)
+    if failed_count:
+        raise RuntimeError(
+            f"{failed_count}/{len(warmup_outputs)} warmup requests failed; "
+            "check server logs before running the benchmark."
+        )
     return list(zip(warmup_requests, warmup_outputs))
 
 
@@ -1455,7 +1461,7 @@ async def benchmark(args):
         print(f"Metrics saved to {args.output_file}")
 
 
-if __name__ == "__main__":
+def build_parser():
     parser = argparse.ArgumentParser(description="Benchmark serving for diffusion models.")
     parser.add_argument(
         "--base-url",
@@ -1637,6 +1643,10 @@ if __name__ == "__main__":
         action="store_true",
         help="Request stage duration metrics from endpoints that support return_stage_metrics.",
     )
+    return parser
 
+
+if __name__ == "__main__":
+    parser = build_parser()
     args = parser.parse_args()
     asyncio.run(benchmark(args))

benchmarks/host_weight_runtime/README.md

@@ -0,0 +1,57 @@
+# Host weight dependency memory diagnostic
+
+`safetensors_retention.py` isolates repeated CPU `get_tensor()` calls from
+access to one cached tensor view. It imports neither vLLM nor HWR and uses a
+synthetic float32 payload controlled by `--tensor-elements` (default: 64).
+Tensor size can affect retention: a negative result for 16 elements does not
+rule out growth for 64 elements. The selected size is recorded in the JSON
+arguments and used for both file creation and the correctness check.
+HWR itself calls `get_tensor()` when
+acquiring a lease, then exposes cached views; this probe does not measure HWR
+requests or repeated lease acquisition.
+
+Run one mode per fresh process in the same Python environment and CPU affinity:
+
+```bash
+git rev-parse HEAD > revision.txt
+git status --short > working-tree.txt
+git diff > working-tree.patch
+# Select an allowed CPU from your process affinity; 0 is only an example.
+CUDA_VISIBLE_DEVICES='' taskset -c 0 timeout 120s python \
+  benchmarks/host_weight_runtime/safetensors_retention.py \
+  --mode get_tensor --iterations 10000 --sample-every 5000 > feasibility.json
+```
+
+For a size comparison, run 16 and 64 elements twice each in fresh processes,
+with cached reuse as a control at both sizes:
+
+```bash
+for repetition in 1 2; do
+  for elements in 16 64; do
+    for mode in get_tensor reuse; do
+      CUDA_VISIBLE_DEVICES='' taskset -c 0 timeout 120s python \
+        benchmarks/host_weight_runtime/safetensors_retention.py \
+        --mode "$mode" --tensor-elements "$elements" \
+        > "$mode-$elements-$repetition.json"
+    done
+  done
+done
+```
+
+The diagnostic sets Torch to one CPU thread, warms up before the baseline,
+and reports preparation separately from timed loop work. Checkpoints run GC
+and report Linux private/anonymous memory, RSS, and open descriptors. It checks
+that transient tensor objects and the final cached view are released. Linux
+`/proc/self/smaps_rollup` must be readable; invalid arguments and missing proc
+support fail explicitly. Keep raw JSON and the repository snapshot with results.
+
+Compare the final loop sample with the warmed baseline, then inspect the closed
+sample. Private-memory growth after Python tensor objects disappear is evidence
+of retention in the dependency/allocator path, not proof of a leak's root cause.
+RSS can include file-backed pages; it is not a private-memory metric. Samples
+also include small diagnostic bookkeeping allocations. Do not assert a portable
+memory threshold or turn this synthetic loop into a per-request service estimate.
+
+Do not drop shared caches, trim allocators, or add serving-time GC to make the
+numbers look smaller. A dependency replacement or version change requires its
+own reproducible comparison and correctness/lifecycle validation.

benchmarks/host_weight_runtime/safetensors_retention.py

@@ -0,0 +1,120 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Linux CPU diagnostic: repeated safetensors materialization versus view reuse."""
+
+from __future__ import annotations
+
+import argparse
+import gc
+import json
+import os
+import platform
+import tempfile
+import time
+import weakref
+from pathlib import Path
+
+
+def positive_int(value: str) -> int:
+    parsed = int(value)
+    if parsed <= 0:
+        raise argparse.ArgumentTypeError("must be positive")
+    return parsed
+
+
+def sample(phase: str, iterations: int, loop_seconds: float) -> dict[str, object]:
+    gc.collect()
+    counters = {}
+    for line in Path("/proc/self/smaps_rollup").read_text().splitlines():
+        fields = line.split()
+        if len(fields) == 3 and fields[2] == "kB":
+            counters[fields[0].removesuffix(":")] = int(fields[1]) * 1024
+    return {
+        "phase": phase,
+        "iterations": iterations,
+        "loop_seconds": loop_seconds,
+        "rss_bytes": counters["Rss"],
+        "private_bytes": counters["Private_Clean"] + counters["Private_Dirty"],
+        "anonymous_bytes": counters["Anonymous"],
+        "fd_count": len(tuple(Path("/proc/self/fd").iterdir())),
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--mode", choices=("get_tensor", "reuse"), required=True)
+    parser.add_argument(
+        "--tensor-elements", type=positive_int, default=64, help="Number of float32 elements in the synthetic tensor"
+    )
+    parser.add_argument("--iterations", type=positive_int, default=1_000_000)
+    parser.add_argument("--sample-every", type=positive_int, default=100_000)
+    parser.add_argument("--warmup", type=positive_int, default=1000)
+    args = parser.parse_args()
+    if not Path("/proc/self/smaps_rollup").is_file():
+        parser.error("requires Linux /proc/self/smaps_rollup")
+
+    preparation_start = time.monotonic()
+    import safetensors
+    import torch
+    from safetensors import safe_open
+    from safetensors.torch import save_file
+
+    torch.set_num_threads(1)
+    torch.set_num_interop_threads(1)
+    samples = [sample("before_file", 0, 0.0)]
+    with tempfile.TemporaryDirectory(prefix="safetensors-retention-") as directory:
+        path = Path(directory) / "synthetic.safetensors"
+        save_file({"weight": torch.arange(args.tensor_elements, dtype=torch.float32)}, str(path))
+        with safe_open(path, framework="pt", device="cpu") as reader:
+            cached = reader.get_tensor("weight")
+            assert torch.equal(cached, torch.arange(args.tensor_elements, dtype=torch.float32))
+            for _ in range(args.warmup):
+                tensor = reader.get_tensor("weight") if args.mode == "get_tensor" else cached
+                del tensor
+            samples.append(sample("baseline", 0, 0.0))
+            preparation_seconds = time.monotonic() - preparation_start
+            completed = 0
+            loop_seconds = 0.0
+            while completed < args.iterations:
+                count = min(args.sample_every, args.iterations - completed)
+                start = time.monotonic()
+                for _ in range(count):
+                    tensor = reader.get_tensor("weight") if args.mode == "get_tensor" else cached
+                last_tensor = weakref.ref(tensor)
+                del tensor
+                loop_seconds += time.monotonic() - start
+                completed += count
+                snapshot = sample("loop", completed, loop_seconds)
+                snapshot["last_tensor_alive"] = last_tensor() is not None
+                if args.mode == "get_tensor":
+                    assert last_tensor() is None, "diagnostic retained a transient tensor"
+                samples.append(snapshot)
+            del cached
+        del reader
+        samples.append(sample("closed", completed, loop_seconds))
+        assert last_tensor() is None, "last tensor survived reader/view teardown"
+
+    print(
+        json.dumps(
+            {
+                "schema_version": 1,
+                "arguments": vars(args),
+                "python": platform.python_version(),
+                "torch": torch.__version__,
+                "safetensors": safetensors.__version__,
+                "kernel": platform.release(),
+                "hostname": platform.node(),
+                "pid": os.getpid(),
+                "cpu_affinity": sorted(os.sched_getaffinity(0)),
+                "torch_threads": torch.get_num_threads(),
+                "pythonmalloc": os.environ.get("PYTHONMALLOC", "default"),
+                "preparation_seconds": preparation_seconds,
+                "samples": samples,
+            },
+            indent=2,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

benchmarks/kernels/bench_qwen_select01_modulation.py

@@ -0,0 +1,333 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Benchmark the Qwen-Image-Edit select01 modulation hot path.
+
+This script intentionally starts with the current PyTorch implementation. It is
+used as the before/after baseline for SGLang PR #20395/#21318 style fusion:
+
+* layernorm + scale/shift + binary select01 gate
+* residual + gated update + layernorm + scale/shift + binary select01 gate
+
+Run on a CUDA machine for actionable numbers, for example:
+
+    python benchmarks/kernels/bench_qwen_select01_modulation.py \
+        --device cuda --dtype bf16 --seq-lens 4096 8192 16384 \
+        --output-json /tmp/qwen_select01_baseline.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import statistics
+import sys
+import time
+from collections.abc import Callable
+from pathlib import Path
+
+import torch
+import torch.nn.functional as F
+
+for candidate in Path(__file__).resolve().parents:
+    if (candidate / "vllm_omni").is_dir():
+        if str(candidate) not in sys.path:
+            sys.path.insert(0, str(candidate))
+        break
+
+from vllm_omni.diffusion.layers.qwen_select01_modulation import (  # noqa: E402
+    fused_layernorm_select01,
+    fused_residual_layernorm_select01,
+)
+
+
+def _dtype(name: str) -> torch.dtype:
+    return {
+        "fp32": torch.float32,
+        "float32": torch.float32,
+        "fp16": torch.float16,
+        "float16": torch.float16,
+        "bf16": torch.bfloat16,
+        "bfloat16": torch.bfloat16,
+    }[name]
+
+
+def _select01_modulation(
+    mod_params: torch.Tensor,
+    index: torch.Tensor | None,
+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+    shift, scale, gate = mod_params.chunk(3, dim=-1)
+
+    if index is None:
+        return scale.unsqueeze(1), shift.unsqueeze(1), gate.unsqueeze(1)
+
+    actual_batch = shift.size(0) // 2
+    shift_0, shift_1 = shift[:actual_batch], shift[actual_batch:]
+    scale_0, scale_1 = scale[:actual_batch], scale[actual_batch:]
+    gate_0, gate_1 = gate[:actual_batch], gate[actual_batch:]
+
+    idx = index.unsqueeze(-1) == 0
+    shift = torch.where(idx, shift_0.unsqueeze(1), shift_1.unsqueeze(1))
+    scale = torch.where(idx, scale_0.unsqueeze(1), scale_1.unsqueeze(1))
+    gate = torch.where(idx, gate_0.unsqueeze(1), gate_1.unsqueeze(1))
+    return scale, shift, gate
+
+
+def _native_layernorm_select01(
+    x: torch.Tensor,
+    mod_params: torch.Tensor,
+    index: torch.Tensor,
+    eps: float,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    scale, shift, gate = _select01_modulation(mod_params, index)
+    out = F.layer_norm(x.float(), (x.shape[-1],), eps=eps).to(x.dtype)
+    return out * (1 + scale) + shift, gate
+
+
+def _native_residual_layernorm_select01(
+    x: torch.Tensor,
+    residual: torch.Tensor,
+    residual_gate: torch.Tensor,
+    mod_params: torch.Tensor,
+    index: torch.Tensor,
+    eps: float,
+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+    scale, shift, gate = _select01_modulation(mod_params, index)
+    residual_out = residual + residual_gate * x
+    out = F.layer_norm(residual_out.float(), (residual_out.shape[-1],), eps=eps).to(residual_out.dtype)
+    return out * (1 + scale) + shift, residual_out, gate
+
+
+def _time_cuda(fn: Callable[[], object], warmup: int, iters: int) -> list[float]:
+    for _ in range(warmup):
+        fn()
+    torch.accelerator.synchronize()
+
+    times: list[float] = []
+    for _ in range(iters):
+        start = torch.cuda.Event(enable_timing=True)
+        end = torch.cuda.Event(enable_timing=True)
+        start.record()
+        fn()
+        end.record()
+        torch.accelerator.synchronize()
+        times.append(float(start.elapsed_time(end)))
+    return times
+
+
+def _reset_peak_memory_stats(device: torch.device) -> None:
+    reset_peak_memory_stats = getattr(torch.accelerator, "reset_peak_memory_stats", None)
+    if reset_peak_memory_stats is None:
+        reset_peak_memory_stats = getattr(getattr(torch, device.type), "reset_peak_memory_stats")
+    reset_peak_memory_stats(device)
+
+
+def _max_memory_allocated(device: torch.device) -> int:
+    max_memory_allocated = getattr(torch.accelerator, "max_memory_allocated", None)
+    if max_memory_allocated is None:
+        max_memory_allocated = getattr(getattr(torch, device.type), "max_memory_allocated")
+    return int(max_memory_allocated(device))
+
+
+def _time_cpu(fn: Callable[[], object], warmup: int, iters: int) -> list[float]:
+    for _ in range(warmup):
+        fn()
+
+    times: list[float] = []
+    for _ in range(iters):
+        t0 = time.perf_counter()
+        fn()
+        times.append((time.perf_counter() - t0) * 1000.0)
+    return times
+
+
+def _summarize(times: list[float]) -> dict[str, float]:
+    ordered = sorted(times)
+    return {
+        "median_ms": statistics.median(ordered),
+        "min_ms": ordered[0],
+        "p20_ms": ordered[max(0, int(len(ordered) * 0.20) - 1)],
+        "p80_ms": ordered[min(len(ordered) - 1, int(len(ordered) * 0.80))],
+    }
+
+
+def _make_inputs(
+    batch: int,
+    seq_len: int,
+    hidden_size: int,
+    dtype: torch.dtype,
+    device: torch.device,
+) -> dict[str, torch.Tensor]:
+    torch.manual_seed(0)
+    return {
+        "x": torch.randn(batch, seq_len, hidden_size, device=device, dtype=dtype),
+        "residual": torch.randn(batch, seq_len, hidden_size, device=device, dtype=dtype),
+        "residual_gate": torch.randn(batch, 1, hidden_size, device=device, dtype=dtype),
+        "mod_params": torch.randn(batch * 2, hidden_size * 3, device=device, dtype=dtype),
+        "index": torch.randint(0, 2, (batch, seq_len), device=device, dtype=torch.int64),
+    }
+
+
+def _bench_one(
+    name: str,
+    fn: Callable[[], object],
+    device: torch.device,
+    warmup: int,
+    iters: int,
+) -> dict[str, object]:
+    if device.type == "cuda":
+        _reset_peak_memory_stats(device)
+        times = _time_cuda(fn, warmup, iters)
+        peak_memory_mb = _max_memory_allocated(device) / 1024**2
+    else:
+        times = _time_cpu(fn, warmup, iters)
+        peak_memory_mb = None
+
+    return {
+        "name": name,
+        **_summarize(times),
+        "peak_memory_mb": peak_memory_mb,
+    }
+
+
+def _run(args: argparse.Namespace) -> dict[str, object]:
+    device = torch.device(args.device)
+    if device.type == "cuda" and not torch.cuda.is_available():
+        raise SystemExit("CUDA is not available on this machine.")
+
+    dtype = _dtype(args.dtype)
+    rows: list[dict[str, object]] = []
+
+    for seq_len in args.seq_lens:
+        tensors = _make_inputs(args.batch_size, seq_len, args.hidden_size, dtype, device)
+        impls = ["native"] if args.impl == "native" else ["fused"] if args.impl == "fused" else ["native", "fused"]
+
+        def first_norm() -> object:
+            return _native_layernorm_select01(
+                tensors["x"],
+                tensors["mod_params"],
+                tensors["index"],
+                args.eps,
+            )
+
+        def residual_norm() -> object:
+            return _native_residual_layernorm_select01(
+                tensors["x"],
+                tensors["residual"],
+                tensors["residual_gate"],
+                tensors["mod_params"],
+                tensors["index"],
+                args.eps,
+            )
+
+        callables: list[tuple[str, Callable[[], object]]] = []
+        if "native" in impls:
+            callables.extend(
+                [
+                    ("native_layernorm_select01", first_norm),
+                    ("native_residual_layernorm_select01", residual_norm),
+                ]
+            )
+        if "fused" in impls:
+
+            def fused_first_norm() -> object:
+                return fused_layernorm_select01(
+                    tensors["x"],
+                    tensors["mod_params"],
+                    tensors["index"],
+                    args.eps,
+                )
+
+            def fused_residual_norm() -> object:
+                return fused_residual_layernorm_select01(
+                    tensors["x"],
+                    tensors["residual"],
+                    tensors["residual_gate"].expand_as(tensors["residual"]),
+                    tensors["mod_params"],
+                    tensors["index"],
+                    args.eps,
+                )
+
+            callables.extend(
+                [
+                    ("fused_layernorm_select01", fused_first_norm),
+                    ("fused_residual_layernorm_select01", fused_residual_norm),
+                ]
+            )
+
+        if args.check and "fused" in impls:
+            ref_norm, ref_gate = first_norm()
+            actual_norm, actual_gate = fused_layernorm_select01(
+                tensors["x"],
+                tensors["mod_params"],
+                tensors["index"],
+                args.eps,
+            )
+            torch.testing.assert_close(actual_norm, ref_norm, atol=args.check_atol, rtol=args.check_rtol)
+            torch.testing.assert_close(actual_gate, ref_gate, atol=0, rtol=0)
+            ref_norm, ref_residual, ref_gate = residual_norm()
+            actual_norm, actual_residual, actual_gate = fused_residual_layernorm_select01(
+                tensors["x"],
+                tensors["residual"],
+                tensors["residual_gate"].expand_as(tensors["residual"]),
+                tensors["mod_params"],
+                tensors["index"],
+                args.eps,
+            )
+            torch.testing.assert_close(actual_norm, ref_norm, atol=args.check_atol, rtol=args.check_rtol)
+            torch.testing.assert_close(actual_residual, ref_residual, atol=args.check_atol, rtol=args.check_rtol)
+            torch.testing.assert_close(actual_gate, ref_gate, atol=0, rtol=0)
+
+        for name, fn in callables:
+            row = _bench_one(name, fn, device, args.warmup, args.iters)
+            row.update(
+                {
+                    "batch_size": args.batch_size,
+                    "seq_len": seq_len,
+                    "hidden_size": args.hidden_size,
+                    "dtype": args.dtype,
+                    "device": str(device),
+                }
+            )
+            rows.append(row)
+            print(
+                f"{name:<38} B={args.batch_size:<2} L={seq_len:<6} C={args.hidden_size:<5} "
+                f"median={row['median_ms']:.4f} ms p20={row['p20_ms']:.4f} p80={row['p80_ms']:.4f}"
+            )
+
+    result = {
+        "benchmark": "qwen_select01_modulation",
+        "torch": torch.__version__,
+        "cuda_available": torch.cuda.is_available(),
+        "gpu": torch.cuda.get_device_name(device) if device.type == "cuda" else None,
+        "rows": rows,
+    }
+
+    if args.output_json:
+        output_path = Path(args.output_json)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        output_path.write_text(json.dumps(result, indent=2), encoding="utf-8")
+
+    return result
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    default_device = "cuda" if torch.cuda.is_available() else "cpu"
+    parser.add_argument("--device", default=default_device)
+    parser.add_argument("--dtype", default="bf16", choices=["fp32", "float32", "fp16", "float16", "bf16", "bfloat16"])
+    parser.add_argument("--batch-size", type=int, default=1)
+    parser.add_argument("--seq-lens", type=int, nargs="+", default=[4096, 8192, 16384])
+    parser.add_argument("--hidden-size", type=int, default=3072)
+    parser.add_argument("--eps", type=float, default=1e-6)
+    parser.add_argument("--warmup", type=int, default=20)
+    parser.add_argument("--iters", type=int, default=100)
+    parser.add_argument("--impl", choices=["native", "fused", "all"], default="native")
+    parser.add_argument("--check", action="store_true")
+    parser.add_argument("--check-atol", type=float, default=5e-2)
+    parser.add_argument("--check-rtol", type=float, default=5e-2)
+    parser.add_argument("--output-json")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    _run(_parse_args())

benchmarks/kernels/benchmark_ming_istft.py

@@ -0,0 +1,218 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Measure Ming ISTFT and optionally its pretrained streaming AudioVAE decoder.
+
+Example (no model weights needed for the operator benchmark):
+    python benchmarks/kernels/benchmark_ming_istft.py --compile --iterations 100
+
+Add --decoder to measure the published decoder with fixed synthetic latents.
+This is a decoder-stage benchmark, not text-to-speech serving latency.
+"""
+
+import argparse
+import json
+import statistics
+import time
+from collections.abc import Callable
+from functools import partial
+
+import torch
+
+from vllm_omni.model_executor.models.common.ming.audio_dsp import ISTFT
+from vllm_omni.model_executor.models.common.ming.fused_istft import fused_istft
+
+
+def measure(operation: Callable, warmup: int, iterations: int) -> dict:
+    for _ in range(warmup):
+        operation()
+    torch.accelerator.synchronize()
+    samples = []
+    for _ in range(iterations):
+        start = time.perf_counter()
+        operation()
+        torch.accelerator.synchronize()
+        samples.append((time.perf_counter() - start) * 1000)
+    torch.accelerator.reset_peak_memory_stats()
+    before = torch.accelerator.memory_allocated()
+    operation()
+    torch.accelerator.synchronize()
+    return {
+        "p50_ms": statistics.median(samples),
+        "min_ms": min(samples),
+        "max_ms": max(samples),
+        "stdev_ms": statistics.pstdev(samples),
+        "extra_peak_mib": (torch.accelerator.max_memory_allocated() - before) / 1024**2,
+    }
+
+
+def native_post_fft(module, frames, audio, window):
+    """The native middle-chunk post-FFT sequence, isolated from cuFFT."""
+    count = frames.shape[-1]
+    size = (count - 1) * module.hop_length + module.win_length
+
+    def fold(value):
+        return torch.nn.functional.fold(
+            value, output_size=(1, size), kernel_size=(1, module.win_length), stride=(1, module.hop_length)
+        )[:, 0, 0, :]
+
+    samples = fold(frames * module.window[None, :, None])
+    samples, audio = module._buffer_process(samples, audio, 0, streaming=True)
+    envelope = fold(module.window.square().expand(1, count, -1).transpose(1, 2))
+    envelope, window = module._buffer_process(envelope, window, 0, streaming=True)
+    if not (envelope > 1e-11).all():
+        raise RuntimeError("ISTFT window envelope underflowed; invalid overlap-add state.")
+    return samples / envelope, audio, window
+
+
+def assert_outputs(actual, reference):
+    maximum = 0.0
+    for value, expected in zip(actual, reference, strict=True):
+        if expected is None:
+            assert value is None
+        else:
+            torch.testing.assert_close(value, expected, atol=5e-7, rtol=1e-5)
+            maximum = max(maximum, (value - expected).abs().max().item())
+    return maximum
+
+
+def benchmark_istft(args):
+    for dtype in (torch.float32, torch.bfloat16):
+        module = ISTFT(3528, 882, 3528).to(device="cuda", dtype=dtype)
+        for count in (16, 100, 400):
+            spec = torch.randn(1, 1765, count, device="cuda", dtype=torch.complex64)
+            _, audio, window = module.forward_native(spec, streaming=True)
+            frames = torch.fft.irfft(spec, 3528, dim=1)
+            for scope in ("post_fft", "istft"):
+                if scope == "post_fft":
+                    native = partial(native_post_fft, module, frames, audio, window)
+                    fused = partial(fused_istft, frames, module.window, 882, audio, window, True, False)
+                else:
+                    native = partial(module.forward_native, spec, audio, window, streaming=True)
+                    fused = partial(module, spec, audio, window, streaming=True)
+                error = assert_outputs(fused(), native())
+                results = {"native": measure(native, args.warmup, args.iterations)}
+                if args.compile:
+                    compiled = torch.compile(native)
+                    compiled_output, reference = compiled(), native()
+                    # Inductor can change BF16 envelope rounding. Report that
+                    # difference without weakening the fused-kernel parity gate.
+                    compiled_matches = all(
+                        torch.allclose(value, expected, atol=5e-7, rtol=1e-5)
+                        for value, expected in zip(compiled_output, reference, strict=True)
+                    )
+                    results["compiled"] = measure(compiled, args.warmup, args.iterations)
+                    results["compiled"]["matches_native_tolerance"] = compiled_matches
+                    results["compiled"]["waveform_max_abs_error"] = (
+                        (compiled_output[0] - reference[0]).abs().max().item()
+                    )
+                results["fused"] = measure(fused, args.warmup, args.iterations)
+                print(
+                    json.dumps(
+                        {
+                            "scope": scope,
+                            "window_dtype": str(dtype),
+                            "frames": count,
+                            "max_abs_error": error,
+                            "results": results,
+                        }
+                    ),
+                    flush=True,
+                )
+
+
+def benchmark_decoder(args):
+    from huggingface_hub import hf_hub_download
+    from safetensors import safe_open
+
+    from vllm_omni.model_executor.models.common.ming.audio_vae import Decoder
+
+    model = "inclusionAI/Ming-omni-tts-0.5B"
+    revision = "9154772e7fbc585907b6237e3190790676f28975"
+    config_path = hf_hub_download(model, "config.json", revision=revision)
+    with open(config_path) as source:
+        config = json.load(source)["audio_tokenizer_config"]
+    kwargs = config["dec_kwargs"]
+    backbone = kwargs["backbone"]
+    backbone["_attn_implementation"] = "sdpa"
+    decoder = Decoder(backbone, kwargs["output_dim"], kwargs["latent_dim"], config["patch_size"])
+    checkpoint = hf_hub_download(model, "model.safetensors", revision=revision)
+    prefix = "audio.decoder."
+    with safe_open(checkpoint, framework="pt") as weights:
+        state = {key.removeprefix(prefix): weights.get_tensor(key) for key in weights.keys() if key.startswith(prefix)}
+    decoder.load_state_dict(state, strict=True)
+    decoder = decoder.eval().to(device="cuda", dtype=torch.bfloat16)
+    chunks = [
+        torch.randn(1, size, kwargs["latent_dim"], device="cuda", dtype=torch.bfloat16) for size in (4, 25, 25, 5)
+    ]
+    istft = decoder.head.istft
+    fused_forward = istft.forward
+
+    def run():
+        cache = state = None
+        outputs = []
+        for index, chunk in enumerate(chunks):
+            output, state, cache = decoder.low_level_reconstruct(
+                chunk, past_key_values=cache, use_cache=True, stream_state=state, last_chunk=index == len(chunks) - 1
+            )
+            outputs.append(output)
+        return torch.cat(outputs, dim=-1)
+
+    istft.forward = istft.forward_native
+    expected = run()
+    istft.forward = fused_forward
+    actual = run()
+    torch.testing.assert_close(actual, expected, atol=5e-7, rtol=1e-5)
+    # Alternate the order to expose drift when the decoder-level gain is small.
+    results = {"native": [], "fused": []}
+    for order in (("native", "fused"), ("fused", "native"), ("native", "fused")):
+        for name in order:
+            istft.forward = istft.forward_native if name == "native" else fused_forward
+            results[name].append(measure(run, args.warmup, args.decoder_iterations))
+    print(
+        json.dumps(
+            {
+                "scope": "pretrained_decoder",
+                "model": model,
+                "revision": revision,
+                "backend": "sdpa",
+                "dtype": "bfloat16",
+                "latent_chunks": [4, 25, 25, 5],
+                "max_abs_error": (actual - expected).abs().max().item(),
+                "results": results,
+            }
+        ),
+        flush=True,
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--warmup", type=int, default=20)
+    parser.add_argument("--iterations", type=int, default=100)
+    parser.add_argument("--compile", action="store_true")
+    parser.add_argument("--decoder", action="store_true")
+    parser.add_argument("--decoder-iterations", type=int, default=30)
+    args = parser.parse_args()
+    if args.warmup < 0 or min(args.iterations, args.decoder_iterations) < 1:
+        parser.error("warmup must be nonnegative and iteration counts must be positive")
+    torch.manual_seed(0)
+    print(
+        json.dumps(
+            {
+                "torch": torch.__version__,
+                "cuda": torch.version.cuda,
+                "gpu": torch.cuda.get_device_name(),
+                "warmup": args.warmup,
+                "iterations": args.iterations,
+                "decoder_iterations": args.decoder_iterations,
+            }
+        )
+    )
+    with torch.inference_mode():
+        benchmark_istft(args)
+        if args.decoder:
+            benchmark_decoder(args)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/qwen2_5_omni/README.md

@@ -0,0 +1,66 @@
+# Qwen2.5-Omni embedding microbenchmark
+
+Measure the host-observed latency of `embed_input_ids` before and after the
+redundant text embedding removal in PR #7477. Both variants use the same real
+checkpoint embedding table, dependencies, inputs and batch shapes. The baseline
+wrapper is extracted from the specified Git commit; the fixed wrapper comes from
+the current checkout. No full model is initialized.
+
+## Run
+
+Use Linux, a CUDA GPU, the project dependencies, and a local
+`Qwen/Qwen2.5-Omni-7B` checkpoint with its safetensors index and weights.
+From the repository root:
+
+```bash
+git fetch origin 01a2f93256975c7ff9565c5414b0fe0225bc4765
+PYTHONPATH="$PWD" OMP_NUM_THREADS=1 python benchmarks/qwen2_5_omni/benchmark_embeddings.py \
+  --model /path/to/Qwen2.5-Omni-7B \
+  --baseline-ref 01a2f93256975c7ff9565c5414b0fe0225bc4765 \
+  --cpu-core 2 --lengths 1172 --warmup 50 --blocks 20 --iterations 20 \
+  --output-dir /tmp/qwen25-embedding-results
+```
+
+Choose a CPU core in your allowed affinity set. For the original CUDA length
+sweep, use `--lengths 512 1172 4096 16384`.
+
+## Method
+
+- BF16 checkpoint embeddings, TP=1, CUDA int32 token IDs, CPU multimodal masks.
+- One pinned CPU core, one PyTorch thread, garbage collection disabled.
+- 50 warmups per variant, then 20 paired blocks of 20 calls with balanced,
+  randomized variant order. Summary: median of block medians.
+- Host latency includes synchronization inside the wrapper. Explicit GPU
+  synchronization before and after the timed call is excluded from host latency.
+  Calling-thread CPU time and synchronized completion time are recorded separately.
+- CPU frequency is sampled at block boundaries; clocks are not locked.
+- Validation checks one text embedding call for the fixed wrapper, two for the
+  multimodal baseline, and `torch.equal` for identical inputs and batch shapes.
+  This does not establish batch invariance or full-model output equality.
+
+Interleaved audio/video uses a valid synthetic multimodal mask. This benchmark
+measures wrapper latency, not E2E throughput.
+
+## Reference results
+
+Original environment: H100 80GB; Python 3.12.3; vLLM 0.29.0+cu129;
+PyTorch 2.13.0+cu129; Transformers 5.14.0; Qwen2.5-Omni-7B BF16.
+At 1172 tokens, the September 13 measurements were:
+
+| Input | Before (μs) | After (μs) | Host latency reduction |
+| --- | ---: | ---: | ---: |
+| Text | 18.81 | 18.66 | 0.80% |
+| Audio | 104.32 | 85.30 | 18.23% |
+| Image | 104.72 | 86.27 | 17.62% |
+| Video | 104.73 | 85.76 | 18.11% |
+| Audio + image + video | 567.92 | 549.54 | 3.24% |
+| Interleaved audio/video | 622.76 | 597.84 | 4.00% |
+
+Results vary between runs and environments. These local savings did not produce
+a consistent E2E throughput improvement in the separate repeated E2E experiment.
+
+## Output
+
+The output directory contains `metadata.json`, `micro-raw.json`,
+`validation.json`, and `summary.json`. Raw records include per-call timings,
+block timestamps and CPU frequency samples. Timer overhead is measured separately.

benchmarks/qwen2_5_omni/benchmark_embeddings.py

@@ -0,0 +1,306 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Reproduce PR #7477 host-observed embedding-wrapper latency on one CUDA GPU.
+
+Run from the PR checkout with PYTHONPATH=$PWD. Baseline and fixed methods share
+weights and inputs. Only the historical wrapper is loaded from --baseline-ref.
+"""
+
+import argparse
+import ast
+import gc
+import json
+import os
+import random
+import statistics
+import subprocess
+import textwrap
+import time
+from pathlib import Path
+
+import torch
+from safetensors import safe_open
+from torch import nn
+from transformers import Qwen2_5OmniThinkerConfig
+from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
+from vllm.multimodal.utils import set_mm_embedding_modality
+
+from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import (
+    Qwen2_5OmniThinkerForConditionalGeneration as Thinker,
+)
+from vllm_omni.platforms import current_omni_platform
+
+
+def baseline_method(cls, ref):
+    path = "vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py"
+    source = subprocess.check_output(
+        ["git", "-C", str(Path(__file__).resolve().parents[2]), "show", f"{ref}:{path}"], text=True
+    )
+    tree = ast.parse(source)
+    owner = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == cls.__name__)
+    node = next(n for n in owner.body if isinstance(n, ast.FunctionDef) and n.name == "embed_input_ids")
+    method = textwrap.dedent(ast.get_source_segment(source, node))
+    # Preserve zero-argument super() against the real model class and MRO.
+    factory = "from __future__ import annotations\ndef make_method(cls):\n    __class__ = cls\n"
+    factory += textwrap.indent(method, "    ") + "\n    return embed_input_ids\n"
+    namespace = dict(cls.embed_input_ids.__globals__)
+    exec(compile(factory, f"{ref}:{path}", "exec"), namespace)
+    return namespace["make_method"](cls)
+
+
+parser = argparse.ArgumentParser(description=__doc__)
+parser.add_argument("--model", type=Path, required=True)
+parser.add_argument("--baseline-ref", default="01a2f93256975c7ff9565c5414b0fe0225bc4765")
+parser.add_argument("--cpu-core", type=int, default=sorted(os.sched_getaffinity(0))[0])
+parser.add_argument("--lengths", type=int, nargs="+", default=[1172])
+parser.add_argument("--warmup", type=int, default=50)
+parser.add_argument("--blocks", type=int, default=20)
+parser.add_argument("--iterations", type=int, default=20)
+parser.add_argument("--output-dir", type=Path, default=Path("micro-results"))
+args = parser.parse_args()
+if min(args.lengths) < 300 or min(args.blocks, args.iterations) < 1 or args.warmup < 0:
+    parser.error("Require lengths >= 300, positive blocks/iterations, and nonnegative warmup")
+if args.cpu_core not in os.sched_getaffinity(0):
+    parser.error("CPU core is outside the allowed affinity set")
+ROOT, MODEL, CORE = args.output_dir, args.model, args.cpu_core
+ROOT.mkdir(parents=True, exist_ok=True)
+CONFIG = json.loads((MODEL / "config.json").read_text())["thinker_config"]
+HIDDEN = CONFIG["text_config"]["hidden_size"]
+VOCAB = CONFIG["text_config"]["vocab_size"]
+IDS = {m: CONFIG[m + "_token_index"] for m in ("audio", "image", "video")}
+METHODS = {"baseline": baseline_method(Thinker, args.baseline_ref), "fixed": Thinker.embed_input_ids}
+RESULTS, VALIDATIONS = [], []
+os.sched_setaffinity(0, {CORE})
+torch.set_num_threads(1)
+torch.set_num_interop_threads(1)
+gc.disable()
+
+
+def cpu_frequency():
+    path = Path(f"/sys/devices/system/cpu/cpu{CORE}/cpufreq/scaling_cur_freq")
+    if path.exists():
+        return float(path.read_text()) / 1000
+    for section in Path("/proc/cpuinfo").read_text().split("\n\n"):
+        fields = dict(line.split(":", 1) for line in section.splitlines() if ":" in line)
+        fields = {k.strip(): v.strip() for k, v in fields.items()}
+        if fields.get("processor") == str(CORE):
+            return float(fields.get("cpu MHz", "nan"))
+
+
+def payload(case, length, device):
+    generator = torch.Generator().manual_seed(7451 + length)
+    ids = torch.randint(2, 10000, (length,), generator=generator)
+    specs = {
+        "text": [],
+        "audio": [("audio", 100)],
+        "image": [("image", 16)],
+        "video": [("video", 128)],
+        "mixed": [("audio", 100), ("image", 16), ("video", 128)],
+        "interleaved": [("video", 64), ("audio", 50), ("video", 64), ("audio", 50)],
+    }[case]
+    offset = 44
+    counts = {}
+    for modality, count in specs:
+        ids[offset : offset + count] = IDS[modality]
+        counts[modality] = counts.get(modality, 0) + count
+        offset += count + (0 if case == "interleaved" else 2)
+    mask = torch.zeros(length, dtype=torch.bool)
+    for token in IDS.values():
+        mask |= ids == token
+    mm = [
+        set_mm_embedding_modality(torch.full((count, HIDDEN), i + 1, device=device, dtype=torch.bfloat16), modality)
+        for i, (modality, count) in enumerate(counts.items())
+    ]
+    return ids.to(device=device, dtype=torch.int32), None if case == "text" else mm, None if case == "text" else mask
+
+
+def once(fn, cuda):
+    # Drain preceding work before the host timer; the production function's
+    # own D2H synchronizations remain inside the timed call.
+    if cuda:
+        current_omni_platform.synchronize()
+    w0 = time.perf_counter_ns()
+    c0 = time.thread_time_ns()
+    value = fn()
+    c1 = time.thread_time_ns()
+    w1 = time.perf_counter_ns()
+    if cuda:
+        current_omni_platform.synchronize()
+    w2 = time.perf_counter_ns() if cuda else w1
+    del value
+    return [(w1 - w0) / 1000, (c1 - c0) / 1000, (w2 - w0) / 1000]
+
+
+def measure(functions, device, case, length, scope, rounds=20, iterations=20, warmups=50):
+    cuda = device == "cuda"
+    for fn in functions.values():
+        for _ in range(warmups):
+            once(fn, cuda)
+    order_rng = random.Random(7451 + length + len(case) + len(scope))
+    directions = [i % 2 for i in range(rounds)]
+    order_rng.shuffle(directions)
+    names = list(functions)
+    for block, direction in enumerate(directions):
+        order = names if direction == 0 else names[::-1]
+        for name in order:
+            frequency_start = cpu_frequency()
+            wall_start = time.time()
+            samples = [once(functions[name], cuda) for _ in range(iterations)]
+            wall_end = time.time()
+            RESULTS.append(
+                dict(
+                    device=device,
+                    case=case,
+                    tokens=length,
+                    scope=scope,
+                    variant=name,
+                    block=block,
+                    iterations=iterations,
+                    samples_us=samples,
+                    wall_start=wall_start,
+                    wall_end=wall_end,
+                    cpu_mhz_start=frequency_start,
+                    cpu_mhz_end=cpu_frequency(),
+                )
+            )
+    (ROOT / "micro-raw.json").write_text(json.dumps(RESULTS))
+    current = [r for r in RESULTS if (r["device"], r["case"], r["tokens"], r["scope"]) == (device, case, length, scope)]
+    print(
+        "RESULT",
+        json.dumps(
+            {
+                "device": device,
+                "case": case,
+                "tokens": length,
+                "scope": scope,
+                "host_median_us": {
+                    v: statistics.median(t[0] for r in current if r["variant"] == v for t in r["samples_us"])
+                    for v in names
+                },
+            }
+        ),
+        flush=True,
+    )
+
+
+def build_model(weights, device):
+    model = Thinker.__new__(Thinker)
+    nn.Module.__init__(model)
+    model.config = Qwen2_5OmniThinkerConfig(**CONFIG)
+    model._has_oov_mm_tokens = False
+    table = VocabParallelEmbedding(VOCAB, HIDDEN, params_dtype=torch.bfloat16, disable_tp=True)
+    table.weight = nn.Parameter(weights.to(device), requires_grad=False)
+
+    class LM(nn.Module):
+        def __init__(self):
+            super().__init__()
+            self.embed_tokens = table
+
+        def embed_input_ids(self, ids):
+            return self.embed_tokens(ids)
+
+    model.language_model = LM()
+    return model
+
+
+def validate(model, args, case, length, device):
+    ids, mm, mask = args
+    lm = model.get_language_model()
+    original = lm.embed_input_ids
+    outputs, counts = {}, {}
+    for variant, method in METHODS.items():
+        calls = []
+
+        def counted(tokens):
+            calls.append(tokens.numel())
+            return original(tokens)
+
+        lm.embed_input_ids = counted
+        try:
+            outputs[variant] = method(model, ids, mm, is_multimodal=mask)
+        finally:
+            lm.embed_input_ids = original
+        counts[variant] = len(calls)
+    assert torch.equal(outputs["baseline"], outputs["fixed"])
+    assert counts == {"baseline": 1 if case == "text" else 2, "fixed": 1}
+    VALIDATIONS.append(dict(case=case, tokens=length, device=device, calls=counts, same_input_wrapper_torch_equal=True))
+    (ROOT / "validation.json").write_text(json.dumps(VALIDATIONS, indent=2))
+
+
+@torch.inference_mode()
+def main():
+    import transformers
+    import vllm
+
+    metadata = dict(
+        baseline_ref=args.baseline_ref,
+        checkout=subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
+        torch=torch.__version__,
+        vllm=vllm.__version__,
+        transformers=transformers.__version__,
+        gpu=torch.cuda.get_device_name(),
+        cpu_core=CORE,
+        torch_cpu_threads=1,
+        warmup=args.warmup,
+        blocks=args.blocks,
+        iterations=args.iterations,
+        lengths=args.lengths,
+        embedding_shape=[VOCAB, HIDDEN],
+        dtype="bfloat16",
+        input_ids_dtype="int32",
+        mask_device="cpu",
+        start_time=time.time(),
+        metric_order=["host_observed_us", "calling_thread_cpu_us", "synchronized_completion_us"],
+    )
+    index = json.loads((MODEL / "model.safetensors.index.json").read_text())["weight_map"]
+    key = "thinker.model.embed_tokens.weight"
+    with safe_open(MODEL / index[key], framework="pt", device="cpu") as f:
+        weights = f.get_tensor(key)
+    assert weights.shape == (VOCAB, HIDDEN) and weights.dtype == torch.bfloat16
+    model = build_model(weights, "cuda")
+    measure({"timer_floor": lambda: None}, "cuda", "timer", 0, "overhead", rounds=10, iterations=100)
+    for length in args.lengths:
+        for case in ("text", "audio", "image", "video", "mixed", "interleaved"):
+            ids, mm, mask = payload(case, length, "cuda")
+            validate(model, (ids, mm, mask), case, length, "cuda")
+            functions = {
+                variant: (lambda method=method: method(model, ids, mm, is_multimodal=mask))
+                for variant, method in METHODS.items()
+            }
+            measure(
+                functions,
+                "cuda",
+                case,
+                length,
+                "whole_wrapper",
+                rounds=args.blocks,
+                iterations=args.iterations,
+                warmups=args.warmup,
+            )
+    summary = []
+    for length in args.lengths:
+        for case in ("text", "audio", "image", "video", "mixed", "interleaved"):
+            medians = {
+                variant: statistics.median(
+                    statistics.median(sample[0] for sample in record["samples_us"])
+                    for record in RESULTS
+                    if record["tokens"] == length and record["case"] == case and record["variant"] == variant
+                )
+                for variant in METHODS
+            }
+            summary.append(
+                dict(
+                    tokens=length,
+                    case=case,
+                    host_us=medians,
+                    reduction_pct=100 * (1 - medians["fixed"] / medians["baseline"]),
+                )
+            )
+    metadata["end_time"] = time.time()
+    (ROOT / "metadata.json").write_text(json.dumps(metadata, indent=2))
+    (ROOT / "summary.json").write_text(json.dumps(summary, indent=2))
+    print(json.dumps(summary, indent=2))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/tts/bench_cosyvoice3_concurrency_matrix.py

@@ -0,0 +1,517 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""CosyVoice3 streaming e2e benchmark: concurrency x utterance-length matrix.
+
+Launches the server once per (concurrency, step-size) combo, streams TTS
+requests cycling through several utterance lengths, and reports
+TTFA/E2E/RTF/per-chunk latency (mean, stdev, range) per cell. "Step size" is
+``codec_chunk_frames``, the token-hop length the first emitted chunk waits
+for; later chunks wait for a doubling hop capped at ``4 * codec_chunk_frames``
+(see ``_token_hop_lengths``). Per-chunk latency is reconstructed from that
+same ramp rather than from raw transport read boundaries, which don't align
+to model chunks (see ``one_request``'s docstring).
+
+Running with no flags reproduces the default matrix. Per-request correctness
+is checked: non-silent, long enough, and (with a fixed ``--seed``)
+byte-identical PCM across requests for the same text.
+"""
+
+import argparse
+import json
+import os
+import signal
+import statistics
+import subprocess
+import sys
+import threading
+import time
+from pathlib import Path
+
+import httpx
+import numpy as np
+
+REPO = Path(__file__).resolve().parents[2]
+DEFAULT_MODEL = "FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
+
+DEFAULT_REF_AUDIO = "https://raw.githubusercontent.com/FunAudioLLM/CosyVoice/main/asset/zero_shot_prompt.wav"
+DEFAULT_REF_TEXT = "希望你以后能够做的比我还好呦。"
+
+DEFAULT_TEXTS = [
+    "Thank you so much for the birthday gift you sent me.",
+    "The weather is lovely today, with warm sunshine and a gentle breeze. "
+    "We decided to take a walk in the park to enjoy the spring flowers and the fresh air.",
+    "The weather is lovely today, with warm sunshine and a gentle breeze. We decided to take a walk in the "
+    "park to enjoy the spring flowers and the fresh air. Many people were out walking, children were "
+    "running and playing on the grass, and older folks sat on benches chatting with one another. We "
+    "strolled slowly along the path and came across a cherry tree in full bloom, its pink petals drifting "
+    "on the wind. It was so beautiful that we stopped to take some photos to remember the moment. Then we "
+    "continued on and reached a small lake, its water so clear you could see straight to the bottom, with "
+    "the blue sky and white clouds reflected on its surface. A few ducks paddled back and forth across the "
+    "water, occasionally letting out a soft quack. We sat on a bench by the lake for a while, enjoying the "
+    "peaceful afternoon. Time passed quickly, and before we knew it evening had arrived, with the setting "
+    "sun painting the sky in shades of gold. Reluctant to leave, we finally headed home, already planning "
+    "our next visit. On the way back we talked about everything we had seen and heard that day, feeling "
+    "wonderfully happy. It was truly an unforgettable weekend.",
+]
+
+# PCM16 mono @ 24 kHz (CosyVoice3 output rate)
+SAMPLE_RATE = 24000
+
+# Well below any real speech's RMS; catches all-zero/near-silent PCM that byte
+# count and duration checks alone pass through.
+MIN_RMS = 50.0
+
+# Overlay: only codec_chunk_frames is overridden; must live next to cosyvoice3.yaml (relative base_config)
+DEPLOY_TEMPLATE = """# Auto-generated by bench_cosyvoice3_concurrency_matrix.py -- do not edit.
+base_config: cosyvoice3.yaml
+
+connectors:
+  connector_of_shared_memory:
+    name: SharedMemoryConnector
+    extra:
+      codec_streaming: true
+      connector_get_sleep_s: 0.01
+      connector_get_max_wait_first_chunk: 3000
+      connector_get_max_wait: 300
+      codec_chunk_frames: {chunk_frames}
+      codec_max_chunk_frames: {max_chunk_frames}
+      codec_stream_scale_factor: {stream_scale_factor}
+      codec_left_context_frames: 25
+      codec_vocab_size: 6561
+"""
+
+# CosyVoice3Config constants (transformers_utils/configs/cosyvoice3.py): tokens
+# are consumed at token_mel_ratio mel frames each, and HiFT emits
+# samples_per_mel PCM16 samples per mel frame. Pinned here (not left to the
+# server's defaults) since one_request's byte-boundary reconstruction below
+# depends on knowing them exactly.
+TOKEN_MEL_RATIO = 2
+SAMPLES_PER_MEL = 480
+STREAM_SCALE_FACTOR = 2
+
+
+def _stats(values: list[float]) -> dict[str, float]:
+    """Mean/median/stdev/p95/min/max/n for a list of per-request measurements."""
+    n = len(values)
+    if n == 0:
+        return {
+            "n": 0,
+            "mean": float("nan"),
+            "median": float("nan"),
+            "stdev": float("nan"),
+            "p95": float("nan"),
+            "min": float("nan"),
+            "max": float("nan"),
+        }
+    return {
+        "n": n,
+        "mean": statistics.mean(values),
+        "median": statistics.median(values),
+        "stdev": statistics.stdev(values) if n >= 2 else 0.0,
+        "p95": statistics.quantiles(values, n=20)[18] if n >= 2 else values[0],
+        "min": min(values),
+        "max": max(values),
+    }
+
+
+def vllm_bin() -> str:
+    for c in (REPO / ".venv/bin/vllm", Path(sys.executable).parent / "vllm"):
+        if c.is_file():
+            return str(c)
+    return "vllm"
+
+
+def wait_ready(url: str, proc: subprocess.Popen, log: Path, timeout: float) -> None:
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if proc.poll() is not None:
+            tail = log.read_text(errors="replace").splitlines()[-40:] if log.exists() else []
+            raise RuntimeError("server exited during startup:\n  " + "\n  ".join(tail))
+        try:
+            if httpx.get(f"{url}/health", timeout=2.0).status_code == 200:
+                return
+        except Exception:
+            pass
+        time.sleep(2.0)
+    raise TimeoutError(f"server not ready within {timeout}s (log: {log})")
+
+
+def _token_hop_lengths(chunk_frames: int, max_chunk_frames: int):
+    """The token-hop-length ramp talker2code2wav_async_chunk uses to decide
+    when to fire the next flow-solve/emission: chunk_frames, then doubling
+    (STREAM_SCALE_FACTOR) each step, capped at max_chunk_frames. This -- not
+    codec_chunk_frames alone -- is what actually paces emitted audio chunks:
+    with the defaults here, hops are chunk_frames, 2x, 4x, 4x, 4x, ...
+
+    The very first emission is consistently one token short of chunk_frames
+    (confirmed against real speech.audio.delta event sizes) even though the
+    ramp's own progression is computed from the nominal chunk_frames, not the
+    actual first hop -- so only the first yield gets the -1 adjustment.
+    """
+    yield chunk_frames - 1
+    hop = chunk_frames
+    while True:
+        hop = min(max_chunk_frames, max(chunk_frames, hop * STREAM_SCALE_FACTOR))
+        yield hop
+
+
+def one_request(client, url, model, text, ref_audio, ref_text, chunk_frames, seed=None):
+    """Run one streaming request, returning TTFA, E2E, per-chunk gaps, and audio bytes.
+
+    ``response_format=pcm`` + ``stream_format=audio`` streams raw bytes with no
+    framing, so a client read boundary alone is a transport artifact, not a
+    model chunk boundary. Reconstructed instead by replicating the server's
+    own token-hop-length ramp (see ``_token_hop_lengths``) to compute each
+    real chunk's expected byte size ahead of time, and slicing the
+    accumulated byte stream at those offsets -- independent of how the
+    transport happens to fragment the underlying reads. The final chunk
+    (finalize, which releases extra held-back frames) is whatever remains
+    when the stream closes, not a predicted hop.
+    """
+    payload = {
+        "model": model,
+        "input": text,
+        "response_format": "pcm",
+        "stream": True,
+        "stream_format": "audio",
+        "ref_audio": ref_audio,
+        "ref_text": ref_text,
+    }
+    if seed is not None:
+        payload["seed"] = seed
+    t0 = time.perf_counter()
+    ttfa = None
+    nbytes = 0
+    audio_bytes = bytearray()
+    chunk_gaps_ms = []  # inter-chunk arrival gaps (per model chunk), by step index
+    prev_t = None
+    step_idx = 0
+
+    bytes_per_token = TOKEN_MEL_RATIO * SAMPLES_PER_MEL * 2
+    hop_gen = _token_hop_lengths(chunk_frames, 4 * chunk_frames)
+    next_boundary = next(hop_gen) * bytes_per_token
+    last_crossed_bytes = 0
+    received = 0
+    last_now = t0
+
+    with client.stream("POST", f"{url}/v1/audio/speech", json=payload, headers={"Authorization": "Bearer EMPTY"}) as r:
+        if r.status_code != 200:
+            raise RuntimeError(f"{r.status_code}: {r.read()[:300]!r}")
+        for chunk in r.iter_bytes():
+            if not chunk:
+                continue
+            now = time.perf_counter()
+            if ttfa is None:
+                ttfa = now - t0
+            last_now = now
+            received += len(chunk)
+            nbytes += len(chunk)
+            audio_bytes += chunk
+            while received >= next_boundary:
+                if prev_t is not None:
+                    chunk_gaps_ms.append((step_idx, (now - prev_t) * 1e3))
+                    step_idx += 1
+                prev_t = now
+                last_crossed_bytes = next_boundary
+                next_boundary += next(hop_gen) * bytes_per_token
+
+    if received > last_crossed_bytes:
+        # Finalize chunk: releases extra held-back frames, so it doesn't
+        # match a predicted hop size -- its arrival is the stream closing.
+        if prev_t is not None:
+            chunk_gaps_ms.append((step_idx, (last_now - prev_t) * 1e3))
+            step_idx += 1
+
+    e2e = time.perf_counter() - t0
+    audio_s = nbytes / 2 / SAMPLE_RATE
+    return {
+        "ttfa_s": ttfa,
+        "e2e_s": e2e,
+        "audio_s": audio_s,
+        "rtf": e2e / audio_s if audio_s > 0 else float("nan"),
+        "chunk_gaps_ms": chunk_gaps_ms,
+        "nbytes": nbytes,
+        "audio_bytes": audio_bytes,
+    }
+
+
+def check_correctness(r: dict, min_audio_s: float) -> list[str]:
+    """Return a list of correctness violations (empty = pass)."""
+    problems = []
+    if r["nbytes"] <= 0:
+        problems.append("empty audio (0 bytes)")
+    if r["audio_s"] < min_audio_s:
+        problems.append(f"audio too short ({r['audio_s']:.2f}s < {min_audio_s:.2f}s)")
+    if r["nbytes"] > 0:
+        samples = np.frombuffer(bytes(r["audio_bytes"]), dtype=np.int16).astype(np.float64)
+        rms = float(np.sqrt(np.mean(samples**2)))
+        if rms < MIN_RMS:
+            problems.append(f"near-silent audio (rms={rms:.1f} < {MIN_RMS})")
+    return problems
+
+
+def drive(url, args, n, conc, label, results, lock, idx, chunk_frames):
+    """Run ``n`` requests at concurrency ``conc``, cycling through ``args.texts``."""
+    texts = args.texts
+
+    def worker():
+        with httpx.Client(timeout=600.0) as c:
+            while True:
+                with lock:
+                    i = idx[0]
+                    idx[0] += 1
+                if i >= n:
+                    return
+                text = texts[i % len(texts)]
+                try:
+                    r = one_request(
+                        c, url, args.model, text, args.ref_audio, args.ref_text, chunk_frames, seed=args.seed
+                    )
+                    r["text"] = text
+                    with lock:
+                        results.append(r)
+                except Exception as e:
+                    print(f"    [{label}] request {i} failed: {e}", flush=True)
+
+    ts = [threading.Thread(target=worker) for _ in range(conc)]
+    t0 = time.perf_counter()
+    for t in ts:
+        t.start()
+    for t in ts:
+        t.join()
+    return time.perf_counter() - t0
+
+
+def run_combo(args, conc: int, chunk_frames: int, out: Path) -> dict:
+    tag = f"c{conc}_chunk{chunk_frames}"
+    run_dir = out / tag
+    run_dir.mkdir(parents=True, exist_ok=True)
+    log = run_dir / "server.log"
+
+    deploy = REPO / "vllm_omni/deploy" / f"_autogen_steps_{tag}.yaml"
+    deploy.write_text(
+        DEPLOY_TEMPLATE.format(
+            chunk_frames=chunk_frames,
+            max_chunk_frames=4 * chunk_frames,
+            stream_scale_factor=STREAM_SCALE_FACTOR,
+        )
+    )
+
+    cmd = [
+        vllm_bin(),
+        "serve",
+        args.model,
+        "--omni",
+        "--deploy-config",
+        str(deploy),
+        "--host",
+        "127.0.0.1",
+        "--port",
+        str(args.port),
+        "--allowed-local-media-path",
+        str(REPO),
+    ]
+    env = dict(os.environ)
+    if args.cuda_visible_devices:
+        env["CUDA_VISIBLE_DEVICES"] = args.cuda_visible_devices
+    if args.trt is not None:
+        env["COSYVOICE3_TRT"] = args.trt
+
+    print(f"\n=== concurrency={conc} step_size={chunk_frames} ===", flush=True)
+    print(f"  launching server (log: {log})", flush=True)
+    with open(log, "w") as lf:
+        proc = subprocess.Popen(cmd, env=env, stdout=lf, stderr=subprocess.STDOUT, start_new_session=True)
+    url = f"http://127.0.0.1:{args.port}"
+    try:
+        wait_ready(url, proc, log, args.server_timeout)
+        print("  server ready", flush=True)
+
+        results: list[dict] = []
+        lock = threading.Lock()
+        idx = [0]
+        if args.warmup:
+            print(f"  warmup ({args.warmup} req)", flush=True)
+            drive(url, args, args.warmup, min(conc, args.warmup), "warmup", results, lock, idx, chunk_frames)
+            results.clear()
+            idx[0] = 0
+
+        print(f"  running {args.num_prompts} req @ concurrency {conc}", flush=True)
+        wall = drive(url, args, args.num_prompts, conc, "bench", results, lock, idx, chunk_frames)
+
+        ok = [r for r in results if r.get("ttfa_s") is not None]
+        problems = []
+        for r in ok:
+            problems += check_correctness(r, args.min_audio_s)
+        # With a fixed seed, same-text requests must produce byte-identical PCM (per text group)
+        if args.seed is not None:
+            by_text: dict[str, bytes] = {}
+            for r in ok:
+                t = r.get("text")
+                ref = by_text.get(t)
+                if ref is None:
+                    by_text[t] = r.get("audio_bytes")
+                elif r.get("audio_bytes") != ref:
+                    problems.append(f"byte-identity violation for text {t!r}: seeded requests produced different PCM")
+        n_ok = len(ok)
+        total_audio_s = sum(r["audio_s"] for r in ok)
+        all_gaps = [g for r in ok for g in r["chunk_gaps_ms"]]  # (step_index, gap_ms) per request
+        gap_vals = [g for _, g in all_gaps]
+
+        # bucket gaps by chunk position within a request, to see any O(n) slope
+        by_idx: dict[int, list[float]] = {}
+        for step_index, g in all_gaps:
+            by_idx.setdefault(step_index, []).append(g)
+        step_latency_by_idx = {i: statistics.median(vals) for i, vals in sorted(by_idx.items())}
+
+        agg = {
+            "concurrency": conc,
+            "step_size": chunk_frames,
+            "ok": n_ok,
+            "errors": args.num_prompts - n_ok,
+            "correctness_problems": len(problems),
+            "correctness_sample": problems[:5],
+            "total_audio_s": total_audio_s,
+            "throughput_audio_s_per_s": total_audio_s / wall if wall > 0 else float("nan"),
+            "ttfa_s": _stats([r["ttfa_s"] for r in ok]),
+            "e2e_s": _stats([r["e2e_s"] for r in ok]),
+            "rtf": _stats([r["rtf"] for r in ok]),
+            "step_latency_ms": _stats(gap_vals),
+            "step_latency_ms_by_idx": step_latency_by_idx,
+            "num_steps": len(all_gaps),
+            # per-utterance-length breakdown; n per text = num_prompts / len(texts)
+            "by_text": {
+                t: {
+                    "audio_s": _stats([r["audio_s"] for r in ok if r.get("text") == t]),
+                    "e2e_s": _stats([r["e2e_s"] for r in ok if r.get("text") == t]),
+                    "rtf": _stats([r["rtf"] for r in ok if r.get("text") == t]),
+                }
+                for t in sorted({r.get("text") for r in ok})
+            },
+        }
+        print(json.dumps(agg, indent=2, ensure_ascii=False), flush=True)
+        return agg
+    finally:
+        print("  shutting down server", flush=True)
+        try:
+            os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+        except Exception:
+            proc.terminate()
+        try:
+            proc.wait(timeout=60)
+        except subprocess.TimeoutExpired:
+            try:
+                os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
+            except Exception:
+                proc.kill()
+            proc.wait(timeout=20)
+        deploy.unlink(missing_ok=True)
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--model", default=DEFAULT_MODEL)
+    p.add_argument(
+        "--text",
+        dest="texts",
+        nargs="+",
+        default=DEFAULT_TEXTS,
+        help="One or more utterance texts. With several, requests cycle through "
+        "them so a single server launch covers all utterance lengths; results "
+        "are bucketed per text.",
+    )
+    p.add_argument("--ref-audio", default=DEFAULT_REF_AUDIO)
+    p.add_argument("--ref-text", default=DEFAULT_REF_TEXT)
+    p.add_argument("--concurrency", type=int, nargs="+", default=[1, 8])
+    p.add_argument(
+        "--step-size",
+        type=int,
+        nargs="+",
+        default=[15],
+        help="codec_chunk_frames values to sweep (the streaming step size).",
+    )
+    p.add_argument("--num-prompts", type=int, default=30)
+    p.add_argument("--warmup", type=int, default=2)
+    p.add_argument("--port", type=int, default=8091)
+    p.add_argument(
+        "--server-timeout",
+        type=float,
+        default=1800.0,
+        help="Startup budget; first run also downloads weights and builds the TRT engine.",
+    )
+    p.add_argument("--output-dir", default="./perf_steps")
+    p.add_argument("--cuda-visible-devices", default="0")
+    p.add_argument(
+        "--trt",
+        choices=["0", "1"],
+        default=None,
+        help="Pin COSYVOICE3_TRT. 0=force torch estimator, 1=allow TensorRT. Default: unset.",
+    )
+    p.add_argument(
+        "--min-audio-s",
+        type=float,
+        default=1.0,
+        help="Minimum expected audio seconds per request for the correctness check.",
+    )
+    p.add_argument(
+        "--seed",
+        type=int,
+        default=42,
+        help="Fixed seed for reproducible (byte-identical) audio. Pass --seed '' "
+        "equivalent (any non-int) to disable -- there is no unset sentinel here "
+        "by design, since the default sweep is meant to be exactly reproducible.",
+    )
+    return p.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    out = Path(args.output_dir).resolve()
+    out.mkdir(parents=True, exist_ok=True)
+
+    all_results = []
+    failed = False
+    for conc in args.concurrency:
+        for chunk_frames in args.step_size:
+            try:
+                result = run_combo(args, conc, chunk_frames, out)
+                all_results.append(result)
+                if result["errors"] > 0 or result["correctness_problems"] > 0:
+                    failed = True
+            except Exception as e:
+                print(f"  combo c={conc} chunk={chunk_frames} FAILED: {e}", flush=True)
+                failed = True
+
+    out_path = out / "steps_results.json"
+    with open(out_path, "w", encoding="utf-8") as f:
+        json.dump({"model": args.model, "texts": args.texts, "runs": all_results}, f, indent=2, ensure_ascii=False)
+    print(f"\nResults written to {out_path}", flush=True)
+
+    print("\n=== SUMMARY ===")
+    header = (
+        f"{'conc':>4} {'step':>4} {'ok':>4} {'corr':>4} "
+        f"{'TTFA med(ms)':>12} {'step med(ms)':>12} {'step p95(ms)':>12} {'RTF mean+-sd':>14}"
+    )
+    print(header)
+    print("-" * len(header))
+    for r in all_results:
+        print(
+            f"{r['concurrency']:>4} {r['step_size']:>4} {r['ok']:>4} {r['correctness_problems']:>4} "
+            f"{r['ttfa_s']['median'] * 1e3:>12.1f} "
+            f"{r['step_latency_ms']['median']:>12.1f} {r['step_latency_ms']['p95']:>12.1f} "
+            f"{r['rtf']['mean']:>6.3f}+-{r['rtf']['stdev']:>5.3f}"
+        )
+        for t, stats in r.get("by_text", {}).items():
+            print(
+                f"      text {t[:24]!r}: e2e {stats['e2e_s']['mean'] * 1e3:8.1f}"
+                f"+-{stats['e2e_s']['stdev'] * 1e3:6.1f} ms  (n={stats['e2e_s']['n']}, "
+                f"audio {stats['audio_s']['mean']:.1f}s)"
+            )
+
+    if failed:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

docker/Dockerfile.cuda

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai:v0.28.0
+ARG BASE_IMAGE=vllm/vllm-openai:v0.29.0
 FROM ${BASE_IMAGE}
 
 ARG COMMON_WORKDIR=/app

docker/Dockerfile.npu

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_VERSION=v0.28.0
+ARG VLLM_ASCEND_VERSION=v0.29.0
 ARG VLLM_ASCEND_DEVICE_SUFFIX=
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_VERSION}${VLLM_ASCEND_DEVICE_SUFFIX}
 

docker/Dockerfile.npu.ci

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_VERSION=v0.28.0
+ARG VLLM_ASCEND_VERSION=v0.29.0
 ARG VLLM_ASCEND_DEVICE_SUFFIX=
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_VERSION}${VLLM_ASCEND_DEVICE_SUFFIX}
 
@@ -44,6 +44,7 @@ RUN python3 -m pip install -U pip && \
       "onnx>=1.16.0" \
       "librosa>=0.10.0" \
       "datasets>=3.0.0" \
+      "torchmetrics==1.9.0" \
       --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
       --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
       --default-timeout=60

docker/Dockerfile.rocm

@@ -1,16 +1,18 @@
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.28.0
+# Keep the ROCm runtime aligned with the vLLM release targeted by
+# docker/Dockerfile.ci. A stale base image can build successfully while failing
+# at runtime when vLLM-Omni imports APIs added by the newer vLLM release.
+ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.29.0
 FROM ${BASE_IMAGE} AS base
 
-# Declare a variable to know if we want to use the nightly build or the stable build.
 # NOTE: REMINDER to vLLM-Omni rebase maintainer
-# Remember to set `USE_NIGHTLY_BUILD` to 0, when switching back to stable vLLM docker image
-# 1. If vLLM-omni maintainer is forced to use custom commits
-#    during rebasing, they can change the two variables
-# 2. Whenever vLLM upstream has released a stable version,
-#    we should swap over to use stable release ASAP.
-#    We should avoid relying on custom commits.
+# BASE_IMAGE pins the default prebuilt runtime when USE_NIGHTLY_BUILD=0.
+# VLLM_VERSION_OR_COMMIT_HASH is consumed only by the opt-in source rebuild when
+# USE_NIGHTLY_BUILD=1. Keep both aligned with docker/Dockerfile.ci's
+# VLLM_BASE_TAG. This is enforced by tests/buildkite/test_rocm_dockerfile.py. An
+# intentional custom source pin must update that regression in the same change,
+# and should return to the stable release as soon as one is available.
 ARG USE_NIGHTLY_BUILD=0
-ARG VLLM_VERSION_OR_COMMIT_HASH=v0.28.0
+ARG VLLM_VERSION_OR_COMMIT_HASH=v0.29.0
 ARG ARG_PYTORCH_ROCM_ARCH
 ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}}
 
@@ -38,6 +40,10 @@ RUN if [ "${USE_NIGHTLY_BUILD}" = "1" ]; then \
         rm -rf vllm/.git; \
     fi
 
+# Fail during image construction instead of letting every AMD test fail at
+# runtime when the base image and vLLM-Omni target incompatible vLLM APIs.
+RUN python3 -c "import vllm; from vllm.v1.kv_cache_interface import compute_layout_strides; from vllm.v1.kv_cache_layout import KVCacheLayout; print(f'Validated vLLM {vllm.__version__}')"
+
 # Build TorchCodec after any optional nightly vLLM reinstall so it targets the
 # final torch/ROCm ABI. Keep this layer before COPY . so source changes do not
 # invalidate the expensive native build. FFmpeg development headers are

docker/Dockerfile.xpu

@@ -1,4 +1,4 @@
-# Target vLLM version: v0.28.0. Keep VLLM_VERSION aligned with the target vLLM
+# Target vLLM version: v0.29.0. Keep VLLM_VERSION aligned with the target vLLM
 # release on every rebase.
 #
 # This file layers vLLM-Omni on top of the published upstream XPU image, which
@@ -18,7 +18,7 @@
 #
 # .buildkite/intel/scripts/run-xpu-test.sh automates that pull-then-build fallback.
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai-xpu
-ARG VLLM_VERSION=v0.28.0
+ARG VLLM_VERSION=v0.29.0
 ARG VLLM_BASE=${VLLM_BASE_IMAGE}:${VLLM_VERSION}
 
 FROM ${VLLM_BASE} AS vllm-omni

docs/.nav.yml

@@ -71,6 +71,7 @@ nav:
       - Overview and Compatibility:
         - Overview: user_guide/diffusion_features.md
         - Feature Compatibility: user_guide/feature_compatibility.md
+      - Device-side Video Postprocessing: user_guide/diffusion/device_side_video_postprocess.md
       - CPU Offloading:
         - Overview: user_guide/diffusion/cpu_offload.md
         - Model-Level: user_guide/diffusion/offloader/module_offload.md
@@ -135,6 +136,7 @@ nav:
           - Mooncake Store Connector: design/feature/omni_connectors/mooncake_store_connector.md
           - Mooncake Transfer Engine Connector: design/feature/omni_connectors/mooncake_transfer_engine_connector.md
           - Mori Transfer Engine Connector: design/feature/omni_connectors/mori_transfer_engine_connector.md
+          - NIXL Connector: design/feature/omni_connectors/nixl_connector.md
           - Shared Memory Connector: design/feature/omni_connectors/shared_memory_connector.md
           - Yuanrong Store Connector: design/feature/omni_connectors/yuanrong_connector.md
           - Yuanrong Transfer Engine Connector: design/feature/omni_connectors/yuanrong_transfer_engine_connector.md

docs/api/README.md

@@ -5,6 +5,9 @@
 Main entry points for vLLM-Omni inference and serving.
 
 - [vllm_omni.entrypoints.async_omni.AsyncOmni][]
+- [vllm_omni.entrypoints.async_omni_base.AsyncOmniBase][]
+- [vllm_omni.entrypoints.duplex_omni.DuplexOmni][]
+- [vllm_omni.entrypoints.duplex_omni.DuplexSessionHandle][]
 - [vllm_omni.engine.cfg_companion_tracker.CfgCompanionTracker][]
 - [vllm_omni.entrypoints.cli.benchmark.base.OmniBenchmarkSubcommandBase][]
 - [vllm_omni.entrypoints.cli.benchmark.main.OmniBenchmarkSubcommand][]
@@ -17,12 +20,13 @@ Main entry points for vLLM-Omni inference and serving.
 
 ## Clients
 
-Client libraries that talk to a running vLLM-Omni server (the full-duplex Realtime client and its per-model session presets).
+Client libraries for full-duplex sessions: the Realtime WebSocket client, the in-process client over `DuplexOmni`, and the per-model session presets.
 
 - [vllm_omni.clients.duplex.AudioDelta][]
 - [vllm_omni.clients.duplex.AudioFormat][]
 - [vllm_omni.clients.duplex.ConnectionResumed][]
 - [vllm_omni.clients.duplex.DuplexClient][]
+- [vllm_omni.clients.duplex.DuplexClientBase][]
 - [vllm_omni.clients.duplex.DuplexClientError][]
 - [vllm_omni.clients.duplex.DuplexConnectionError][]
 - [vllm_omni.clients.duplex.DuplexEvent][]
@@ -44,6 +48,7 @@ Client libraries that talk to a running vLLM-Omni server (the full-duplex Realti
 - [vllm_omni.clients.duplex.TextDelta][]
 - [vllm_omni.clients.duplex.TranscriptDelta][]
 - [vllm_omni.clients.duplex.WebSocketTransport][]
+- [vllm_omni.clients.inline_duplex.InlineDuplexClient][]
 
 ## Inputs
 

docs/cli/bench/serve.md

@@ -316,8 +316,12 @@ arrives, checkpointing the response's history position so a later committed user
 outputs are ineligible and omitted from the official manifest; `audio_clipped_bytes` records output beyond the rounded video
 horizon.
 
-TTFT, TTFP, and RTF start at client receipt of `response.created`. TPOT/ITL use engine stage-0 timing; ITL is emitted only when
-every token interval is present.
+Per-response TTFT and TTFP start when the server begins executing the native model-turn request that owns the response. RTF
+continues to use client receipt of `response.created` through the last audio packet, divided by emitted audio duration. Global
+TTFT, TTFP, and RTF cover the complete input-stream window. TPOT/ITL use engine stage-0 timing; ITL is emitted only when every
+token interval is present within a continuous generation segment. Model-unit pacing and gaps between generation segments are
+excluded from TPOT/ITL. Raw request metrics retain `response_created_to_first_text_ms` and
+`response_created_to_first_audio_ms` as client-envelope diagnostics.
 
 The checked-in local performance configuration measures four deterministic cases from each OmniInteract subset (12 videos
 total), with no benchmark warmups and a maximum concurrency of two. Each subset also sends one readiness request before its

docs/cli/serve.md

@@ -1,5 +1,67 @@
 # vllm-omni serve
 
+## Multiple API frontends
+
+For local EngineCore pipelines with one or more stages, `--api-server-count` starts
+multiple API frontend processes that share one parent-owned set of stage
+engines:
+
+```bash
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+    --api-server-count 2 \
+    --port 8091
+```
+
+This mode is intended for workloads whose request parsing, multimodal input
+processing, or response encoding can bottleneck a single frontend. It
+currently requires local multi-process EngineCore stages with one local
+process group per replica. A stage may still use multiple `num_replicas`.
+It cannot be combined with diffusion, headless or
+remote stages, intra-stage data parallelism, Ray, fault tolerance, elastic
+expert parallelism, sleep mode, or runtime LoRA updating. Runtime voice upload
+and deletion are also disabled because those mutations are process-local;
+built-in voices, inline reference audio, and voices restored at startup remain
+available. The `/v1/omni/sleep` and `/v1/omni/wakeup` control routes return HTTP
+409 for the same reason: their bookkeeping is process-local while stage engines
+are shared.
+
+A single-stage model pipeline is supported through the normal launch command;
+this is distinct from the distributed `--stage-id` / `--headless` launch mode.
+Every stage in the pipeline must be an EngineCore stage. Mixed AR-to-DiT
+pipelines, including MiniMax-H3, remain unsupported even when their first
+stage is an EngineCore stage. Diffusion support requires per-frontend channels
+and shared-engine lifecycle handling in the diffusion backend.
+
+### Workloads and performance validation
+
+Each API process owns its frontend processor, Orchestrator, request state,
+and per-replica EngineCore channels. Requests return through the channel of
+the originating API process; model weights and GPU stage engines are shared.
+This distributes frontend and orchestration work across CPU processes.
+
+| Workload | Potential benefit | Limitation |
+| ---------- | ------------------- | ------------ |
+| Concurrent cold image/video inputs on supported EngineCore pipelines | Parallel frontend processing when one API process is CPU-limited | A single cold request and GPU computation may be unchanged |
+| Many replicas producing frequent text/audio outputs | Less API/Orchestrator work and GIL contention per process | Routing remains serial within each Orchestrator |
+| Warm inputs or low-concurrency, GPU-limited generation | Additional frontend capacity | More API processes may add overhead without improving throughput |
+
+Here, a cold input means media or preprocessing caches miss, not that model
+weights must be loaded or GPU kernels compiled. Each frontend has its own
+processor/cache state, so cache duplication, CPU thread contention, and memory
+usage also matter. Multi-API serving does not itself change the output queue,
+move response encoding to an executor, or change the model's numerical scheduler.
+
+Compare `--api-server-count 1`, `2`, and higher counts with the same hardware,
+model/stage configuration, media, sampling parameters, output lengths, and
+offered load. Keep cold-cache and warmed-cache runs separate and report repeated
+measurements. Record CPU and request counts for every API worker, input and
+output queue delay, GPU utilization, first-output latency, and end-to-end latency.
+Confirm requests reach every worker: connection reuse can concentrate traffic
+on one process. Queue-only or single-API experiments do not establish a
+multi-API throughput gain. The multi-replica workload in
+[#4680](https://github.com/vllm-project/vllm-omni/issues/4680) remains a relevant
+validation target.
+
 ## Stage-based CLI quickstart
 
 The stage-based CLI is designed for deployments that require launching each pipeline stage in an isolated process
@@ -61,3 +123,10 @@ CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
 ## Arguments
 
 --8<-- "docs/generated/argparse_omni/omni_serve.inc.md"
+
+Async video jobs use process-local metadata and task handles. With multiple API
+workers, creating, listing, retrieving, downloading, and deleting these jobs
+returns HTTP 409. This restriction is independent of the current diffusion
+stage launch restriction: supporting multi-worker diffusion also requires a
+shared job lifecycle. Missing frontend topology state returns HTTP 503 for
+operations that require process-local state protection.

docs/configuration/environment_variables.md

@@ -82,6 +82,7 @@ depends on the installed kernels and model path.
 | --- | --- | --- | --- | --- |
 | `SPEAKER_SAMPLES_DIR` | Filesystem path; default `~/.cache/vllm-omni/speakers` | Speech server; read when speaker storage initializes | Environment-only setting. The directory is created; filesystem errors propagate. | Stable |
 | `SPEAKER_MAX_UPLOADED` | Integer; default `1000` | Speech server; read when speaker storage initializes | Environment-only setting. A non-integer logs a warning and uses `1000`; range is not otherwise validated. | Stable |
+| `VLLM_OMNI_ABORT_TIMEOUT` | Float seconds; default `2` | Engine abort wait for Videos DELETE and `generate()` cancel/error cleanup; read when `async_omni` imports | Environment-only setting. A non-float raises `ValueError` during import. | Experimental |
 | `VLLM_OMNI_ASYNC_OUTPUT_TIMEOUT` | Float seconds; default `600` | Diffusion engine async-output wait in `step_streaming`; resolved per call on the request path, not at import | Environment-only setting. A non-float or `<=0` value warns once and uses the default. | Experimental |
 | `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `1`, `true`, `yes` or `on` enables; default `0` (off) | Orchestration loop and the serving-side final-output drain; read once when the `Orchestrator` is constructed | Environment-only setting. Values are stripped and case-normalized; any unrecognized value leaves the legacy poll loop selected. | Experimental |
 | `VLLM_OMNI_INPUT_WAIT_TIMEOUT_S` | Float seconds; default `600`; `<=0` disables | Full-payload input coordinator, not async-chunk transfer; read when the scheduler module imports in each worker | Environment-only setting. A non-float logs a warning and uses `600`. | Stable operational control |
@@ -90,6 +91,13 @@ depends on the installed kernels and model path.
 | `VLLM_VIDEO_ASYNC_CHUNK` | `on` or `off`; default `on` | Streaming video output; read on attribute access | Environment-only setting. Values are trimmed and case-normalized; an invalid value warns once and uses `on`. | Experimental |
 | `VLLM_VIDEO_AUDIO_DELTA_MODE` | `fast` or `slow`; default `fast` | Streaming video audio deltas; read on attribute access | Environment-only setting. Values are trimmed and case-normalized; an invalid value warns once and uses `fast`. | Experimental |
 
+### NIXL stage transfer
+
+| Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
+| --- | --- | --- | --- | --- |
+| `VLLM_OMNI_NIXL_LEASE_S` | Float seconds; default `3600` | Unclaimed producer payload expiry; connector construction | Nonempty environment value overrides `lease_seconds`; empty falls back to config/default. Invalid floats raise `ValueError`. Claimed READ allocations never expire by time. | Experimental |
+| `VLLM_OMNI_NIXL_XFER_TIMEOUT_S` | Float seconds; default `300` | Receiver READ wait; connector construction | Nonempty environment value overrides `transfer_timeout_s`; empty falls back to config/default. Invalid floats raise `ValueError`. Timeout does not cancel DMA or release active allocations. | Experimental |
+
 ### Server storage
 
 Storage names use Pydantic's nested-settings delimiter (`__`). They are read
@@ -112,6 +120,7 @@ settings.
 
 | Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
 | --- | --- | --- | --- | --- |
+| `VLLM_OMNI_FUSED_QK_NORM_ROPE_MIN_TOKENS` | Non-negative integer; unset or empty means each consumer's own default (Boogu-Image: `2048`) | Token gate of the shared `fused_qk_norm_rope` op: a rotary table is packed for the fused kernel only when it spans at least this many positions (`B*S`); read at forward time | Environment-only. `0` = always fuse; a very large value disables the fused path. Any other value raises `ValueError` on the first forward. The crossover is host-dependent; re-benchmark before overriding. | Experimental performance control |
 | `VLLM_OMNI_SKIP_NVFP4_NAN_CLAMP` | Boolean truthy spellings: `1`, `true`, `yes`, `on`; default false | ModelOpt NVFP4 compatibility patch; read when `vllm_omni.patch` imports | Environment-only escape hatch. Any other value means false. Set only to diagnose the upstream NaN-scale issue. | Diagnostic and temporary |
 | `VLLM_OMNI_USE_QUACK_FP8` | Boolean truthy spellings: `1`, `true`, `yes`, `on`; unset means hardware auto-detection | FP8 scaled matrix multiplication; evaluated when quack capability is selected | A set value overrides auto-detection. Any non-truthy value forces quack off. If quack cannot load, vLLM-Omni warns and falls back to FlashInfer. | Experimental performance control |
 

docs/configuration/stage_configs.md

@@ -17,6 +17,7 @@ Common `PipelineConfig` fields include:
 | ------- | ------------- |
 | `model_type` | Pipeline identifier used during model and config resolution. |
 | `default_deploy_config_name` | Bundled deploy YAML loaded when the user does not pass `deploy_config`. |
+| `duplex_plugin` | Dotted path of the model's `DuplexModelPlugin`. Set only for full-duplex models; it makes `vllm-omni serve` run the model through `DuplexOmni` (a server whose every surface runs on a duplex session) and the engine host a `DuplexOrchestrator` with the plugin loaded. |
 | `model_arch` | Default Hugging Face architecture for the pipeline. |
 | `hf_architectures` | Architecture names used to identify checkpoints whose `model_type` is shared. |
 | `hf_config_predicate` | Optional predicate used to select between pipelines with otherwise identical HF metadata. |
@@ -38,7 +39,7 @@ Common `StagePipelineConfig` fields include:
 | `custom_process_input_func` | Processor applied to this stage's incoming payload. |
 | `custom_process_next_stage_input_func` | Processor used for full-payload handoff to the next stage. |
 | `async_chunk_process_next_stage_input_func` | Processor used for async chunk handoff. |
-| `sampling_constraints` | Model-owned sampling constraints that deploy defaults cannot override. |
+| `sampling_constraints` | Model-owned sampling constraints that deploy defaults cannot override. Scalar values replace deploy defaults; required `stop_token_ids` extend and deduplicate them. |
 
 To add or change topology, define and register a new pipeline variant. Use
 deploy YAML only for runtime placement, resource sizing, connectors, and other
@@ -52,7 +53,7 @@ The new deploy schema lives under `vllm_omni/deploy/` and is paired with a froze
 | ------- | ------ | ---------- | --------- | ------------- |
 | `base_config` | str (path) | optional | — | Overlay parent (relative or absolute). `stages:` / `platforms:` deep-merged by stage_id; other scalars overlay-wins. Intended for user-authored overlays; prod yamls stay flat. |
 | `async_chunk` | bool | optional | `true` | Enable chunked streaming between stages. Pin to `false` if the pipeline runs end-to-end. |
-| `session_mode` | str | optional | `"turn"` | Session behavior. MiniCPM-o 4.5 deploy YAMLs set `"duplex"` so `/v1/realtime` and chat share the same profile. |
+| `session_mode` | str | optional | `"turn"` | Session behavior. For pipelines with a `duplex_plugin`, explicitly select `"duplex"` for the duplex engine or `"turn"` for the ordinary online serving stack. The shipped MiniCPM-o 4.5 default remains `"duplex"`. See [Full Duplex](../serving/full_duplex_api.md#enable-full-duplex). |
 | `active_stream_window` | int | optional | `0` | Number of active downstream stream slots; `0` preserves all-stream cycling. |
 | `duplex_session` | dict | optional | runtime defaults | Full-duplex session lifecycle, buffering, replay, and capacity limits. |
 | `connectors` | dict | optional | `null` | Named connector specs (`{name, extra}`). Referenced by each stage's `input_connectors` / `output_connectors`. See [Connector schema](#connector-schema). |
@@ -97,7 +98,7 @@ Each entry under `stages:` accepts any `StageDeployConfig` field directly (no ne
 | `devices` | str \| null | optional | `null` | Device list assigned to this stage. The number of device ids must equal this stage's local world size (`tensor_parallel_size` × local data-parallel size × `pipeline_parallel_size`, or `num_replicas` × that product for a replica pool); a mismatch fails early — see the note below. |
 | `output_connectors` | dict \| null | optional | `null` | Keyed by `to_stage_<n>`; values are names registered under top-level `connectors:`. |
 | `input_connectors` | dict \| null | optional | `null` | Keyed by `from_stage_<n>`; values are names registered under top-level `connectors:`. |
-| `default_sampling_params` | dict \| null | optional | `null` | Baseline sampling params. Deep-merged with pipeline `sampling_constraints` (pipeline wins). |
+| `default_sampling_params` | dict \| null | optional | `null` | Baseline sampling params. Merged with pipeline `sampling_constraints`; scalar constraints win, while required `stop_token_ids` are appended and deduplicated. |
 | `engine_extras` | dict | optional | `{}` | Catch-all for engine fields not listed above; deep-merged across overlays and forwarded to the stage engine. |
 
 **Note:** a stage's `devices` count must equal its local world size (`tensor_parallel_size` × `data_parallel_size_local` × `pipeline_parallel_size`, falling back to global `data_parallel_size` when the local size is unset), or `num_replicas` × that product for a replica pool. A mismatch fails early and names the offending stage. A top-level `--tensor-parallel-size` is broadcast to every stage, so it can make a single-GPU stage violate this contract ([issue #5003](gh-issue:5003)); fix that case with `--stage-overrides` (set `tensor_parallel_size` and `devices` together per stage) or set TP only on the multi-GPU stage.

docs/contributing/ci/ci_settings.md

@@ -15,7 +15,8 @@ Canonical layout (prefer these paths for new changes):
 │   │   ├── skip_ci.py               # skip-ci decision (docs / skip-mark / CI YAML paths)
 │   │   ├── upload_pipeline.py       # Bootstrap + test-pipeline uploader (CUDA/NPU)
 │   │   └── resolve_skip_ci.sh       # Shell helpers for AMD/Intel bootstrap
-│   └── ci_mirror_hardwares.yml      # CUDA uploader presets (referenced by name only)
+│   ├── ci_mirror_hardwares.yml      # CUDA/NPU hardware presets (referenced by name only)
+│   └── ci_source_file_dependencies.yml  # L2–L5 path-prefix presets (referenced by name only)
 ├── cuda/                            # Primary NVIDIA CUDA CI
 │   ├── pipeline.yml                 # Bootstrap entry (hook upload)
 │   ├── bootstrap-upload-steps.yml   # Bootstrap child steps (upload_pipeline --upload)
@@ -34,6 +35,7 @@ Canonical layout (prefer these paths for new changes):
 ├── amd/
 │   ├── test-amd-ready.yml           # L2 job definitions (template input)
 │   ├── test-amd-merge.yml           # L3 job definitions
+│   ├── test-amd-nightly.yml         # Nightly full-model jobs (experimental)
 │   ├── test-template-amd-omni.j2    # Renders final pipeline.yaml
 │   └── scripts/
 │       ├── bootstrap-amd-omni.sh    # Entry: skip-ci → Jinja → upload
@@ -63,7 +65,7 @@ Canonical layout (prefer these paths for new changes):
 | -------- | ---------------- | -------------- | ---------------- | -------------------- |
 | **CUDA** | `cuda/pipeline.yml` | `test-ready.yml`, `test-merge.yml`, `test-nightly.yml`, `test-weekly.yml` | `upload_pipeline.py --upload` (expands uploader-only keys) | omit `mirror_hardwares` (from `-m`) / preset string + `MIRROR_HW` |
 | **NPU** | `npu/pipeline-npu.yml` | `test-npu-ready.yml`, `test-npu-nightly.yml` | `upload_pipeline.py --upload` | `mirror_hardwares: a2b3_npu_1` / `a2b3_npu_4` / `a3_npu_2` |
-| **AMD** | `amd/scripts/bootstrap-amd-omni.sh` | `test-amd-ready.yml`, `test-amd-merge.yml` | Jinja (`test-template-amd-omni.j2`) → `pipeline upload` | `agent_pool` + `mirror_hardwares: [amdproduction]` (array, template filter) |
+| **AMD** | `amd/scripts/bootstrap-amd-omni.sh` | `test-amd-ready.yml`, `test-amd-merge.yml`, `test-amd-nightly.yml` | Jinja (`test-template-amd-omni.j2`) → `pipeline upload` | `agent_pool` + `mirror_hardwares: [amdproduction]` (array, template filter) |
 | **Intel** | `intel/scripts/bootstrap-intel-omni.sh` | `intel/pipeline-intel.yml` (steps inline) | Direct `pipeline upload` | Inline `agents.queue` on each step |
 
 ## Platform configuration style
@@ -108,19 +110,21 @@ Canonical layout (prefer these paths for new changes):
 
     Inferred form: unset/`empty` `MIRROR_HW` matches `H100` or `L4` in `-m` (both present → H100); no match skips the step. `MIRROR_HW=b200` must appear in `-m` or the step is skipped. The uploader does **not** rewrite pytest `-m`; B200 collection comes from the YAML expression and from tests that declare `B200` in `hardware_test` / `hardware_marks` (for example `res={"cuda": ["H100", "B200"]}`). `MIRROR_HW` must be empty or `b200` (case-insensitive); unknown values (for example `b20o`) **fail the upload**. A CUDA preset string such as `mirror_hardwares: h100_4` is skipped when `MIRROR_HW=b200`. Unset `MIRROR_HW` keeps string presets. NPU presets ignore `MIRROR_HW`. A `mirror_hardwares` name that is not a key in `ci_mirror_hardwares.yml` **fails the upload**.
 
+    **Path filter in YAML:** `source_file_dependencies: <preset>` (string, or a list of preset names)—preset names in [`common/ci_source_file_dependencies.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/ci_source_file_dependencies.yml). Required on **E2E Test** leaf jobs in L2/L3, and on L4/L5 leaf jobs that should be diff-gated on PR labels; see [Step filtering](#step-filtering).
+
     **Conventions**
 
     - **`depends_on`:** leaf jobs depend on `upload-ready-pipeline`, `upload-merge-pipeline`, etc.
     - **`group` / `label`:** `:card_index_dividers:` groups; labels like `Diffusion · Qwen Image Test`.
     - **`commands`:** `timeout … pytest …` with markers and `--run-level` for the pipeline level.
-    - **`source_file_dependencies`:** required on **E2E Test** leaf jobs in L2/L3; see [Step filtering](#step-filtering).
+    - **`source_file_dependencies`:** required on **E2E Test** leaf jobs in L2/L3, and on L4/L5 leaf jobs that should be diff-gated on PR labels; name a job key from `ci_source_file_dependencies.yml` (see [Step filtering](#step-filtering)). The key must list every script the step runs; `commands` is not scanned.
 
     **Adding a job**
 
     1. Pick the level file (`test-ready.yml` / `test-merge.yml` / `test-nightly.yml` / `test-weekly.yml`).
     2. Add a step under the right **group** (usually **E2E Test** for model pytest).
-    3. Set `label`, `commands`, `mirror_hardwares`, `depends_on: upload-ready-pipeline` (or `upload-merge-pipeline` / `upload-nightly-pipeline` / `upload-weekly-pipeline`).
-    4. For L2/L3 E2E, add `source_file_dependencies` (pytest + model + deploy YAML prefixes).
+    3. Set `label`, `commands`, `mirror_hardwares`, `depends_on: upload-<level>-pipeline`.
+    4. For L2–L5 jobs that should be diff-gated, add a job key in `common/ci_source_file_dependencies.yml` and set `source_file_dependencies: <key>` on the step. YAML anchors hold model business-code paths only. The job key (`{model}_function` / `_perf` / `_accuracy` / `_reliability` / `_doc` / `_cov`) aliases that anchor and lists the pytest / example / benchmark scripts the step actually runs. The uploader does **not** parse `commands` for path filters—omitting a script from the key means editing that script will not select the job.
     5. Dry-run:
 
     ```bash
@@ -144,6 +148,8 @@ Canonical layout (prefer these paths for new changes):
     - A5: `a5_npu_2`, `a5_npu_4`, `a5_npu_8`, `a5_npu_16`
     - 310P: `310p_npu_1`, `310p_npu_2`, `310p_npu_4`
 
+    **Path filter in YAML:** same `source_file_dependencies` presets as CUDA, on **L4** leaf jobs in `test-npu-nightly.yml`. Filtering applies on the PR `nightly-test` label; `main` + `NIGHTLY=1` keeps every job. If no job-key prefix matches, a change to the pipeline YAML being uploaded, or to any path under `source_filter_fallback`, keeps every job in that upload. See [Step filtering](#step-filtering).
+
     **Conventions**
 
     - **`depends_on: upload-ready-pipeline`** (or `upload-nightly-pipeline`) ties jobs to bootstrap upload keys.
@@ -153,7 +159,7 @@ Canonical layout (prefer these paths for new changes):
 
     1. Edit `test-npu-ready.yml` (L2) or `test-npu-nightly.yml` (L4).
     2. Add a step with `mirror_hardwares` (add a new preset in `ci_mirror_hardwares.yml` first if needed).
-    3. Set `commands` to your pytest file and markers.
+    3. Set `commands` to your pytest file and markers. For L4, set `source_file_dependencies` to a job key from `ci_source_file_dependencies.yml` that aliases the model anchor and lists that pytest file.
     4. Dry-run:
 
     ```bash
@@ -166,9 +172,9 @@ Canonical layout (prefer these paths for new changes):
 
     **Bootstrap:** [`amd/scripts/bootstrap-amd-omni.sh`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/amd/scripts/bootstrap-amd-omni.sh)—skip-ci, diff filtering, Jinja render, then `buildkite-agent pipeline upload`.
 
-    **Test YAML (data):** `amd/test-amd-ready.yml` (L2 / `ready`), `test-amd-merge.yml` (L3 / `merge-test` and `main`). On a PR carrying both labels, AMD combines both suites behind one image build. PR builds without either tier label retain the legacy L2 fallback for compatibility with the `amd-test` trigger. `DEBUG_TEST_YAML` remains an explicit override. AMD has no L4 file yet, so `nightly-test` does not select an additional suite.
+    **Test YAML (data):** `amd/test-amd-ready.yml` (L2 / `ready`), `test-amd-merge.yml` (L3 / `merge-test` and ordinary `main`), and experimental `test-amd-nightly.yml` (`full_model` / `nightly-test` or `main` with `NIGHTLY=1`). Multiple PR tier labels combine their suites behind one image build. PR builds that reach the bootstrap without a tier label retain the legacy ready-suite fallback. `DEBUG_TEST_YAML` accepts `ready`, `merge`, and `nightly` as an explicit override. During nightly burn-in every AMD nightly leaf is non-blocking.
 
-    **Trigger boundary:** AMD label handling has two layers. First, the external `vllm-omni-amd-ci` Buildkite pipeline condition decides whether a GitHub label event creates a build. Only after that build starts does this repository's bootstrap inspect all current PR labels and select L2, L3, or both. Repository-side selection therefore cannot make a `merge-test` event start AMD CI by itself. The external condition must admit both `ready` and `merge-test`, while preserving the legacy `amd-test` trigger and non-PR `main` / scheduled builds. For PR builds without a `DEBUG_TEST_YAML` override, an unreadable label set fails the bootstrap instead of silently selecting the wrong tier. Debug overrides skip label lookup and are validated directly; invalid values, including whitespace-only values, fail the bootstrap. Keep `nightly-test` out of the AMD condition until an AMD L4 suite exists; otherwise it would start a build without providing nightly coverage.
+    **Trigger boundary:** AMD label handling has two layers. First, the external `vllm-omni-amd-ci` Buildkite pipeline condition decides whether a GitHub label event creates a build. Only after that build starts does this repository's bootstrap inspect all current PR labels and select the ready, merge, nightly, or combined suites. Repository-side selection therefore cannot start AMD CI by itself. AMD-only nightly validation uses both `amd-test` and `nightly-test`: the external condition should accept an event for either label only when the complete PR label set contains both. `amd-test` admits the AMD pipeline, while `nightly-test` makes the bootstrap select `NIGHTLY_TESTS:test-amd-nightly.yml`. Existing `ready` / `merge-test` behavior and non-PR `main` / scheduled builds remain separate. For PR builds without a `DEBUG_TEST_YAML` override, an unreadable label set fails the bootstrap instead of silently selecting the wrong tier. Debug overrides skip label lookup and are validated directly; invalid values, including whitespace-only values, fail the bootstrap. Until the paired-label condition is configured, use an authorized manual Buildkite build with `DEBUG_TEST_YAML=nightly` and verify that the bootstrap reports only the nightly suite.
 
     **Rendering:** [`test-template-amd-omni.j2`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/amd/test-template-amd-omni.j2) wraps data steps with `amd-build` image build and `amd_<agent_pool>` queues. Do **not** hand-edit generated `pipeline.yaml`.
 
@@ -184,7 +190,7 @@ Canonical layout (prefer these paths for new changes):
 
     **Adding a job**
 
-    1. Edit `test-amd-ready.yml` or `test-amd-merge.yml`.
+    1. Edit `test-amd-ready.yml`, `test-amd-merge.yml`, or `test-amd-nightly.yml`.
     2. Copy a neighboring block: `label`, `agent_pool`, `mirror_hardwares`, `commands`, optional `grade`.
     3. Regenerate via bootstrap / Jinja; update `skip_ci.py` if you add a new YAML path.
 
@@ -213,7 +219,7 @@ PR diffs drive **two independent skip layers**. Both read changed files from git
 | Layer | Script | When | What is skipped | Where you configure |
 | ----- | ------ | ---- | --------------- | ------------------- |
 | **Bootstrap** | [`skip_ci.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/skip_ci.py) + [`upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/upload_pipeline.py) | Before child test pipelines upload (`cuda/pipeline.yml`, `npu/pipeline-npu.yml`, AMD/Intel bootstraps) | Entire default CI, or whole L2/L3 upload for a platform | Whitelists in `skip_ci.py`; bootstrap `if` injected by step `key` in `upload_pipeline.py` |
-| **Step filter** | [`upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/upload_pipeline.py) | While uploading CUDA L2/L3 YAML | Individual Buildkite steps inside `test-ready.yml` / `test-merge.yml` | `source_file_dependencies` on each step or group |
+| **Step filter** | [`upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/upload_pipeline.py) | While uploading CUDA L2–L5 YAML | Individual Buildkite steps inside `test-ready.yml` / `test-merge.yml` / `test-nightly.yml` / `test-weekly.yml` | `source_file_dependencies` key on each step (presets in `ci_source_file_dependencies.yml`) |
 
 **Changed files** (both layers):
 
@@ -240,7 +246,7 @@ Label triggers (`ready`, `merge-test`) are unchanged—diff-aware logic only red
 | Non-qualifying skip-mark and no CI YAML | normal CI | all on |
 | Diff unavailable | normal CI | all on |
 
-**`skip_all` exceptions (CUDA/NPU bootstrap only):** PR labels (`nightly-test`, `merge-test`, `weekly-test`, `ready`, …) do **not** revive jobs under `skip_all`. On `main`, scheduled `NIGHTLY=1` still builds the image and uploads L4; `WEEKLY=1` or `NON_CRITICAL=1` still builds the image and uploads L5 (CUDA). `WEEKLY=1` on `main` also uploads L2/L3 with `--e2e`.
+**`skip_all` exceptions:** For CUDA/NPU, PR labels (`nightly-test`, `merge-test`, `weekly-test`, `ready`, …) do **not** revive jobs under `skip_all`; on `main`, scheduled `NIGHTLY=1` still builds the image and uploads L4, while `WEEKLY=1` or `NON_CRITICAL=1` still builds the image and uploads L5 (CUDA). `WEEKLY=1` on `main` also uploads L2/L3 with `--e2e`. AMD applies `skip_ci.py gate` only to selected ready/merge specs, so an explicitly selected AMD nightly spec (`nightly-test`, `NIGHTLY=1`, or a nightly debug override) remains runnable even for a docs-only diff.
 
 **Yaml-gated nightly/weekly:** L4/L5 upload steps keep their normal label / `NIGHTLY` / `WEEKLY` / `NON_CRITICAL` conditions (for example PR `nightly-test` still uploads nightly). Only L2/L3 upload steps are matrix-gated.
 
@@ -257,6 +263,7 @@ Register new files in `L2_YAML_FILES`, `L3_YAML_FILES`, or `L45_YAML_FILES` in `
 | L3 | `.buildkite/cuda/test-merge.yml` | cuda |
 | L3 | `.buildkite/amd/test-amd-merge.yml` | amd |
 | L4/L5 | `.buildkite/cuda/test-nightly.yml`, `test-weekly.yml` | cuda |
+| L4/L5 | `.buildkite/amd/test-amd-nightly.yml` | amd |
 | L4/L5 | `.buildkite/npu/test-npu-nightly.yml` | npu |
 
 ##### Yaml-gated category branches
@@ -289,7 +296,7 @@ Docs/skip-mark mixed with any row below follows the same matrix.
 | CUDA ready + AMD merge | G4 | `cuda/l2` + `amd/l3` |
 | CUDA ready + AMD ready | G2 | `cuda/l2` + `amd/l2` |
 | CUDA merge + AMD merge | G3 | `cuda/l3` + `amd/l3` |
-| Only nightly / weekly / npu-nightly YAML | G1 | (none) |
+| Only nightly / weekly / amd-nightly / npu-nightly YAML | G1 | (none) |
 | CUDA ready + nightly | G5 | `cuda/l2` |
 | CUDA merge + nightly | G6 | `cuda/l3` |
 | CUDA ready + merge + nightly | G7 | `cuda/l2` + `cuda/l3` |
@@ -306,40 +313,38 @@ Unit coverage: `tests/buildkite/test_skip_ci.py`.
 
 #### Step filtering {#step-filtering}
 
-CUDA **L2** (`.buildkite/cuda/test-ready.yml`) and **L3** (`.buildkite/cuda/test-merge.yml`) only. Bootstrap upload entry: `upload_pipeline.py --upload .buildkite/cuda/bootstrap-upload-steps.yml`.
+CUDA **L2** (`.buildkite/cuda/test-ready.yml`), **L3** (`.buildkite/cuda/test-merge.yml`), **L4** (`.buildkite/cuda/test-nightly.yml`), **L5** (`.buildkite/cuda/test-weekly.yml`), and NPU **L4** (`.buildkite/npu/test-npu-nightly.yml`). Bootstrap upload entry: `upload_pipeline.py --upload .buildkite/cuda/bootstrap-upload-steps.yml` (or `npu/bootstrap-upload-steps.yml`).
+
+**When filtering applies:** **PR label** uploads (`ready`, `merge-test`, `nightly-test`, `weekly-test`) and **post-merge `main` L3** (`test-merge.yml`). Scheduled **`main` + env** uploads that pass `--all` (`NIGHTLY=1`, `WEEKLY=1` / `NON_CRITICAL=1`) or `--e2e` (`WEEKLY=1` on L2/L3) keep every selected job and still strip the key. If **no** job-key prefix matches the diff, a change to the pipeline YAML being uploaded, or any path under the `source_filter_fallback` key in `ci_source_file_dependencies.yml`, keeps every job in that upload. `source_filter_fallback` is not a job key — do not attach it to a step. When any job-key prefix already matches, normal filtering wins (the fallback does not expand the selection).
 
 **Uploader-only keys** — removed before Buildkite sees the YAML; never used at runtime on agents:
 
 | Key | Purpose |
 | --- | ------- |
-| `source_file_dependencies` | List of path **prefixes**. If any changed file equals a prefix or starts with `prefix/`, keep the step (or group); otherwise omit it. |
+| `source_file_dependencies` | Named preset (or list of presets) from [`ci_source_file_dependencies.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/ci_source_file_dependencies.yml). The uploader expands only that key; it does **not** scan `commands`. If any changed file equals a listed prefix or starts with `prefix/`, keep the step (or group); otherwise omit it. Inline path-prefix lists are still accepted. |
 | `mirror_hardwares` | Expand to `agents` + `plugins` (+ optional `image`) from `ci_mirror_hardwares.yml`. |
 
 ### Policy
 
-- **Always uploaded** (no key): groups outside **E2E Test**—Simple Test, Diffusion unit tests, Engine/Model Executor, Distributed, Custom Pipeline, Entrypoints (L2), LoRA / Entrypoints (L3).
-- **Diff-gated**: every **E2E Test** leaf job. List the smallest prefix set per step—pytest file(s), model code under `vllm_omni/model_executor/models/` or `vllm_omni/diffusion/models/`, plus `stage_input_processors/` and `vllm_omni/deploy/*.yaml` when applicable. A **group** may define the key instead; the whole group drops if no prefix matches.
+- **Always uploaded** (no key): L2/L3 groups outside **E2E Test**—Simple Test, Diffusion unit tests, Engine/Model Executor, Distributed, Custom Pipeline, Entrypoints (L2), LoRA / Entrypoints (L3).
+- **Diff-gated**: every **E2E Test** leaf job in L2/L3, L4 nightly leaf jobs (CUDA and NPU), and L5 weekly jobs whose `if` includes the `weekly-test` label (Reliability / Perf). Jobs gated only by `WEEKLY=1` or `NON_CRITICAL=1` have no `source_file_dependencies`. Registry layout: YAML anchors are model business code only (`vllm_omni/model_executor/models/`, `vllm_omni/diffusion/models/`, plus `stage_input_processors/` and `vllm_omni/deploy/*.yaml` when applicable). Job keys alias the anchor and **also** list the scripts that job runs (pytest files, `run_cov_split.sh --offline` / `--online` targets, example README runners). Feature keys stay cross-cutting. A **group** may define the key instead; the whole group drops if no prefix matches.
 
 ### YAML examples
 
 ```yaml
       - label: "Diffusion · Qwen Image Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen_image.py
-          - vllm_omni/diffusion/models/qwen_image/
+        source_file_dependencies: diffusion_qwen_image_function
+        # diffusion_qwen_image_function: *qwen_image + tests/e2e/online_serving/test_qwen_image.py
         commands:
           - pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' ...
         mirror_hardwares: h100_1
 
       - label: "TTS · Qwen3-TTS CustomVoice Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
-          - vllm_omni/model_executor/models/qwen3_tts/
-          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
-          - vllm_omni/deploy/qwen3_tts.yaml
+        source_file_dependencies: tts_qwen3_tts_function
+        # tts_qwen3_tts_function: *qwen3_tts + tests/e2e/online_serving/test_qwen3_tts_customvoice.py
         commands:
           - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py ...
-        mirror_hardwares: l4_4
+        mirror_hardwares: l4_1
 ```
 
 ### Local dry-run
@@ -443,15 +448,16 @@ plugin, which provides the agent. `l4_*` jobs run on the EKS `l4-k8s` queue
 still needs `mount-buildkite-agent: true`.
 
 List both `run_cov_split.sh` and `pyproject.toml` in every opted-in job's
-`source_file_dependencies` — both change what the job measures, so without them a
+`source_file_dependencies` preset — both change what the job measures, so without them a
 change there is filtered out of normal PR builds and only surfaces in a later
 nightly. `tests/buildkite/test_upload_pipeline.py` covers the filter behavior with
 a synthetic job (it does not pin real merge labels).
-Editing only the surrounding CI YAML still does not schedule them, so a PR that
-touches just the wiring needs a full E2E run (or the commands run on a GPU host)
-to produce artifacts. When checking a new model's
-artifacts, compare `lines-covered` between the online and offline XML rather than
-just confirming both files exist.
+If no job-key prefix matches, editing the pipeline YAML being uploaded, or any
+path under the `source_filter_fallback` registry key, keeps every job in that
+upload (including coverage jobs) so the wiring can be validated before merge.
+When a job-key prefix already matches, bypass does not expand the selection. A
+change to a different pipeline YAML still does not schedule them. When checking a new model's artifacts, compare `lines-covered`
+between the online and offline XML rather than just confirming both files exist.
 
 ### Validation checklist
 

docs/contributing/ci/test_system_overview.md

@@ -182,6 +182,8 @@ CUDA **L4** (`full_model`) jobs are defined once in [`.buildkite/cuda/test-night
 - **Nightly:** scheduled `main` builds with `NIGHTLY=1` (and PR labels such as `nightly-test` / `omni-test`). Default fleet is typically H100.
 - **Pre-release:** the same L4 suite runs again on extra GPU SKUs before a release (for example B200 via `MIRROR_HW=b200`), so `full_model` coverage is not limited to the nightly machine type.
 
+AMD has an experimental, non-blocking `full_model` nightly vertical slice in [`.buildkite/amd/test-amd-nightly.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/amd/test-amd-nightly.yml). It selects ROCm Qwen3-Omni function-expansion, accuracy, documentation-example, and AITER-on smoke coverage on the MI300 pool. AMD-only PR validation uses `amd-test` to admit the external AMD pipeline and `nightly-test` to select this suite; scheduled `main` builds select it with `NIGHTLY=1`. Keep it non-blocking while the lane accumulates stability and completion-time evidence.
+
 How count-form `mirror_hardwares` and `MIRROR_HW` pick a SKU: [CI Settings](./ci_settings.md).
 
 ### L2 / L3 diff-aware CI (CUDA)

docs/design/architecture_overview.md

@@ -131,13 +131,13 @@ for the validated batching, attention, and quantization combinations.
 
 ### MiniMax-H3: conditioning, denoising, and decode
 
-MiniMax-H3 illustrates logical component and phase boundaries between text
-encoding, task-specific denoising, and media decoding:
-`Text Encoder → DiT → VAE Decoder`. The current registered topology co-locates
-these components in one diffusion stage. These boundaries are useful for
-profiling and offload; independent placement would require a future
-multi-stage `PipelineConfig`. The stage mapping follows the implementation and
-serving recipe; the [release overview](https://minimaxi.com/blog/minimax-h3)
+MiniMax-H3 combines text and reference-media encoding, task-specific denoising,
+and media decoding. Standard serving co-locates these components in one diffusion
+stage. The opt-in
+[disaggregated topology](https://github.com/vllm-project/vllm-omni/blob/main/recipes/MiniMaxAI/MiniMax-H3-Disaggregated.md)
+runs Qwen and the video/audio VAE encoders in Stage 0, then sends their conditioning
+to a Stage 1 containing the DiTs and VAE decoders. The stage mapping follows the
+implementation and serving recipe; the [release overview](https://minimaxi.com/blog/minimax-h3)
 notes that a detailed technical report is forthcoming.
 
 #### Model architecture by component
@@ -149,16 +149,15 @@ notes that a detailed technical report is forthcoming.
 | **VAE Decoder** | Video VAE and audio VAE decode the generated latents and assemble the synchronized output | Decoded video frames and stereo waveform, with FPS and audio sample rate metadata |
 
 Both DiT partitions remain behind one task-selection point; `task` chooses
-which partition runs. In the current implementation, the three logical
-components execute inside one diffusion stage. This is different from
-Qwen3-Omni's three independently scheduled stages, even though both models
-produce synchronized audio and video-like outputs.
+which partition runs.
 
 #### Representative component execution
 
+The following table describes the standard single-stage deployment.
+
 | Logical component | Batching | Attention and execution | Parallelism | Quantization |
 | --- | --- | --- | --- | --- |
-| **Text Encoder** | The current co-located pipeline follows the diffusion request scheduler; independent prompt/reference batching would require a separate multi-stage topology | Qwen3-VL attention and multimodal preprocessing | `--text-encoder-tp-size N` shards the encoder across the first `N` DiT ranks; independent placement is not part of the current topology | BF16/FP32 baseline; the H3 DiT FP8 path does not quantize the text encoder |
+| **Text Encoder** | Follows the diffusion request scheduler | Qwen3-VL attention and multimodal preprocessing | `--text-encoder-tp-size N` shards the encoder across the first `N` DiT ranks | BF16/FP32 baseline; the H3 DiT FP8 path does not quantize the text encoder |
 | **DiT** | The current H3 implementation executes one generation request per diffusion batch; use concurrency for service-level throughput | cuDNN attention for the validated two-GPU consumer profile; TRTLLM attention or FlashAttention-4 on supported Blackwell profiles | TP2 plus text-encoder TP and Ulysses/VAE parallel groups on multi-GPU profiles; DLO/CPU offload trade memory for transfer time | Online FP8 applies to eligible DiT linears and is incompatible with layerwise offload |
 | **VAE Decoder** | Decode follows each generation request; tile/patch work can be distributed even when denoising is not batched | VAE decode kernels rather than DiT attention | VAE patch parallelism and native tiled decode within the diffusion stage | BF16/FP32 baseline; the documented H3 FP8 path leaves both VAEs unchanged |
 
@@ -324,7 +323,7 @@ inputs, fields, and ownership rules are described below.
 flowchart TB
     layer1["Layer 1 · Authoring inputs<br/>PipelineConfig + DeployConfig"]
     layer2["Layer 2 · Production resolution boundary<br/>resolve_omni_config()<br/>(StageConfigFactory + VllmOmniConfig internally)"]
-    layer3["Layer 3 · Startup hand-off<br/>OmniConfigResolution<br/>PipelineConfig + temporary OmegaConf stage bridge"]
+    layer3["Layer 3 · Startup hand-off<br/>OmniConfigResolution<br/>PipelineConfig + typed stage configs"]
     layer4["Layer 4 · Runtime launch planning<br/>StageRuntime"]
     layer5["Layer 5 · Engine materialization<br/>VllmConfig / OmniDiffusionConfig"]
 
@@ -336,10 +335,10 @@ The production resolution boundary is `resolve_omni_config()` in
 It delegates typed construction to `StageConfigFactory.create_from_model()` and
 `VllmOmniConfig.from_pipeline_config()`, then returns an
 `OmniConfigResolution` consumed by both `AsyncOmniEngine` and headless startup.
-Until `StageRuntime` consumes typed stage configs directly, this envelope carries
-the effective `PipelineConfig` alongside OmegaConf-compatible `stage_configs` as
-a temporary runtime bridge; both views describe the same resolved topology,
-including injected stages.
+This envelope carries the effective `PipelineConfig` alongside typed
+`stage_configs`, which `StageRuntime` consumes directly. Both views describe the
+same resolved topology, including injected stages. Backend arguments are
+projected from these typed stages when each engine is initialized.
 
 The legacy `stage_args` YAML path has been removed. Model topology now resolves
 through `PipelineConfig`, with runtime overrides supplied by `DeployConfig`.
@@ -361,7 +360,7 @@ The important ownership rules are:
 3. CLI and Python overrides are applied at the resolution boundary, with
    per-stage overrides taking precedence over global values where supported.
 4. `OmniConfigResolution` is the sole startup hand-off. Its `pipeline_config`
-   and temporary `stage_configs` compatibility view must describe the same
+   and typed `stage_configs` must describe the same
    topology.
 5. `StageRuntime` owns launch planning and replica lifecycle. `ReplicaInitPlan`
    is runtime-private state, not a user configuration object.

docs/design/feature/attention_backend_selection.md

@@ -76,6 +76,34 @@ preserve common `self` or `cross` policies. A model also decides whether its
 path is compatible with automatic TRTLLM selection, because only the model
 knows whether masking and packing satisfy the kernel contract.
 
+### Model-owned implementation specializations
+
+Use `Attention(..., impl_overrides={backend_name: implementation_class})`
+when a model needs different computation within an existing backend contract.
+The model defines the implementation in its own directory and passes it when
+constructing the layer:
+
+```python
+# MyModelVSAImpl is a model-owned subclass of FastVideoVSAImpl.
+self.attention = Attention(
+    num_heads=num_heads,
+    head_size=head_size,
+    softmax_scale=head_size**-0.5,
+    causal=False,
+    role="self",
+    impl_overrides={"FASTVIDEO_VSA": MyModelVSAImpl},
+)
+```
+
+The override is applied after role/platform selection, keyed by the selected
+backend's `get_name()`. It must subclass the **selected implementation**;
+incompatible platform implementations raise `TypeError`. Unselected overrides
+have no effect. Backend capabilities, constructor options, and shared
+parallel dispatch remain unchanged and must be honored by the specialization.
+
+Use a new backend for a new provider/capability contract. `custom_attention`
+instead owns communication and requires `skip_sequence_parallel=True`.
+
 ## Adding or changing a backend
 
 An implementation change is complete only when it:
@@ -91,8 +119,9 @@ An implementation change is complete only when it:
 
 ## FastVideo VSA model metadata
 
-FastVideo VSA support is scoped to `FastVideo/FastWan2.2-TI2V-5B-Diffusers`, whose text-to-video and image-to-video modes both use `Wan22Pipeline`. It operates on a flattened DiT sequence but partitions tokens in
-the original latent video grid. Wan integrations therefore attach the
+FastWan's text-to-video and image-to-video modes both use `Wan22Pipeline`.
+It operates on a flattened DiT sequence but partitions tokens in the original
+latent video grid. Wan integrations therefore attach the
 post-patch `(T, H, W)` shape as `vsa_dit_seq_shape` attention metadata. The
 backend validates that `T * H * W` equals the sequence length, derives the
 runtime block count, and selects the configured top-k key/value blocks for
@@ -104,3 +133,8 @@ the checkpoint contains those weights. Checkpoints without the projection do
 not allocate or execute it. When top-k selects every block, native checkpoints
 route to SDPA, while FastVideo DMD checkpoints preserve the VSA all-block path
 to retain checkpoint semantics.
+
+MiniMax-H3 supplies `MiniMaxH3VSAImpl` through `impl_overrides`.
+`models/minimax_h3/attention/fastvideo_h3.py` owns its prefix layout, video tiling, and
+learned compression gate. Pooling, block-map construction, and tile64 provider
+calls are shared through `attention/ops/block_sparse.py`.

docs/design/feature/disaggregated_inference.md

@@ -9,6 +9,7 @@ Backend-specific setup lives in separate docs:
 - [MooncakeStoreConnector](omni_connectors/mooncake_store_connector.md)
 - [MooncakeTransferEngineConnector](omni_connectors/mooncake_transfer_engine_connector.md)
 - [MoriTransferEngineConnector](omni_connectors/mori_transfer_engine_connector.md)
+- [NixlConnector](omni_connectors/nixl_connector.md)
 - [YuanrongConnector](omni_connectors/yuanrong_connector.md)
 - [YuanrongTransferEngineConnector](omni_connectors/yuanrong_transfer_engine_connector.md)
 
@@ -25,6 +26,7 @@ Current connectors operate in D2H2D (device to host to device) mode.
 | Multi node (Mooncake Store) | MooncakeStoreConnector | TCP-based, requires Mooncake Master + metadata server. |
 | Multi node (Mooncake RDMA) | MooncakeTransferEngineConnector | RDMA/TCP direct transfer with managed memory pool. Fastest. |
 | Multi node (Mori RDMA) | MoriTransferEngineConnector | RDMA direct transfer via Mori IOEngine. |
+| Multi node (NIXL) | NixlConnector | NIXL `READ` over UCX; the only backend covering Intel XPU. |
 | Multi node (Yuanrong) | YuanrongConnector | Requires Yuanrong Datasystem + etcd. |
 | Ascend NPU P2P (Yuanrong TE) | YuanrongTransferEngineConnector | Uses Yuanrong TransferEngine directly. Configure NPU device IPv4 and `memory_pool_device: "npu"`. |
 

docs/design/feature/host_weight_runtime.md

@@ -42,6 +42,11 @@ V1 does not include:
 
 ## Motivation and use cases
 
+For investigating CPU memory retained by dependency tensor materialization,
+see the [standalone safetensors diagnostic](https://github.com/vllm-project/vllm-omni/blob/main/benchmarks/host_weight_runtime/README.md).
+It distinguishes repeated dependency calls from reuse of cached views and does
+not establish per-request HWR leakage.
+
 Model loading can create the same final host representation repeatedly. This
 is especially expensive when loading performs checkpoint decoding, tensor
 renaming, TP slicing, quantization, packing, or scale construction before GPU
@@ -313,8 +318,15 @@ One process per exact identity owns a build; other workers wait and then acquire
 leases for the published artifact. Publication is invisible until all payloads
 and metadata are validated, hashed, fsynced, and atomically renamed.
 
-`coordination_timeout_seconds` bounds filesystem lock acquisition. It does not
-cancel synchronous validation, a producer that has already started, or atomic
+`coordination_timeout_seconds` bounds domain-initialization and lookup/build
+lock acquisition. Store construction and each later resolution or publication
+operation have separate budgets from the same wait policy, rather than one
+end-to-end startup deadline. A domain-init timeout follows the retryable domain
+failure policy below. After contention ends, a fresh runtime construction can
+retry initialization; the timed-out runtime retains its original failure.
+
+The coordination budget does not cancel filesystem I/O, synchronous validation,
+a producer that has already started, or atomic
 publication. A hung in-process producer therefore blocks its owning process and
 must be handled by external process supervision. Enforceable producer
 cancellation requires a future process-isolated producer contract.

docs/design/feature/model_local_kv_caches.md

@@ -0,0 +1,132 @@
+# Model-local KV caches
+
+Some models keep their attention KV as HuggingFace `transformers` cache objects
+instead of using the engine's paged KV manager. Each such model declares what it
+holds by implementing `model_local_kv_specs()`, and
+`OmniGPUModelRunner.load_model` sums and logs the result after load.
+
+## What is and is not already accounted for
+
+This distinction decides what the protocol is for, and an earlier revision of
+this page got it wrong by claiming none of this memory was accounted for.
+
+CUDA-graph capture happens *during* `load_model` -- MiMo in its constructor,
+Qwen3-TTS in `load_weights`. `determine_available_memory()` then samples
+per-process GPU memory through NVML and subtracts it from the requested budget
+(`vllm_omni/worker/base.py`). So the **graph-resident** portion is already
+charged. Subtracting the declared total from the KV budget would double-count
+it.
+
+What is *not* charged is the **per-request** portion, which is allocated after
+profiling. That half is not small: Qwen3-TTS's codec decoder is 4.44 MiB per
+request, so a server at `max_num_seqs=256` reaches roughly 1.1 GiB that no
+per-stage number predicted.
+
+The consumer therefore reports and does not subtract.
+
+## What the four models declare
+
+| Model | Cache | Bounded by | Per row | Rows |
+|---|---|---|---|---|
+| Qwen3-TTS codec decoder (async-chunk only) | sliding `DynamicCache` | `sliding_window - 1 = 71` | 4.44 MiB | `max_num_seqs`, x2 for the working copy |
+| Qwen3-TTS graph pool | retained captures | same | 4.44 MiB | fixed, from captured shapes |
+| MiMo-Audio local transformer | `DynamicCache` | `group_size + max(delay_pattern) = 11` | 704 KiB | `max_num_seqs` |
+| MiMo-Audio graph pool | captured | same | 704 KiB | fixed 261 (bucket sum) |
+| MiniCPM-o Whisper encoder (streaming only) | `EncoderDecoderCache` | `embed_positions` rows = 1500 | 140.62 MiB | fixed 1, per session |
+| ming_flash_omni talker | `StaticCache` | hardcoded `max_cache_len = 2048` | 24.00 MiB | fixed 1 |
+
+**No cache is bounded by `max_model_len`.** Each has its own mechanism: a
+sliding window, a decode-loop trip count, an encoder frame limit, a constant.
+
+**ming's geometry does not exist in this repo.** `self._llm_config` is a
+`Qwen2Config` from the checkpoint, so layers, kv-heads, head-dim and dtype are
+only knowable after load. That is what makes this a runtime query rather than a
+table.
+
+## Lifetime is not multiplicity
+
+`scope` records when memory appears and goes. It does **not** decide how wide
+the allocation is, and an earlier revision that derived scaling from it was
+wrong in two different directions at once:
+
+- ming's talker is per-call but serialized -- `forward` takes
+  `runtime_additional_information[0]` and runs the whole AR loop inline -- so
+  scaling it by `max_num_seqs` over-reported it by that factor.
+- MiniCPM-o's encoder is per *session*, not per sequence, so `max_num_seqs` is
+  simply the wrong number for it.
+
+So width is declared explicitly, by naming the driver. `RowDriver` has two
+values, because two are all the known caches need:
+
+- `FIXED` -- a count the model controls, with `rows_fixed` and a required
+  `rows_reason`. `rows_fixed=1` on a cache that looks per-request is a claim a
+  reviewer must be able to check.
+- `MAX_NUM_SEQS` -- one row per in-flight sequence; only the engine knows the
+  value.
+
+A third value belongs here when a model actually has a cache the engine widens
+by some other number. An earlier revision added one speculatively, for a single
+declarer, and paid for it with a class hierarchy, an engine-capacity object and
+a conditional-annotation field.
+
+`rows` counts row-equivalents at peak and deliberately does not distinguish
+"one allocation N rows wide" from "N allocations of one row". Those cost the
+same, and a revision that tried to carry both got the object topology wrong in
+three of four declarers: MiniCPM-o keeps one cache object per session while
+asserting batch size 1, and both graph pools keep one distinct object per
+captured bucket. The layout belongs in `allocation_note`, where it cannot be
+mistaken for arithmetic.
+
+## Declaring nothing is a valid declaration
+
+A declaration must be inert when its path is. Qwen3-TTS holds KV only in
+async-chunk mode -- stateless decoding runs `_forward_exact`, which calls the
+transformer without `use_cache` -- so the stateless path declares nothing.
+Getting this wrong is easy in both directions, and both have happened here: one
+revision hung the declaration off the CUDA-graph wrapper and reported zero under
+`enforce_eager`, where the eager async-chunk path does allocate; the next
+declared unconditionally and invented roughly a gigabyte for stateless
+deployments.
+
+Where the model cannot see the deciding setting, it declares the honest unit
+and says so in `allocation_note`. MiniCPM-o's encoder cache exists only on the
+streaming path and is one object per session; the session cap is not visible
+from the model, so it declares one session rather than scaling by a number it
+would have to guess at.
+
+## Finding the declarers
+
+`collect_model_local_kv_specs()` walks `named_modules()`. Two wrinkles:
+
+- The runner's `self.model` may be a `CUDAGraphWrapper`, which is not an
+  `nn.Module` but forwards attribute access to its runnable. The walk therefore
+  works either way, but entering through both the wrapper and its runnable would
+  count a root-level declarer twice. The collector calls the supported
+  `unwrap()` accessor first.
+- An owner that is not a submodule at all (Qwen3-TTS's graph wrapper, ming's
+  `MingAudioGenerator`) is reached by one explicit hop from the module that
+  holds it.
+
+## Adding a model
+
+Return one entry per distinct allocation. Prefer `spec_from_hf_config()` and
+pass the same config object the cache is built from; it raises rather than
+guessing when an attribute is missing. `capacity_source` is diagnostic text --
+never branch on it.
+
+## Known limits
+
+Row counts taken from live state after warmup miss caches `transformers` has
+not yet materialized, which makes Qwen3-TTS's graph-pool figure a floor for the
+xvec path.
+
+MiMo's graph-pool cache has no reachable Python object -- it is a local inside
+the captured frame -- so its width comes from the captured bucket list rather
+than from tensors. Its two entries are also mutually exclusive per call: a call
+either replays a bucket or runs eagerly. The sum is the right ceiling for the
+process and the wrong number for a single request.
+
+MiMo captures every bucket gated only on `torch.cuda.is_available()`
+(`mimo_audio_llm.py:670`), so those 179.44 MiB stay resident even when the
+deploy config sets `enforce_eager: true`. Gating it is a one-line change that
+needs a model boot to verify, so it is noted rather than bundled here.

docs/design/feature/offloader/README.md

@@ -41,7 +41,8 @@ Every backend implements `OffloadBackend`:
 - `is_enabled()` reports lifecycle state.
 
 `disable()` does not promise to restore the pipeline's original device
-placement. The caller owns any subsequent rematerialization.
+placement. It must restore usable parameter and buffer storage before releasing
+backend-owned host weights. The caller owns subsequent device placement.
 
 An enable failure must remove every partially installed hook. Rank-local
 backends restore ordinary tensors so the pipeline can be retried. A failed
@@ -54,9 +55,19 @@ must use distinct hook names and remove only hooks they own.
 ## Discovery and topology
 
 `resolve_offload_plan(pipeline, config)` is the single topology entry point.
-Every backend calls it once when `enable()` starts and then consumes the
-resolved components, block stacks, resident heads, and staged-residency flags
-it returns. Backends do not re-read model declarations.
+The generic model-level and ordinary layerwise paths call it once when
+`enable()` starts and consume its component selection and block stacks.
+Ordinary layerwise offload also respects a resolved `skip_reason`, preserving
+the legacy missing-DiT no-op without placing auxiliary components. Non-block
+DiT and encoder state is placed by resolved block tensor identity, so the
+backend does not rediscover block containers by attribute name.
+
+Two compatibility paths remain outside this completed generic cutover:
+model-level pipelines implementing `SupportsModelCpuOffload` still own their
+custom phase lifecycle (RFC #6648 J4), and distributed layerwise offload still
+reads declarations for nested-submodule staging and loader-owned mmap paths
+(J3). The custom model-level protocol is delegated to before generic plan
+resolution; it must not be subjected to the generic encoder/DiT swap checks.
 
 Pipeline component discovery prefers `SupportsComponentDiscovery` declarations:
 
@@ -102,3 +113,17 @@ remains responsible for transfer, synchronization, and storage ownership.
 
 The diffusion [Offloader module design](../../module/diffusion/offloader.md)
 describes how these feature contracts fit into the larger diffusion runtime.
+
+## Backend acceptance tests
+
+`tests/diffusion/offloader/test_backend_plan_contract.py` covers the two
+behaviors the generic cutover changed. A DiT attribute aliasing a streamed
+block must keep the ring's host residency, so placement cannot follow attribute
+names; the CUDA case fails against the previous name-based placement. Both
+backends must then run with their selector helpers and the model declaration
+made unreadable, proving execution consumes the resolved plan alone.
+
+Selection errors, rollback, residency, and enable/disable cycles stay in
+`test_plan_resolver.py`, `test_layerwise_backend.py` and
+`test_sequential_backend.py`. Model-specific phases and executed tied aliases
+remain the J4 contract suite.

docs/design/feature/omni_connectors/nixl_connector.md

@@ -0,0 +1,135 @@
+# NixlConnector
+
+## When to Use
+
+Multi-node or intra-node stage transfer over NIXL, which brokers RDMA/shared-memory
+transports through a single API. Useful when the deployment already standardises on
+NIXL for KV transfer, or when the target accelerator has no Mooncake/Mori backend --
+NIXL reaches Intel XPU through UCX's Level Zero support.
+
+## Mechanism
+
+Uses vLLM's `NixlWrapper` (`vllm.distributed.nixl_utils`) to register the producer's
+tensors and let the consumer pull them with a NIXL `READ`.
+
+- Data Plane: NIXL agent-to-agent `READ`, GPU-to-GPU where the backend allows it.
+- Control Plane: a ZMQ ROUTER socket atomically claims the source allocation before
+  returning descriptors. Forwarded `put()` metadata includes the generation and
+  endpoint; the consumer claims that generation before submitting any READ.
+
+Transfer metadata uses schema version 1. Tensor descriptors are grouped by NIXL
+memory type (`DRAM`, `VRAM`, and so on), while each descriptor retains its global
+tensor index so mixed CPU/accelerator payloads can be reconstructed in order.
+
+Payloads are not restricted to tensors. A single tensor or a list of tensors is
+transferred as-is; a nested structure has its tensor leaves extracted and shipped
+alongside a msgpack-encoded skeleton; anything else is msgpack-encoded into one
+uint8 tensor. Mapping and sequence structure is preserved. `msgspec.Struct` values,
+including `OmniPayloadStruct`, are normalised to mappings so their tensor leaves use
+the direct NIXL path rather than CPU object serialisation.
+
+Each NIXL agent uses strict thread synchronization. A consumer performs one short
+metadata query per scheduler poll (10ms by default), which prevents an unavailable
+producer from blocking unrelated requests. The regular transfer-completion handshake
+keeps the configured transfer timeout.
+
+## Installation
+
+CUDA hosts can install the published wheel:
+
+```bash
+pip install nixl
+```
+
+For Intel XPU, use `docker/Dockerfile.xpu`. Its upstream vLLM XPU base image
+provides a matching torch-xpu, UCX with Level Zero support, and NIXL runtime.
+
+## Configuration
+
+```yaml
+connectors:
+  nixl_connector:
+    name: NixlConnector
+    extra:
+      host: "auto"
+      zmq_port: 50061
+      backends: ["UCX"]
+
+stages:
+  - stage_id: 0
+    output_connectors:
+      to_stage_1: nixl_connector
+
+  - stage_id: 1
+    input_connectors:
+      from_stage_0: nixl_connector
+```
+
+Parameters:
+
+- `host`: address the producer binds its handshake socket to (`"auto"` to detect).
+- `zmq_port`: handshake port. If a producer omits it, an ephemeral port is opened
+  for ownership claims and returned in `put()` metadata. The value in the deploy YAML is a
+  base port. Purpose, replica, rank, and producer-stage offsets are added centrally,
+  so colocated connectors and tensor-parallel workers receive distinct endpoints
+  without one config entry per rank.
+- `sender_host` / `sender_zmq_port`: consumer-side override naming the producer's
+  handshake endpoint. Only needed when the consumer cannot learn it from metadata.
+  In replicated deployments, the producer endpoint is carried with each request and
+  selected at `get()` time; it is not mutable connector-wide state.
+  Explicit sender ports are preserved, not recomputed using the receiver replica.
+  Intermediate NIXL stages retain their incoming receiver configuration and a
+  separate outgoing bind configuration, so one worker can receive and publish.
+- `backends`: NIXL backends to register memory with. Defaults to `["UCX"]`.
+- `receive_device`: forces where received tensors land. By default the consumer
+  keeps the producer's device *type* but uses its own current device of that type.
+- `memory_type`: overrides the NIXL memory type, otherwise `DRAM` for CPU tensors
+  and `VRAM` for accelerator tensors.
+- `lease_seconds`: how long a `put()` payload stays registered while waiting to be
+  read (default 3600). The consumer reports completion, so this only bounds payloads
+  nobody has claimed. An internal reaper releases only unclaimed registrations.
+  `VLLM_OMNI_NIXL_LEASE_S` overrides it.
+- `transfer_timeout_s`: how long a `get()` waits for its `READ` to complete
+  (default 300). NIXL 1.3 has no transfer cancellation API, so a timed-out transfer's
+  buffers and registrations remain owned by the connector until NIXL reports a
+  terminal state; `close()` waits for that state before releasing them.
+  `VLLM_OMNI_NIXL_XFER_TIMEOUT_S` overrides it.
+
+### Source ownership and failure limits
+
+Claims and expiry use the same lock. Completion identifies both a payload
+generation and a unique READ claim; duplicate or stale ACKs cannot release a
+different reader or replacement payload. Replacing a claimed key is rejected.
+Terminal errors also release claims; timeout, unknown status, and active sibling
+transfers retain the complete destination bundle and source claim until terminal.
+Zero-element tensors retain their shape/dtype/skeleton positions but never create
+DMA descriptors or registrations.
+
+A lost metadata response, lost completion ACK, or abandoned consumer can retain a
+claim indefinitely. NIXL 1.3 cannot prove remote cancellation, so neither TTL nor
+`cleanup()` frees those allocations. Producer `close()` rejects new work and
+retains its agent, listener and claimed allocations; a later `close()` finishes
+teardown after claims drain. Permanently abandoned claims remain until process
+exit. This favors memory safety over bounded shutdown/memory usage. Only trusted
+peers may access this unauthenticated control plane. Both endpoints must use the
+claim-aware protocol; rolling compatibility with older GET_META clients is not
+provided. Legacy externally-owned metadata without a generation is accepted only
+under the caller's lifetime guarantee, not managed as a leased allocation here.
+
+## Validation
+
+The native CUDA smoke test requires NIXL and at least two visible CUDA devices:
+
+```bash
+UCX_RCACHE_MAX_UNRELEASED=1024 pytest -o addopts='' -q -s \
+  tests/distributed/omni_connectors/test_nixl_connector_native.py
+```
+
+It starts independent producer and consumer processes, loads the real UCX backend,
+performs the ZMQ `GET_META` handshake and a native NIXL `READ` from GPU 0 to GPU 1,
+checks mixed CPU/GPU tensor leaves from an `OmniPayloadStruct`, sends `XFER_DONE`,
+and verifies that producer registrations are released. This validates same-host,
+cross-GPU operation. Inter-node UCX transport is supported by the design but was not
+validated by this test.
+
+For more details, refer to the [NIXL repository](https://github.com/ai-dynamo/nixl).

docs/design/feature/realtime_ar_diffusion.md

@@ -143,8 +143,20 @@ fraction to select additional resident sessions up to the model-declared cap.
 All model-owned reservations are deducted before the paged self-attention pool
 is allocated.
 
-LingBot reports its persistent full-horizon image-condition tensor. DreamZero
-reports the measured 603 MiB per-session Wan VAE causal-convolution state.
+LingBot advances a session-owned causal Wan VAE encoder for each condition
+block: the opening pixel frame is followed by four zero pixel frames per later
+latent frame. It retains one current condition block, with separate committed
+and in-flight encoder histories so a failed block cannot overwrite committed
+context. The state reservation includes both encoder histories, derived from
+the encoder's convolution input grids, and the streaming decoder cache. These
+caches are released through the same close/reset hooks. Condition encoding
+currently requires the unpatched, non-tiled Wan encoder path; the existing
+decoder's tiling fallback remains independent. Real-weight GPU parity with a
+full-horizon encode still requires validation because the pointwise posterior
+projection now runs per block.
+
+DreamZero reports the measured 603 MiB per-session Wan VAE causal-convolution
+state.
 
 ## Concurrency and routing limits
 
@@ -178,6 +190,24 @@ For LingBot World v2:
 - `lingbot_world/pipeline.py` constructs conditioning, owns small non-KV session state, and
   produces one block plus the standard metadata envelope.
 
+## Ulysses sequence parallelism
+
+LingBot supports pure Ulysses sequence parallelism for both direct and
+AR-Diffusion execution. With Ulysses degree greater than one, hidden tokens,
+camera features, token-expanded timestep modulation, and RoPE tables are sharded
+together. Without SP, timestep modulation retains the frame-broadcast path.
+Self-attention performs the sequence-to-head all-to-all before reading or writing
+paged KV, while static text K/V uses the same local head shard. Text K/V shards
+own compact storage; cross-attention exchanges query/output layouts to use these
+shards. This retains the shared cache geometry and reduces text K/V storage at
+the cost of two all-to-all calls per layer.
+
+The output head projects local tokens before gathering the flow values. For
+the 14B model this reduces the gathered width from 5120 to 64; frame modulation
+uses each shard's global token offset, including shards that split a frame. Only
+`ulysses_mode="strict"` is supported; `advanced_uaa`, Ring, and AllGather-KV modes
+remain unsupported for this model.
+
 ## Non-goals
 
 This contract does not currently provide:

docs/design/feature/sequence_parallel.md

@@ -77,7 +77,9 @@ Use it when plain Ulysses-SP would otherwise fail because:
 
 - `auto_pad=True` pads sequence tokens in `_sp_plan` and requires attention backends that support `attention_mask`.
 - `advanced_uaa` does not depend on mask-based token padding inside Ulysses attention. It is therefore a better fit for non-divisible head counts and uneven Ulysses shard sizes.
-- `auto_pad=True` remains incompatible with Ring attention because the ring backend does not consume `attention_mask`.
+- `auto_pad=True` remains incompatible with Ring attention because the ring backend does not consume arbitrary
+  `attention_mask` values. A model with contiguous suffix padding may instead publish `valid_kv_length`; Ring trims
+  each circulated K/V block against that global valid prefix.
 - `advanced_uaa` is still experimental and hybrid mode remains limited by Ring's equal-shape requirement.
 
 ---
@@ -416,7 +418,7 @@ _sp_plan = {
 | Constraint | Description |
 |------------|-------------|
 | **Attention Backend Compatibility** | The attention backends must support `attention_mask`. Currently only `TORCH_SDPA` and `FLASH_ATTN` (default for diffusion models) are supported. |
-| **Ring Attention Limitation** | Ring attention does not support `attention_mask`. Therefore, when using `auto_pad=True`, the combination of Ulysses + Ring attention is not feasible. |
+| **Ring Attention Limitation** | Ring attention does not support arbitrary `attention_mask` values, so `auto_pad=True` remains incompatible. Contiguous suffix padding is supported when the model publishes the global `valid_kv_length` prefix. |
 
 1. Enable `auto_pad=True` for all sequence-dimension inputs in `_sp_plan`:
 ```python

docs/design/fullduplex-personaplex.md

@@ -20,7 +20,7 @@ The target is the engine-native path:
   -> PersonaPlex Talker
   -> streaming PersonaPlex Code2Wav
   -> PersonaPlex data-plane projector
-  -> response.audio.delta + response.audio_transcript.delta
+  -> response.output_audio.delta + response.output_audio_transcript.delta
 ```
 
 The standalone `/api/chat` and `/v1/audio/duplex` server that accompanied the
@@ -197,7 +197,7 @@ model-neutral native results:
 
 It owns per-request audio and text cursors so a cumulative output cannot replay
 old audio. The generic projector turns each native result into the Realtime
-`response.audio.delta` / `response.audio_transcript.delta` pair (and
+`response.output_audio.delta` / `response.output_audio_transcript.delta` pair (and
 `response.output_text.delta` for text) under one `response_id`; the full
 mapping is the name map in the
 [Realtime Duplex API](../serving/realtime_duplex_api.md) serving guide.
@@ -271,3 +271,111 @@ The required evidence is:
 
 Audio that is empty, all zero, non-finite, or only a protocol `listen` event is
 not a successful end-to-end result.
+
+### Paced multi-session load driver
+
+The existing driver keeps its two-session admission and slot-reuse validation
+when `--sessions` is omitted. Passing `--sessions N` selects a separate,
+bounded load mode; it does not replace the lifecycle test or change server
+admission capacity. Start an existing PersonaPlex server configured to admit
+the requested number of sessions, then run from a source checkout with the
+vLLM-Omni client dependencies installed:
+
+```bash
+python -m tests.e2e.online_serving.personaplex_realtime_duplex \
+  --model nvidia/personaplex-7b-v1 \
+  --input-wav /path/to/input.wav \
+  --sessions 2 --load-frames 1000 --drain-s 2 \
+  --server-revision SERVER_COMMIT --server-hardware A100-80GB \
+  --output-dir /tmp/personaplex-load-n2
+```
+
+Use a new output directory for every run. The driver refuses to overwrite an
+existing `load-result.json`. The server labels are supplied by the operator,
+not detected or verified by the client. Record the server command, package
+versions, checkpoint revision and hardware alongside a published measurement.
+
+All attempted sessions finish admission before admitted sessions share a
+common input start. A rejected session remains a failed row in the report;
+it is not removed from the denominator. Each session receives at most
+`--load-frames` input frames, without looping the WAV, and uses 80 ms pacing.
+A stalled send cannot cause a catch-up burst. The timeline records send
+schedule lateness so an overloaded client is visible rather than mistaken
+for a faster server. This mode measures steady concurrency, not join/leave
+interference; the original lifecycle mode still checks slot reuse.
+Load mode does not probe an additional N+1 session for `resource_exhausted`;
+the lifecycle mode owns that admission-limit check. A per-session
+`client_pacing_warning` and the summary's `client_pacing_warning_sessions`
+identify send-lateness p99 above one nominal frame (80 ms). This is a warning,
+not an extra acceptance gate: read it alongside server-facing measurements
+so a slow client is not mistaken for a slow server.
+
+The fixed drain window is part of the measurement configuration. Audio
+arriving during cleanup is excluded, so closing cannot rescue a session that
+missed its observation deadline. Cleanup is attempted for admitted sessions
+even when audio validation fails. Protocol errors, missing/invalid audio,
+shared response IDs and cleanup failures make the report fail and the CLI
+exit nonzero. Cancellation closes transports and propagates to the caller.
+`--cleanup-timeout-s` (default 5 s) bounds session-close send and acknowledgement
+together, independently of `--timeout-s`. The subsequent WebSocket close
+handshake has the same separate bound; an unresponsive close acknowledgement
+cannot hold cancellation for the full inference timeout.
+
+`load-result.json` retains per-session send and audio-packet timelines using
+the client's monotonic clock. It does not export handshake credentials or
+raw audio payloads. Metric definitions are explicit:
+
+Both modes validate each nonempty audio packet's strict base64 encoding, PCM16
+sample alignment, 24 kHz sample rate and response identity. Valid aggregate
+audio cannot hide a malformed packet. In load mode, rejected packets make the session fail
+and appear in `invalid_audio_packets` with their event index, receive time and
+fixed diagnostic code; they do not contribute samples or timing intervals.
+Empty audio deltas remain legal and contribute no samples. This check does
+not change the public client's decoding. The lifecycle scenario and dispatch
+remain the same, but malformed packets that aggregate checks once tolerated
+now fail that mode as well.
+
+The probe accepts both the current OpenAI Realtime audio event name
+`response.output_audio.delta` and the legacy `response.audio.delta`. Validation
+uses the wire payload directly rather than relying on a particular client
+library's alias table, so a server-side event-name migration cannot silently
+turn real audio into a zero-output measurement.
+
+| Field | Meaning |
+| --- | --- |
+| `client_audio_packet_interval_ms` | Consecutive nonempty packet arrivals: median, linear p99, max, and zero-based index of the first maximal interval. Interval index 0 ends at packet 1. One packet can contain multiple codec frames; this is not server tick latency. |
+| `client_first_audio_after_stream_start_ms` | First audio receipt minus first input send start; admission is excluded. |
+| `client_stream_rtf` | First input send start to last audio receipt, divided by received audio duration. This includes real-time input pacing and is not an inference-only RTF. |
+| `client_send_lateness_ms` | Send start minus its original shared schedule, clipped at zero. |
+| `frame_deficit` | Integer input-frame minus output-frame count from the shared session validation; missing when valid whole-frame accounting is unavailable. |
+| `output_frames`, `voiced_frames`, `silent_frames`, `frame_coverage_ratio`, `audio_rms`, `audio_chunks` | Session-validation statistics retained on success and on acceptance failure once computed. |
+| `acceptance_check` | Failed local check, with numeric diagnostics in `error`; other exceptions retain phase/type only, without remote handshake or metadata text. |
+| `audio_underrun_s`, `audio_underrun_event_count`, `audio_continuity_ok` | Shared `compute_continuity_stats` on valid PCM16 packet arrivals, with its default 100 ms underrun threshold; absent audio gives missing values rather than a continuity pass. |
+
+No output gives missing latency/RTF values, not zero. By default RTF is
+diagnostic; `--max-client-rtf` adds an explicit ceiling. The existing frame
+coverage, audible-frame and scheduler-data-plane checks still apply. A longer
+`--drain-s` can demonstrate eventual completion under backlog, but it must not
+be reported as real-time capacity when the fixed-window run failed or client
+RTF exceeded the stated target. Likewise, the first live session can pay
+one-time model/codec/JIT costs; either use the server's configured duplex
+warmup path or label that run as cold-start evidence before comparing steady
+capacity. Passing client-only metrics does not certify model quality or a
+real-time capacity target on another GPU. Begin with N=1 and N=2; higher N is a
+capacity experiment, not a promised property of an A100 or of this patch.
+Continuity is a client-arrival diagnostic, not proof of speaker playback and
+not an additional pass/fail gate. Packet-interval and send-lateness summaries
+retain their documented linear p99; the older client interval helper reports
+rounded nearest-rank p95 instead and is not substituted for that schema.
+
+Deterministic driver and localhost WebSocket checks need no model or GPU:
+
+```bash
+python -m pytest tests/e2e/online_serving/test_personaplex_load_driver.py \
+  -m 'core_model and cpu' --run-level core_model -q
+```
+
+These checks test the driver against controlled protocol responses. Run the
+load-driver command against real weights separately before reporting model-serving
+performance. Server tick timestamps and generic TTFT/TPOT aggregation remain
+the responsibility of the existing duplex metrics paths, not this driver.

docs/design/index.md

@@ -26,6 +26,7 @@ implementation contract; it is not, by itself, a general support claim.
 - [Async Omni Output Materialization](feature/omni_async_output_materialization.md)
 - [Runner-to-model Prefill/Decode Phase Contract](feature/preprocess_phase_contract.md)
 - [Automatic Prefix Caching in Omni Models](feature/prefix_caching.md)
+- [Model-local KV Caches](feature/model_local_kv_caches.md)
 - [Realtime AR-Diffusion Sessions](feature/realtime_ar_diffusion.md)
 
 ### Communication
@@ -35,6 +36,7 @@ implementation contract; it is not, by itself, a general support claim.
 - [Mooncake Store Connector](feature/omni_connectors/mooncake_store_connector.md)
 - [Mooncake Transfer Engine Connector](feature/omni_connectors/mooncake_transfer_engine_connector.md)
 - [Mori Transfer Engine Connector](feature/omni_connectors/mori_transfer_engine_connector.md)
+- [NIXL Connector](feature/omni_connectors/nixl_connector.md)
 - [Shared Memory Connector](feature/omni_connectors/shared_memory_connector.md)
 - [Yuanrong Store Connector](feature/omni_connectors/yuanrong_connector.md)
 - [Yuanrong Transfer Engine Connector](feature/omni_connectors/yuanrong_transfer_engine_connector.md)

docs/design/metrics.md

@@ -60,7 +60,7 @@ Upstream per-engine metrics retain the `vllm:` prefix but are now registered by
 
 There are five independent paths for metric collection.
 
-**Path 1: Pipeline-level metrics (`vllm_omni:*`)**
+#### Path 1: Pipeline-level metrics (`vllm_omni:*`)
 
 `OmniPrometheusMetrics` registers the Gauge / Counter / Histogram collectors at import time. It is instantiated once per entrypoint, labeled with the model name. The entrypoint calls its methods as requests progress:
 
@@ -69,7 +69,7 @@ There are five independent paths for metric collection.
 - `request_failed()` — recorded by the cleanup path when a request exits without natural completion. Internally maps to `finished_reason="abort"` on the existing completion Counter.
 - `inc_requests_failed(reason)` — increments the dedicated failure-attribution Counter once per request. Reasons are normalized to the bounded set `client_abort`, `client_disconnect`, `stage_error`, and `unknown`.
 
-**Path 2: Image and diffusion metrics**
+#### Path 2: Image and diffusion metrics
 
 Image and diffusion metrics use two cooperating collectors. `OmniPrometheusMetrics` owns pipeline/stage workload families, while `OmniModalityMetrics` owns per-replica diffusion breakdowns. Finished stage messages are consumed once in `OmniBase._process_single_result`, which prevents replayed terminal messages from incrementing Counters or Histograms twice.
 
@@ -81,15 +81,15 @@ Image and diffusion metrics use two cooperating collectors. `OmniPrometheusMetri
 
 Zero and missing values have different meanings. Valid zero-duration queue observations are recorded, while unavailable optional measurements are omitted instead of being exported as synthetic zeroes. `peak_memory_mb`, `diffusion_forward_s`, `diffusion_kv_load_s`, `vae_decode_s`, and `kv_wait_s` therefore appear only when their data source is active.
 
-**Path 3: Audio modality metrics (`vllm_omni:audio_*`)**
+#### Path 3: Audio modality metrics (`vllm_omni:audio_*`)
 
 `OmniModalityMetrics` registers seven audio families with `{model_name, stage, replica}` (plus an extra `threshold_ms` / `reason` label on the two extra-cardinality Counters). Three observation sites:
 
 - `observe_modality_at_finalize(...)` — called from `omni_base._process_single_result` inside the existing `e2e_done` finalize guard. For `output_type == "audio"` it emits `audio_frames_total`, `audio_duration_s`, `audio_rtf` (or `audio_skipped_requests_total{reason="no_audio_data"}` when no audio was produced). Sample rate is resolved from `engine_outputs.multimodal_output` via `definitions.resolve_audio_sample_rate(...)` (fallback chain mirrors `serving_chat.py`'s audio response path).
-- `observe_audio_first_packet(...)` — called from the OpenAI SSE audio branch in `serving_chat.py` on the first audio packet for a request. The once-per-request guard is held by `ClientRequestState.first_audio_ts`. The `request_arrival_ts` anchor is stored in `ClientRequestState` by `async_omni.generate()`, computed at request entry.
-- `observe_audio_streaming_finalize(...)` — called from `serving_chat.py` after the streaming chunk loop exhausts. It runs the per-chunk player simulation from `vllm_omni/benchmarks/audio_continuity.py` to compute the worst-case underrun and emits `audio_underrun_s` plus (when the request stayed below the threshold) `audio_continuity_ok_total{threshold_ms}`. Per-chunk PCM byte counts and arrival timestamps are recorded by the same audio branch that updates `first_audio_ts`.
+- `observe_audio_first_packet(...)` — called from the OpenAI SSE audio branch in `serving_chat.py` and the streaming TTS output paths in `serving_speech.py` on the first non-empty audio payload for a request. Streaming Speech observes the first PCM payload rather than a header-only WAV chunk. HTTP carries the middleware request timestamp through response generation; WebSocket uses the start of each sentence synthesis request. Non-streaming Speech does not emit TTFP and uses `e2e_request_latency_s` for overall latency instead.
+- `observe_audio_streaming_finalize(...)` — called from `serving_chat.py` and `serving_speech.py` after a streaming chunk loop exhausts normally. It runs the per-chunk player simulation from `vllm_omni/benchmarks/audio_continuity.py` to compute the worst-case underrun and emits `audio_underrun_s` plus (when the request stayed below the threshold) `audio_continuity_ok_total{threshold_ms}`. Per-chunk PCM byte counts and arrival timestamps are recorded by the same output paths that update TTFP. Speech records raw PCM payloads before raw/SSE/WebSocket encoding, excludes WAV headers and empty chunks, and does not emit continuity samples for failed, cancelled, or non-streaming requests.
 
-**Path 4: Cross-stage transfer metrics (`vllm_omni:transfer_*`)**
+#### Path 4: Cross-stage transfer metrics (`vllm_omni:transfer_*`)
 
 `OmniTransferMetrics` registers four Histogram families with `{model_name, from_stage, from_replica, to_stage, to_replica}` labels. Each observation corresponds to one physical transfer hop (one chunk between adjacent stages), not the per-request accumulated total — so the histograms track per-transfer distribution.
 
@@ -100,7 +100,7 @@ The hook lives in `OrchestratorAggregator.record_transfer_tx` and `record_transf
 
 Defensive fail-safe: if `transfer_emitter` or `replica_resolver` is missing, or the resolver returns `None` for either side, the emit is skipped silently (the underlying `TransferEdgeStats` accumulation is unaffected).
 
-**Path 5: Per-engine metrics (`vllm:*`, stage/replica wrap)**
+#### Path 5: Per-engine metrics (`vllm:*`, stage/replica wrap)
 
 The Orchestrator instantiates `OmniPrometheusStatLogger` (a thin subclass of upstream `vllm.v1.metrics.loggers.PrometheusStatLogger`) and feeds it scheduler stats and iteration stats after processing each batch of engine outputs. This populates the standard ~37 vLLM metric families (TTFT, ITL, TPOT, KV cache usage, etc.) using the same upstream code path — but with the `engine` label reshaped into `stage` + `replica` so multi-replica deployments produce distinct series per replica. See the next section for the wrap mechanics.
 
@@ -174,7 +174,7 @@ second.
 ### Pipeline (4)
 
 | Metric | Type | Labels | Description |
-|--------|------|--------|-------------|
+| -------- | ------ | -------- | ------------- |
 | `vllm_omni:num_requests_running` | Gauge | `model_name` | Requests currently executing across all stages |
 | `vllm_omni:num_requests_waiting` | Gauge | `model_name` | Requests queued but not yet scheduled |
 | `vllm_omni:requests_success_total` | Counter | `model_name`, `finished_reason` | Total requests by completion reason ({stop, length, abort, ...}); aborts cover client-disconnect / cancellation paths in addition to upstream `FinishReason.ABORT` |
@@ -183,7 +183,7 @@ second.
 ### Image and diffusion service-level metrics
 
 | Metric | Type | Labels | Description |
-|--------|------|--------|-------------|
+| -------- | ------ | -------- | ------------- |
 | `vllm_omni:stage_gen_time_s` | Histogram | `model_name`, `stage`, `stage_type` | Stage submit to finished output; includes in-stage queueing |
 | `vllm_omni:request_queue_wait_s` | Histogram | `model_name` | Orchestration-layer queue wait; a present zero is recorded |
 | `vllm_omni:stage_waiting_requests` | Gauge | `model_name`, `stage` | Sum of the latest waiting snapshots across live replicas in the stage |
@@ -200,7 +200,7 @@ second.
 Labels: `{model_name, stage, replica}`.
 
 | Metric | Type | Description |
-|--------|------|-------------|
+| -------- | ------ | ------------- |
 | `vllm_omni:diffusion_exec_s` | Histogram | Core diffusion-step execution time per request |
 | `vllm_omni:diffusion_exec_per_step_s` | Histogram | Core diffusion execution divided by inference-step count |
 | `vllm_omni:diffusion_preprocess_s` | Histogram | Diffusion input preprocessing time |
@@ -218,7 +218,7 @@ The optional breakdown and memory families are sparse by design: when an engine
 Labels: `{model_name, stage, replica}` plus the listed extra label.
 
 | Metric | Type | Extra label | Description |
-|--------|------|-------------|-------------|
+| -------- | ------ | ------------- | ------------- |
 | `vllm_omni:audio_ttfp_s` | Histogram | — | Time from request arrival to first audio packet/frame |
 | `vllm_omni:audio_duration_s` | Histogram | — | Audio content duration (`audio_frames / sample_rate`) |
 | `vllm_omni:audio_rtf` | Histogram | — | Real-time factor `stage_gen_time_s / audio_duration_s` (SLO `< 1`); uses `RTF_BUCKETS` |
@@ -227,12 +227,29 @@ Labels: `{model_name, stage, replica}` plus the listed extra label.
 | `vllm_omni:audio_continuity_ok_total` | Counter | `threshold_ms` | Incremented when the request's worst underrun stayed below `threshold_ms` |
 | `vllm_omni:audio_skipped_requests_total` | Counter | `reason` | Silent-loss counter — code2wav rejected malformed codec input and returned `200 OK` with empty audio |
 
+### Speech streaming (2)
+
+Labels: `{model_name}` plus the listed extra label.
+
+| Metric | Type | Extra label | Description |
+| -------- | ------ | ------------- | ------------- |
+| `vllm_omni:speech_stream_aborted_total` | Counter | `reason` | Interrupted audio generators, including before first PCM; reasons: `cancelled`, `closed`, `engine_dead`, `error` |
+| `vllm_omni:speech_stream_completed_total` | Counter | — | Normally completed audio generators; does not confirm client receipt |
+
+Scope: started Speech audio generators (raw/SSE/WebSocket), counted once per
+generation; WebSocket counts each sentence. Chat, non-streaming Speech, and
+failures before generator execution are excluded. Interrupted streams do not
+emit continuity or underrun samples.
+
+Interruption ratio: `aborted / (aborted + completed)`, using increments over the
+same time window and summing all reasons per model.
+
 ### Cross-stage transfer (4)
 
 Labels: `{model_name, from_stage, from_replica, to_stage, to_replica}`.
 
 | Metric | Type | Description |
-|--------|------|-------------|
+| -------- | ------ | ------------- |
 | `vllm_omni:transfer_size_bytes` | Histogram | Per-transfer payload size in bytes |
 | `vllm_omni:transfer_tx_s` | Histogram | Sender-side time (serialize + submit to connector) |
 | `vllm_omni:transfer_rx_s` | Histogram | Receiver-side time (recv + deserialize) |
@@ -245,8 +262,8 @@ After the wrap, every upstream `vllm:*` family — TTFT, ITL, TPOT, e2e latency,
 ## Naming Convention
 
 - All time-bearing metrics use the `_s` suffix (values in seconds). Two bucket families are used:
-  - `SECONDS_BUCKETS` (0.05 s – 300 s) for e2e / generation / TTFP style values.
-  - `SECONDS_FAST_BUCKETS` (0.001 s – 60 s) for fine-grained cross-stage transfer and audio-underrun values that need millisecond-level resolution.
+    - `SECONDS_BUCKETS` (0.05 s – 300 s) for e2e / generation / TTFP style values.
+    - `SECONDS_FAST_BUCKETS` (0.001 s – 60 s) for fine-grained cross-stage transfer and audio-underrun values that need millisecond-level resolution.
 - Counters use the `_total` suffix (auto-appended by `prometheus_client`).
 - Sizes use the `_bytes` suffix.
 - All omni-specific families are prefixed `vllm_omni:`. The upstream `unregister_vllm_metrics()` function is monkey-patched to a scoped version that still strips upstream `vllm:*` collectors (so multi-engine init within one process does not crash on duplicate registration) but preserves anything prefixed `vllm_omni:` / `vllm_omni`.

docs/design/module/engine_orchestration.md

@@ -16,6 +16,7 @@ required_reviewers:
   - "@yenuo26"
   - "@NickCao"
 primary_code_paths:
+  - vllm_omni/engine/omni_engine_base.py
   - vllm_omni/engine/async_omni_engine.py
   - vllm_omni/engine/async_engine_utils.py
   - vllm_omni/engine/orchestrator.py
@@ -28,8 +29,12 @@ related_code_paths:
   - vllm_omni/engine/stage_pool.py
   - vllm_omni/outputs/output_processor.py
   - vllm_omni/distributed/omni_coordinator/**
-  # Duplex kernel design is owned by ../fullduplex.md; listed here because the
-  # orchestrator hosts the duplex control plane.
+  # The duplex siblings (DuplexOmniEngine / DuplexOrchestrator) and the
+  # engine-resident session framework are owned by ../fullduplex.md; listed
+  # here because they fill the template seams of OmniEngineBase and
+  # OrchestratorBase.
+  - vllm_omni/engine/duplex_omni_engine.py
+  - vllm_omni/engine/duplex_orchestrator.py
   - vllm_omni/engine/duplex/**
 depends_on:
   - input_output_modality_contracts.md
@@ -48,6 +53,8 @@ validation_paths:
   - tests/engine/test_cfg_companion_lifecycle.py
   - tests/engine/test_rpc_result_router.py
   - tests/engine/duplex/**
+  - tests/engine/test_duplex_orchestrator.py
+  - tests/engine/test_duplex_omni_engine.py
   - tests/engine/test_duplex_import_boundary.py
 upstream_refs:
   - vllm.v1.engine.EngineCoreRequest

docs/design/module/entrypoints.md

@@ -19,7 +19,9 @@ required_reviewers:
 primary_code_paths:
   - vllm_omni/entrypoints/omni.py
   - vllm_omni/entrypoints/async_omni.py
+  - vllm_omni/entrypoints/async_omni_base.py
   - vllm_omni/entrypoints/omni_base.py
+  - vllm_omni/entrypoints/duplex_omni.py
   - vllm_omni/entrypoints/cli/**
   - vllm_omni/entrypoints/openai/**
   - vllm_omni/entrypoints/openpi/**
@@ -27,14 +29,13 @@ primary_code_paths:
   - vllm_omni/entrypoints/stage_utils.py
   - vllm_omni/entrypoints/utils.py
   - vllm_omni/entrypoints/duplex/**
-  - vllm_omni/entrypoints/duplex_request_client.py
   - vllm_omni/clients/**
 primary_path_exceptions:
   - path: vllm_omni/entrypoints/openai/errors.py
     owner: error_contracts.md
   - path: vllm_omni/entrypoints/duplex/**
     owner: ../fullduplex.md
-  - path: vllm_omni/entrypoints/duplex_request_client.py
+  - path: vllm_omni/entrypoints/duplex_omni.py
     owner: ../fullduplex.md
   - path: vllm_omni/clients/**
     owner: ../fullduplex.md
@@ -42,7 +43,9 @@ related_code_paths:
   - vllm_omni/errors.py
   - vllm_omni/inputs/**
   - vllm_omni/outputs/**
+  - vllm_omni/engine/omni_engine_base.py
   - vllm_omni/engine/async_omni_engine.py
+  - vllm_omni/engine/duplex_omni_engine.py
   - vllm_omni/config/**
   - vllm_omni/deploy/**
 depends_on:
@@ -53,7 +56,9 @@ validation_paths:
   - tests/entrypoints/test_omni_entrypoints.py
   - tests/entrypoints/test_async_omni.py
   - tests/entrypoints/test_async_omni_pause_sleep_routing.py
-  - tests/entrypoints/test_async_omni_duplex.py
+  - tests/entrypoints/test_duplex_omni.py
+  - tests/entrypoints/duplex/**
+  - tests/entrypoints/openai_api/test_duplex_api_server.py
   - tests/entrypoints/test_serve.py
   - tests/entrypoints/test_stream_finish_reason.py
   - tests/entrypoints/openai/**
@@ -89,6 +94,13 @@ This document owns offline API semantics, CLI and serve composition,
 supported OpenAI-compatible routes, request validation and normalization,
 response conversion, streaming/session behavior, and engine handoff.
 
+When `--api-server-count` is greater than one, the serve composition root
+starts the frontend process manager while a parent-owned `StageRuntime`
+launches the shared local stage engines. Each frontend receives only its own
+stage-client channel configuration and does not launch or retire backend
+processes. CLI validation rejects unsupported distributed, diffusion,
+fault-tolerant, elastic-EP, Ray, and runtime-LoRA combinations before startup.
+
 It does not own configuration precedence, cross-stage routing, stage
 placement, payload implementation, or semantic error classification.
 `entrypoints/openai/errors.py` is an explicit primary-path exception owned by
@@ -136,6 +148,9 @@ stage so `wake_up(stage_ids=[0])` does not skip a later `wake_up(stage_ids=[1])`
 Streaming input pumps take an admission slot immediately before each EngineCore
 ADD or update, not while waiting for the next client chunk. Frontend abort
 keeps `request_states` until `generate()` consumes the terminal output.
+Multi-API serving must preserve one client rank and one channel set per
+frontend, include every frontend in readiness/failure observation, and keep
+stage lifecycle ownership in the parent composition root.
 
 ## Promotion gate
 

docs/design/module/host_weight_runtime.md

@@ -136,8 +136,16 @@ so only when it is marked retryable; unsupported capabilities, identity
 collisions, producer failures, and publication failures remain visible as
 startup failures.
 
-`coordination_timeout_seconds` bounds acquisition of lookup and build locks; it
-is not a hard wall-clock deadline for in-process validation, production, or
+`coordination_timeout_seconds` bounds acquisition of the domain-initialization
+lock and lookup/build locks. Store construction and each later resolution or
+publication operation receive separate coordination budgets from the same
+`WaitPolicy`; they do not share an end-to-end startup deadline. A domain-init
+lock timeout is a retryable domain failure: preferred mode may fall back and
+required mode fails. Retrying initialization requires constructing a fresh
+runtime after contention ends; a waiter never cancels or unlocks the owner.
+
+The coordination budget is not a hard wall-clock deadline for filesystem I/O,
+in-process validation, production, or
 atomic publication. Once this V1 implementation becomes the producer, the
 synchronous producer and publication run to completion. A hung producer blocks
 its owning process and requires external process supervision; enforceable

docs/design/module/stage_runtime.md

@@ -77,6 +77,22 @@ readiness, one pool per logical stage, replica identity and membership,
 selection and affinity, stage-client/process lifecycle, liveness, draining,
 and shutdown.
 
+Local EngineCore initialization and multi-API serving use
+`StageRuntime.launch_stage_engines(num_api_servers=1)` as their common launch
+entry. It resolves plans when not supplied by the runtime's device-group
+scheduler, coordinates device locks and environment overlays, starts engines,
+and rolls back resources on startup or attachment failure. Single-API clients
+attach locally after readiness and take ownership of the engine resources;
+there is no additional API subprocess. Diffusion and remote attachment retain
+their backend-specific initialization.
+
+For local multi-API serving, the parent runtime owns and launches each stage
+engine once. It supplies a distinct input/output channel for every API
+frontend and stage replica; frontend workers attach clients to those engines
+without acquiring stage-process ownership. Readiness observation and shutdown
+remain parent-owned. This topology currently excludes diffusion and
+remote/headless stages, intra-stage data parallelism, and Ray backends.
+
 It does not own cross-stage request policy, model scheduler policy, connector
 transfer semantics, configuration precedence, or public error rendering.
 
@@ -114,6 +130,10 @@ affinity, membership changes, abort routing, process failure, draining, and
 repeated shutdown. Cross-stage policy changes belong in
 `engine_orchestration.md`.
 
+For multi-API changes, additionally exercise per-frontend channel assignment,
+shared-engine startup handshakes, frontend failure observation, and parent
+cleanup without duplicate stage shutdown.
+
 ## Promotion gate
 
 - Reconcile terminology and ownership after #5441 reaches a final state.

docs/features/comfyui.md

@@ -13,7 +13,6 @@ It can send model inference requests to either a locally running vLLM-Omni servi
 !!! tip
     If you run both ComfyUI and vLLM-Omni on the same device, you can create separate virtual environments and use different Python versions for them.
 
-
 ## Installation
 
 Copy the `apps/ComfyUI-vLLM-Omni` folder to the `custom_nodes` subfolder of your ComfyUI installation. Your directory should look like `ComfyUI/custom_nodes/ComfyUI-vLLM-Omni`.
@@ -21,9 +20,10 @@ Copy the `apps/ComfyUI-vLLM-Omni` folder to the `custom_nodes` subfolder of your
 If you are running ComfyUI during copying, you should restart ComfyUI to load this extension.
 
 !!! tip
-    You can use utility websites such as https://download-directory.github.io/ to download a subdirectory of a repo. Also checkout community discussions (e.g., https://stackoverflow.com/questions/7106012/download-a-single-folder-or-directory-from-a-github-repository) for more info.
+    You can use utility websites such as <https://download-directory.github.io/> to download a subdirectory of a repo. Also checkout community discussions (e.g., <https://stackoverflow.com/questions/7106012/download-a-single-folder-or-directory-from-a-github-repository>) for more info.
 
 On the device and virtual environment you run ComfyUI, launch ComfyUI with
+
 ```bash
 cd ComfyUI
 
@@ -35,6 +35,7 @@ python main.py --cpu
 ```
 
 On the device and virtual environment you run vLLM-Omni, start a model service with
+
 ```bash
 vllm serve The_Model_ID_to_Serve --omni --port 8000
 ```
@@ -50,12 +51,27 @@ This extension offers the following nodes based on the output modalities:
 - **Generate Video** for text-to-video and image-to-video tasks
 - **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks
 - **TTS** and **TTS Voice Clone** for TTS tasks
+- **Generate Music** for text-to-music tasks
 
 This extension also offers example workflows (at **ComfyUI sidebar -> Templates -> vLLM-Omni**)
 
 !!! info
     The node UI and feature designs are intended to match vLLM-Omni online serving interfaces. It cannot offer more than what the interfaces support.
 
+Every node carries the vLLM-Omni mark in its title bar and is tinted by what it outputs, so a graph is readable at a glance:
+
+| Colour | Nodes | What they produce |
+| --- | --- | --- |
+| Blue | Generate Image, Generate Video, Multimodality Understanding, TTS, TTS Voice Clone | A generated image, video, audio, or text. These are the only nodes that reach a server. |
+| Amber | AR / Diffusion / Multi-Stage Sampling Params | Sampling parameters that apply to any model |
+| Purple | Qwen TTS Params, Wan Video Params, MiniMax-H3 Video Params | Parameters that only one model family accepts |
+| Red | LoRA, FastH3 Deployment | Which weights the server is expected to have loaded |
+| Teal | Video References | Reference media |
+
+Recolouring a node by hand (right click -> Colors) overrides its tint, and the choice is kept.
+
+**Generate Video** takes a clip length in seconds (`duration`), not a frame count. Frames stay the wire unit and are derived with the node's `fps`, so the length is always measured against the rate that is actually served; models that accept only certain frame counts still round to their own lattice server-side. Graphs saved before this widget existed stored `num_frames` in its place and are converted on load, using the fps recorded alongside it -- the browser console names every node it rewrites.
+
 To build a simple workflow yourself,
 
 - Drag a generation node onto the canvas.
@@ -66,6 +82,18 @@ To build a simple workflow yourself,
     - For some multi-stage models like BAGEL, [only one stage's sampling parameters are exposed and tunable via vLLM-Omni's online serving API](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/examples/online_serving/bagel/). Thus, these models are treated as single-stage ones. Please check the vLLM-Omni documentation on how to correctly set each model's sampling parameters.
     - For multi-stage models where all stages are either autoregression or diffusion, you can also connect only a single Sampling Params node, indicating that this set of sampling parameters will be used for all stages.
 
+## MiniMax H3 text-to-video workflow
+
+The **MiniMax H3 Text to Video** template uses the existing remote Generate Video,
+H3 Params, sampling, and Remote LoRA nodes. It includes Base settings and an
+optional Turbo preset, with native resolution, 24 FPS, and `17k+5` frame counts.
+ComfyUI receives the generated video and audio from the server; it does not load
+H3 weights locally. Configure the server-side LoRA path before enabling Turbo.
+
+See the [H3 workflow guide](https://github.com/vllm-project/vllm-omni/blob/main/apps/ComfyUI-vLLM-Omni/docs/minimax-h3-t2v.md)
+for server setup, template import, Turbo configuration, recorded validation,
+and saved-video/audio checks.
+
 ## Examples & Screenshots
 
 Please read the [ComfyUI integration's README](https://github.com/vllm-project/vllm-omni/tree/main/apps/ComfyUI-vLLM-Omni) for more info.

docs/getting_started/installation/gpu/rocm.inc.md

@@ -1,11 +1,23 @@
+<!-- markdownlint-disable MD001 MD025 -->
+
 # --8<-- [start:requirements]
 
 - GPU: Validated on gfx942 (It should be supported on the AMD GPUs that are supported by vLLM.)
 
 # --8<-- [end:requirements]
+
 # --8<-- [start:set-up-using-python]
 
-vLLM-Omni current recommends the steps in under setup through Docker Images.
+For ROCm, vLLM-Omni currently recommends the setup steps through Docker Images.
+
+vLLM-Omni depends on the matching major/minor release of vLLM. The 0.29
+development line uses vLLM 0.29.x. Published 0.28.0 wheels and images use vLLM
+0.28.x.
+
+The Dockerfile's `BASE_IMAGE` pin applies only to Docker builds. The
+`vllm-omni` package does not install vLLM as a dependency, so non-Docker source
+installs must install the matching ROCm vLLM release explicitly before
+installing vLLM-Omni, as shown below.
 
 # --8<-- [start:pre-built-wheels]
 
@@ -14,6 +26,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
 These pre-built wheel instructions install the published vLLM-Omni 0.28.0 release. For the 0.29 development line, use the source-install instructions below.
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
+
 ```bash
 uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.28.0/rocm723
 ```
@@ -36,6 +49,7 @@ uv pip install onnxruntime-rocm
 # --8<-- [start:build-wheel-from-source]
 
 #### Installation of vLLM
+
 If you do not need to modify source code of vLLM, you can directly install the stable 0.29.0 release version of the library
 
 ```bash
@@ -45,7 +59,9 @@ uv pip install vllm==0.29.0+rocm723 --extra-index-url https://wheels.vllm.ai/roc
 The pre-built 0.29.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
 
 #### Installation of vLLM-Omni
+
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
+
 ```bash
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
@@ -71,15 +87,31 @@ python3 setup.py develop
 
 #### Build docker image
 
+The source build defaults to the published upstream base image
+`vllm/vllm-openai-rocm:v0.29.0`, aligned with the vLLM release used by CI.
+Older or custom bases must provide the vLLM APIs checked by the Dockerfile's
+image-build canary. This upstream base is distinct from the prebuilt
+`vllm/vllm-omni-rocm` images discussed below; their published-tag availability
+is tracked in [#7405](https://github.com/vllm-project/vllm-omni/issues/7405).
+
 ```bash
 DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.rocm -t vllm-omni-rocm .
 ```
 
+To select the upstream ROCm base explicitly, pass `BASE_IMAGE`:
+
+```bash
+DOCKER_BUILDKIT=1 docker build \
+  -f docker/Dockerfile.rocm \
+  --build-arg BASE_IMAGE=vllm/vllm-openai-rocm:v0.29.0 \
+  -t vllm-omni-rocm .
+```
+
 #### Launch the docker image
 
 ##### Launch with OpenAI API Server
 
-```
+```bash
 docker run --rm \
 --group-add=video \
 --ipc=host \
@@ -97,7 +129,7 @@ vllm-omni-rocm \
 
 ##### Launch with interactive session for development
 
-```
+```bash
 docker run --rm -it \
 --network=host \
 --group-add=video \
@@ -119,7 +151,9 @@ vllm-omni-rocm
 vLLM-Omni offers an official docker image for deployment. These images are built on top of vLLM docker images and available on Docker Hub as [vllm/vllm-omni-rocm](https://hub.docker.com/r/vllm/vllm-omni-rocm/tags). The version of vLLM-Omni indicates which release of vLLM it is based on.
 
 #### Launch vLLM-Omni Server
+
 Here's an example deployment command that has been verified on 2 x MI300's:
+
 ```bash
 docker run --rm \
   --group-add=video \
@@ -136,8 +170,10 @@ docker run --rm \
   --model Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```
 
-#### Launch an interactive terminal with prebuilt docker image.
+#### Launch an interactive terminal with prebuilt docker image
+
 If you want to run in dev environment you can launch the docker image as follows:
+
 ```bash
 docker run --rm -it \
   --network=host \

docs/getting_started/installation/npu/npu.inc.md

@@ -10,25 +10,25 @@ The recommended way to use vLLM-Omni on NPU is through the vLLM-Ascend pre-built
 === "A2"
 
     ```bash
-    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.28.0
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.29.0
     ```
 
 === "A3"
 
     ```bash
-    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.28.0-a3
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.29.0-a3
     ```
 
 === "A5"
 
     ```bash
-    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.28.0-a5
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.29.0-a5
     ```
 
 === "310P (Experimental)"
 
     ```bash
-    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.28.0-310p
+    export IMAGE=quay.io/atlas-ci/vllm-ascend:v0.29.0-310p
     ```
 
 ```bash
@@ -53,7 +53,7 @@ docker run --rm \
 
 # Inside the container, install vLLM-Omni from source
 cd /vllm-workspace
-git clone -b v0.28.0 https://github.com/vllm-project/vllm-omni.git
+git clone -b v0.29.0rc1 https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
 pip install -v -e . --no-build-isolation
 # or VLLM_OMNI_TARGET_DEVICE=npu pip install -v -e .
@@ -71,10 +71,10 @@ We are keeping [issue #886](https://github.com/vllm-project/vllm-omni/issues/886
 
 # --8<-- [start:installation-main]
 
-You can also install vLLM-Omni from the latest main branch if you want to use the latest features or bug fixes. Use the aligned vLLM-Ascend v0.28.0 image described above, and check [issue #886](https://github.com/vllm-project/vllm-omni/issues/886) for the status of the latest vLLM-Omni commit on NPU.
+You can also install vLLM-Omni from the latest main branch if you want to use the latest features or bug fixes. Use the aligned vLLM-Ascend v0.29.0 image described above, and check [issue #886](https://github.com/vllm-project/vllm-omni/issues/886) for the status of the latest vLLM-Omni commit on NPU.
 
 ```bash
-# Inside the vLLM-Ascend v0.28.0 container, install vLLM-Omni from main
+# Inside the vLLM-Ascend v0.29.0 container, install vLLM-Omni from main
 cd /vllm-workspace
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
@@ -103,7 +103,7 @@ Here's an example deployment command that has been verified on 4 x NPUs:
 === "A2"
 
     ```bash
-    export IMAGE=quay.io/ascend/vllm-omni:v0.28.0
+    export IMAGE=quay.io/ascend/vllm-omni:v0.29.0
     docker run --rm \
         --name vllm-omni-a2 \
         --shm-size=64g \
@@ -127,7 +127,7 @@ Here's an example deployment command that has been verified on 4 x NPUs:
 === "A3"
 
     ```bash
-    export IMAGE=quay.io/ascend/vllm-omni:v0.28.0-a3
+    export IMAGE=quay.io/ascend/vllm-omni:v0.29.0-a3
     docker run --rm \
         --name vllm-omni-a3 \
         --shm-size=64g \
@@ -151,7 +151,7 @@ Here's an example deployment command that has been verified on 4 x NPUs:
 === "A5"
 
     ```bash
-    export IMAGE=quay.io/ascend/vllm-omni:v0.28.0-a5
+    export IMAGE=quay.io/ascend/vllm-omni:v0.29.0-a5
     docker run --rm \
         --name vllm-omni-a5 \
         --shm-size=64g \
@@ -175,7 +175,7 @@ Here's an example deployment command that has been verified on 4 x NPUs:
 === "310P (Experimental)"
 
     ```bash
-    export IMAGE=quay.io/ascend/vllm-omni:v0.28.0-310p
+    export IMAGE=quay.io/ascend/vllm-omni:v0.29.0-310p
     docker run --rm \
         --name vllm-omni-310p \
         --shm-size=64g \

docs/mkdocs/hooks/generate_api_readme.py

@@ -25,8 +25,8 @@ CATEGORIES = {
     "clients": {
         "name": "Clients",
         "description": (
-            "Client libraries that talk to a running vLLM-Omni server "
-            "(the full-duplex Realtime client and its per-model session presets)."
+            "Client libraries for full-duplex sessions: the Realtime WebSocket client, "
+            "the in-process client over `DuplexOmni`, and the per-model session presets."
         ),
     },
     "inputs": {

docs/mkdocs/hooks/generate_argparse.py

@@ -2,6 +2,7 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
 import ast
+import json
 import logging
 import re
 import sys
@@ -124,6 +125,7 @@ def extract_omni_serve_subparser_init():
                         "make_arg_parser": lambda parser: parser,  # no-op for doc
                         "_ensure_vllm_platform": lambda: None,  # no-op for doc
                         "_nonneg_finite_float": float,
+                        "_json_object": json.loads,
                         "_parse_stage_overrides": __import__("json").loads,
                         "VLLM_SUBCMD_PARSER_EPILOG": "",
                         "logger": logger,

docs/mkdocs/hooks/search_index.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """Reduce browser search-index memory for generated API reference pages."""
 
 from material.plugins.search.plugin import SearchIndex

docs/models/supported_models.md

@@ -68,8 +68,10 @@ th {
 | `StableDiffusionXLPipeline` | Stable-Diffusion-XL | `stabilityai/stable-diffusion-xl-base-1.0` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ | — |
+| `AuKForConditionalGeneration` | AuK / AuK-Flash (instruction-driven speech generation and editing) | `tencent/AuK`, `tencent/AuK-Flash` (with `Qwen/Qwen2.5-Omni-3B`; assemble with `tools/prepare_auk_checkpoint.py`) | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/AuK-H100.md) |
 | `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | ✅︎ | — |
 | `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | ✅︎ | — |
+| `BreezeForConditionalGeneration` | Breeze-TTS-2 (TTS, voice clone, voice direction) | `BreezeBlue/Breeze-TTS-2` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/BreezeBlue/Breeze-TTS-2.md) |
 | `DotsTTSForConditionalGeneration` | dots.tts | `dots-studio/dots.tts-soar` | ✅︎ | | | | — |
 | `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | | — |
 | `MammothModa2ForConditionalGeneration` | MammothModa2-Dev (AR-only image understanding) | `bytedance-research/MammothModa2-Dev` | ✅︎ | | | | — |
@@ -85,7 +87,7 @@ th {
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `MingTTSForConditionalGeneration` | Ming-omni-tts dense 0.5B | `inclusionAI/Ming-omni-tts-0.5B` | ✅︎ | | | ✅︎ | — |
 | `GLMTTSForConditionalGeneration` | GLM-TTS | `zai-org/GLM-TTS` | ✅︎ | | | | — |
-| `MossTTSNanoForCausalLM` | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✅︎ | | | | — |
+| `MossTTSNanoForCausalLM` | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✅︎ | | ✅︎ | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/OpenMOSS/MOSS-TTS-Nano-NPU.md) |
 | `MossTTSDelayModel` | MOSS-TTS, MOSS-TTSD, MOSS-SoundEffect, MOSS-VoiceGenerator | `OpenMOSS-Team/MOSS-VoiceGenerator` | ✅︎ | | | | — |
 | `MossTTSRealtime` | MOSS-TTS-Realtime | `OpenMOSS-Team/MOSS-TTS-Realtime` | ✅︎ | | | | — |
 | `HiggsAudioV2ForConditionalGeneration` | Higgs-Audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` | ✅︎ | | | | — |
@@ -105,7 +107,7 @@ th {
 | `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | | — |
 | `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | | — |
 | `DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | ✅︎ | — |
-| `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | | — |
+| `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/OpenBMB/MiniCPM-o-4_5.md) |
 | `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `GepardTalkerForConditionalGeneration` | Gepard-1.0 | `nineninesix/gepard-1.0` | ✅︎ | | | ✅︎ | — |
 | `HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | | — |

docs/serving/README.md

@@ -145,15 +145,22 @@ interchangeably.
 | Incremental text input for speech synthesis | `WS /v1/audio/speech/stream` | Send text events and receive audio | [Streaming Text to Speech](speech_api.md#streaming-text-input-websocket) |
 | Live video understanding | `WS /v1/video/chat/stream` | Send video frames and receive text/audio | [Streaming Video Input](video_stream_api.md) |
 | Turn-based realtime audio | `WS /v1/realtime` | Stream one audio input and receive transcript/audio events | [Realtime Audio](realtime_api.md) |
-| Continuous speech-to-speech interaction | `WS /v1/realtime?duplex=1` or `WS /v1/duplex` | Listen and speak concurrently with session control | [Full Duplex](full_duplex_api.md) |
+| Continuous speech-to-speech interaction | `WS /v1/realtime?duplex=1` (alias `WS /v1/duplex`) | Listen and speak concurrently with session control | [Full Duplex](full_duplex_api.md) |
 | Generated video chunks | `WS /v1/realtime/video` | Start a diffusion request and receive fragmented MP4 | [Streaming Video Output](streaming_video_output_api.md) |
 | Robot policy inference | `WS /v1/realtime/robot/openpi` | Send MessagePack observations and receive action arrays | [OpenPI Robot Policy](openpi_api.md) |
 
 All six routes are model- or configuration-dependent. In particular,
-`/v1/realtime` is not full duplex unless the client sets `duplex=1` and the
-deployment explicitly enables duplex sessions. Clients should also verify the
-duplex capability payload because the query-parameter form falls back to the
-ordinary realtime handler when duplex is unavailable.
+`/v1/realtime` is full duplex when the model is a duplex model (its pipeline
+declares a `duplex_plugin` and the deploy configuration sets
+`session_mode: duplex`); a stock Realtime client needs no vendor query
+parameter, and `duplex=0` is the explicit opt-out. Such a server serves the
+websocket route plus `POST /v1/chat/completions`, and no other turn-based
+HTTP route. On a duplex server that chat route is not the turn-based path:
+each request runs on a short-lived duplex session, so it holds one of
+`duplex_session.max_sessions` for its lifetime and answers at the model's
+real-time pace -- see [Full Duplex](full_duplex_api.md). Clients should
+also verify the duplex capability payload because the query-parameter form
+falls back to the ordinary realtime handler when duplex is unavailable.
 
 ## Related Endpoints
 

docs/serving/full_duplex_api.md

@@ -15,40 +15,68 @@ capability gates, see the [Realtime Duplex API guide](realtime_duplex_api.md).
 
 | Endpoint | Protocol | Recommended use |
 | --- | --- | --- |
-| `WS /v1/realtime?duplex=1` | OpenAI Realtime-style event projection | Applications and browser clients |
-| `WS /v1/duplex` | Native vLLM-Omni duplex events | Runtime integration and low-level testing |
-
-Both endpoints use the same duplex engine and require the same model-side
-adapter. Prefer the Realtime projection unless the native lifecycle events are
-specifically required.
+| `WS /v1/realtime?duplex=1` | OpenAI Realtime-style events (the normative contract) | Applications and browser clients |
+| `WS /v1/duplex` | Alias of `/v1/realtime?duplex=1` (same protocol, same handler) | Clients that prefer a dedicated path |
+| Python: `DuplexOmni` / `InlineDuplexClient` | Typed commands and events in-process | Embedding the model without a server |
 
 ## Enable Full Duplex
 
-The route becomes usable only when the deployment configuration explicitly
-sets:
+A model is served full duplex when its registered pipeline declares a
+`duplex_plugin` (the model's `DuplexModelPlugin`) and its deploy configuration
+sets `session_mode: duplex`. `vllm serve --omni` constructs `DuplexOmni`
+instead of `AsyncOmni`. It serves
+`/v1/realtime?duplex=1` (and its alias `/v1/duplex`),
+`POST /v1/chat/completions`, `/v1/models` and `/health`; every other
+turn-based HTTP route (speech, batch, embeddings, video, ...) answers "not
+available". Set `session_mode: turn` to select the ordinary online serving
+stack instead. The model's supported tasks and endpoint restrictions still apply.
+
+`/v1/chat/completions` is the ordinary chat service running on the duplex
+engine: a request is a turn-based generation on the same stages, served
+alongside the live websocket sessions, with the request options the chat
+service supports. It is wired only when the model's plugin declares
+`DuplexCapabilities.supports_chat_completions`; on any other duplex model the
+route reports "not available". A model that should not serve the route at all
+lists it in the deploy configuration's `endpoint_restrictions`.
+
+The deploy configuration of such a model must agree:
 
 ```yaml
 session_mode: duplex
 ```
 
-The selected model pipeline must also provide a duplex serving adapter.
-Model-native deployments configure the corresponding engine runtime extension
-and control plane as part of their registered pipeline.
+A duplex-capable model must explicitly set `session_mode` to `duplex` or `turn`
+in its deploy configuration, possibly through `base_config` inheritance.
+Missing or invalid values fail at startup. The default MiniCPM-o deployment
+continues to use duplex mode.
+
+To run MiniCPM-o 4.5 with the turn-based engine:
+
+```bash
+vllm serve openbmb/MiniCPM-o-4_5 --omni \
+  --deploy-config vllm_omni/deploy/minicpmo_4_5_turn.yaml \
+  --trust-remote-code \
+  --port 8091
+```
+
+This profile inherits the default model and stage settings and overrides only
+`session_mode`. It supports ordinary HTTP requests without creating a duplex
+session handler. Mode selection happens at startup; a WebSocket query parameter
+does not switch engines. The Python `Omni` / `AsyncOmni` APIs are unchanged.
 
 !!! warning
 
-    `WS /v1/duplex` fails with `Duplex API is not available` when duplex is not
-    enabled. By contrast, `/v1/realtime?duplex=1` falls back to the ordinary
-    turn-based realtime handler when no duplex handler exists. Confirm that
+    On a deployment that is *not* duplex, `WS /v1/duplex` fails with
+    `Duplex API is not available` and `/v1/realtime?duplex=1` falls back to the
+    ordinary turn-based realtime handler. Confirm that
     `session.created.session.capabilities` is present before treating the
     connection as full duplex.
 
-The current unified-runtime integrations are:
-
-- MiniCPM-o 4.5, using `vllm_omni/deploy/minicpmo_4_5.yaml`
-  (`session_mode: duplex`);
-- PersonaPlex, whose default `vllm_omni/deploy/personaplex.yaml` enables duplex;
-- Nemotron VoiceChat, via its registered duplex plugin package.
+**MiniCPM-o 4.5** (`vllm_omni/deploy/minicpmo_4_5.yaml`) is the only model
+served over this endpoint today. PersonaPlex and Nemotron VoiceChat still carry
+their pre-framework duplex code: their pipelines declare no `duplex_plugin`, so
+they run turn-based until the follow-up PRs port them to the plugin contract
+(RFC [vllm-omni#7181](https://github.com/vllm-project/vllm-omni/issues/7181)).
 
 JoyVL is a separate HTTP interaction orchestrator and does not use these
 WebSocket endpoints. See [Standalone Experimental Servers](standalone_servers.md).
@@ -86,7 +114,7 @@ A typical `/v1/realtime?duplex=1` session follows this lifecycle:
 3. Send `input_audio_buffer.append` events while microphone audio arrives.
 4. Send `input_audio_buffer.commit` at a user-turn boundary when required by
    the model policy.
-5. Consume `response.created`, transcript deltas, `response.audio.delta`, and
+5. Consume `response.created`, transcript deltas, `response.output_audio.delta`, and
    `response.done` or `response.listen` events.
 6. Send `playback.ack` after audio has been played when the session advertises
    playback acknowledgement support.
@@ -101,22 +129,23 @@ deferred, treated as a short acknowledgement, or used to interrupt output.
 The `session.created` payload includes capability fields such as
 `supports_barge_in`, `supports_playback_ack`, `supports_multi_session`,
 `supports_session_resume`, and `chunk_period_ms`. Treat this payload as the
-runtime contract.
+runtime contract and branch on the flags, never on the model name: a model
+that supports native overlapping speech may still advertise
+`supports_barge_in=false` when destructive output interruption and model-state
+rewind are not validated for it. Capacity and session-resume behavior also
+depend on the selected deployment configuration. The per-model table lives in
+the [Realtime Duplex API guide](realtime_duplex_api.md#capability-negotiation-by-model).
 
-For example, PersonaPlex supports native overlapping speech but currently
-advertises `supports_barge_in=false`; destructive output interruption and
-model-state rewind have not been validated for that integration. Capacity and
-session-resume behavior also depend on the selected deployment configuration.
+## Python API
 
-## Native Protocol
+`vllm_omni.entrypoints.duplex_omni.DuplexOmni` runs the same engine-resident
+sessions in-process: `open_session()` returns a `DuplexSessionHandle` whose
+`submit()` takes typed `DuplexCommand` objects and whose `events()` yields
+typed `DuplexEvent` objects (each with a `to_realtime()` wire rendering).
+`vllm_omni.clients.inline_duplex.InlineDuplexClient` exposes that handle
+behind the `DuplexClient` API. See the
+[Realtime Duplex API guide](realtime_duplex_api.md#using-the-python-api).
 
-`WS /v1/duplex` exposes lower-level lifecycle names including
-`session.create`, `input_audio_buffer.append`, `turn.signal`, `playback.ack`,
-and `session.close`. It returns native session, input, response, overlap, and
-error events. This protocol is experimental and may evolve with the runtime;
-applications should use the provided Realtime client where possible.
-
-See the [MiniCPM-o example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/minicpmo),
-[PersonaPlex example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/personaplex),
+See the [MiniCPM-o example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/minicpmo)
 and the [full-duplex runtime design](../design/fullduplex.md)
 for model-specific validation and architecture details.

docs/serving/realtime_api.md

@@ -43,8 +43,8 @@ event is base64-encoded raw PCM16.
 | Client to server | `input_audio_buffer.commit` with `final: true` | Marks the end of input |
 | Server to client | `transcription.delta` | Carries incremental response text |
 | Server to client | `transcription.done` | Carries final text and token usage |
-| Server to client | `response.audio.delta` | Carries incremental PCM16 response audio |
-| Server to client | `response.audio.done` | Marks the end of response audio |
+| Server to client | `response.output_audio.delta` | Carries incremental PCM16 response audio |
+| Server to client | `response.output_audio.done` | Marks the end of response audio |
 | Server to client | `error` | Reports an invalid event, model, or audio payload |
 
 A minimal client sends events in this order:
@@ -63,7 +63,7 @@ engine is consuming the stream.
 ## Audio Handling
 
 - Input is mono PCM16 at 16 kHz for the Qwen3-Omni example.
-- `response.audio.delta.audio` contains base64-encoded PCM16 bytes.
+- `response.output_audio.delta.audio` contains base64-encoded PCM16 bytes.
 - Read `sample_rate_hz` from each audio event instead of assuming an output
   rate. Qwen3-Omni output is typically 24 kHz.
 - Concatenate audio deltas in receive order to construct the output waveform.

docs/serving/realtime_duplex_api.md

@@ -1,17 +1,18 @@
 # Realtime Duplex API
 
 vLLM-Omni serves full-duplex speech models over a WebSocket endpoint,
-`/v1/realtime`, that speaks the OpenAI Realtime vocabulary plus a
+`/v1/realtime?duplex=1`, that speaks the OpenAI Realtime vocabulary plus a
 set of duplex extensions: the model decides when to listen and when to speak,
 the user can talk over the assistant, playback progress is reported back so
 history stays honest, and a dropped connection can resume the same session.
 This page covers how to run a duplex deployment, how to drive it from Python
 with `vllm_omni.clients.duplex.DuplexClient`, and the complete wire contract.
 
-The handler is enabled by either a model-native `session_mode: duplex`
-deployment or an explicit `duplex_session` configuration for turn-based
-Server VAD. The runtime architecture is described in
-[Full-Duplex Runtime (MiniCPM-o 4.5)](../design/fullduplex.md).
+The endpoint is served for models that ship a duplex plugin — currently
+MiniCPM-o 4.5 only — and just for deploy configurations that declare
+`session_mode: duplex`. PersonaPlex and Nemotron VoiceChat are ported to the
+plugin contract in follow-up PRs and are not served over this endpoint yet. The runtime architecture
+is described in [Full-Duplex Runtime (MiniCPM-o 4.5)](../design/fullduplex.md).
 
 ## Quick Start
 
@@ -26,11 +27,24 @@ vllm-omni serve openbmb/MiniCPM-o-4_5 \
 ```
 
 `vllm_omni/deploy/minicpmo_4_5.yaml` declares `session_mode: duplex` and
-`duplex_session.max_sessions: 4`, which mounts the WebSocket routes
-`ws://<host>:8099/v1/realtime` (this page) and `ws://<host>:8099/v1/duplex`
-(the native dialect for raw-protocol clients) next to the usual HTTP API.
-PersonaPlex and Nemotron VoiceChat use `vllm_omni/deploy/personaplex.yaml` and
-`vllm_omni/deploy/nemotron_labs_voicechat_duplex.yaml` in the same way.
+`duplex_session.max_sessions: 4`. Because the MiniCPM-o 4.5 pipeline declares a
+`duplex_plugin`, `vllm-omni serve` runs it through `DuplexOmni`: the server
+mounts `ws://<host>:8099/v1/realtime?duplex=1` (this page; `ws://<host>:8099/v1/duplex`
+is an alias of the same route), `POST /v1/chat/completions`, `/v1/models` and
+`/health`, and nothing else. The remaining turn-based HTTP routes (speech,
+batch, embeddings, video, ...) are not served in duplex mode.
+
+`/v1/chat/completions` uses the ordinary chat service on the same engine
+when the plugin declares `supports_chat_completions`. HTTP chat requests do
+not create duplex sessions or consume session admission slots. Their options
+follow the [Chat Completions API](chat_completions_api.md) and the model's
+capabilities and endpoint restrictions.
+
+For a turn-based online deployment, use `minicpmo_4_5_turn.yaml` or explicitly
+set `session_mode: turn` in a deploy configuration derived from the model's
+profile. This selects `AsyncOmni` and the ordinary API initialization instead
+of the duplex handler. See [Full Duplex](full_duplex_api.md#enable-full-duplex)
+for the command and configuration requirements.
 
 ### Run the Example Client
 
@@ -81,25 +95,24 @@ ships a preset:
 
 | Preset | Input / output audio | What it sets |
 | --- | --- | --- |
-| `vllm_omni.clients.minicpmo_4_5.create_duplex_session_config(ref_audio=...)` | 16 kHz `pcm16` / 24 kHz `pcm16` | `extra_body.native_duplex=True` (opt in to the model-native lane), `force_listen_count=0`, `overlap_policy="listen_only"`, `playback_commit_policy="ack_only"`; `ref_audio` is the assistant voice clip |
+| `vllm_omni.clients.minicpmo_4_5.create_duplex_session_config(ref_audio=...)` | 16 kHz `pcm16` / 24 kHz `pcm16` | `force_listen_count=0`, `overlap_policy="listen_only"`, `playback_commit_policy="ack_only"`; `ref_audio` is the assistant voice clip |
 | `vllm_omni.clients.personaplex.create_duplex_session_config(voice="NATF2.pt", persona="")` | 24 kHz `pcm_f32le` / 24 kHz `pcm16` | bundled `.pt` voice prompt and the persona as `instructions` |
 | `SessionConfig(...)` | 16 kHz `pcm16` / 24 kHz `pcm16` | model-neutral defaults; pass `extra_body`, `turn_detection`, `overlap_policy`, `playback_commit_policy`, `instructions`, `voice`, `temperature` yourself |
 
 Keyword overrides on a preset replace the corresponding `SessionConfig`
 field, e.g. `create_duplex_session_config(ref_audio=..., temperature=0.6)`.
 
-Constructor options: `session_id` (names the session the client creates; it
-does not resume or take over an existing session — entering the client always
-performs the `session.update` handshake), `reconnect` (a `ReconnectPolicy`,
-default five attempts with 0.25–4 s jittered backoff; `None` disables
-auto-resume), `heartbeat_interval_s` (default 30 s; `None` disables the lease
-heartbeat), `handshake_timeout_s`, and `connect` (a custom transport factory,
-used by tests and benchmarks). Resume is supported only as automatic reconnect
-within the same `DuplexClient` instance (see
-[Reconnect and resume](#reconnect-and-resume)); re-attaching from a new client
-or process requires the wire-level `session.resume` handshake
-(`resume_token`, `incarnation`, `last_received_server_event_seq`), which this
-client does not expose.
+Constructor options: `reconnect` (a `ReconnectPolicy`, default five
+attempts with 0.25–4 s jittered backoff; `None` disables auto-resume),
+`heartbeat_interval_s` (default 30 s; `None` disables the lease heartbeat),
+`handshake_timeout_s`, and `connect` (a custom transport factory, used by
+tests and benchmarks). The session id is allocated by the server and becomes
+`client.session_id` once `session.created` arrives; a client cannot choose
+it. Resume is supported only as automatic reconnect within the same
+`DuplexClient` instance (see [Reconnect and resume](#reconnect-and-resume));
+re-attaching from a new client or process requires the wire-level
+`session.resume` handshake (`resume_token`,
+`last_received_server_event_seq`), which this client does not expose.
 
 ### Stream audio and commit a turn
 
@@ -195,7 +208,7 @@ the stamped `event_id`.
 On a transport drop the client reconnects with the `ReconnectPolicy` backoff
 and sends `session.resume` from the last acknowledged `server_event_seq`.
 Replayed events are deduplicated, consumers see one `ConnectionResumed`
-event, and `client.incarnation` / `client.resume_token` are refreshed. The
+event, and `client.resume_token` is refreshed. The
 server keeps a detached session alive for `disconnect_grace_s` (30 s by
 default); after that a resume fails with `session_resume_expired`. Resume is
 gated by `capabilities.supports_session_resume` (MiniCPM-o 4.5 and Nemotron
@@ -238,9 +251,67 @@ definitions out of the dependency-free client. `audio_bytes(response_id)`, `coun
 `openai-realtime-duplex` benchmark backend (`vllm-omni bench serve`) is built
 on the same helpers.
 
+## Using the Python API
+
+`vllm_omni.entrypoints.duplex_omni.DuplexOmni` is the in-process Python API
+of a duplex model: sessions run inside the engine, and the class only opens,
+resumes and closes them and pipes typed commands in and typed events out
+through a `DuplexSessionHandle`.
+
+```python
+from vllm_omni.engine.duplex.commands import AppendAudio, Commit
+from vllm_omni.engine.duplex.events import AudioDelta, SessionClosed
+from vllm_omni.entrypoints.duplex_omni import DuplexOmni
+
+omni = DuplexOmni(model="openbmb/MiniCPM-o-4_5", trust_remote_code=True)
+async with await omni.open_session({"modalities": ["audio", "text"], "ref_audio": ref_audio_data_url}) as session:
+    async def consume():
+        async for event in session.events():          # typed DuplexEvent objects
+            if isinstance(event, AudioDelta):
+                play(event.audio)
+            elif isinstance(event, ResponseDone):
+                break                                  # one turn is enough here
+    task = asyncio.create_task(consume())
+    await session.submit(AppendAudio(audio=pcm16_bytes, format="pcm16", sample_rate_hz=16000))
+    await session.submit(Commit())
+    await task
+# leaving the block closes the session (the last event is SessionClosed)
+```
+
+`open_session` takes a `DuplexSessionConfig` or the same Realtime `session`
+object the WebSocket handshake takes; the session id is always allocated by
+`DuplexOmni`. `DuplexSessionHandle` offers `submit(command)` plus one
+convenience wrapper per command (`append_audio`, `commit`, `create_response`,
+`cancel_response`, `barge_in`, `ack_playback`, `update`, `create_item`,
+`delete_item`, `truncate_item`, `heartbeat`, ...). `events()` is a
+single-consumer async iterator that ends after `session.closed` /
+`session.expired`; rejected commands come back as `ErrorEvent`s on it.
+Every event renders the wire JSON with `to_realtime()`, so anything written
+against the WebSocket protocol works unchanged on the typed stream.
+
+`vllm_omni.clients.inline_duplex.InlineDuplexClient` wraps a `DuplexOmni`
+behind the `DuplexClient` API, so the same application code runs in-process
+(no server) or over the network:
+
+```python
+from vllm_omni.clients.inline_duplex import InlineDuplexClient
+from vllm_omni.clients.minicpmo_4_5 import create_duplex_session_config
+
+config = create_duplex_session_config(ref_audio=audio_data_url(Path("reference_voice.wav")))
+async with InlineDuplexClient(omni, model="openbmb/MiniCPM-o-4_5", config=config) as client:
+    await client.stream_pcm(question)
+    await client.commit()
+    async for response in client.responses():
+        ...
+```
+
+`examples/online_serving/barge_in_client.py --inline` drives the barge-in
+scenario this way. The inline client has no transport, so it has no
+reconnect, resume, heartbeat or event-acknowledgement machinery.
+
 ## API Reference
 
-This section is the complete wire contract of `/v1/realtime`: how
+This section is the complete wire contract of `/v1/realtime?duplex=1`: how
 the dialect relates to the OpenAI Realtime protocol, which surfaces each
 model's capability flags gate, a catalogue of every client-to-server and
 server-to-client event, and a JSON example for each. `DuplexClient` speaks
@@ -253,21 +324,26 @@ its protocol tests.
 
 ### Endpoint and transport
 
-The endpoint is `ws(s)://<host>/v1/realtime`. Bare connections select the
-session handler when the deployment declares either `session_mode: duplex`
-or `duplex_session`. `?duplex=1` remains a compatibility alias, while
-`?duplex=0` selects the legacy Realtime handler. Optional query parameters
-are `model`, `session_id`,
-`autostart` (`0` means resume-only), `resume`, and `native_duplex`
-(`minicpmo45_native_duplex` is accepted as a deprecated alias and folded
-into the canonical name). Every message is one JSON object per WebSocket
+The endpoint is `ws(s)://<host>/v1/realtime`. On a deployment whose pipeline
+declares a `duplex_plugin` and whose deploy configuration declares
+`session_mode: duplex`, a bare connection is already a duplex session, so a
+stock Realtime client needs no vendor query parameter. `?duplex=1` (`1`,
+`true` or `on`) is the spelling this page uses and stays valid;
+`?duplex=0` (`0`, `false`, `off`) explicitly opts out and selects the
+turn-based Realtime handler, which a duplex server does not mount --
+that connection is answered with `Realtime API is not available`.
+`ws(s)://<host>/v1/duplex` is an alias that always selects the duplex
+handler. Optional query parameters are `model`, `autostart` (`0` means
+resume-only) and `resume`. Every message is one JSON object per WebSocket
 text frame, discriminated by `type`. Inbound events are applied in arrival
 order through one per-session mailbox; outbound events preserve that order
 and carry a monotonically increasing `server_event_seq`, which is the replay
 cursor for `session.resume`. Client events may carry an `event_id`, echoed in
 `error.error.event_id` when the event is rejected.
 
-Session identity is `(session_id, incarnation)`; `attachment_generation`
+Session identity is the server-allocated `session_id` (`duplex-<uuid>`,
+announced in `session.created`; a `session_id` the client puts in the session
+object is ignored, and ids are never reused); `attachment_generation`
 names the current socket; `epoch` is the interruption fence that advances on
 every cancel, clear, or barge-in; `turn_id` counts committed model turns;
 `response_id` (item id `item_<response_id>`) names one assistant utterance.
@@ -281,12 +357,13 @@ with no OpenAI counterpart.
 
 The event vocabulary is uniform, but several surfaces are gated by the
 `capabilities` object the server returns in `session.created`; a client must
-branch on those flags rather than on the model name. The current model
-plugins advertise:
+branch on those flags rather than on the model name. MiniCPM-o 4.5 is the
+only model on the plugin contract today; the other two columns record what
+their integrations advertise once the follow-up PRs port them:
 
 | Capability | MiniCPM-o 4.5 | PersonaPlex | Nemotron VoiceChat | Gated surface |
 | --- | --- | --- | --- | --- |
-| `implementation_level` | `model_native_duplex` (when `extra_body.native_duplex=true`) | `model_native_duplex` | `model_native_duplex` | model-owned `response.listen` / `response.speak`; the chat-fallback lane otherwise |
+| `implementation_level` | `model_native_duplex` | `model_native_duplex` | `model_native_duplex` | model-owned `response.listen` / `response.speak` (constant: every duplex model is model-native) |
 | `chunk_period_ms` | 1000 | 80 | 80 | the model unit that `response.listen` decisions and camera frames align to |
 | `supports_session_resume` | yes | no | yes | `session.resume`, `session.resumed`, `session.replaced`, replay after a transport drop |
 | `supports_barge_in` | yes | no | no | `barge_in`, `turn.signal{event:"barge_in"}`, `overlap_policy=barge_in_on_speech`, `turn_detection.server_vad` |
@@ -311,13 +388,13 @@ work unmodified. The overlap falls into the three tiers used by the event catalo
 | Direction | Messages |
 | --- | --- |
 | Client → server | `session.update`, `input_audio_buffer.append` / `.commit` / `.clear`, `conversation.item.create` / `.retrieve` / `.truncate` / `.delete`, `response.create`, `response.cancel`, `output_audio_buffer.clear` (OpenAI: WebRTC-only; here also a WebSocket event) |
-| Server → client | `error` (same `{type, code, message, event_id, param}` envelope and the same classes `invalid_request_error` / `server_error` / `rate_limit_error`), `session.created`, `session.updated`, `input_audio_buffer.committed` / `.cleared` / `.speech_started` / `.speech_stopped`, `conversation.item.created` / `.added` / `.done` / `.deleted` / `.retrieved` / `.truncated`, `conversation.item.input_audio_transcription.completed`, `response.created`, `response.done`, `response.output_item.added` / `.done`, `response.content_part.added` / `.done`, `response.audio.delta` / `.done`, `response.audio_transcript.delta` / `.done`, `response.output_text.delta` / `.done`, `response.function_call_arguments.delta` / `.done`, `rate_limits.updated`, `output_audio_buffer.cleared` |
+| Server → client | `error` (same `{type, code, message, event_id, param}` envelope and the same classes `invalid_request_error` / `server_error` / `rate_limit_error`), `session.created`, `session.updated`, `input_audio_buffer.committed` / `.cleared` / `.speech_started` / `.speech_stopped`, `conversation.item.created` / `.added` / `.done` / `.deleted` / `.retrieved` / `.truncated`, `conversation.item.input_audio_transcription.completed`, `response.created`, `response.done`, `response.output_item.added` / `.done`, `response.content_part.added` / `.done`, `response.output_audio.delta` / `.done`, `response.output_audio_transcript.delta` / `.done`, `response.output_text.delta` / `.done`, `response.function_call_arguments.delta` / `.done`, `rate_limits.updated`, `output_audio_buffer.cleared` |
 | Session fields | `model`, `modalities` / `output_modalities`, `instructions`, `voice`, `input_audio_format` / `output_audio_format` (beta spelling) **and** `audio.input.format` / `audio.output.format` objects (GA spelling), `turn_detection.server_vad {threshold, prefix_padding_ms, silence_duration_ms}`, `input_audio_transcription`, `input_audio_noise_reduction`, `tools`, `tool_choice`, `temperature`, `max_response_output_tokens`, `speed`, `tracing` |
 | Object shapes | `realtime.session`, `realtime.item`, `realtime.response`; item content parts `input_text` / `input_audio` / `output_text` / `output_audio`; `function_call` and `function_call_output` items; `previous_item_id` chaining; `output_index` / `content_index` addressing |
 | Sequencing | `response.created → output_item.added → content_part.added → deltas → *.done → content_part.done → output_item.done → response.done → rate_limits.updated`; `speech_started → speech_stopped → committed` |
 
 The server deliberately emits and accepts **both** the beta and the GA
-spellings at once (`response.audio.delta` alongside `output_modalities`,
+spellings at once (`response.output_audio.delta` alongside `output_modalities`,
 `conversation.item.added` alongside `conversation.item.created`) so either
 generation of OpenAI client parses the stream.
 
@@ -327,9 +404,9 @@ A stock client ignores the extra keys; the extensions are additive.
 
 | Message | Extension |
 | --- | --- |
-| `session.created` / `session.updated` | top-level `incarnation`, `attachment_generation`, `resume_token`; inside `session`: `state`, `turn_state`, `epoch`, `turn_id`, `active_request_id`, `active_response_id`, `active_response_turn_id`, `overlap_policy`, `overlap_*_ms/rms`, `playback_commit_policy`, `playback`, `capabilities`, `ref_audio`, `extra_body`, `idle_timeout_s`, `response_format` |
+| `session.created` / `session.updated` | top-level `attachment_generation`, `resume_token`; inside `session`: `state`, `turn_state`, `epoch`, `turn_id`, `active_request_id`, `active_response_id`, `active_response_turn_id`, `overlap_policy`, `overlap_*_ms/rms`, `playback_commit_policy`, `playback`, `capabilities`, `ref_audio`, `extra_body`, `idle_timeout_s`, `response_format` |
 | `response.created` / `response.done` / `response.listen` | `response_id` at top level; the raw duplex event under `response.metadata` (`duplex_event`); `status_details.reason` uses vLLM reasons (`barge_in`, `client_cancelled`, `new_response`, …) |
-| `response.audio.delta` | `format`, `sample_rate_hz`, `metadata{session_id, epoch, model_speak, end_of_turn, audio_duration_ms, audio_text_marks, playback}` |
+| `response.output_audio.delta` | `format`, `sample_rate_hz`, `metadata{session_id, epoch, model_speak, end_of_turn, audio_duration_ms, audio_text_marks, playback}` |
 | `response.speak` (inserted before the first delta) | not an OpenAI event, but rides the OpenAI response envelope (`response_id`, `item_id`, `output_index`, `content_index`) |
 | `input_audio_buffer.append` | `is_speech`, `video_frames` (+ `max_slice_nums`), `duration_ms`, `audio_end_ms`, per-event `format` / `sample_rate_hz` |
 | `input_audio_buffer.commit` | `final`, `response_create`, `is_speech:false` (silence declaration) |
@@ -343,14 +420,9 @@ Semantic divergences hidden behind shared names:
   before any commit; a commit may end in `response.listen` and no
   response, and the model may open a response with no commit at all.
   OpenAI: commit ⇒ item, `response.create` ⇒ exactly one response.
-- With `turn_detection={"type":"server_vad"}`, omitted `interrupt_response`
-  defaults to `false` on turn-based models such as Qwen3-Omni, enabling
-  endpointing and automatic responses without interruption. Explicit `true`
-  is rejected there. Model-native duplex keeps the `true` default and rejects
-  `false`; its VAD implies `overlap_policy=barge_in_on_speech`.
-  This capability-dependent default differs from OpenAI's `true` default;
-  the effective configuration is returned in `session.created` / `session.updated`.
-  `semantic_vad` is unsupported.
+- `turn_detection.interrupt_response=false` is rejected;
+  `create_response` is ignored; `semantic_vad` is unsupported; the VAD
+  runs per session (Silero) and implies `overlap_policy=barge_in_on_speech`.
 - `rate_limits.updated` is always an empty list (compatibility only).
 - Cancellation never reuses a `response_id`; truncation is driven by
   `playback.ack` as well as `conversation.item.truncate`.
@@ -359,11 +431,11 @@ Semantic divergences hidden behind shared names:
 
 | Area | Messages / fields |
 | --- | --- |
-| Session lifetime | `session.heartbeat` / `session.heartbeat_ack`, `session.event_ack`, `session.close` / `session.closed`, `session.resume` / `session.resumed` / `session.replaced` / `session.expired` / `session.resync_required`; `server_event_seq`, `resume_token`, `incarnation`, `attachment_generation`; query params `?duplex=1`, `autostart`, `resume`, `native_duplex` |
+| Session lifetime | `session.heartbeat` / `session.heartbeat_ack`, `session.event_ack`, `session.close` / `session.closed`, `session.resume` / `session.resumed` / `session.replaced` / `session.expired` / `session.resync_required`; `server_event_seq`, `resume_token`, `attachment_generation`; query params `?duplex=1`, `autostart`, `resume` |
 | Turn-taking | `response.speak`, `response.listen`, `overlap.decision`, `overlap_policy`, `barge_in`, `turn.signal`, `input.cancel`, `input.text.append`, `epoch` / `turn_id` fencing, `force_listen` |
 | Playback truth | `playback.ack` / `playback.acknowledged`, `playback_commit_policy` (`ack_only` \| `commit_all_on_done`), `playback{generated_ms, sent_ms, played_ms, committed_ms}`, `history_committed`, `audio_text_marks` |
-| Model negotiation | `capabilities{implementation_level, supports_input_append, supports_barge_in, supports_session_resume, chunk_period_ms, input_modes, …}`, `ref_audio`, `extra_body` (`auto_response`, `native_duplex` — deprecated alias `minicpmo45_native_duplex` is folded into it, `force_listen_count`, `duplex_initial_user_text`) |
-| Diagnostics | `runtime.control`, `duplex.function_call.done`, the full error-code vocabulary (`REALTIME_ERROR_TYPES_BY_CODE` in `vllm_omni/entrypoints/duplex/realtime_state.py`) |
+| Model negotiation | `capabilities{implementation_level, supports_input_append, supports_barge_in, supports_session_resume, chunk_period_ms, input_modes, …}`, `ref_audio`, `extra_body` (`auto_response`, `force_listen_count`, `duplex_initial_user_text`) |
+| Diagnostics | `duplex.function_call.done`, the full error-code vocabulary (`REALTIME_ERROR_TYPES_BY_CODE` in `vllm_omni/engine/duplex/events.py`) |
 
 #### Consequences for clients
 
@@ -381,14 +453,61 @@ Semantic divergences hidden behind shared names:
   the Tier 3 surface on top of the same Tier 1 events, so both kinds of
   client can share one deployment and one event log.
 
+### Turn-based Chat audio metadata
+
+For turn-based models served through Chat fallback, non-empty audio choices
+produced by the Omni Chat audio encoder include `audio_metadata`, in both
+streaming and complete Chat responses. Text-only choices do not require it.
+
+| Field | Meaning |
+| --- | --- |
+| `format` | Encoding of the returned audio, such as `pcm` or `wav`; `pcm16` requests normalize to `pcm`. |
+| `sample_rate_hz` | Sample rate used by the encoder, in samples per second. |
+| `frame_count` | Samples per channel in this chunk, or in the complete waveform for a non-streaming choice. Not a byte count. |
+| `channels` | Number of audio channels. |
+
+These values describe the waveform after audio transformations. Compressed
+formats may add codec padding that is not included in `frame_count`.
+The encoder uses a model-reported sample rate when available; Qwen3-Omni
+currently omits it and uses the existing 24 kHz default. This metadata does
+not independently determine a model's sample rate.
+
+Chat fallback requests the session's output format explicitly. Its internal
+`response.output_audio.delta` carries `format`, `sample_rate_hz`, `channels`,
+and `audio_duration_ms`. The Realtime endpoint exposes output `format` and
+`sample_rate_hz` on `response.audio.delta`, with duration in
+`metadata.audio_duration_ms`; it does not currently forward `channels`.
+The duration is cumulative for the response:
+integer frames are summed before converting to milliseconds and rounding down.
+Non-empty audio without valid metadata, or a sample-rate change within one
+response, fails that fallback response with `response_error`.
+
+In this Chat fallback path, `response.done.response.metadata.playback` reports:
+
+- `generated_ms`: cumulative source-waveform duration.
+- `sent_ms`: cumulative audio accepted into the server output path, including
+  queued or journaled audio. A journaled event remains accepted if the
+  subsequent send fails; this is not proof of socket delivery.
+- `played_ms`: cumulative playback reported by the client.
+- `committed_ms`: playback position used for history reconciliation.
+
+The native model path still records audio before sending; it does not yet
+share Chat fallback's output-acceptance timing. Only client playback reports
+establish what was played. The existing duration-proportional text truncation
+remains an approximation, not word-level audio alignment.
+
 ### Event catalogue
 
 Every event type the Realtime route accepts or emits, with a worked example in the next section: 21
-client→server and 42 server→client. Aliases (`push_chunk`,
-`input.audio.append`, `input_text.append`, `push_text`, `signal_turn`,
-`audio.playback_ack`, `close_session`, `close`, `session_close`) share the
-payload of their canonical event and are not listed separately; the alias
-table lives in the normative contract section of [Full-Duplex Runtime (MiniCPM-o 4.5)](../design/fullduplex.md).
+client→server and 42 server→client. Aliases (`input_text.append`,
+`push_text`, `signal_turn`, `audio.playback_ack`, `close_session`, `close`)
+share the payload of their canonical event and are not listed
+separately; the alias table is `translate_realtime_command` in
+`vllm_omni/engine/duplex/realtime_commands.py`. The pre-Realtime WAV-append
+aliases `push_chunk` and `input.audio.append` are **not** accepted on this
+route: send `input_audio_buffer.append` with one of the supported input
+formats (`pcm16`, `pcm_s16le`, `s16le`, `pcm_f32le`, `g711_ulaw`,
+`g711_alaw`).
 
 #### Client to server
 
@@ -398,18 +517,18 @@ table lives in the normative contract section of [Full-Duplex Runtime (MiniCPM-o
 | `session.heartbeat` | 3 | Keepalive; refreshes the engine session lease. |
 | `session.event_ack` | 3 | Acknowledges received server events by `server_event_seq`; trims the resume replay journal. |
 | `session.close` | 3 | Graceful close; answered with `session.closed`. |
-| `session.resume` | 3 | Re-attach a live session on a new socket (`resume_token`, `incarnation`, `last_received_server_event_seq`). |
+| `session.resume` | 3 | Re-attach a live session on a new socket (`session_id`, `resume_token`, `last_received_server_event_seq`). |
 | `input_audio_buffer.append` | 2 | Append one audio chunk. OpenAI name; extended with `is_speech`, `video_frames`, `duration_ms`, `audio_end_ms`, per-event `format`/`sample_rate_hz`. |
 | `input_audio_buffer.commit` | 2 | Seal the buffered utterance into a user item. OpenAI name; extended with `final`, `response_create`, `is_speech:false`. |
 | `input.commit` | 3 | Alias of `input_audio_buffer.commit` accepted by the runner. |
 | `input_audio_buffer.clear` | 1 | Drop un-committed input audio. |
 | `input.cancel` | 3 | Cancel pending input; advances `epoch`. |
-| `input.text.append` | 3 | Append text to the open input item (chat-fallback lane only). |
+| `input.text.append` | 3 | Append text input; model-native sessions reject it with `native_text_append_unsupported`. |
 | `conversation.item.create` | 1 | Add a completed user item (text, audio, or `function_call_output`) to history. |
 | `conversation.item.retrieve` | 1 | Fetch a stored item by id. |
 | `conversation.item.delete` | 1 | Delete a stored item by id. |
 | `conversation.item.truncate` | 1 | Truncate an assistant item's audio/transcript at `audio_end_ms`. |
-| `response.create` | 1 | Explicitly request a response with per-response overrides. |
+| `response.create` | 1 | Explicitly request a response with per-response overrides. Requires committed audio input; otherwise `response_create_without_input`. |
 | `response.cancel` | 1 | Cancel the active (or named) response; advances `epoch`. |
 | `output_audio_buffer.clear` | 1 | Discard queued output audio; advances `epoch` (OpenAI: WebRTC-only, here also WebSocket). |
 | `barge_in` | 3 | Explicit hard interrupt; requires `capabilities.supports_barge_in`. |
@@ -420,10 +539,10 @@ table lives in the normative contract section of [Full-Duplex Runtime (MiniCPM-o
 
 | Event type | Tier | Description |
 | --- | --- | --- |
-| `session.created` | 2 | Session opened. OpenAI name; adds `incarnation`, `attachment_generation`, `resume_token` and vLLM-Omni keys inside `session` (`epoch`, `turn_id`, `playback`, `capabilities`, ...). |
+| `session.created` | 2 | Session opened. OpenAI name; adds `attachment_generation`, `resume_token` and vLLM-Omni keys inside `session` (`id` is the server-allocated session id, `epoch`, `turn_id`, `playback`, `capabilities`, ...). |
 | `session.updated` | 2 | Echo of the effective session config after every `session.update` (same extended `session` object). |
 | `session.heartbeat_ack` | 3 | Reply to `session.heartbeat`. |
-| `session.closed` | 3 | Last event on the socket; `reason` ∈ `session_close`, `timeout`, `disconnect`, ... |
+| `session.closed` | 3 | Last event on the socket, emitted once the engine released the session; `reason` ∈ `client_close`, `disconnect`, `timeout`, `transport_error`, `shutdown`, ... |
 | `session.resumed` | 3 | Resume accepted; carries the new `attachment_generation` and rotated `resume_token`; journaled events are replayed after it. |
 | `session.replaced` | 3 | Sent to the superseded socket when another socket resumes the session. |
 | `session.expired` | 3 | Engine lease reaped (`disconnect_grace_expired`, `idle_ttl_expired`); socket closes after it. |
@@ -443,12 +562,12 @@ table lives in the normative contract section of [Full-Duplex Runtime (MiniCPM-o
 | `response.output_item.added` | 1 | Assistant item (or `function_call` item) attached to the response. |
 | `response.content_part.added` | 1 | Audio/text content part opened on the assistant item. |
 | `response.speak` | 3 | Model chose to speak (native lane); at most once per response, before the first audio delta. |
-| `response.audio.delta` | 2 | One ordered audio chunk. OpenAI name; adds `format`, `sample_rate_hz`, `metadata{epoch, model_speak, end_of_turn, audio_text_marks, playback}`. |
-| `response.audio_transcript.delta` | 1 | Transcript text paired one-to-one with each audio delta. |
+| `response.output_audio.delta` | 2 | One ordered audio chunk. OpenAI name; adds `format`, `sample_rate_hz`, `metadata{epoch, model_speak, end_of_turn, audio_text_marks, playback}`. |
+| `response.output_audio_transcript.delta` | 1 | Transcript text paired one-to-one with each audio delta. |
 | `response.output_text.delta` | 1 | Text delta for text-modality responses. |
 | `response.output_text.done` | 1 | Final text of a text-modality response. |
-| `response.audio.done` | 1 | Audio stream closed for the response. |
-| `response.audio_transcript.done` | 1 | Full transcript (concatenation of the deltas). |
+| `response.output_audio.done` | 1 | Audio stream closed for the response. |
+| `response.output_audio_transcript.done` | 1 | Full transcript (concatenation of the deltas). |
 | `response.content_part.done` | 1 | Content part finalized with its transcript/text. |
 | `response.output_item.done` | 1 | Assistant item finalized. |
 | `response.done` | 2 | Terminal event; `status` ∈ `completed` \| `cancelled` \| `failed`. OpenAI name; vLLM-Omni `status_details.reason` values and raw duplex event under `metadata`. |
@@ -459,8 +578,8 @@ table lives in the normative contract section of [Full-Duplex Runtime (MiniCPM-o
 | `duplex.function_call.done` | 3 | Raw fallback when a model tool call is malformed. |
 | `output_audio_buffer.cleared` | 1 | Reply to `output_audio_buffer.clear`. |
 | `playback.acknowledged` | 3 | Reply to `playback.ack`; returns the playback ledger and `history_committed`. |
-| `overlap.decision` | 3 | How an append that overlapped an active response was handled (`drop` \| `listen` \| `barge_in`). |
-| `runtime.control` | 3 | Diagnostic: an engine control signal was partially unsupported. |
+| `overlap.decision` | 3 | How an append that overlapped an active response was handled (`drop` \| `listen` \| `barge_in`); details under `metadata`. |
+| `turn.event` | 3 | A turn-taking signal the session accepted (`event`, resulting `turn_state`). |
 | `error` | 2 | Error envelope. OpenAI shape and classes; vLLM-Omni error codes. |
 
 ### Message examples
@@ -479,7 +598,6 @@ additionally carries `"server_event_seq": <int>` (omitted below for brevity).
   "event_id": "evt_0001",
   "session": {
     "model": "openbmb/MiniCPM-o-4_5",
-    "session_id": "sess_demo_01",
     "modalities": ["audio", "text"],
     "instructions": "You are a concise voice assistant.",
     "input_audio_format": "pcm16",
@@ -492,7 +610,6 @@ additionally carries `"server_event_seq": <int>` (omitted below for brevity).
     "idle_timeout_s": 300,
     "extra_body": {
       "auto_response": true,
-      "native_duplex": true,
       "force_listen_count": 0
     }
   }
@@ -504,13 +621,12 @@ additionally carries `"server_event_seq": <int>` (omitted below for brevity).
 ```json
 {
   "type": "session.created",
-  "incarnation": 0,
   "attachment_generation": 1,
   "resume_token": "rt_3f9c…",
   "session": {
     "object": "realtime.session",
     "type": "realtime",
-    "id": "sess_demo_01",
+    "id": "duplex-5f2a…",
     "model": "openbmb/MiniCPM-o-4_5",
     "state": "open",
     "turn_state": "idle",
@@ -563,7 +679,7 @@ additionally carries `"server_event_seq": <int>` (omitted below for brevity).
 `session.created` and every later `session.update`)
 
 ```json
-{"type": "session.updated", "session": {"object": "realtime.session", "id": "sess_demo_01", "…": "…"}}
+{"type": "session.updated", "session": {"object": "realtime.session", "id": "duplex-5f2a…", "…": "…"}}
 ```
 
 `C→S session.heartbeat` / `S→C session.heartbeat_ack`
@@ -573,7 +689,7 @@ additionally carries `"server_event_seq": <int>` (omitted below for brevity).
 ```
 
 ```json
-{"type": "session.heartbeat_ack", "session_id": "sess_demo_01"}
+{"type": "session.heartbeat_ack", "session_id": "duplex-5f2a…"}
 ```
 
 `C→S session.event_ack`
@@ -589,7 +705,7 @@ additionally carries `"server_event_seq": <int>` (omitted below for brevity).
 ```
 
 ```json
-{"type": "session.closed", "session_id": "sess_demo_01", "reason": "session_close"}
+{"type": "session.closed", "session_id": "duplex-5f2a…", "epoch": 3, "reason": "client_close", "event": {"type": "session.closed", "reason": "client_close"}}
 ```
 
 `C→S session.resume` (new socket, `?duplex=1&autostart=0`)
@@ -597,8 +713,7 @@ additionally carries `"server_event_seq": <int>` (omitted below for brevity).
 ```json
 {
   "type": "session.resume",
-  "session_id": "sess_demo_01",
-  "incarnation": 0,
+  "session_id": "duplex-5f2a…",
   "resume_token": "rt_3f9c…",
   "last_received_server_event_seq": 40
 }
@@ -610,8 +725,8 @@ events with `server_event_seq > 40` are replayed in order)
 ```json
 {
   "type": "session.resumed",
-  "session_id": "sess_demo_01",
-  "incarnation": 0,
+  "session_id": "duplex-5f2a…",
+  "session": {"id": "duplex-5f2a…", "state": "open", "epoch": 3, "turn_id": 2, "...": "the engine's current session object"},
   "attachment_generation": 2,
   "resume_token": "rt_a71e…"
 }
@@ -620,19 +735,19 @@ events with `server_event_seq > 40` are replayed in order)
 `S→C session.replaced` (sent to the superseded socket, which is then closed)
 
 ```json
-{"type": "session.replaced", "session_id": "sess_demo_01", "attachment_generation": 1}
+{"type": "session.replaced", "session_id": "duplex-5f2a…", "attachment_generation": 1}
 ```
 
 `S→C session.expired`
 
 ```json
-{"type": "session.expired", "session_id": "sess_demo_01", "incarnation": 0, "reason": "disconnect_grace_expired"}
+{"type": "session.expired", "session_id": "duplex-5f2a…", "reason": "disconnect_grace_expired"}
 ```
 
 `S→C session.resync_required`
 
 ```json
-{"type": "session.resync_required", "session_id": "sess_demo_01", "reason": "journal_gap"}
+{"type": "session.resync_required", "session_id": "duplex-5f2a…", "reason": "journal_gap"}
 ```
 
 #### Input
@@ -723,7 +838,7 @@ silence/noise declaration (no runtime append; answered with `response.listen`)
   "item_id": "item_9b2d…",
   "event": {
     "type": "input.committed",
-    "session_id": "sess_demo_01",
+    "session_id": "duplex-5f2a…",
     "turn_id": 1,
     "epoch": 0,
     "history_len": 1,
@@ -753,7 +868,7 @@ deferred commit during an active response (`event.response_create_deferred`)
   "type": "input_audio_buffer.committed",
   "previous_item_id": "item_9b2d…",
   "item_id": "item_c044…",
-  "event": {"type": "input.committed", "session_id": "sess_demo_01", "turn_id": 2, "epoch": 0, "history_len": 3, "response_create_deferred": true, "message": {"…": "…"}}
+  "event": {"type": "input.committed", "session_id": "duplex-5f2a…", "turn_id": 2, "epoch": 0, "history_len": 3, "response_create_deferred": true, "message": {"…": "…"}}
 }
 ```
 
@@ -862,7 +977,7 @@ deferred commit during an active response (`event.response_create_deferred`)
     "status_details": null,
     "output": [],
     "modalities": ["audio", "text"],
-    "metadata": {"duplex_event": {"type": "response.created", "session_id": "sess_demo_01", "response_id": "resp_01", "epoch": 0, "turn_id": 1}}
+    "metadata": {"duplex_event": {"type": "response.created", "session_id": "duplex-5f2a…", "response_id": "resp_01", "epoch": 0, "turn_id": 1}}
   }
 }
 ```
@@ -898,15 +1013,15 @@ deferred commit during an active response (`event.response_create_deferred`)
   "item_id": "item_resp_01",
   "output_index": 0,
   "content_index": 0,
-  "metadata": {"session_id": "sess_demo_01", "epoch": 0, "model_speak": true}
+  "metadata": {"session_id": "duplex-5f2a…", "epoch": 0, "model_speak": true}
 }
 ```
 
-`S→C response.audio.delta` (one ~1 s unit, 24 kHz PCM16)
+`S→C response.output_audio.delta` (one ~1 s unit, 24 kHz PCM16)
 
 ```json
 {
-  "type": "response.audio.delta",
+  "type": "response.output_audio.delta",
   "response_id": "resp_01",
   "item_id": "item_resp_01",
   "output_index": 0,
@@ -915,7 +1030,7 @@ deferred commit during an active response (`event.response_create_deferred`)
   "format": "pcm16",
   "sample_rate_hz": 24000,
   "metadata": {
-    "session_id": "sess_demo_01",
+    "session_id": "duplex-5f2a…",
     "epoch": 0,
     "model_speak": true,
     "end_of_turn": false,
@@ -926,10 +1041,10 @@ deferred commit during an active response (`event.response_create_deferred`)
 }
 ```
 
-`S→C response.audio_transcript.delta` (exactly one per audio delta)
+`S→C response.output_audio_transcript.delta` (exactly one per audio delta)
 
 ```json
-{"type": "response.audio_transcript.delta", "response_id": "resp_01", "item_id": "item_resp_01", "output_index": 0, "content_index": 0, "delta": "It is sunny"}
+{"type": "response.output_audio_transcript.delta", "response_id": "resp_01", "item_id": "item_resp_01", "output_index": 0, "content_index": 0, "delta": "It is sunny"}
 ```
 
 `S→C response.output_text.delta` / `response.output_text.done` (text modality)
@@ -942,14 +1057,14 @@ deferred commit during an active response (`event.response_create_deferred`)
 {"type": "response.output_text.done", "response_id": "resp_02", "item_id": "item_resp_02", "output_index": 0, "content_index": 0, "text": "Tomorrow looks cloudy."}
 ```
 
-`S→C response.audio.done` / `response.audio_transcript.done`
+`S→C response.output_audio.done` / `response.output_audio_transcript.done`
 
 ```json
-{"type": "response.audio.done", "response_id": "resp_01", "item_id": "item_resp_01", "output_index": 0, "content_index": 0}
+{"type": "response.output_audio.done", "response_id": "resp_01", "item_id": "item_resp_01", "output_index": 0, "content_index": 0}
 ```
 
 ```json
-{"type": "response.audio_transcript.done", "response_id": "resp_01", "item_id": "item_resp_01", "output_index": 0, "content_index": 0, "transcript": "It is sunny, 24 degrees."}
+{"type": "response.output_audio_transcript.done", "response_id": "resp_01", "item_id": "item_resp_01", "output_index": 0, "content_index": 0, "transcript": "It is sunny, 24 degrees."}
 ```
 
 `S→C response.content_part.done`
@@ -980,7 +1095,7 @@ deferred commit during an active response (`event.response_create_deferred`)
     "status": "completed",
     "status_details": null,
     "output": [{"id": "item_resp_01", "object": "realtime.item", "type": "message", "role": "assistant", "status": "completed", "content": [{"type": "output_audio", "transcript": "It is sunny, 24 degrees."}]}],
-    "metadata": {"type": "response.done", "session_id": "sess_demo_01", "response_id": "resp_01", "epoch": 0, "committed": true, "playback": {"generated_ms": 2400, "sent_ms": 2400, "played_ms": 2400, "committed_ms": 2400}}
+    "metadata": {"type": "response.done", "session_id": "duplex-5f2a…", "response_id": "resp_01", "epoch": 0, "committed": true, "playback": {"generated_ms": 2400, "sent_ms": 2400, "played_ms": 2400, "committed_ms": 2400}}
   }
 }
 ```
@@ -1006,9 +1121,9 @@ deferred commit during an active response (`event.response_create_deferred`)
 ```json
 {
   "type": "response.listen",
-  "session_id": "sess_demo_01",
+  "session_id": "duplex-5f2a…",
   "epoch": 0,
-  "response": {"object": "realtime.response", "status": "listening", "metadata": {"type": "response.listen", "session_id": "sess_demo_01", "epoch": 0, "reason": "model_listen"}}
+  "response": {"object": "realtime.response", "status": "listening", "metadata": {"type": "response.listen", "session_id": "duplex-5f2a…", "epoch": 0, "reason": "model_listen"}}
 }
 ```
 
@@ -1055,7 +1170,7 @@ function-call response (Nemotron VoiceChat tools)
 raw fallback when a model tool call is malformed (`call_id`/`name` missing)
 
 ```json
-{"type": "duplex.function_call.done", "event": {"type": "function_call.done", "session_id": "sess_demo_01", "epoch": 0, "call_id": null, "name": null, "arguments": "{"}}
+{"type": "duplex.function_call.done", "event": {"type": "function_call.done", "session_id": "duplex-5f2a…", "epoch": 0, "call_id": null, "name": null, "arguments": "{"}}
 ```
 
 #### Playback
@@ -1071,7 +1186,7 @@ raw fallback when a model tool call is malformed (`call_id`/`name` missing)
   "type": "playback.acknowledged",
   "event": {
     "type": "playback.acknowledged",
-    "session_id": "sess_demo_01",
+    "session_id": "duplex-5f2a…",
     "epoch": 0,
     "item_id": "item_resp_01",
     "played_ms": 1000,
@@ -1096,20 +1211,17 @@ truncating ack after an interruption (what was actually heard)
 ```json
 {
   "type": "overlap.decision",
-  "session_id": "sess_demo_01",
+  "session_id": "duplex-5f2a…",
   "epoch": 0,
   "policy": "listen_only",
   "action": "listen",
   "reason": "long_overlap_speech",
-  "overlap_ms": 1400,
-  "buffer_audio": true,
-  "defer_runtime_append": true,
-  "force_listen": true
+  "metadata": {"overlap_ms": 1400, "buffer_audio": true, "defer_runtime_append": true, "force_listen": true}
 }
 ```
 
 ```json
-{"type": "overlap.decision", "session_id": "sess_demo_01", "epoch": 0, "policy": "barge_in_on_speech", "action": "barge_in", "reason": "server_vad_utterance_active"}
+{"type": "overlap.decision", "session_id": "duplex-5f2a…", "epoch": 0, "policy": "barge_in_on_speech", "action": "barge_in", "reason": "server_vad_utterance_active"}
 ```
 
 server-VAD session (`session.update` fragment) and the annotated append the
@@ -1123,12 +1235,6 @@ translator forwards internally (shown for reference; clients never see it)
 {"type": "input_audio_buffer.append", "audio": "…", "format": "pcm16", "sample_rate_hz": 16000, "is_speech": true, "force_listen": true, "vad": {"backend": "silero", "is_speech": true, "speech_active": true, "speech_started": true, "speech_stopped": false, "speech_probability": 0.93}}
 ```
 
-`S→C runtime.control` (only when an engine signal was partially unsupported)
-
-```json
-{"type": "runtime.control", "session_id": "sess_demo_01", "epoch": 1, "result": {"…": "redacted engine control result"}}
-```
-
 #### Errors
 
 `S→C error` — Realtime shape
@@ -1154,71 +1260,80 @@ translator forwards internally (shown for reference; clients never see it)
 ```
 
 ```json
-{"type": "error", "error": {"type": "rate_limit_error", "code": "duplex_session_busy", "message": "duplex session capacity reached (max_sessions=4)"}}
+{"type": "error", "error": {"type": "rate_limit_error", "code": "resource_exhausted", "message": "duplex_session_capacity_exhausted: limit=4"}}
 ```
 
 Handshake-stage errors (before the Realtime projector is bound) use the
 flat native shape:
 
 ```json
-{"type": "error", "error": "Unknown or expired duplex session: sess_demo_01", "code": "session_resume_expired"}
+{"type": "error", "error": "Unknown or expired duplex session: duplex-5f2a…", "code": "session_resume_expired"}
 ```
 
 #### Coverage
 
 The examples above cover every event type the Realtime route emits or
 accepts (verified by diffing the `type` literals in
-`vllm_omni/entrypoints/duplex/` against this document). Not shown, because
-they never reach a `/v1/realtime?duplex=1` client unprojected: the native
-`/v1/duplex` dialect (`session.create` / `open_session` / `session.config`,
-`input.committed`, `input.cancelled`, `audio.cancelled`,
-`response.output_audio.delta`, `response.text.delta`, `response.message`,
-`function_call.done`) — their Realtime projections are the examples above
-(see the name map at the end of this section) — and the runner-internal markers `__timeout__`,
-`__disconnect__`, `__replaced_attachment__`. Input aliases (`push_chunk`,
-`input.audio.append`, `input_text.append`, `push_text`, `signal_turn`,
-`audio.playback_ack`, `close_session`, `close`, `session_close`) share the
-payload of their canonical event (see the alias table in [Full-Duplex Runtime (MiniCPM-o 4.5)](../design/fullduplex.md)).
+`vllm_omni/engine/duplex/events.py` and `vllm_omni/engine/duplex/commands.py`
+against this document). Not shown, because they never reach a client
+unprojected: the session-internal events the runner produces
+(`input.committed`, `input.cancelled`, `audio.cancelled`,
+`response.output_audio.delta`, `response.text.delta`, `function_call.done`)
+— their Realtime projections are the examples above (see the name map at the
+end of this section). Every public event is a typed `DuplexEvent` whose
+`to_realtime()` is the wire object; every client event is parsed into a
+typed `DuplexCommand` by `command_from_realtime()`.
 
-#### Native to Realtime name map
+#### Internal to Realtime name map
 
-The native `/v1/duplex` dialect is what the session runner produces; the
-Realtime projector renames and fans out these events before they reach a
-`/v1/realtime?duplex=1` client.
+The session runner emits these session-internal events; the Realtime
+projector (`vllm_omni/engine/duplex/realtime_events.py`) renames and fans
+them out into typed events before they reach a client.
 
-| Native / internal event | Realtime projection |
+| Internal event | Realtime projection |
 | --- | --- |
-| `session.create` (`open_session`, `session.config`) | first `session.update` |
 | `input.committed` | `conversation.item.added` / `conversation.item.created`, `input_audio_buffer.committed`, `conversation.item.input_audio_transcription.completed`, `conversation.item.done` |
 | `input.cancelled` | `input_audio_buffer.cleared` |
-| `response.output_audio.delta` (with `audio_transcript`) | optional `response.speak`, then `response.audio.delta` and `response.audio_transcript.delta` |
-| `response.output_audio.done` / `response.output_text.done` | `response.audio.done` / `response.output_text.done` |
+| `response.output_audio.delta` (with `audio_transcript`) | optional `response.speak`, then `response.output_audio.delta` and `response.output_audio_transcript.delta` |
+| `response.output_audio.done` / `response.output_text.done` | `response.output_audio.done` / `response.output_text.done` |
 | `response.text.delta` | `response.output_text.delta` |
-| `response.message` (chat-fallback raw chunk) | passed through |
 | `audio.cancelled` | optional `output_audio_buffer.cleared`, then the cancelled terminal events and `response.done` with `status: "cancelled"` |
 | `function_call.done` | `response.created`, `response.output_item.added` (`function_call` item), `response.function_call_arguments.delta` / `.done`, `response.output_item.done`, `response.done` |
 | `playback.acknowledged` | `playback.acknowledged` (wrapped verbatim under `event`) |
-| `runtime.control`, `session.resumed`, `session.replaced`, `session.expired`, `session.resync_required`, `session.heartbeat_ack` | passed through unchanged |
+| `session.resumed`, `session.replaced`, `session.expired`, `session.resync_required`, `session.heartbeat_ack` | passed through unchanged |
 
 ## Known Limitations
 
+- Only MiniCPM-o 4.5 is served over this endpoint today; PersonaPlex and
+  Nemotron VoiceChat arrive with the follow-up PRs that port them to the
+  plugin contract.
 - Several surfaces are capability-gated per model (see *Capability
   negotiation by model* above): PersonaPlex does not support session resume,
   barge-in, or audio truncation; Nemotron VoiceChat does not support barge-in
   or audio truncation; camera frames are consumed only by MiniCPM-o 4.5; tool
   calls are produced only by Nemotron VoiceChat.
-- `turn_detection` supports `server_vad` or `null`, not `semantic_vad`.
-  Turn-based models support endpointing without interruption, including
-  `create_response=false` for explicit response creation. Model-native duplex
-  requires `interrupt_response=true` and `create_response=true`.
+- A response is generated from committed **audio**, so the OpenAI text-prompt
+  shape does not drive one: `conversation.item.create` with an `input_text`
+  part adds the item to history, but a following `response.create` with no
+  committed audio is rejected with `response_create_without_input`, and
+  `input.text.append` is rejected with `native_text_append_unsupported`.
+  Text-to-speech has its own shape here — put the text in
+  `extra_body.duplex_initial_user_text` on `session.update`, then stream audio
+  units (silence is enough). The seeded turn is what the model answers.
+- `turn_detection` supports only `server_vad` with `interrupt_response=true`;
+  `semantic_vad`, `interrupt_response=false`, and `create_response` are not
+  supported.
 - `input_audio_transcription` and `input_audio_noise_reduction` are accepted
   and echoed but no separate transcription or noise-reduction stage runs;
   transcripts come from the model.
 - `rate_limits.updated` is emitted for compatibility only and always carries
   an empty list.
 - Session capacity is bounded by `duplex_session.max_sessions` in the deploy
-  configuration; admission beyond it fails with `duplex_session_busy` or
-  `resource_exhausted`.
-- The native `/v1/duplex` route speaks the internal dialect and does not
-  support `session.resume`; use `/v1/realtime?duplex=1` for the contract on
-  this page.
+  configuration; admission beyond it fails with `resource_exhausted`
+  (`rate_limit_error`, retryable).
+- A duplex deployment serves the websocket route and, when supported by the
+  model, ordinary `/v1/chat/completions` requests on the same engine without a
+  session per request (see [Start the Server](#start-the-server)). Other
+  turn-based HTTP routes report "not available" in duplex mode. To use the
+  ordinary online serving stack, select `session_mode: turn` as described in
+  [Full Duplex](full_duplex_api.md#enable-full-duplex).

docs/serving/speech_api.md

@@ -6,8 +6,9 @@ vLLM-Omni provides an OpenAI-compatible API for text-to-speech (TTS) generation.
 - **Fish Speech S2 Pro** (`fishaudio/s2-pro`) -- Dual-AR TTS with DAC codec. Supports text-to-speech and voice cloning via reference audio. Output: 44.1 kHz.
 - **Voxtral TTS** (`mistralai/Voxtral-4B-TTS-2603`) -- AR + FlowMatching TTS with preset voices. Output: 24 kHz.
 - **CosyVoice3** (`FunAudioLLM/Fun-CosyVoice3-0.5B-2512`) -- 2-stage talker + flow-matching code2wav. Voice cloning via `ref_audio` + `ref_text` (no presets). Output: 24 kHz.
+- **Gepard-1.0** (`nineninesix/gepard-1.0`) -- single-stage native-AR TTS with a 22.05 kHz NanoCodec. Zero-shot default voice only.
 
-See the [Supported Models](#supported-models) section below for the full list, including OmniVoice, VoxCPM2, and MOSS-TTS-Nano.
+See the [Supported Models](#supported-models) section below for the full list, including OmniVoice, VoxCPM2, MOSS-TTS-Nano, and Breeze-TTS-2.
 
 !!! tip "Deployment recipes"
     TTS deployment recipes are published at
@@ -40,11 +41,16 @@ vllm serve mistralai/Voxtral-4B-TTS-2603 --omni --port 8091
 # CosyVoice3 (voice cloning only — supply ref_audio + ref_text per request)
 vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 \
     --omni --port 8091 --trust-remote-code
+
+# Gepard-1.0 (zero-shot default voice; packaged gepard.yaml)
+vllm-omni serve nineninesix/gepard-1.0 --omni --port 8091 --trust-remote-code \
+    --stage-init-timeout 900 \
+    --deploy-config vllm_omni/deploy/gepard.yaml
 ```
 
 ### Generate Speech
 
-**Using curl:**
+**Qwen3-TTS CustomVoice, using curl:**
 
 ```bash
 curl -X POST http://localhost:8091/v1/audio/speech \
@@ -56,7 +62,24 @@ curl -X POST http://localhost:8091/v1/audio/speech \
     }' --output output.wav
 ```
 
-**Using Python:**
+**Gepard-1.0, using curl:**
+
+Gepard accepts `input` (required), `voice` (`"default"`), `response_format`,
+`stream` / `stream_format`, `max_new_tokens`, and `seed`. Unsupported fields
+include `speed`, `language`, `instructions`, `task_type`, `ref_audio`,
+`ref_text`, `extra_params`, and `word_timestamps`.
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Hello, this is Gepard speaking.",
+        "voice": "default",
+        "seed": 7
+    }' --output output.wav
+```
+
+**Qwen3-TTS CustomVoice, using Python:**
 
 ```python
 import httpx
@@ -761,6 +784,28 @@ Fish Speech uses `ref_audio` and `ref_text` for voice cloning (no `task_type` ne
 | ------- | ------------- |
 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | Voice cloning from `ref_audio` + `ref_text`. No built-in voice presets — upload a voice or pass `ref_audio`/`ref_text` per request. |
 
+### Gepard-1.0
+
+| Model | Description |
+| ----- | ----------- |
+| `nineninesix/gepard-1.0` | Zero-shot native-AR TTS. 22.05 kHz mono. `voice` must be omitted or `"default"`. |
+
+Gepard request fields:
+
+| Field | Behavior |
+| ----- | -------- |
+| `input` | Required. Empty/whitespace-only returns 400. |
+| `voice` | Omitted or `"default"` only. Other values 400. |
+| `response_format` | `wav` default. Non-streaming: `wav`/`pcm`/`flac`/`mp3`. `opus` returns 400 (22.05 kHz is not an Opus sample rate). Streaming: `pcm`/`wav` only. |
+| `stream` / `stream_format` | SSE (`speech.audio.*`) and raw `audio` byte streaming. |
+| `speed` | Must be `1.0`. Gepard has no native speed control. |
+| `max_new_tokens` | Frame budget (1 token = 1 frame = 1024 samples ≈ 46.4 ms). Default 1000 from deploy YAML; adapter bounds 1..4096. |
+| `seed` | Optional. Reaches the in-model 32-head sampler. The packaged YAML currently pins `seed: 42`, so serving is deterministic by default until that pin is removed. |
+| `extra_params` | Any key returns 400, including `temperature`/`top_p`/`top_k`. |
+| Cloning / style fields | `ref_audio`, `ref_text`, `speaker_embedding`, `instructions`, `language`, `task_type`, `word_timestamps`, and similar declared-but-unsupported fields return 400. |
+
+See the [Gepard section of the online TTS hub](../user_guide/examples/online_serving/text_to_speech.md#gepard-10) for launch commands. Native-AR recompute preemption is a known limitation under concurrency.
+
 ### OmniVoice
 
 | Model | Description |
@@ -777,12 +822,52 @@ hidden-state dtype before the output projection. Consequently, a float32 stage c
 | ------- | ------------- |
 | `openbmb/VoxCPM2` | TTS + voice cloning with built-in speaker presets and uploaded-voice support. Accepts `voice` (preset or uploaded) or `ref_audio` + optional `ref_text`. |
 
+#### Startup LoRA adapter
+
+To serve a fine-tuned VoxCPM2 voice, set
+`voxcpm2_runtime_config.startup_lora_path` in the stage's `hf_overrides`.
+Copy `vllm_omni/deploy/voxcpm2.yaml` to a local deploy config and add this key
+alongside its existing runtime settings:
+
+```yaml
+# Under stages[0].engine_extras.hf_overrides.voxcpm2_runtime_config:
+startup_lora_path: /absolute/path/to/adapter
+```
+
+Then launch with the modified config:
+
+```bash
+vllm serve openbmb/VoxCPM2 --omni --deploy-config /absolute/path/to/voxcpm2-lora.yaml
+```
+
+The directory must be accessible to the worker and contain the native VoxCPM2
+training export: `lora_config.json` (with a `lora_config` object containing
+`r`, `alpha`, enabled groups, and target module names) and
+`lora_weights.safetensors`. PEFT checkpoints and pickle checkpoints are not
+accepted. Missing, unexpected, non-finite, or incorrectly shaped adapter
+tensors fail model loading rather than silently loading a partial adapter.
+
+The adapter is merged into the base LM, residual LM, LocDiT, and optional
+projection layers selected by its configuration, after base weight loading
+and before compilation or CUDA graph capture. All requests use that adapter;
+`voice` still selects a reference voice, not a LoRA adapter. Changing adapters
+requires restarting the server. Runtime loading/unloading, per-request
+multi-LoRA selection, and `load_format=dummy` are not supported by this path.
+Weight fusion rounds to the base model's dtype, so numerical and speech-quality
+parity should be checked against the upstream adapter on your deployment.
+
 ### MOSS-TTS-Nano
 
 | Model | Description |
 | ------- | ------------- |
 | `OpenMOSS-Team/MOSS-TTS-Nano` | Voice cloning only. Requires `ref_audio` (or an uploaded `voice`); no built-in voice presets. `ref_text` is accepted but ignored — upstream's `voice_clone` mode does not consume a transcript. |
 
+### Breeze-TTS-2
+
+| Model | Description |
+| ------- | ------------- |
+| `BreezeBlue/Breeze-TTS-2` | Two-stage AR TTS (T5Gemma2 + Qwen3 talker with a depth decoder, bundled Qwen3-TTS codec) at 24 kHz. Four modes are selected from the request fields: plain (`input` + speaker tag `voice`, `S0`..`S9`), voice design (`instructions`), voice clone (`ref_audio` + `ref_text`, exactly one clip), and voice direction (reference + `instructions`). Greedy decoding only: `sample_rate` must be `24000`, `speed` must be `1.0`, and `guidance_scale`/`cfg_scale` other than `1.0`, `negative_prompt`, `temperature`/`top_p`/`top_k` overrides, `language`, and `speaker_embedding` are rejected. Streaming returns PCM `speech.audio.delta` events. See [`recipes/BreezeBlue/Breeze-TTS-2.md`](https://github.com/vllm-project/vllm-omni/blob/main/recipes/BreezeBlue/Breeze-TTS-2.md). |
+
 ## Error Responses
 
 ### 400 Bad Request

docs/serving/video_stream_api.md

@@ -45,11 +45,19 @@ WebSocket /v1/video/chat/stream
 | Server -> Client | `response.start` | none | Query generation started. |
 | Server -> Client | `response.text.delta` | `delta` | Incremental text output. |
 | Server -> Client | `response.text.done` | `text` | Final text output for the query. |
-| Server -> Client | `response.audio.delta` | `data`, `format` | Incremental generated audio, base64 WAV. |
-| Server -> Client | `response.audio.done` | none | Audio output finished. |
+| Server -> Client | `response.output_audio.delta` | `data`, `format` | Incremental generated audio, base64 WAV. |
+| Server -> Client | `response.output_audio.done` | none | Audio output finished. |
 | Server -> Client | `session.done` | none | Session closed. |
 | Server -> Client | `error` | `message` | Recoverable protocol or generation error. |
 
+### Frame Consumption Reporting
+
+When buffered frames include client-supplied `frame_id` values, the server emits `video.frames.consumed` after the first engine output. Its `frame_ids`, `frames`, and `latest_pts_ms` describe the images included in that query's prompt.
+
+Queries stride-sample the buffered frames, keeping the last frame, then exclude frames already known to have failed decoding. Excluded frames are not replaced with other buffered frames. Frames whose background decoding has not finished remain eligible through the image URL path.
+
+**Bugfix compatibility note:** `video.frames.consumed` now excludes known decode failures that were already excluded from the prompt. Older versions could report those frames and their timestamps as consumed. The event name and fields are unchanged; an empty selection reports empty lists and `latest_pts_ms: null`.
+
 ### `session.config` Fields
 
 | Field | Type | Default | Description |
@@ -60,10 +68,24 @@ WebSocket /v1/video/chat/stream
 | `max_frames` | integer, 1-256 | `50` | Maximum retained frame buffer size. Oldest frames are evicted first. |
 | `system_prompt` | string or null | null | Optional custom system prompt. |
 | `use_audio_in_video` | bool | `true` | Include streamed audio chunks in multimodal video understanding when audio is present. |
-| `sampling_params_list` | list or null | null | Optional per-stage sampling parameter overrides. |
+| `sampling_params_list` | list or null | null | Optional per-stage parameter dictionaries. Each provided entry replaces that stage's deployment sampling settings. |
 | `enable_frame_filter` | bool | `true` | Enable EVS near-duplicate frame filtering. |
 | `frame_filter_threshold` | float, 0.0-1.0 | `0.95` | EVS similarity threshold. Higher keeps more frames; lower drops more near-duplicates. |
 
+Sampling parameter bugfix: `sampling_params_list` is now forwarded to the engine;
+earlier versions accepted this field but silently used deployment defaults instead.
+Each provided entry constructs a fresh `SamplingParams` for that stage in pipeline
+order; it is not merged with that stage's deployment defaults. Fields absent from a
+provided entry use `SamplingParams` constructor defaults. Only omitted trailing
+stages keep their deployment defaults. An omitted, `null`, or empty list keeps the
+engine defaults. Invalid sampling parameters return an `error` when the query is
+submitted.
+
+For example, `[{"temperature": 0.2, "max_tokens": 64}]` configures the thinker while
+the talker and code2wav retain their deployment defaults. With only
+`[{"temperature": 0.2}]`, the thinker's `max_tokens` and `top_p` use constructor
+defaults, not its YAML values.
+
 ### Legacy Aliases
 
 The server accepts these legacy field names and rewrites them before validation. New clients should send the canonical names above.

docs/serving/videos_api.md

@@ -109,6 +109,16 @@ curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4
 The final content is available from `/v1/videos/{video_id}/content` after the
 job status becomes `completed`.
 
+`queued` means the request is still waiting for diffusion scheduler admission.
+The status changes to `in_progress` when the scheduler first selects the
+request for execution.
+
+`DELETE /v1/videos/{video_id}` issues a bounded engine abort
+(`VLLM_OMNI_ABORT_TIMEOUT`, default 2s), then cancels the frontend
+task. Cancellation cleanup is also bounded and best-effort: it confirms
+the abort was queued, and the current request batch may still drain.
+The job is then re-read so a completed save is not orphaned.
+
 ### Synchronous Response
 
 `POST /v1/videos/sync` blocks until generation finishes and returns raw video
@@ -234,6 +244,34 @@ curl -X POST http://localhost:8091/v1/videos/sync \
   -o output.mp4
 ```
 
+## Output Encoding
+
+These `extra_params` control how the server turns decoded frames into MP4 bytes.
+
+| Field | Type | Default | Description |
+| --- | --- | --- | --- |
+| `preencode_mp4` | boolean | false | Encode the MP4 on the worker while the VAE is still decoding, instead of after the full video is materialized |
+| `video_codec_options` | object | null | Encoder options passed through to the H.264 encoder, such as `{"preset": "ultrafast", "threads": "0"}` |
+
+With `preencode_mp4` enabled, each committed VAE chunk leaves the accelerator and
+is encoded while later chunks are still decoding, so host transfer and CPU
+encoding overlap the remaining decode instead of following it. The response is
+unchanged: the same complete MP4, byte-for-byte equivalent frames.
+
+```bash
+curl -X POST http://localhost:8091/v1/videos/sync \
+  -F "prompt=A small robot walking through a neon city" \
+  -F 'extra_params={"preencode_mp4": true, "video_codec_options": {"preset": "ultrafast"}}' \
+  -o output.mp4
+```
+
+`preencode_mp4` applies to the complete-MP4 response paths only. The
+`/v1/realtime/video` WebSocket endpoint rejects it, because that path already
+overlaps encoding through its own incremental fragmented-MP4 encoder.
+
+Support is per model: MiniMax-H3 implements it, and other models ignore the flag
+and take the full-decode path.
+
 ## Storage
 
 Set `VLLM_OMNI_SERVER_STORAGE__PATH` to control where asynchronous video outputs are

docs/usage/metrics.md

@@ -12,7 +12,7 @@ curl http://localhost:8000/metrics
 ## Metric Namespaces
 
 | Prefix | Source | Present when |
-|--------|--------|--------------|
+| -------- | -------- | -------------- |
 | `vllm_omni:` | vLLM-Omni orchestrator / audio modality / cross-stage transfer | Pipeline-dependent |
 | `vllm:` | Upstream vLLM engine, wrapped by `OmniPrometheusStatLogger` to expose `{stage, replica}` | Pipeline includes an LLM (AR) stage |
 | `http_` / `process_` | Uvicorn / Python runtime | Always |
@@ -24,7 +24,7 @@ Defined in `vllm_omni/metrics/prometheus.py`. Track request lifecycle across the
 ### Request counts
 
 | Metric | Type | Labels | Description |
-|--------|------|--------|-------------|
+| -------- | ------ | -------- | ------------- |
 | `vllm_omni:num_requests_running` | Gauge | `model_name` | Pipeline-global in-flight requests (dispatched to engine, not yet finalized) |
 | `vllm_omni:num_requests_waiting` | Gauge | `model_name` | Requests waiting in the Orchestrator queue |
 | `vllm_omni:requests_success_total` | Counter | `model_name`, `finished_reason` | Total requests by completion reason. `finished_reason` ∈ {`stop`, `length`, `abort`, ...} mirroring upstream `vllm:request_success_total`; aborts cover client disconnect / cancellation paths in addition to upstream `FinishReason.ABORT` |
@@ -32,15 +32,15 @@ Defined in `vllm_omni/metrics/prometheus.py`. Track request lifecycle across the
 ### Latency
 
 | Metric | Type | Labels | Description |
-|--------|------|--------|-------------|
+| -------- | ------ | -------- | ------------- |
 | `vllm_omni:e2e_request_latency_s` | Histogram | `model_name` | Pipeline-global end-to-end request latency in seconds |
 
 ## Audio Modality Metrics (`vllm_omni:`)
 
-Emitted at request finalize, except for `audio_ttfp_s` (streaming-hook at the first audio packet) and `audio_underrun_s` / `audio_continuity_ok_total` (streaming finalize, after the chunk stream is exhausted). All carry `{model_name, stage, replica}` plus the listed extra label.
+Emitted at request finalize, except for `audio_ttfp_s` (the first audio packet/frame from streaming Chat or Speech APIs) and `audio_underrun_s` / `audio_continuity_ok_total` (Chat or Speech streaming finalize, after the chunk stream is exhausted). Non-streaming Speech latency is represented by `e2e_request_latency_s`, not TTFP. All audio metrics carry `{model_name, stage, replica}` plus the listed extra label.
 
 | Metric | Type | Extra label | Description |
-|--------|------|-------------|-------------|
+| -------- | ------ | ------------- | ------------- |
 | `vllm_omni:audio_ttfp_s` | Histogram | — | Time from request arrival to first audio packet/frame |
 | `vllm_omni:audio_duration_s` | Histogram | — | Audio content duration (`audio_frames / sample_rate`) |
 | `vllm_omni:audio_rtf` | Histogram | — | Real-time factor (`stage_gen_time_s / audio_duration_s`); streaming TTS SLO red line `< 1`; uses `RTF_BUCKETS` |
@@ -50,13 +50,22 @@ Emitted at request finalize, except for `audio_ttfp_s` (streaming-hook at the fi
 | `vllm_omni:audio_skipped_requests_total` | Counter | `reason` | Silent-loss counter — code2wav rejected malformed codec input and returned `200 OK` with empty audio |
 
 The continuity math comes from `vllm_omni/benchmarks/audio_continuity.py::compute_continuity_stats` so the server-side observation aligns with the bench-side definition.
+For Speech, continuity applies to raw audio, SSE, and WebSocket streaming. WAV headers and empty chunks are excluded; non-streaming Speech requests do not emit continuity samples.
+
+Continuity rate for the default 100 ms threshold can be derived from the successful-continuity counter and the underrun Histogram's request count:
+
+```promql
+sum by (model_name, stage, replica) (rate(vllm_omni:audio_continuity_ok_total{threshold_ms="100"}[5m]))
+/
+sum by (model_name, stage, replica) (rate(vllm_omni:audio_underrun_s_count[5m]))
+```
 
 ## Diffusion Metrics (`vllm_omni:`)
 
 Per-request timing breakdowns for diffusion (image/video) stages. Emitted at request finalize when `stage_metrics.diffusion_metrics` is present. All carry `{model_name, stage, replica}`.
 
 | Metric | Type | Description |
-|--------|------|-------------|
+| -------- | ------ | ------------- |
 | `vllm_omni:diffusion_exec_s` | Histogram | DiT forward pass execution time per request in seconds |
 | `vllm_omni:diffusion_exec_per_step_s` | Histogram | DiT forward pass execution time per denoising step in seconds (`exec_s / num_inference_steps`) |
 | `vllm_omni:diffusion_preprocess_s` | Histogram | Diffusion input preprocessing time per request in seconds |
@@ -80,7 +89,7 @@ rate(vllm_omni:diffusion_preprocess_s_sum[5m])
 Per-physical-transfer histograms tracking the data hop between adjacent stages. Labels `{model_name, from_stage, from_replica, to_stage, to_replica}` let dashboards attribute latency to specific replica edges. `from_replica` / `to_replica` are resolved from the orchestrator's sticky-routing binding (`stage_pool.get_bound_replica_id(request_id)`), so no extra plumbing through `TransferEdgeStats` is needed.
 
 | Metric | Type | Description |
-|--------|------|-------------|
+| -------- | ------ | ------------- |
 | `vllm_omni:transfer_size_bytes` | Histogram | Per-transfer payload size in bytes |
 | `vllm_omni:transfer_tx_s` | Histogram | Sender-side time (serialize + submit to connector) |
 | `vllm_omni:transfer_rx_s` | Histogram | Receiver-side time (recv + deserialize) |
@@ -106,7 +115,7 @@ For the full list of upstream metrics, see [the vLLM docs](https://github.com/vl
 ## Metric Availability by Pipeline Type
 
 | Metric group | Multi-stage LLM (Qwen3-Omni) |
-|---|---|
+| --- | --- |
 | `vllm_omni:` request tracking + latency | With `--log-stats` |
 | `vllm_omni:` audio modality | With `--log-stats`, if pipeline has a talker stage |
 | `vllm_omni:` transfer | With `--log-stats`, if pipeline has ≥ 2 stages |

docs/user_guide/diffusion/attention_backends/fastvideo_vsa.md

@@ -1,24 +1,33 @@
 # FastVideo VSA
 
-FastVideo Variable Sparse Attention (VSA) accelerates the `FastVideo/FastWan2.2-TI2V-5B-Diffusers` model's
-self-attention by partitioning the post-patch latent grid into spatiotemporal
+FastVideo Variable Sparse Attention (VSA) partitions the post-patch latent grid into spatiotemporal
 blocks. For every query block, VSA scores the key/value blocks and computes
 attention only against the selected top-k blocks.
 
-The supported checkpoint provides both text-to-video and image-to-video modes through `Wan22Pipeline`; the separate Wan I2V-14B, S2V, and VACE pipelines are outside this backend's supported scope.
+## Supported models
 
-MiniMax-H3 served with a FastH3 VSA adapter uses a second, H3-specific route
-through the same backend. See the
-[MiniMax-H3 recipe](https://recipes.vllm.ai/MiniMaxAI/MiniMax-H3) for its geometry,
-supported topologies, and commands.
+| Model / checkpoint | Required adapter | Tasks | Sequence parallelism |
+| --- | --- | --- | --- |
+| `FastVideo/FastWan2.2-TI2V-5B-Diffusers` | None | T2V, I2V through `Wan22Pipeline` | Disabled |
+| `MiniMaxAI/MiniMax-H3` | [FastH3 VSA (recipe)](https://github.com/vllm-project/vllm-omni/blob/main/recipes/MiniMaxAI/MiniMax-H3.md#fasth3-adapter) | T2VA | Disabled or pure Ulysses |
+| Wan I2V-14B, S2V, VACE | — | Unsupported | — |
 
-VSA is a CUDA-only, explicitly selected backend. It requires the
-`fastvideo-kernel` package and currently supports non-causal self-attention
-with equal query and key/value sequence lengths. Unsupported shapes, masks,
-dtypes, sequence-parallel execution, or kernel failures fall back to
-`TORCH_SDPA` and emit a warning with the reason. On the Wan route, an active
-sequence-parallel context is one of those fallbacks; the H3 route supports pure
-Ulysses and rejects ring or all-gather sequence parallelism at startup.
+## Installation
+
+Install vLLM-Omni from this checkout with the `vsa` extra:
+
+```bash
+uv pip install -e '.[vsa]'
+```
+
+The extra installs the tested kernel dependency automatically. Prebuilt kernels require
+Linux, Python 3.12, and glibc 2.34 or newer (x86-64 or aarch64). The full
+FastVideo framework and provider environment variables are not required.
+
+`FASTVIDEO_VSA` selects the attention algorithm. On SM120, the H3 integration
+currently executes FastVideo's 64-token Triton block-sparse kernel; it does not
+dispatch to FlashInfer. The `vsa` extra installs `fastvideo-kernel==0.3.4` for
+this execution path.
 
 ## Enable the backend
 
@@ -27,24 +36,18 @@ flag. Use `--fastvideo-vsa-topk` to set the number of key/value blocks retained
 for every query block:
 
 ```bash
-vllm-omni serve <model> \
+vllm serve <model> --omni \
   --diffusion-attention-backend FASTVIDEO_VSA \
   --fastvideo-vsa-topk 64
 ```
 
-The backwards-compatible environment variable selects the backend with the
-default `topk=64`:
+<details markdown="1">
+<summary>Alternative configuration formats</summary>
+
+Equivalent structured configuration:
 
 ```bash
-export DIFFUSION_ATTENTION_BACKEND=FASTVIDEO_VSA
-vllm-omni serve <model>
-```
-
-To tune top-k, pass the CLI backend and top-k flags together as shown above,
-or use the equivalent structured configuration:
-
-```bash
-vllm-omni serve <model> \
+vllm serve <model> --omni \
   --diffusion-attention-config \
   '{"default":{"backend":"FASTVIDEO_VSA","fastvideo_vsa_topk":64}}'
 ```
@@ -73,8 +76,15 @@ stages:
         fastvideo_vsa_topk: 64
 ```
 
+</details>
+
 ## Choose top-k
 
+H3 uses 64-token video blocks and keeps
+all prefix blocks; see the [FastH3 VSA recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/MiniMaxAI/MiniMax-H3.md#fasth3-adapter).
+
+### Wan top-k behavior
+
 At runtime the backend logs the sequence shape and derived block count:
 
 ```text
@@ -117,8 +127,16 @@ selecting VSA does not turn a native checkpoint into a distilled model.
 
 ## Verify routing and fallback
 
-Check the startup and first-forward logs instead of assuming that selecting
-the backend guarantees sparse execution:
+VSA is a CUDA-only, explicitly selected backend. It requires the
+`fastvideo-kernel` package and currently supports non-causal self-attention
+with equal query and key/value sequence lengths. Unsupported shapes, masks,
+dtypes, sequence-parallel execution, or kernel failures fall back to
+`TORCH_SDPA` and emit a warning with the reason. On the Wan route, an active
+sequence-parallel context is one of those fallbacks; the H3 route supports pure
+Ulysses and rejects ring or all-gather sequence parallelism at startup.
+H3 accelerator faults propagate instead of attempting dense recovery.
+
+On the Wan route, check the startup and first-forward logs:
 
 - `route=VSA` means top-k block selection is active.
 - `route=VSA_ALL_BLOCKS` means the FastVideo DMD checkpoint retained all
@@ -126,6 +144,10 @@ the backend guarantees sparse execution:
 - `route=SDPA` or `FASTVIDEO_VSA falling back to SDPA: ...` means dense SDPA
   executed; the warning includes the reason.
 
+For H3, check `FastH3 adapter active` at startup and
+`FASTVIDEO_VSA H3 routing` during DiT execution, as described in the
+[model recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/MiniMaxAI/MiniMax-H3.md#fasth3-adapter).
+
 The Wan route requires CUDA tensors in FP16 or BF16, 256-token blocks,
 standard `head_size**-0.5` scaling, equal Q/K/V head counts, no attention mask,
 and no active sequence-parallel context. The MiniMax-H3 route uses 64-token

docs/user_guide/diffusion/device_side_video_postprocess.md

@@ -0,0 +1,72 @@
+# Device-side video postprocessing
+
+vLLM-Omni can convert decoded WAN2.2 video from normalized floating-point
+`[B, C, T, H, W]` tensors to contiguous `uint8` `[B, T, H, W, C]` frames before
+the worker-to-engine device-to-host transfer. This reduces the transported video
+payload by four times when the original IPC path widens bfloat16 to float32.
+
+The optimization is disabled by default. Enable it with a structured diffusion
+configuration:
+
+```python
+from vllm_omni import Omni
+
+engine = Omni(
+    model="Wan-AI/Wan2.2-TI2V-5B-Diffusers",
+    video_output_transport={"enable_device_postprocess": True},
+)
+```
+
+For the server CLI, pass the same object as JSON:
+
+```bash
+vllm serve Wan-AI/Wan2.2-TI2V-5B-Diffusers --omni \
+  --video-output-transport '{"enable_device_postprocess": true}'
+```
+
+For a deploy configuration, place it on the diffusion stage:
+
+```yaml
+stages:
+  - stage_id: 0
+    video_output_transport:
+      enable_device_postprocess: true
+```
+
+## Runtime contract
+
+WAN2.2 emits decoded video through the typed `DiffusionOutput.media` field. The
+model runner validates and splits the batch into request-local tensors before
+preparing each video for transport. The worker then performs the D2H copy and
+shared-memory IPC using the prepared representation. The engine uses the generic
+media finalizer instead of a model-specific postprocessor.
+
+The runtime leaves the video in normalized floating-point form when:
+
+- `enable_device_postprocess` is false;
+- the requested output type is not `np`; or
+- frame interpolation still requires the floating-point tensor.
+
+These are policy fallbacks, not malformed-contract errors. If the device cannot
+allocate the temporary conversion buffer, the runner logs a warning and prepares
+the request-local float representation instead. Invalid tensor layout, encoding,
+value range, lifecycle state, and non-memory runtime failures still fail before
+IPC.
+
+## Precision
+
+Device preparation converts to float32 before denormalization and quantization.
+For bfloat16 WAN output this is more precise than the historical host path,
+which denormalizes in bfloat16. Relative to that path, values may differ by at
+most one level out of 255. Float32 inputs remain byte-identical after
+quantization.
+
+## Scope
+
+WAN2.2 is the reference migration for the first contract version. Other video
+pipelines continue using their existing output paths until they can declare the
+same typed media contract without bypassing model-specific float consumers such
+as safety checks or audio/video packaging.
+
+See [RFC #6541](https://github.com/vllm-project/vllm-omni/issues/6541) for the
+contract, lifecycle, batching rules, and migration plan.

docs/user_guide/diffusion/parallelism/cfg_parallel.md

@@ -19,6 +19,12 @@ CFG-Parallel accelerates diffusion models by distributing positive and negative
 
 See supported models list in [Supported Models](../../diffusion_features.md#supported-models).
 
+Distilled Cosmos3 checkpoints, including `nvidia/Cosmos3-Super-Image2Video-4Step`,
+run without CFG and reject `--cfg-parallel-size` greater than 1 at startup.
+Use `--ulysses-degree 2` for two-GPU inference. These checkpoints always use
+`guidance_scale=1.0`; other explicitly requested values produce a warning, and
+`negative_prompt` does not affect generation.
+
 ---
 
 ## Quick Start

docs/user_guide/diffusion/regional_compilation.md

@@ -34,5 +34,34 @@ that provide their own `setup_compile()` implementation manage their compilation
 policy independently. Compilation is lazy, so backend or graph errors can first
 surface on the initial request.
 
+## Pinning one packed shape (MiniMax-H3)
+
+Compiled regions are keyed by input shape. MiniMax-H3 packs a request into a
+row count that follows the request -- a longer prompt, more frames, a larger
+frame -- and rounds it up to the next 64-row boundary, so two requests can land
+on two different packed lengths.
+
+Under the default `diffusion_compile_dynamic: true` a new packed length is
+absorbed by the dynamic shape, so it is not by itself a recompilation. It still
+hands the backend a shape it has not autotuned, which is the risk this knob
+exists to remove: a fixed-shape deployment (`--no-diffusion-compile-dynamic`)
+recompiles the transformer blocks per shape, and a first-seen large shape can
+push autotuning into its worst-case memory use.
+
+A request pins the length with `extra_args["pad_seq_len"]`:
+
+```python
+sampling_params = SamplingParams(extra_args={"pad_seq_len": 54080})
+```
+
+The value must be a positive multiple of 64 and must cover the rows the request
+actually uses; the packed sequence is then padded to it instead of to the next
+64-row boundary. Pick a bucket that covers every request dimension the
+deployment accepts -- prompt length, frame count, frame size and reference
+blocks all change the used row count. The server logs the effective length as
+`MiniMax H3 packed sequence: ... pad_seq_len=... used=... seq_len=...` whenever
+a request pins it. The padding rows are masked out, so their cost is the
+attention and feed-forward work on those rows.
+
 Use `--enforce-eager` to disable the model runner's generic compile setup.
 Pipelines that compile internally define their own eager-mode behavior.

docs/user_guide/examples/offline_inference/hunyuan_image3.md

@@ -1,35 +0,0 @@
-# HunyuanImage-3.0 Image-to-Text Inference
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/hunyuan_image3>.
-
-
-This example demonstrates how to run HunyuanImage-3.0 Image-to-Text with the vLLM-Omni.
-
-## Local CLI Usage
-
-Download the example image:
-
-```bash
-wget https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg
-```
-
-Run example:
-
-```bash
-python image_to_text.py \
-  --image cherry_blossom.jpg \
-  --prompt "<|startoftext|>You are an assistant that understands images and outputs text.<img>Describe the content of the picture."
-```
-
-Key arguments:
-
-- `--model`: Model used. Default is: tencent/HunyuanImage-3.0-Instruct (Optional).
-- `--image`: Path to input image (required).
-- `--prompt`: Text description used to guide image understanding (required).
-
-## Example materials
-
-??? abstract "image_to_text.py"
-    ``````py
-    --8<-- "examples/offline_inference/hunyuan_image3/image_to_text.py"
-    ``````

docs/user_guide/examples/online_serving/qwen3_omni.md

@@ -2,7 +2,6 @@
 
 Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/qwen3_omni>.
 
-
 ## 🛠️ Installation
 
 Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md)
@@ -15,6 +14,19 @@ Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```
 
+For a local deployment with multiple API frontend processes sharing one set
+of stage engines, add `--api-server-count`:
+
+```bash
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+    --api-server-count 2 --port 8091
+```
+
+This mode supports local EngineCore stages with one local process group per
+replica, including stages configured with multiple `num_replicas`. Headless,
+remote, diffusion, intra-replica data-parallel, and Ray stage deployments
+remain single-frontend.
+
 The default deployment configuration situated at `vllm_omni/deploy/qwen3_omni_moe.yaml` is resolved and loaded
 automatically via the model registry, obviating the necessity for the `--deploy-config` flag in standard deployment topologies.
 The bundled Qwen3-Omni setup explicitly sets `moe_backend: triton` for the Thinker and Talker stages. This keeps
@@ -22,7 +34,12 @@ both stages on vLLM's Triton unquantized MoE path and avoids the unstable FlashI
 some GPUs.
 Asynchronous chunk streaming is **enabled by default** within the bundled configuration.
 
+For Thinker pipeline parallelism, configure `pipeline_parallel_size` and `devices`
+on the Thinker stage. Keep Talker and Code2Wav at `pipeline_parallel_size: 1`. The
+default asynchronous scheduling and audio chunk streaming can remain enabled.
+
 To explicitly utilize a custom deployment YAML, specify the configuration path:
+
 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
     --deploy-config /path/to/deploy_config_file
@@ -46,7 +63,7 @@ Adopt the stage-based CLI architecture to independently instantiate execution pr
 The example below pins Stage 0 to GPU 0 and Stage 1/2 to GPU 1 via
 `CUDA_VISIBLE_DEVICES`.
 
-**1. Stage 0 (Thinker + API server)**
+#### 1. Stage 0 (Thinker + API server)
 
 ```bash
 CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
@@ -56,7 +73,7 @@ CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --omni-master-port 26000
 ```
 
-**2. Stage 1 (Talker)**
+#### 2. Stage 1 (Talker)
 
 ```bash
 CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
@@ -66,7 +83,7 @@ CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --omni-master-port 26000
 ```
 
-**3. Stage 2 (Code2Wav)**
+#### 3. Stage 2 (Code2Wav)
 
 ```bash
 CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
@@ -91,6 +108,9 @@ To experiment with another supported MoE backend, set `moe_backend` in the
 deployment YAML for both the Thinker and Talker stages. For the stage-based
 CLI, pass `--moe-backend` directly on each stage command.
 
+For other stage settings, pass the option on that stage command. For example,
+to change its GPU memory utilization:
+
 ```bash
 CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --stage-id 1 \
@@ -103,11 +123,12 @@ CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
 ### Send Multi-modal Request
 
 Get into the example folder
+
 ```bash
 cd examples/online_serving/qwen3_omni
 ```
 
-####  Send request via python
+#### Send request via python
 
 ```bash
 python openai_chat_completion_client_for_multimodal_generation.py --query-type use_image --port 8091 --host "localhost"
@@ -122,7 +143,6 @@ The Python client supports the following command-line arguments:
 - `--audio-path` (or `-a`): Path to local audio file or URL. If not provided and query-type is `use_audio`, uses default audio URL. Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs and common audio formats: MP3, WAV, OGG, FLAC, M4A. Example: `--audio-path /path/to/audio.wav` or `--audio-path https://example.com/audio.mp3`
 - `--prompt` (or `-p`): Custom text prompt/question. If not provided, uses default prompt for the selected query type. Example: `--prompt "What are the main activities shown in this video?"`
 
-
 For example, to use a local video file with custom prompt:
 
 ```bash
@@ -132,19 +152,20 @@ python openai_chat_completion_client_for_multimodal_generation.py \
     --prompt "What are the main activities shown in this video?"
 ```
 
-####  Send request via curl
+#### Send request via curl
 
 ```bash
 bash run_curl_multimodal_generation.sh use_image
 ```
 
 ## Modality control
+
 You can control output modalities to specify which types of output the model should generate. This is useful when you only need text output and want to skip audio generation stages for better performance.
 
 ### Supported modalities
 
 | Modalities | Output |
-|------------|--------|
+| ------------ | -------- |
 | `["text"]` | Text only |
 | `["audio"]` | Audio only |
 | `["text", "audio"]` | Text + Audio |
@@ -222,7 +243,9 @@ print(response.choices[1].message.audio)    # Audio response
 ```
 
 ## Streaming Output
+
 If you want to enable streaming output, please set the argument as below. The final output will be obtained just after generated by corresponding stage. We support both text streaming output and audio streaming output. Other modalities can output normally.
+
 ```bash
 python openai_chat_completion_client_for_multimodal_generation.py \
     --query-type use_image \
@@ -255,12 +278,14 @@ The convenience script launches both the vLLM server and Gradio demo together:
 ```
 
 This script will:
+
 1. Start the vLLM server in the background
 2. Wait for the server to be ready
 3. Launch the Gradio demo
 4. Handle cleanup when you press Ctrl+C
 
 The script supports the following arguments:
+
 - `--model`: Model name/path (default: Qwen/Qwen3-Omni-30B-A3B-Instruct)
 - `--server-port`: Port for vLLM server (default: 8091)
 - `--gradio-port`: Port for Gradio demo (default: 7861)
@@ -271,18 +296,19 @@ The script supports the following arguments:
 
 #### Option 2: Manual Launch (Two-Step Process)
 
-**Step 1: Launch the vLLM API server**
+##### Step 1: Launch the vLLM API server
 
 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```
 
 If you have a custom deploy config file:
+
 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --deploy-config /path/to/deploy_config_file
 ```
 
-**Step 2: Run the Gradio demo**
+##### Step 2: Run the Gradio demo
 
 In a separate terminal:
 
@@ -295,7 +321,7 @@ Then open `http://localhost:7861/` on your local browser to interact with the we
 The gradio script supports the following arguments:
 
 - `--model`: Model name/path (should match the server model)
-- `--api-base`: Base URL for the vLLM API server (default: http://localhost:8091/v1)
+- `--api-base`: Base URL for the vLLM API server (default: <http://localhost:8091/v1>)
 - `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
 - `--port`: Port for Gradio server (default: 7861)
 - `--share`: Share the Gradio demo publicly (creates a public link)

docs/user_guide/examples/online_serving/text_to_speech.md

@@ -18,6 +18,7 @@ For the full list of supported architectures across all modalities, see
 | --- | --- | --- | --- | --- | --- |
 | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | — |
 | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
+| Gepard-1.0 | `nineninesix/gepard-1.0` | — (zero-shot default voice) | ✓ (PCM / WAV stream) | `default` only | — |
 | higgs-audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` | ✓ (`ref_audio`+`ref_text`) | ✓ (codec_streaming) | — | — |
 | GLM-TTS | `zai-org/GLM-TTS` | ✓ (`ref_audio`+`ref_text`, required) | ✓ (PCM stream) | — | ✓ |
 | OmniVoice | `k2-fsa/OmniVoice` | (offline only) | — | — | — |
@@ -92,7 +93,7 @@ curl -X POST http://localhost:8091/v1/audio/speech \
     }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 -
 ```
 
-Adjust the player's sample rate to match the model (44.1 kHz for Fish Speech, 48 kHz for VoxCPM2, 24 kHz for the others).
+Adjust the player's sample rate to match the model (44.1 kHz for Fish Speech, 48 kHz for VoxCPM2, 22.05 kHz for Gepard, 24 kHz for the others).
 
 For full request-shape documentation (all parameters, response formats, error codes), see the [Speech API reference](https://github.com/vllm-project/vllm-omni/tree/main/docs/serving/speech_api.md).
 
@@ -120,6 +121,15 @@ vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 --omni --port 8091 --trust-remot
 
 Streaming is on by default via `async_chunk: true` in `vllm_omni/deploy/cosyvoice3.yaml`. Pass `--no-async-chunk` (or `NO_ASYNC_CHUNK=1 ./cosyvoice3/run_server.sh`) for the legacy synchronous path.
 
+Cross-request Stage-1 flow batching is opt-in. Enable it in the server environment when concurrent requests should share a flow-estimator call:
+
+```bash
+export COSYVOICE3_BATCH_FLOW=1
+vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 --omni --port 8091 --trust-remote-code
+```
+
+Batching preserves output lengths and streaming cache alignment, but the different GEMM shapes can produce small waveform differences compared with processing each request separately. Leave `COSYVOICE3_BATCH_FLOW` unset (or set it to `0`) when request-independent numerical behavior is required. Set `COSYVOICE3_BATCH_FLOW_DEBUG=1` to log the observed group-size distribution and enable detailed Stage-1 profiler scopes; diagnostics are disabled by default to avoid per-step profiling overhead.
+
 ### CLI client
 
 The client defaults to the official upstream zero-shot prompt, so it runs without extra flags:
@@ -203,6 +213,61 @@ python fish_speech/gradio_demo.py --api-base http://localhost:8091  # if server
 
 ---
 
+## Gepard-1.0
+
+Single-stage native AR TTS at 22.05 kHz mono. Zero-shot only: omit `voice` or pass `"default"`. Voice cloning from reference audio is not available yet.
+
+### Prerequisites
+
+Same NeMo NanoCodec install as the [offline Gepard section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#gepard-10). On a host whose CUDA toolkit cannot build kernels — no `nvcc`/`ninja`, or a consumer Blackwell (`sm_120`) card — also `export VLLM_USE_FLASHINFER_SAMPLER=0` before launch. That env is not a deploy-YAML field.
+
+### Launch
+
+```bash
+vllm-omni serve nineninesix/gepard-1.0 --omni --port 8091 --trust-remote-code \
+    --stage-init-timeout 900 \
+    --deploy-config vllm_omni/deploy/gepard.yaml
+# or:
+./gepard/run_server.sh
+```
+
+`--stage-init-timeout 900` matches the online e2e fixture and the offline example. Serve defaults to 300s, which is often too short for a cold download of the talker plus NanoCodec.
+
+The packaged `vllm_omni/deploy/gepard.yaml` must be passed with `--deploy-config`. The checkpoint self-identifies as `qwen3_5_text`, so omitting the YAML launches a diffusion fallback instead of the Gepard pipeline. The YAML sets `async_chunk: false`, `max_num_seqs: 4`, and currently pins `seed: 42`, so serving is deterministic by default until that YAML seed is removed. Pass an explicit per-request `seed` in tests and clients rather than depending on either default.
+
+### Sending requests
+
+```bash
+python examples/online_serving/text_to_speech/gepard/speech_client.py \
+    --text "Hello, this is Gepard speaking."
+
+python examples/online_serving/text_to_speech/gepard/speech_client.py \
+    --text "Hello, this is Gepard speaking." --seed 7 --stream --output output.pcm
+```
+
+Gepard accepts `input` (required), `voice` (`"default"`), `response_format`, `stream` / `stream_format`, `max_new_tokens`, and `seed`. Unsupported fields include `speed`, `language`, `instructions`, `task_type`, `ref_audio`, `ref_text`, `extra_params`, and `word_timestamps`.
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Hello, this is Gepard speaking.",
+        "voice": "default",
+        "seed": 7
+    }' --output output.wav
+```
+
+### Notes
+
+- Output: 22.05 kHz mono. `max_new_tokens` is a **frame** budget (1 token = 1 frame = 1024 samples ≈ 46.4 ms at 21.5 fps; adapter bounds 1..4096).
+- Supported request fields: `input` (required), `voice` (`default` only), `response_format` (`wav` default; `wav/pcm/flac/mp3` non-streaming; `opus` 400 because 22.05 kHz is not an Opus sample rate; streaming `pcm/wav` only), `stream` / `stream_format`, `max_new_tokens`, `seed`.
+- Unsupported: `speed`, `extra_params` (including `temperature`/`top_p`/`top_k`), `ref_audio`, `ref_text`, `speaker_embedding`, `task_type`, `instructions`, `language`, and `word_timestamps`.
+- Concurrent requests at `max_num_seqs: 4` are supported. Native-AR recompute preemption is a known limitation of this architecture (a request that is preempted mid-generation can resume incorrectly); keep concurrency at or below `max_num_seqs` and treat preemption as out of scope until the platform fix lands.
+- Optional comparison against the upstream Gepard reference server needs Blackwell/Hopper + CUDA 13 + Postgres and is not part of CI.
+- For the request-field table, see the [Speech API Gepard section](https://github.com/vllm-project/vllm-omni/blob/main/docs/serving/speech_api.md#gepard-10).
+
+---
+
 ## higgs-audio v2
 
 2-stage TTS at 24 kHz: a vLLM-native Llama-3.2-3B talker with a DualFFN audio expert (Stage 0) feeding a HiggsAudio codec decoder (Stage 1) that streams chunks back to the client.
@@ -348,6 +413,22 @@ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni --port 8091
 ./qwen3_tts/run_server.sh Base
 ```
 
+For a local deployment with multiple API frontend processes sharing one set
+of TTS stage engines, add `--api-server-count`:
+
+```bash
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
+    --omni --api-server-count 2 --port 8091
+```
+
+This mode supports local EngineCore stages with one local process group per
+replica, including stages configured with multiple `num_replicas`. Headless,
+remote, intra-replica data-parallel, and Ray stage deployments are not
+supported. Runtime voice upload and deletion are disabled with multiple
+API frontends because their registries are process-local. Built-in voices,
+inline `ref_audio`, and voices restored from `custom_voice_dir` at startup are
+supported.
+
 ### Executor backend
 
 Single-GPU serves now default to the uniproc executor (lower IPC overhead, the Base cloning use case from [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) / [#2604](https://github.com/vllm-project/vllm-omni/pull/2604)). `vllm_omni/deploy/qwen3_tts.yaml` is the only Qwen3-TTS deploy config; pass `--deploy-config <path>` to override.
@@ -413,6 +494,10 @@ checkpoint. When a request names an uploaded voice, the server infers
 `task_type="Base"`. Built-in presets such as `vivian` and `ryan` remain
 CustomVoice speakers and require a CustomVoice checkpoint.
 
+The runtime upload and delete routes require a single API frontend; with
+`--api-server-count > 1`, use inline `ref_audio` or restore precomputed voices
+from `custom_voice_dir` at startup.
+
 ### Precomputed custom voices
 
 For reused Base voice-cloning speakers, precompute the reference artifacts once

docs/user_guide/quantization/modelopt.md

@@ -204,6 +204,91 @@ omni = Omni(
 | `--linear-backend cutlass` | str | auto | Select the validated CUTLASS linear backend for supported ModelOpt NVFP4 or mixed FP8/NVFP4 diffusion stages |
 | `--moe-backend cutlass` | str | auto | Select the validated CUTLASS MoE backend for supported ModelOpt mixed MoE checkpoints |
 
+## Cosmos3 Mixed-Precision Schedule
+
+The experimental Cosmos3 schedule can use the checkpoint-native quantized GEMM for middle denoising
+steps and dense 16-bit activation execution for selected first and last steps.
+The schedule supports serialized ModelOpt FP8 and NVFP4 checkpoints:
+
+| Detected weight format | Native middle steps | A16 boundary steps |
+|------------------------|---------------------|--------------------|
+| FP8 | W8A8 | W8A16 |
+| NVFP4 | W4A4 | W4A16 |
+
+The native and A16 paths share one quantized weight representation. The A16
+path dequantizes the live backend weight and calls `F.linear`; it does not keep
+a second checkpoint-weight snapshot. Scheduled FP8 currently requires
+serialized tensorwise scales and a backend that retains canonical FP8 weights.
+Scheduled NVFP4 requires a supported CUTLASS-compatible native or FlashInfer
+layout. SmoothQuant, Marlin-repacked FP8, and per-channel/per-token FP8
+schedules fail closed.
+
+The reasoner uses dense A16 execution by default when its weights are FP8 or
+NVFP4. Set the nested `reasoner` field to `native` to retain the checkpoint-native
+W8A8/W4A4 path. A BF16 reasoner remains BF16 in either mode. The schedule
+currently requires tensor parallel size 1 and one active request per worker,
+does not support HSDP, and does not support block-scaled FP8. Its live-weight
+design is compatible with model-level and standard layer-wise offload, but
+those paths still require GPU end-to-end validation. Distributed layer-wise
+offload is rejected because its direct loader bypasses ModelOpt post-load
+transformations.
+
+A checkpoint can opt in by adding a versioned policy to its authoritative
+`transformer/config.json`. The same policy schema is used for FP8 and NVFP4:
+
+```json
+{
+  "quantization_config": {
+    "quant_method": "modelopt",
+    "quant_algo": "FP8",
+    "runtime": {
+      "diffusion_step_policy": {
+        "schema_version": 1,
+        "type": "first_last_n",
+        "index_space": "denoising_loop_iteration",
+        "scope": ["transformer"],
+        "default_mode": "native",
+        "first_steps": {"count": 3, "mode": "a16"},
+        "last_steps": {"count": 3, "mode": "a16"},
+        "overlap": "a16",
+        "reasoner": "a16"
+      }
+    }
+  }
+}
+```
+
+The example preserves the checkpoint's existing `quant_method` and
+`quant_algo`; only `runtime` is added.
+
+When present and valid, the checkpoint policy is the default. An explicit
+runtime object overrides it. To retain ordinary checkpoint-native execution
+for every step, disable only the schedule:
+
+```bash
+vllm serve /path/to/Cosmos3-Nano-modelopt \
+  --omni \
+  --additional-config '{"cosmos3_mixed_precision":{"enabled":false}}'
+```
+
+This does not disable checkpoint quantization; it prevents installation of the
+mixed-step runtime.
+
+```bash
+vllm serve /path/to/Cosmos3-Nano-modelopt \
+  --omni \
+  --additional-config \
+  '{"cosmos3_mixed_precision":{"first_steps":3,"last_steps":3,"reasoner":"a16"}}'
+```
+
+The presence of `cosmos3_mixed_precision` supplies an explicit runtime
+override; an empty object uses the defaults shown above.
+The schedule uses vLLM's selected native backend and fails during loading if
+its live weight layout cannot support A16 dequantization.
+The runtime selects FP8 or NVFP4 from the checkpoint's ModelOpt configuration.
+BF16 linears are untouched. Mixed FP8/NVFP4 checkpoints are not supported by
+this schedule.
+
 ## Validation and Notes
 
 1. Compare the ModelOpt checkpoint against the BF16 baseline with the same