[Doc] Add Qwen-Image-2.1 recipe to recipes index (#8070)

Signed-off-by: SamitHuang <285365963@qq.com>

.buildkite/amd/scripts/bootstrap-amd-omni.sh

@@ -60,7 +60,8 @@ resolve_test_specs() {
     if ! selected_specs=$(python3 "$AMD_SUITE_SELECTOR" \
         --branch "$BUILDKITE_BRANCH" \
         --labels "$PR_LABELS" \
-        --debug-test-yaml "${DEBUG_TEST_YAML:-}"); then
+        --debug-test-yaml "${DEBUG_TEST_YAML:-}" \
+        --nightly "$NIGHTLY"); then
         exit 1
     fi
 
@@ -80,6 +81,12 @@ filter_test_specs_by_skip_ci() {
         case "$suite_spec" in
             READY_TESTS:*) level="l2" ;;
             MERGE_TESTS:*) level="l3" ;;
+            NIGHTLY_TESTS:*)
+                # Explicit PR and scheduled nightly selections must survive
+                # L2/L3 diff gating, including docs-only main commits.
+                runnable_specs+=("$suite_spec")
+                continue
+                ;;
             *)
                 echo "ERROR: unknown AMD test suite spec '$suite_spec'" >&2
                 exit 1
@@ -101,7 +108,7 @@ filter_test_specs_by_skip_ci() {
                 ":memo: CI skipped — docs or pytest skip-mark changes only" \
                 --style "info" 2>/dev/null || true
         fi
-        echo "No AMD L2/L3 suites remain after skip-ci filtering."
+        echo "No AMD suites remain after skip-ci filtering."
         exit 0
     fi
 }
@@ -219,8 +226,8 @@ if [[ $BUILDKITE_BRANCH == "main" ]]; then
     file_diff=$(get_diff_main)
 fi
 
-# Resolve PR tier labels before skip-ci so ready and merge suites can be
-# filtered independently for CI-YAML-only changes.
+# Resolve PR tier labels before skip-ci so ready/merge suites can be filtered
+# independently while an explicitly selected nightly suite passes through.
 resolve_test_specs
 filter_test_specs_by_skip_ci
 

.buildkite/amd/scripts/build-ci-image.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+set -euo pipefail
+
+# Build and push the ROCm CI image with a registry-backed BuildKit cache. The
+# AMD builders are ephemeral, so a local Docker cache cannot survive between
+# jobs. Keeping the cache beside the runtime image also reuses the registry
+# credentials already configured on the amd-cpu agents.
+
+REGISTRY="${ROCM_CI_IMAGE_REGISTRY:-rocm/vllm-omni}"
+DOCKERFILE="docker/Dockerfile.rocm"
+BUILDER_NAME="vllm-omni-rocm-builder"
+ROCM_ARCH="${PYTORCH_ROCM_ARCH:-gfx942}"
+
+if [[ -z "${BUILDKITE_COMMIT:-}" ]]; then
+    echo "ERROR: BUILDKITE_COMMIT is not set" >&2
+    exit 1
+fi
+
+dependency_files=(
+    "$DOCKERFILE"
+    "${DOCKERFILE}.dockerignore"
+    pyproject.toml
+    setup.py
+    tools/install_torchcodec_rocm.sh
+)
+while IFS= read -r dependency_file; do
+    dependency_files+=("$dependency_file")
+done < <(find requirements -maxdepth 1 -type f -name '*.txt' -print | LC_ALL=C sort)
+
+for dependency_file in "${dependency_files[@]}"; do
+    if [[ ! -f "$dependency_file" ]]; then
+        echo "ERROR: cache-key input does not exist: $dependency_file" >&2
+        exit 1
+    fi
+done
+
+CACHE_KEY=$(
+    {
+        printf 'rocm-arch=%s\n' "$ROCM_ARCH"
+        for dependency_file in "${dependency_files[@]}"; do
+            printf 'path=%s\n' "$dependency_file"
+            sha256sum "$dependency_file"
+        done
+    } | sha256sum | cut -c1-16
+)
+CACHE_REF="${REGISTRY}:rocm-deps-cache-${CACHE_KEY}"
+IMAGE_REF="${REGISTRY}:${BUILDKITE_COMMIT}"
+
+echo "ROCm dependency cache: ${CACHE_REF}"
+echo "ROCm CI image: ${IMAGE_REF}"
+
+# The docker-container driver supports exporting the full cache to a registry;
+# the default docker driver on CI does not.
+if ! docker buildx inspect "$BUILDER_NAME" >/dev/null 2>&1; then
+    docker buildx create --name "$BUILDER_NAME" --driver docker-container
+fi
+docker buildx use "$BUILDER_NAME"
+docker buildx inspect --bootstrap
+
+docker buildx build --push --pull --progress=plain \
+    --target test \
+    --build-arg "ARG_PYTORCH_ROCM_ARCH=${ROCM_ARCH}" \
+    --cache-from "type=registry,ref=${CACHE_REF}" \
+    --cache-to "type=registry,ref=${CACHE_REF},mode=max,compression=zstd,ignore-error=true" \
+    --file "$DOCKERFILE" \
+    --tag "$IMAGE_REF" .

.buildkite/amd/scripts/select_test_suites.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
-"""Select AMD L2/L3 suites after Buildkite has created an AMD build.
+"""Select AMD ready/merge/nightly suites after Buildkite creates a build.
 
 This repository-side selector does not decide which GitHub label events start
 the external ``vllm-omni-amd-ci`` pipeline. Its Buildkite PR build condition
@@ -17,6 +17,7 @@ from collections.abc import Iterable
 SUITE_SPECS = {
     "ready": "READY_TESTS:test-amd-ready.yml",
     "merge": "MERGE_TESTS:test-amd-merge.yml",
+    "nightly": "NIGHTLY_TESTS:test-amd-nightly.yml",
 }
 
 
@@ -28,7 +29,7 @@ def _parse_debug_suites(value: str) -> tuple[str, ...]:
     seen: set[str] = set()
     for suite in suites:
         if suite not in SUITE_SPECS:
-            raise ValueError(f"DEBUG_TEST_YAML entries must be 'merge' or 'ready', got {suite!r}")
+            raise ValueError(f"DEBUG_TEST_YAML entries must be 'merge', 'nightly', or 'ready', got {suite!r}")
         if suite in seen:
             raise ValueError(f"duplicate DEBUG_TEST_YAML suite {suite!r}")
         seen.add(suite)
@@ -40,18 +41,27 @@ def select_amd_test_suites(
     branch: str,
     labels: Iterable[str],
     debug_test_yaml: str = "",
+    nightly: bool = False,
 ) -> tuple[str, ...]:
     """Return ordered suite names while preserving legacy unlabeled PRs."""
 
     if debug_test_yaml:
         return _parse_debug_suites(debug_test_yaml)
     if branch == "main":
-        return ("merge",)
+        return ("nightly",) if nightly else ("merge",)
 
     label_set = {label.strip() for label in labels if label.strip()}
-    selected = tuple(suite for label, suite in (("ready", "ready"), ("merge-test", "merge")) if label in label_set)
-    # AMD historically uploaded L2 for every PR build. Keep that behavior for
-    # triggers such as amd-test while adding tier selection for ready/merge-test.
+    selected = tuple(
+        suite
+        for label, suite in (
+            ("ready", "ready"),
+            ("merge-test", "merge"),
+            ("nightly-test", "nightly"),
+        )
+        if label in label_set
+    )
+    # AMD historically uploaded the ready suite for every admitted PR build.
+    # Preserve that fallback when no explicit tier label is present.
     return selected or ("ready",)
 
 
@@ -60,6 +70,7 @@ def main() -> int:
     parser.add_argument("--branch", required=True)
     parser.add_argument("--labels", default="")
     parser.add_argument("--debug-test-yaml", default="")
+    parser.add_argument("--nightly", type=int, choices=(0, 1), default=0)
     args = parser.parse_args()
 
     labels = tuple(args.labels.splitlines())
@@ -68,19 +79,15 @@ def main() -> int:
             branch=args.branch,
             labels=labels,
             debug_test_yaml=args.debug_test_yaml,
+            nightly=bool(args.nightly),
         )
     except ValueError as exc:
         parser.error(str(exc))
 
     label_set = {label.strip() for label in labels if label.strip()}
-    if not args.debug_test_yaml and args.branch != "main" and not ({"ready", "merge-test"} & label_set):
+    if args.branch != "main" and not args.debug_test_yaml and not ({"ready", "merge-test", "nightly-test"} & label_set):
         print(
-            "No AMD L2/L3 tier label found; preserving the legacy ready-suite fallback.",
-            file=sys.stderr,
-        )
-    if "nightly-test" in label_set:
-        print(
-            "AMD nightly-test has no suite yet and does not affect L2/L3 selection.",
+            "No AMD ready/merge/nightly label found; preserving the legacy ready-suite fallback.",
             file=sys.stderr,
         )
 

.buildkite/amd/test-amd-merge.yml

@@ -9,15 +9,26 @@ steps:
   steps:
     - label: "Simple · Model Executor Test · Shard %N/%t"
       agent_pool: mi300_1
-      parallelism: 2
+      parallelism: 3
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      timeout_in_minutes: 90
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+        - "pytest -sv tests/model_executor -m 'core_model and cpu and not omni' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+
+    - label: "Simple · Omni Processor Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 60
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/model_executor/models/test_omni_processing.py -m 'core_model and cpu and omni'"
 
     - label: "Simple · Diffusion Test · Shard %N/%t"
       agent_pool: mi300_1
@@ -25,6 +36,7 @@ steps:
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      timeout_in_minutes: 45
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM

.buildkite/amd/test-amd-nightly.yml

@@ -0,0 +1,84 @@
+# AMD MI300 `full_model` nightly coverage during burn-in.
+# All jobs are non-blocking until the lane has stable runtime evidence.
+# Historical build #11646 was red in Function Expansion and Accuracy (one
+# async_chunk similarity mismatch; Seed-TTS WER 0.376480 > 0.35).
+
+env:
+  VLLM_WORKER_MULTIPROC_METHOD: spawn
+  HF_HUB_DOWNLOAD_TIMEOUT: 300
+  HF_HUB_ETAG_TIMEOUT: 60
+
+steps:
+  # MI325 is the repository's legacy ROCm pytest marker. The AMD template maps
+  # agent_pool to the current MI300 Buildkite queues independently of that mark.
+  - group: ":full_moon: Qwen3-Omni MI300 Nightly"
+    steps:
+      - label: "Qwen3-Omni Function Expansion"
+        agent_pool: mi300_2
+        depends_on: amd-build
+        mirror_hardwares: [amdproduction]
+        grade: NonBlocking
+        timeout_in_minutes: 180
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_expansion.py -m "full_model and rocm and MI325 and cards_2" --run-level "full_model"
+
+      - label: "Qwen3-Omni Accuracy"
+        agent_pool: mi300_2
+        depends_on: amd-build
+        mirror_hardwares: [amdproduction]
+        grade: NonBlocking
+        timeout_in_minutes: 180
+        artifact_paths:
+          - tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json
+        commands:
+          - export SEED_TTS_WER_EVAL=1
+          # PyTorch ROCm uses the torch.cuda device namespace.
+          - export SEED_TTS_EVAL_DEVICE=cuda:1
+          - pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model and rocm and MI325 and cards_2" --run-level "full_model"
+          # Native tests run from the image copy under /app/vllm-omni, while
+          # Buildkite's artifact hook searches the checkout workspace.
+          - |
+            if [[ "$$PWD" != "$${BUILDKITE_BUILD_CHECKOUT_PATH:?}" ]]; then
+              artifact_dir="$${BUILDKITE_BUILD_CHECKOUT_PATH}/tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc"
+              mkdir -p "$$artifact_dir"
+              cp -v tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json "$$artifact_dir/"
+            fi
+
+      - label: "Qwen3-Omni Documentation Examples"
+        agent_pool: mi300_2
+        depends_on: amd-build
+        mirror_hardwares: [amdproduction]
+        grade: NonBlocking
+        timeout_in_minutes: 120
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - pytest -s -v tests/examples/online_serving/test_qwen3_omni.py -m "full_model and rocm and MI325 and cards_2" --run-level "full_model"
+
+      - label: "Qwen3-Omni AITER-on Smoke"
+        agent_pool: mi300_2
+        depends_on: amd-build
+        mirror_hardwares: [amdproduction]
+        grade: NonBlocking
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ROCM_USE_AITER=1
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_expansion.py::test_text_to_text_audio_001 -m "full_model and rocm and MI325 and cards_2" --run-level "full_model"
+
+  - group: ":speaker: CosyVoice3 MI300 Nightly"
+    steps:
+      # Exercise all sync/async and Chinese/English cases with the platform's
+      # default attention backend. On ROCm this retains end-to-end AITER
+      # coverage without putting its cold JIT build on the READY critical path.
+      - label: "CosyVoice3-TTS E2E Test"
+        agent_pool: mi300_1
+        depends_on: amd-build
+        mirror_hardwares: [amdproduction]
+        grade: NonBlocking
+        timeout_in_minutes: 90
+        retry:
+          automatic:
+            - exit_status: 134
+              limit: 1
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"

.buildkite/amd/test-amd-ready.yml

@@ -9,26 +9,39 @@ steps:
   steps:
     - label: "Simple · Model Executor Test · Shard %N/%t"
       agent_pool: mi300_1
-      parallelism: 2
+      parallelism: 3
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      timeout_in_minutes: 90
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+        - "pytest -sv tests/model_executor -m 'core_model and cpu and not omni' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
 
-    - label: "Simple · Diffusion Test"
+    - label: "Simple · Omni Processor Test"
       agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      timeout_in_minutes: 60
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/model_executor/models/test_omni_processing.py -m 'core_model and cpu and omni'"
+
+    - label: "Simple · Diffusion Test · Shard %N/%t"
+      agent_pool: mi300_1
+      parallelism: 4
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 45
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "pytest -sv tests/diffusion -m 'core_model and cpu'  --ignore=tests/diffusion/cache/test_teacache_extractors.py"
+        - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
 
     - label: "Simple · Engine&Entrypoints Test"
       agent_pool: mi300_1
@@ -69,6 +82,27 @@ steps:
 
 - group: ":card_index_dividers: Diffusion Test"
   steps:
+    # Keep real FlashAttention execution out of the CPU-marked Simple lane.
+    # AITER may need to build several kernels on a cold worker, so run one
+    # pytest worker with job-local build directories and a bounded timeout.
+    - label: "ROCm · FlashAttention/AITER GPU Coverage"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: NonBlocking
+      timeout_in_minutes: 60
+      commands:
+        - export VLLM_ROCM_USE_AITER=1
+        - export AITER_JIT_DIR="/tmp/vllm-omni-aiter-$$BUILDKITE_JOB_ID"
+        - export TORCH_EXTENSIONS_DIR="/tmp/vllm-omni-torch-extensions-$$BUILDKITE_JOB_ID"
+        - mkdir -p "$$AITER_JIT_DIR" "$$TORCH_EXTENSIONS_DIR"
+        - >-
+          timeout --signal=TERM --kill-after=2m 55m
+          pytest -s -v -n 1
+          tests/diffusion/attention/
+          -m 'core_model and rocm and MI325 and cards_1'
+          --run-level "core_model"
+
     - label: "Diffusion · Batch Test"
       agent_pool: mi300_1
       depends_on: amd-build
@@ -160,6 +194,39 @@ steps:
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
 
+- label: "ROCm · Engine & Model Executor GPU Coverage (R2-01)"
+  agent_pool: mi300_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: NonBlocking
+  timeout_in_minutes: 20
+  artifact_paths:
+    - "artifacts/rocm-r2-01/**/*"
+  commands:
+    # Native AMD tests run from /app/vllm-omni, while Buildkite uploads
+    # artifacts relative to its checkout directory.
+    - export R2_01_ARTIFACT_DIR="$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/rocm-r2-01"
+    - mkdir -p "$$R2_01_ARTIFACT_DIR"
+    # The CUDA ready job also covers tests/worker/test_batched_omni_output.py.
+    # Keep the initial ROCm burn-in scoped to the requested directories.
+    # Verbose pytest output and JUnit retain node IDs without a second
+    # collection-only pass over the same GPU-backed test tree.
+    # The CUDA/L4-only CosyVoice3 ragged-flow parity test cold-JITs multiple
+    # AITER kernels on ROCm and is covered by the CUDA ready lane.
+    - >-
+      pytest -s -v -ra
+      tests/engine/ tests/model_executor/
+      -m 'core_model and cuda and not
+      (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
+      -k 'not test_code2wav_streaming_batch_matches_ragged_flow_numerics'
+      --run-level "core_model"
+      --durations=0
+      --junitxml="$$R2_01_ARTIFACT_DIR/pytest.xml"
+      2>&1 | tee "$$R2_01_ARTIFACT_DIR/pytest.log"
+    - >-
+      tail -n 100 "$$R2_01_ARTIFACT_DIR/pytest.log" >
+      "$$R2_01_ARTIFACT_DIR/pytest-summary.txt"
+
 - label: "Engine Test"
   agent_pool: mi300_1
   depends_on: amd-build
@@ -172,6 +239,71 @@ steps:
               pytest -s -v tests/engine/test_async_omni_engine_abort.py
       '
 
+- label: "ROCm · Entrypoints GPU Coverage (R2-02)"
+  agent_pool: mi300_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: NonBlocking
+  timeout_in_minutes: 30
+  env:
+    VLLM_WORKER_MULTIPROC_METHOD: spawn
+  artifact_paths:
+    - "artifacts/rocm-r2-02/**/*"
+  commands:
+    # Native AMD tests run from /app/vllm-omni, while Buildkite uploads
+    # artifacts relative to its checkout directory.
+    - export R2_02_ARTIFACT_DIR="$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/rocm-r2-02"
+    - mkdir -p "$$R2_02_ARTIFACT_DIR"
+    - >-
+      ps -eo pid=,ppid=,lstart=,stat=,comm= >
+      "$$R2_02_ARTIFACT_DIR/processes-before.txt"
+    # Verbose pytest output and JUnit retain selected node IDs without a
+    # second collection-only pass over the same GPU-backed test tree.
+    # The inner timeout leaves five minutes for teardown verification and
+    # artifact generation before the Buildkite step timeout.
+    - >-
+      timeout --signal=TERM --kill-after=1m 25m
+      pytest -s -v -ra
+      tests/entrypoints/
+      -m 'core_model and cuda and not
+      (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
+      --run-level "core_model"
+      --durations=0
+      --junitxml="$$R2_02_ARTIFACT_DIR/pytest.xml"
+      2>&1 | tee "$$R2_02_ARTIFACT_DIR/pytest.log"
+    - >-
+      tail -n 100 "$$R2_02_ARTIFACT_DIR/pytest.log" >
+      "$$R2_02_ARTIFACT_DIR/pytest-summary.txt"
+    - |
+      sleep 5
+      ps -eo pid=,ppid=,lstart=,stat=,comm= > "$$R2_02_ARTIFACT_DIR/processes-after.txt"
+      # Compare PID plus process start time so PID reuse cannot hide a leak.
+      # Double dollars preserve awk field references through Buildkite interpolation.
+      awk '
+        NR == FNR {
+          if ($$9 != "ps") {
+            baseline[$$1 "|" $$3 "|" $$4 "|" $$5 "|" $$6 "|" $$7] = 1
+          }
+          next
+        }
+        $$9 != "ps" {
+          identity = $$1 "|" $$3 "|" $$4 "|" $$5 "|" $$6 "|" $$7
+          if (!(identity in baseline)) {
+            print
+            leaked = 1
+          }
+        }
+        END {
+          if (leaked) {
+            print "status=FAIL leaked_processes_detected=1"
+            exit 1
+          }
+          print "status=PASS leaked_processes=0"
+        }
+      ' "$$R2_02_ARTIFACT_DIR/processes-before.txt" \
+        "$$R2_02_ARTIFACT_DIR/processes-after.txt" \
+        2>&1 | tee "$$R2_02_ARTIFACT_DIR/process-cleanup.txt"
+
 # - label: "Omni Model Test Qwen3-Omni"
 #   agent_pool: mi300_2
 #   depends_on: amd-build
@@ -196,13 +328,23 @@ steps:
         - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
         - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"
 
-    - label: "CosyVoice3-TTS E2E Test"
+    # Keep a fast model/server integration check in the blocking lane. The
+    # full default-backend suite remains in the nightly pipeline because a
+    # cold ROCm AITER MHA build can add more than 25 minutes to this job.
+    - label: "CosyVoice3-TTS E2E Smoke (SDPA)"
       agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      # A ROCm hardware hang aborts the interpreter with 134. Let Buildkite
+      # create one fresh pod instead of retrying on the poisoned GPU process.
+      retry:
+        automatic:
+          - exit_status: 134
+            limit: 1
       commands:
-        - pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"
+        - export DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA
+        - pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py::test_voice_clone_zh_002 -m "slow" --run-level "core_model"
 
 # TODO: Bagel test on ROCm is very unstable. @tjtanaa
 # Need to debug before reneable numerical changes across large PRs

.buildkite/amd/test-template-amd-omni.j2

@@ -3,7 +3,8 @@
    Last synced: 2026-08-04
    Modifications: Use the shared MI300 queues and Omni command-passing contract
 #}
-{% set docker_image_amd = "rocm/vllm-omni:$BUILDKITE_COMMIT" %}
+{% set docker_registry_amd = "rocm/vllm-omni" %}
+{% set docker_image_amd = docker_registry_amd ~ ":$BUILDKITE_COMMIT" %}
 {% set default_working_dir = "/app/vllm-omni" %}
 
 {# MI300 runs tests natively in Kubernetes; Docker-in-Docker is unavailable. #}
@@ -74,11 +75,12 @@ steps:
         depends_on: ~
         soft_fail: false
         commands:
-          - "docker build -f docker/Dockerfile.rocm -t {{ docker_image_amd }} --target test --build-arg ARG_PYTORCH_ROCM_ARCH=gfx942 --progress plain ."
-          - "docker push {{ docker_image_amd }}"
+          - "bash .buildkite/amd/scripts/build-ci-image.sh"
         key: "amd-build"
         env:
           DOCKER_BUILDKIT: "1"
+          PYTORCH_ROCM_ARCH: "gfx942"
+          ROCM_CI_IMAGE_REGISTRY: "{{ docker_registry_amd }}"
         retry:
           automatic:
             - exit_status: -1  # Agent was lost
@@ -156,6 +158,14 @@ steps:
 {% for path in step.artifact_paths %}
           - "{{ path }}"
 {% endfor %}
+{% endif %}
+{% if step.retry %}
+        retry:
+          automatic:
+{% for retry_rule in step.retry.automatic %}
+            - exit_status: {{ retry_rule.exit_status }}
+              limit: {{ retry_rule.limit }}
+{% endfor %}
 {% endif %}
         {% if step.timeout_in_minutes %}
         timeout_in_minutes: {{ step.timeout_in_minutes }}
@@ -224,6 +234,14 @@ steps:
 {% for path in step.artifact_paths %}
       - "{{ path }}"
 {% endfor %}
+{% endif %}
+{% if step.retry %}
+    retry:
+      automatic:
+{% for retry_rule in step.retry.automatic %}
+        - exit_status: {{ retry_rule.exit_status }}
+          limit: {{ retry_rule.limit }}
+{% endfor %}
 {% endif %}
     {% if step.timeout_in_minutes %}
     timeout_in_minutes: {{ step.timeout_in_minutes }}

.buildkite/common/ci_source_file_dependencies.yml

@@ -0,0 +1,474 @@
+# Uploader-only path-prefix presets for CUDA L2–L5 and NPU L4 test YAML.
+# Referenced as ``source_file_dependencies: omni_qwen3_omni_function`` (or a list of
+# keys) on a step; ``upload_pipeline.py`` expands the key(s) only.
+#
+# YAML anchors above the registry hold model business-code paths. Job keys
+# ({model}_function / _perf / _accuracy / _reliability / _doc / _cov) alias
+# those anchors and add the scripts the job actually runs. Feature keys below
+# are unchanged. ``source_filter_fallback`` is not a job key: when no job-key
+# prefix matches, a matching diff keeps every job for that upload.
+#
+# YAML anchors are nested lists; the loader flattens them.
+
+x-qwen3-tts: &qwen3_tts
+  - vllm_omni/engine/async_omni_engine.py
+  - vllm_omni/entrypoints/omni_base.py
+  - vllm_omni/worker/gpu_model_runner.py
+  - vllm_omni/model_executor/models/qwen3_tts/
+  - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+  - vllm_omni/model_executor/models/common/snake_activation.py
+  - vllm_omni/model_executor/models/common/alias_free_activation.py
+  - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+  - vllm_omni/deploy/qwen3_tts.yaml
+  - vllm_omni/entrypoints/openai/serving_speech.py
+  - vllm_omni/entrypoints/openai/tts_adapters/
+  - vllm_omni/engine/stage_pool.py
+  - vllm_omni/worker_v2/
+  - vllm_omni/core/sched/
+  - vllm_omni/distributed/omni_connectors/
+  - vllm_omni/config/stage_config.py
+  - vllm_omni/config/omni_config.py
+  - vllm_omni/engine/orchestrator.py
+  - vllm_omni/engine/stage_engine_core_proc.py
+  - vllm_omni/deploy/qwen3_tts_mrv2.yaml
+  - vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2.yaml
+  - vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2_b4.yaml
+
+x-minicpmo-4-5: &minicpmo_4_5
+  - vllm_omni/engine/async_omni_engine.py
+  - vllm_omni/entrypoints/omni_base.py
+  - vllm_omni/model_executor/models/minicpmo_4_5/
+  - vllm_omni/model_executor/models/step_audio2/
+  - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
+  - vllm_omni/model_executor/models/cosyvoice3/utils.py
+  - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+  - vllm_omni/deploy/minicpmo_4_5.yaml
+  - vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja
+
+x-minicpmo-4-5-duplex: &minicpmo_4_5_duplex
+  - vllm_omni/deploy/minicpmo_4_5_duplex.yaml
+  - vllm_omni/engine/duplex/
+  - vllm_omni/engine/duplex_omni_engine.py
+  - vllm_omni/engine/duplex_orchestrator.py
+  - vllm_omni/engine/omni_engine_base.py
+  - vllm_omni/entrypoints/duplex/
+  - vllm_omni/entrypoints/duplex_omni.py
+  - vllm_omni/entrypoints/async_omni_base.py
+  - vllm_omni/protocol/
+  - vllm_omni/clients/
+  - vllm_omni/outputs/duplex.py
+  - vllm_omni/model_executor/duplex_sampling.py
+  - vllm_omni/experimental/fullduplex/
+  - vllm_omni/core/sched/
+  - vllm_omni/outputs/multimodal_accumulation.py
+  - vllm_omni/engine/stage_init_utils.py
+  - vllm_omni/engine/orchestrator.py
+  - vllm_omni/entrypoints/cli/serve.py
+  - vllm_omni/entrypoints/openai/api_server.py
+  - vllm_omni/entrypoints/openai/duplex_capability.py
+  - vllm_omni/config/stage_config.py
+
+x-qwen2-5-omni: &qwen2_5_omni
+  - vllm_omni/engine/async_omni_engine.py
+  - vllm_omni/entrypoints/omni_base.py
+  - vllm_omni/model_executor/models/qwen2_5_omni/
+  - vllm_omni/model_executor/stage_input_processors/qwen2_5_omni.py
+  - vllm_omni/deploy/qwen2_5_omni.yaml
+
+x-qwen3-omni: &qwen3_omni
+  - vllm_omni/engine/async_omni_engine.py
+  - vllm_omni/entrypoints/omni_base.py
+  - vllm_omni/entrypoints/openai/serving_chat.py
+  - vllm_omni/model_executor/models/qwen3_omni/
+  - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+  - vllm_omni/model_executor/models/common/snake_activation.py
+  - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
+  - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
+  - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
+  - vllm_omni/deploy/qwen3_omni_moe.yaml
+  - vllm_omni/deploy/qwen3_omni_moe_thinking.yaml
+
+x-wan22: &wan22
+  - vllm_omni/diffusion/models/wan2_2/
+  - vllm_omni/diffusion/models/dmd2/
+
+x-cosmos3: &cosmos3
+  - vllm_omni/diffusion/models/cosmos3/
+  - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_wan.py
+  - vllm_omni/experimental/world_models/
+  - vllm_omni/model_extras/cosmos3.py
+
+x-qwen-image: &qwen_image
+  - vllm_omni/diffusion/models/qwen_image/
+  - vllm_omni/diffusion/models/dmd2/
+  - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py
+
+x-hunyuan-image3: &hunyuan_image3
+  - vllm_omni/diffusion/models/hunyuan_image3/
+  - vllm_omni/model_executor/models/hunyuan_image3/
+  - vllm_omni/deploy/hunyuan_image3_dit.yaml
+
+x-minimax-h3: &minimax_h3
+  - vllm_omni/diffusion/models/minimax_h3/
+  - vllm_omni/diffusion/layers/indexed_modulation.py
+
+x-hunyuan-video: &hunyuan_video
+  - vllm_omni/diffusion/models/hunyuan_video/
+  - vllm_omni/model_executor/models/hunyuan_video/
+
+x-ltx2: &ltx2
+  - vllm_omni/diffusion/models/ltx2/
+  - vllm_omni/model_extras/ltx2.py
+
+x-sana-video: &sana_video
+  - vllm_omni/diffusion/models/sana_video/
+  - vllm_omni/model_extras/sana_video.py
+
+x-z-image: &z_image
+  - vllm_omni/diffusion/models/z_image/
+
+x-stable-audio: &stable_audio
+  - vllm_omni/diffusion/models/stable_audio/
+
+x-lingbot-video: &lingbot_video
+  - vllm_omni/diffusion/models/lingbot_video/
+  - vllm_omni/model_extras/lingbot_video.py
+
+x-lingbot-world: &lingbot_world
+  - vllm_omni/diffusion/models/lingbot_world/
+
+
+source_file_dependencies:
+  # Hardware presets and shared test helpers. Not a job key — do not attach this
+  # to a step. Used only when no job-key prefix matches: then a hit keeps every job.
+  source_filter_fallback:
+    - .buildkite/common/ci_mirror_hardwares.yml
+    - tests/helpers/
+
+  # --- features ---
+  diffusion_distributed_attention:
+    - tests/diffusion/distributed/
+    - tests/diffusion/attention/test_attention_sp.py
+    - vllm_omni/diffusion/distributed/
+    - vllm_omni/diffusion/attention/
+    - vllm_omni/diffusion/data.py
+    - vllm_omni/diffusion/models/
+
+  custom_pipeline:
+    - tests/e2e/features/custom_pipeline/
+    - tests/e2e/features/helpers/custom_pipeline.py
+    - vllm_omni/diffusion/
+    - vllm_omni/engine/
+    - vllm_omni/entrypoints/
+    - vllm_omni/inputs/
+    - vllm_omni/outputs/
+
+  rlhf_verl_omni:
+    - tests/e2e/features/rlhf_test/test_verl_omni_e2e.py
+    - tests/e2e/features/helpers/verl_omni_server.py
+    - tests/e2e/features/helpers/custom_pipeline.py
+    - vllm_omni/diffusion/
+    - vllm_omni/engine/
+    - vllm_omni/entrypoints/
+    - vllm_omni/inputs/
+    - vllm_omni/outputs/
+
+  # Tiny-model diffusion jobs (shared across many model trees under models/).
+  diffusion_tiny_model:
+    - tests/model_tests/diffusion/
+    - vllm_omni/diffusion/models/
+
+  diffusion_offloader:
+    - tests/diffusion/offloader/
+    - vllm_omni/diffusion/offloader/
+
+  diffusion_quantization:
+    - tests/diffusion/quantization/
+    - vllm_omni/diffusion/quantization/
+
+  # --- models ---
+  # --- omni / tts jobs ---
+  omni_qwen3_omni_function:
+    - *qwen3_omni
+    - tests/e2e/offline_inference/test_qwen3_omni.py
+    - tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py
+    - tests/e2e/offline_inference/test_qwen3_omni_autoround_w4a16_expansion.py
+    - tests/e2e/offline_inference/test_qwen3_omni_modelopt_nvfp4_w4a4.py
+    - tests/e2e/online_serving/test_qwen3_omni.py
+    - tests/e2e/online_serving/test_qwen3_omni_expansion.py
+    - tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
+
+  omni_qwen3_omni_accuracy:
+    - *qwen3_omni
+    - tests/e2e/accuracy/qwen3_omni/
+
+  omni_qwen3_omni_perf:
+    - *qwen3_omni
+    - tests/dfx/perf/scripts/run_benchmark.py
+    - tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json
+    - tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
+    - tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
+    - tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json
+
+  omni_qwen3_omni_reliability:
+    - *qwen3_omni
+    - tests/dfx/reliability/test_reliability_qwen3_omni.py
+
+  omni_qwen3_omni_doc:
+    - *qwen3_omni
+    - examples/online_serving/qwen3_omni/
+    - examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py
+    - tests/examples/online_serving/test_qwen3_omni.py
+
+  omni_qwen2_5_omni_doc:
+    - *qwen2_5_omni
+    - examples/online_serving/qwen2_5_omni/
+    - examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py
+    - tests/examples/online_serving/test_qwen2_5_omni.py
+
+  omni_minicpmo_4_5_function:
+    - *minicpmo_4_5
+    - tests/e2e/offline_inference/test_minicpmo_4_5.py
+    - tests/e2e/online_serving/test_minicpmo_4_5.py
+    - tests/e2e/online_serving/test_minicpmo_4_5_expansion.py
+
+  omni_minicpmo_4_5_accuracy:
+    - *minicpmo_4_5
+    - tests/e2e/accuracy/minicpmo_4_5/
+
+  omni_minicpmo_4_5_perf:
+    - *minicpmo_4_5
+    - tests/dfx/perf/scripts/run_benchmark.py
+    - tests/dfx/perf/tests/test_minicpmo_4_5_ready.json
+    - tests/dfx/perf/tests/test_minicpmo_4_5.json
+
+  omni_minicpmo_4_5_doc:
+    - *minicpmo_4_5
+    - examples/online_serving/minicpmo/realtime_web/
+    - examples/online_serving/minicpmo/realtime_duplex_demo.py
+    - tests/examples/test_minicpmo_realtime_web_server.py
+    - tests/examples/test_minicpmo_realtime_web_static.py
+    - tests/examples/test_minicpmo_realtime_duplex_simple_demo.py
+
+  omni_minicpmo_4_5_duplex_function:
+    - *minicpmo_4_5
+    - *minicpmo_4_5_duplex
+    - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
+    - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+    - tests/e2e/online_serving/test_minicpmo_4_5_window.py
+    - tests/e2e/online_serving/helpers/minicpmo_window_e2e.py
+    - tests/e2e/online_serving/test_duplex_client_live.py
+    - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
+    - tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
+    - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
+    - tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py
+    - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
+    - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
+    - tests/assets/minicpmo_4_5/
+    - tests/clients/
+    - tests/engine/duplex/
+    - tests/engine/test_duplex_omni_engine.py
+    - tests/engine/test_duplex_orchestrator.py
+    - tests/entrypoints/duplex/
+    - tests/entrypoints/test_duplex_omni.py
+    - tests/entrypoints/openai_api/test_duplex_api_server.py
+    - tests/model_executor/models/minicpmo_4_5/duplex/
+    - tests/protocol/
+
+  omni_minicpmo_4_5_duplex_perf:
+    - *minicpmo_4_5
+    - *minicpmo_4_5_duplex
+    - tests/dfx/perf/scripts/run_benchmark.py
+    - tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json
+
+  tts_qwen3_tts_function:
+    - *qwen3_tts
+    - tests/e2e/features/prefill_phase/
+    - tests/e2e/offline_inference/test_qwen3_tts_base.py
+    - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
+    - tests/e2e/online_serving/test_qwen3_tts_base.py
+    - tests/e2e/online_serving/test_qwen3_tts_base_expansion.py
+    - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
+    - tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py
+
+  tts_qwen3_tts_perf:
+    - *qwen3_tts
+    - tests/dfx/perf/scripts/run_benchmark.py
+    - tests/dfx/perf/tests/test_tts.json
+
+  tts_qwen3_tts_cov:
+    - *qwen3_tts
+    - .buildkite/common/scripts/run_cov_split.sh
+    - pyproject.toml
+    - tests/e2e/offline_inference/test_qwen3_tts_base.py
+    - tests/e2e/online_serving/test_qwen3_tts_base.py
+
+  # --- diffusion jobs ---
+  diffusion_wan22_function:
+    - *wan22
+    - tests/e2e/offline_inference/test_wan22_t2v.py
+    - tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py
+    - tests/e2e/offline_inference/test_device_postprocess_equivalence.py
+    - tests/e2e/online_serving/test_wan22_t2v.py
+    - tests/e2e/online_serving/test_wan22_expansion.py
+
+  diffusion_wan22_accuracy:
+    - *wan22
+    - tests/e2e/accuracy/wan22_i2v/
+    - tests/e2e/accuracy/test_diffusers_backend_similarity.py
+
+  diffusion_wan22_perf:
+    - *wan22
+    - tests/dfx/perf/scripts/run_diffusion_benchmark.py
+    - tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json
+
+  diffusion_wan22_reliability:
+    - *wan22
+    - tests/dfx/reliability/test_reliability_wan22.py
+
+  diffusion_image_to_video_doc:
+    - *wan22
+    - *hunyuan_video
+    - *ltx2
+    - *sana_video
+    - *lingbot_video
+    - examples/offline_inference/image_to_video/
+    - tests/examples/offline_inference/test_image_to_video.py
+
+  diffusion_cosmos3_function:
+    - *cosmos3
+    - tests/e2e/online_serving/test_cosmos3.py
+
+  diffusion_cosmos3_accuracy:
+    - *cosmos3
+    - tests/e2e/accuracy/test_cosmos3_similarity.py
+
+  diffusion_cosmos3_perf:
+    - *cosmos3
+    - tests/dfx/perf/scripts/run_diffusion_benchmark.py
+    - tests/dfx/perf/tests/test_cosmos3_vllm_omni.json
+
+  diffusion_qwen_image_function:
+    - *qwen_image
+    - tests/e2e/online_serving/test_qwen_image.py
+    - tests/e2e/online_serving/test_qwen_image_expansion.py
+    - tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py
+
+  diffusion_qwen_image_accuracy:
+    - *qwen_image
+    - tests/e2e/accuracy/test_qwen_image.py
+    - tests/e2e/accuracy/test_diffusers_backend_similarity.py
+
+  diffusion_qwen_image_perf:
+    - *qwen_image
+    - tests/dfx/perf/scripts/run_diffusion_benchmark.py
+    - tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
+
+  diffusion_text_to_image_doc:
+    - *qwen_image
+    - *z_image
+    - examples/offline_inference/text_to_image/
+    - examples/online_serving/text_to_image/
+    - tests/examples/offline_inference/test_text_to_image.py
+    - tests/examples/online_serving/test_text_to_image.py
+    - vllm_omni/diffusion/models/nextstep_1_1/
+    - vllm_omni/diffusion/models/longcat_image/
+    - vllm_omni/diffusion/models/ovis_image/
+    - vllm_omni/diffusion/models/omnigen2/
+    - vllm_omni/diffusion/models/sd3/
+    - vllm_omni/diffusion/models/flux/
+    - vllm_omni/diffusion/models/flux2/
+    - vllm_omni/diffusion/models/flux2_klein/
+    - vllm_omni/diffusion/models/hidream_image/
+    - vllm_omni/diffusion/models/krea2/
+    - vllm_omni/diffusion/models/lingbot_video/
+
+  diffusion_hunyuan_image3_function:
+    - *hunyuan_image3
+    - tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py
+
+  diffusion_hunyuan_image3_accuracy:
+    - *hunyuan_image3
+    - tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py
+
+  diffusion_hunyuan_image3_perf:
+    - *hunyuan_image3
+    - tests/dfx/perf/scripts/run_diffusion_benchmark.py
+    - tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json
+    - tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json
+    - tests/dfx/perf/tests/test_hunyuan_image_tp4.json
+
+  diffusion_hunyuan_image3_reliability:
+    - *hunyuan_image3
+    - tests/dfx/reliability/test_reliability_hunyuan_image.py
+
+  diffusion_minimax_h3_function:
+    - *minimax_h3
+    - tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py
+    - tests/e2e/online_serving/minimax_h3/
+    - tests/diffusion/models/minimax_h3/
+    - vllm_omni/diffusion/lora/
+    - vllm_omni/diffusion/offloader/
+    - vllm_omni/diffusion/distributed/
+    - vllm_omni/entrypoints/openai/
+
+  diffusion_minimax_h3_accuracy:
+    - *minimax_h3
+    - tests/e2e/accuracy/minimax_h3/
+
+  diffusion_minimax_h3_perf:
+    - *minimax_h3
+    - tests/dfx/perf/scripts/run_diffusion_benchmark.py
+    - tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json
+
+  diffusion_hunyuan_video_function:
+    - *hunyuan_video
+    - tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
+
+  diffusion_hunyuan_video_accuracy:
+    - *hunyuan_video
+    - tests/e2e/accuracy/hunyuanvideo15_t2v/
+    - tests/e2e/accuracy/hunyuanvideo15_i2v/
+
+  diffusion_hunyuan_video_perf:
+    - *hunyuan_video
+    - tests/dfx/perf/scripts/run_diffusion_benchmark.py
+    - tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
+    - tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json
+
+  diffusion_z_image_function:
+    - *z_image
+    - tests/e2e/online_serving/test_z_image.py
+    - tests/e2e/online_serving/test_zimage_expansion.py
+
+  diffusion_sana_video_function:
+    - *sana_video
+    - tests/e2e/online_serving/test_sana_video_expansion.py
+    - tests/e2e/accuracy/sana_video/
+
+  diffusion_stable_audio_function:
+    - *stable_audio
+    - tests/e2e/offline_inference/test_stable_audio_expansion.py
+    - tests/e2e/online_serving/test_stable_audio_online_expansion.py
+
+  diffusion_lingbot_function:
+    - *lingbot_video
+    - *lingbot_world
+    - tests/e2e/online_serving/test_lingbot_video.py
+    - tests/e2e/online_serving/test_lingbot_video_expansion.py
+    - tests/e2e/online_serving/test_lingbot_video_moe.py
+    - tests/e2e/offline_inference/test_lingbot_world_v2.py
+
+
+  # Invalid-param jobs: model business code only, plus the reliability scripts.
+  reliability_invalid_param_h100:
+    - *qwen3_omni
+    - *minicpmo_4_5
+    - *qwen_image
+    - *wan22
+    - tests/dfx/reliability/invalid_param_test/
+
+  reliability_invalid_param_l4:
+    - *qwen3_tts
+    - *stable_audio
+    - tests/dfx/reliability/invalid_param_test/

.buildkite/common/scripts/skip_ci.py

@@ -44,6 +44,7 @@ L3_YAML_FILES: dict[str, str] = {
 L45_YAML_FILES: dict[str, str] = {
     ".buildkite/cuda/test-nightly.yml": "cuda",
     ".buildkite/cuda/test-weekly.yml": "cuda",
+    ".buildkite/amd/test-amd-nightly.yml": "amd",
     ".buildkite/npu/test-npu-nightly.yml": "npu",
 }
 

.buildkite/common/scripts/upload_pipeline.py

@@ -10,8 +10,17 @@ Bootstrap mode (``bootstrap-upload-steps.yml``):
   - Detect docs-only, pytest skip-mark-only, or combined skip-ci from git diff.
   - When only CI level YAML changes, enable **L2/L3** upload steps for affected levels only.
 
-Test pipeline mode (e.g. test-merge.yml):
-  - Drop steps whose ``source_file_dependencies`` do not match changed files.
+Test pipeline mode (e.g. test-merge.yml, test-nightly.yml, test-weekly.yml):
+  - Drop steps whose ``source_file_dependencies`` do not match changed files
+    (string/list keys from ci_source_file_dependencies.yml, or inline path
+    prefixes). Filtering applies on **PR label** uploads and on post-merge
+    ``main`` L3 uploads. Scheduled ``main`` uploads that pass ``--all``
+    (NIGHTLY/WEEKLY) or ``--e2e`` keep every selected job and still strip
+    the field. If no job-key prefix matches, a change to the pipeline YAML
+    being uploaded or to a path under the ``source_filter_fallback`` registry
+    key keeps every job so command, env, and hardware edits can be validated
+    before merge. When any job-key prefix already matches, normal filtering
+    wins.
   - Expand uploader-only ``mirror_hardwares`` into ``agents`` (+ optional ``image``
     for NPU) + ``plugins`` (see ci_mirror_hardwares.yml).
   - Omit ``mirror_hardwares`` to compose ``{chip}_{n}`` from pytest ``-m`` SKU
@@ -87,6 +96,10 @@ BOOTSTRAP_UPLOAD_IF_KEYS = {
 }
 E2E_GROUP_MARKER = "E2E Test"
 CI_MIRROR_HARDWARES_PATH = ROOT / ".buildkite/common/ci_mirror_hardwares.yml"
+CI_SOURCE_FILE_DEPENDENCIES_PATH = ROOT / ".buildkite/common/ci_source_file_dependencies.yml"
+# Registry key whose prefixes bypass source filtering. Paths live in
+# ci_source_file_dependencies.yml; do not attach this key to a job.
+SOURCE_FILTER_FALLBACK_KEY = "source_filter_fallback"
 CUDA_HF_TOKEN_ENV = "VLLM_CI_HF_TOKEN"
 CUDA_HF_TOKEN_EXPORT = f'if [ -n "$${{{CUDA_HF_TOKEN_ENV}:-}}" ]; then export HF_TOKEN="$${{{CUDA_HF_TOKEN_ENV}}}"; fi'
 
@@ -95,14 +108,7 @@ CUDA_HF_TOKEN_EXPORT = f'if [ -n "$${{{CUDA_HF_TOKEN_ENV}:-}}" ]; then export HF
 # ``*_UPLOAD_IF``: full gate for uploading that child pipeline.
 NIGHTLY_MAIN_IF = 'build.branch == "main" && build.env("NIGHTLY") == "1"'
 NIGHTLY_LABEL_IF = (
-    f"({NIGHTLY_MAIN_IF}) || "
-    '(build.branch != "main" && ('
-    'build.pull_request.labels includes "nightly-test" || '
-    'build.pull_request.labels includes "omni-test" || '
-    'build.pull_request.labels includes "tts-test" || '
-    'build.pull_request.labels includes "diffusion-x2iat-test" || '
-    'build.pull_request.labels includes "diffusion-x2v-test"'
-    "))"
+    f'({NIGHTLY_MAIN_IF}) || (build.branch != "main" && build.pull_request.labels includes "nightly-test")'
 )
 WEEKLY_E2E_IF = 'build.branch == "main" && build.env("WEEKLY") == "1"'
 WEEKLY_MAIN_IF = 'build.branch == "main" && (build.env("WEEKLY") == "1" || build.env("NON_CRITICAL") == "1")'
@@ -248,7 +254,7 @@ def _render_bootstrap_pipeline(
     return yaml.safe_dump(doc, sort_keys=False)
 
 
-# --- Test pipeline (test-ready.yml, test-merge.yml) ---
+# --- Test pipeline (test-ready.yml, test-merge.yml, test-nightly.yml) ---
 
 
 @lru_cache(maxsize=1)
@@ -264,6 +270,117 @@ def _load_mirror_hardwares() -> dict[str, dict[str, Any]]:
     return presets
 
 
+@lru_cache(maxsize=1)
+def _load_source_file_dependencies() -> dict[str, list[str]]:
+    def flatten(value: Any, *, key: str) -> list[str]:
+        """Flatten YAML-anchor nested lists into a de-duplicated prefix list."""
+        if isinstance(value, str):
+            if not value.strip():
+                raise ValueError(f"empty path in source_file_dependencies[{key!r}]")
+            return [value]
+        if isinstance(value, list):
+            flattened: list[str] = []
+            seen: set[str] = set()
+            for item in value:
+                for path in flatten(item, key=key):
+                    if path not in seen:
+                        seen.add(path)
+                        flattened.append(path)
+            return flattened
+        raise ValueError(
+            f"source_file_dependencies[{key!r}] must be a list of path prefixes, got {type(value).__name__}",
+        )
+
+    if not CI_SOURCE_FILE_DEPENDENCIES_PATH.is_file():
+        raise FileNotFoundError(
+            f"missing CI source_file_dependencies registry: {CI_SOURCE_FILE_DEPENDENCIES_PATH}",
+        )
+    doc = yaml.safe_load(CI_SOURCE_FILE_DEPENDENCIES_PATH.read_text(encoding="utf-8"))
+    if not isinstance(doc, dict):
+        raise ValueError(f"invalid CI source_file_dependencies registry: {CI_SOURCE_FILE_DEPENDENCIES_PATH}")
+    presets = doc.get("source_file_dependencies")
+    if not isinstance(presets, dict):
+        raise ValueError(f"source_file_dependencies must be a mapping in {CI_SOURCE_FILE_DEPENDENCIES_PATH}")
+    loaded: dict[str, list[str]] = {}
+    for key, value in presets.items():
+        if not isinstance(key, str) or not key.strip():
+            raise ValueError(
+                f"source_file_dependencies keys must be non-empty strings in {CI_SOURCE_FILE_DEPENDENCIES_PATH}",
+            )
+        loaded[key] = flatten(value, key=key)
+    return loaded
+
+
+_REGISTRY_KEY_RE = re.compile(r"^[a-z][a-z0-9_]*$")
+
+
+def _parse_command_text(step: dict[str, Any]) -> str:
+    """Parse step ``commands`` / ``command`` into a single string."""
+    raw = step.get("commands", step.get("command"))
+    if raw is None:
+        return ""
+    if isinstance(raw, str):
+        return raw
+    if isinstance(raw, list):
+        return "\n".join(str(item) for item in raw if item is not None)
+    return str(raw)
+
+
+def _resolve_source_file_dependencies(step: dict[str, Any]) -> list[str] | None:
+    """Expand a registry key (or list of keys / inline path prefixes)."""
+    deps = step.get("source_file_dependencies")
+    if deps is None:
+        return None
+
+    def dedupe(*groups: list[str]) -> list[str]:
+        merged: list[str] = []
+        seen: set[str] = set()
+        for group in groups:
+            for path in group:
+                if path not in seen:
+                    seen.add(path)
+                    merged.append(path)
+        return merged
+
+    def lookup(key: str) -> list[str]:
+        if key == SOURCE_FILTER_FALLBACK_KEY:
+            raise ValueError(
+                f"source_file_dependencies {key!r} bypasses filtering and cannot be attached to a step "
+                f"({_get_step_label(step)!r})",
+            )
+        registry = _load_source_file_dependencies()
+        paths = registry.get(key)
+        if paths is None:
+            known = ", ".join(sorted(registry))
+            raise ValueError(
+                f"unknown source_file_dependencies {key!r} in step {_get_step_label(step)!r}; known: {known}",
+            )
+        return list(paths)
+
+    prefixes: list[str]
+    if isinstance(deps, str):
+        prefixes = lookup(deps) if _REGISTRY_KEY_RE.fullmatch(deps) else [deps]
+    elif isinstance(deps, list):
+        if not all(isinstance(item, str) for item in deps):
+            raise ValueError(
+                f"source_file_dependencies must be a string key or list of strings in step {_get_step_label(step)!r}",
+            )
+        keyish = [bool(_REGISTRY_KEY_RE.fullmatch(item)) for item in deps]
+        if deps and all(keyish):
+            prefixes = dedupe(*(lookup(key) for key in deps))
+        elif any(keyish):
+            raise ValueError(
+                f"source_file_dependencies in step {_get_step_label(step)!r} mixes registry keys and path prefixes",
+            )
+        else:
+            prefixes = list(deps)
+    else:
+        raise ValueError(
+            f"source_file_dependencies must be a string key or list in step {_get_step_label(step)!r}",
+        )
+    return prefixes
+
+
 @lru_cache(maxsize=1)
 def _cuda_mirror_chips() -> tuple[str, ...]:
     """Lowercase CUDA SKUs that have at least one ``{chip}_{n}`` preset.
@@ -278,7 +395,7 @@ def _cuda_mirror_chips() -> tuple[str, ...]:
     return tuple(sorted(chips, key=lambda chip: (-len(chip), chip)))
 
 
-def _read_cards_marks(expr: str) -> tuple[set[int], bool]:
+def _parse_cards_marks(expr: str) -> tuple[set[int], bool]:
     """Parse registered ``cards_n`` / ``not cards_n`` from a pytest ``-m`` expr.
 
     Returns positive card counts and whether any ``not cards_*`` is present.
@@ -295,8 +412,8 @@ def _read_cards_marks(expr: str) -> tuple[set[int], bool]:
     return positives, has_not_cards
 
 
-def _read_hardware_marks(expr: str) -> set[str]:
-    """Return lowercase positive CUDA SKUs that have a mirror preset.
+def _parse_hardware_marks(expr: str) -> set[str]:
+    """Parse lowercase positive CUDA SKUs that have a mirror preset.
 
     ``not H100`` is ignored. SKUs with no ``{chip}_*`` preset (e.g. H200) are dropped.
     """
@@ -338,36 +455,22 @@ def _get_mirror_hw_selector() -> str:
     return selector
 
 
-def _read_pytest_marks(commands: Any) -> tuple[set[str], int | Literal["max"] | None]:
-    """Read pytest ``-m`` from step commands: CUDA SKU chips and cards count.
+def _parse_pytest_marks(step: dict[str, Any]) -> tuple[set[str], int | Literal["max"] | None]:
+    """Parse pytest ``-m`` from step commands: CUDA SKU chips and cards count.
 
     *chips* are lowercase SKUs from positive markers (``not H100`` is ignored).
     *cards* is the max positive ``cards_n``, ``"max"`` when only ``not cards_*``
     is present, or ``None``.
     """
-    chunks: list[str] = []
-
-    def _collect(value: Any) -> None:
-        if value is None:
-            return
-        if isinstance(value, str):
-            chunks.append(value)
-        elif isinstance(value, list):
-            for part in value:
-                _collect(part)
-        else:
-            chunks.append(str(value))
-
-    _collect(commands)
-    text = "\n".join(chunks)
+    text = _parse_command_text(step)
 
     chips: set[str] = set()
     positives: set[int] = set()
     has_not_cards = False
     for match in _PYTEST_MARKER_ARG.finditer(text):
         expr = match.group(1) or match.group(2) or match.group(3) or ""
-        chips |= _read_hardware_marks(expr)
-        card_counts, not_cards = _read_cards_marks(expr)
+        chips |= _parse_hardware_marks(expr)
+        card_counts, not_cards = _parse_cards_marks(expr)
         positives.update(card_counts)
         has_not_cards = has_not_cards or not_cards
 
@@ -484,7 +587,7 @@ def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any] | None:
     if "mirror_hardwares" not in step:
         if has_pool:
             return step
-        chips, cards = _read_pytest_marks(step.get("commands"))
+        chips, cards = _parse_pytest_marks(step)
         if not chips and cards is None:
             return step
         preset_name = _compose_mirror_hardware_name(chips, cards, step_label=step_label)
@@ -552,11 +655,7 @@ def _process_test_steps(
             processed.append(step)
             continue
 
-        deps = step.get("source_file_dependencies")
-        if deps is not None and not isinstance(deps, list):
-            raise ValueError(
-                f"source_file_dependencies must be a list in step {_get_step_label(step)!r}",
-            )
+        deps = _resolve_source_file_dependencies(step)
         if changed_files is not None and deps is not None and not _match_source_file(changed_files, deps):
             _log(f"skip {_get_step_label(step)!r} (no changes under {deps})")
             continue
@@ -595,10 +694,25 @@ def _select_e2e_group_steps(steps: list[Any]) -> list[Any]:
     return selected
 
 
+def _any_source_dependency_match(steps: list[Any], changed_files: list[str]) -> bool:
+    """True when any step with ``source_file_dependencies`` matches *changed_files*."""
+    for step in steps:
+        if not isinstance(step, dict):
+            continue
+        deps = _resolve_source_file_dependencies(step)
+        if deps is not None and _match_source_file(changed_files, deps):
+            return True
+        nested = step.get("steps")
+        if isinstance(nested, list) and _any_source_dependency_match(nested, changed_files):
+            return True
+    return False
+
+
 def _render_test_pipeline(
     doc: dict[str, Any],
     changed_files: list[str] | None,
     *,
+    pipeline_path: Path | None = None,
     e2e_only: bool = False,
 ) -> dict[str, Any]:
     """Filter steps by PR diff and strip uploader-only ``source_file_dependencies`` metadata."""
@@ -611,6 +725,13 @@ def _render_test_pipeline(
         return doc
     if e2e_only:
         steps = _select_e2e_group_steps(steps)
+    # Bypass is a fallback: only when no job-key prefix matched.
+    if changed_files is not None and pipeline_path is not None:
+        if not _any_source_dependency_match(steps, changed_files):
+            bypass = _source_filter_fallback_reason(changed_files, pipeline_path)
+            if bypass is not None:
+                _log(f"keep all jobs (no source_file_dependencies match; bypassed by {bypass})")
+                changed_files = None
     steps = _process_test_steps(steps, changed_files)
     return {**doc, "steps": steps}
 
@@ -618,6 +739,60 @@ def _render_test_pipeline(
 # --- Entry (read file → bootstrap or test render → YAML string) ---
 
 
+def _repo_relative_posix(path: Path) -> str:
+    resolved = path.resolve()
+    try:
+        return resolved.relative_to(ROOT).as_posix()
+    except ValueError:
+        return path.as_posix().replace("\\", "/")
+
+
+def _source_filter_fallback_prefixes() -> list[str]:
+    """Shared uploader/registry prefixes from ``source_filter_fallback``. Not a job key."""
+    registry = _load_source_file_dependencies()
+    prefixes = registry.get(SOURCE_FILTER_FALLBACK_KEY)
+    if not prefixes:
+        raise ValueError(
+            f"source_file_dependencies[{SOURCE_FILTER_FALLBACK_KEY!r}] must list shared uploader paths "
+            f"in {CI_SOURCE_FILE_DEPENDENCIES_PATH}",
+        )
+    return prefixes
+
+
+def _source_filter_fallback_reason(changed_files: list[str] | None, pipeline_path: Path) -> str | None:
+    """Return the changed path that should disable source filtering, if any."""
+    if not changed_files:
+        return None
+    changed = set(changed_files)
+    pipeline_rel = _repo_relative_posix(pipeline_path)
+    if pipeline_rel in changed:
+        return pipeline_rel
+    prefixes = _source_filter_fallback_prefixes()
+    for path in changed_files:
+        if _match_source_file([path], prefixes):
+            return path
+    return None
+
+
+def _changed_files_for_source_filter(
+    ctx,
+    *,
+    force_all: bool,
+    e2e_only: bool,
+) -> list[str] | None:
+    """Return the diff to filter against, or None to keep every step.
+
+    ``--all`` / ``--e2e`` disable filtering (scheduled NIGHTLY/WEEKLY full
+    uploads and weekly E2E sweeps). Post-merge ``main`` L3 and PR-label
+    uploads filter against the commit/PR diff. Pipeline YAML /
+    ``source_filter_fallback`` matches are applied later in
+    ``_render_test_pipeline`` only when no job-key prefix already matched.
+    """
+    if force_all or e2e_only:
+        return None
+    return ctx.changed_files
+
+
 def _render_pipeline(
     path: Path,
     *,
@@ -635,13 +810,18 @@ def _render_pipeline(
 
     text = path.read_text(encoding="utf-8")
     ctx = resolve_ci_context_from_git()
-    changed_files = None if force_all or e2e_only else ctx.changed_files
+    changed_files = _changed_files_for_source_filter(ctx, force_all=force_all, e2e_only=e2e_only)
 
     doc = yaml.safe_load(text)
     if not isinstance(doc, dict):
         raise ValueError(f"invalid pipeline YAML: {path}")
 
-    doc = _render_test_pipeline(doc, changed_files, e2e_only=e2e_only)
+    doc = _render_test_pipeline(
+        doc,
+        changed_files,
+        pipeline_path=path,
+        e2e_only=e2e_only,
+    )
     return yaml.safe_dump(doc, sort_keys=False)
 
 

.buildkite/cuda/bootstrap-upload-steps.yml

@@ -22,7 +22,8 @@ steps:
     agents:
       queue: "cpu_queue_premerge"
 
-  # L3 Test — PR merge-test label (diff-aware) or main+WEEKLY=1 (--e2e, full E2E group)
+  # L3 Test — PR merge-test label or post-merge on main (both diff-aware);
+  # main+WEEKLY=1 uploads the E2E group only (--e2e).
   - label: "Upload Merge Pipeline"
     depends_on: image-build
     key: upload-merge-pipeline
@@ -36,20 +37,30 @@ steps:
     agents:
       queue: "cpu_queue_premerge"
 
-  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
+  # L4 Test — main+NIGHTLY=1 (full pipeline), or PR nightly-test label (diff-aware)
   - label: "Upload Nightly Pipeline"
     depends_on: image-build
     key: upload-nightly-pipeline
     commands:
-      - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-nightly.yml
+      - |
+        EXTRA=""
+        if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
+          EXTRA="--all"
+        fi
+        python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-nightly.yml
     agents:
       queue: "cpu_queue_premerge"
 
-  # L5 Test — main+WEEKLY=1 / NON_CRITICAL=1 (scheduled), or PR with weekly-test label
+  # L5 Test — main+WEEKLY=1 / NON_CRITICAL=1 (full pipeline), or PR weekly-test label (diff-aware)
   - label: "Upload Weekly Pipeline"
     depends_on: image-build
     key: upload-weekly-pipeline
     commands:
-      - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-weekly.yml
+      - |
+        EXTRA=""
+        if [ "${BUILDKITE_BRANCH}" = "main" ]; then
+          EXTRA="--all"
+        fi
+        python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-weekly.yml
     agents:
       queue: "cpu_queue_premerge"

.buildkite/cuda/test-merge.yml

@@ -1,5 +1,6 @@
 # CUDA CI L3 (`advanced_model` e2e) plus L1 CPU Simple Test (`core_model and cpu`).
-# Uploaded by Upload Merge Pipeline: PR label `merge-test`, or post-merge on main.
+# Uploaded by Upload Merge Pipeline: PR label `merge-test`, or post-merge on main
+# (both diff-aware via source_file_dependencies).
 
 env:
   VLLM_WORKER_MULTIPROC_METHOD: spawn
@@ -65,7 +66,9 @@ steps:
         # 0.26 startup regression is understood and fixed.
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"
+          # Turn-based server VAD over /v1/realtime is deferred to RFC #7181 PR 5
+          # (see the P0 thread on #7413); the test stays in the tree and runs again there.
+          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model" --deselect tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py::TestQwen3OmniRealtimeWebSocket::test_server_vad_multi_turn_without_client_commit
         mirror_hardwares: h100_2
 
       - label: "Entrypoint Test with L4"
@@ -78,21 +81,7 @@ steps:
     depends_on: upload-merge-pipeline
     steps:
       - label: "TTS · Qwen3-TTS CustomVoice Test"
-        source_file_dependencies:
-          - tests/e2e/features/prefill_phase/
-          - vllm_omni/worker/gpu_model_runner.py
-          - tests/helpers/media.py
-          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
-          - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
-          - vllm_omni/model_executor/models/qwen3_tts/
-          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-          - vllm_omni/model_executor/models/common/snake_activation.py
-          - vllm_omni/model_executor/models/common/alias_free_activation.py
-          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
-          - vllm_omni/deploy/qwen3_tts.yaml
-          - vllm_omni/entrypoints/openai/serving_speech.py
-          - vllm_omni/entrypoints/openai/tts_adapters/
-          - vllm_omni/engine/stage_pool.py
+        source_file_dependencies: tts_qwen3_tts_function
         commands:
           - |
             timeout 40m bash -c "
@@ -103,21 +92,7 @@ steps:
         mirror_hardwares: l4_1
 
       - label: "TTS · Qwen3-TTS Base Test"
-        source_file_dependencies:
-          - tests/helpers/media.py
-          - tests/e2e/online_serving/test_qwen3_tts_base.py
-          - tests/e2e/offline_inference/test_qwen3_tts_base.py
-          - vllm_omni/model_executor/models/qwen3_tts/
-          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-          - vllm_omni/model_executor/models/common/snake_activation.py
-          - vllm_omni/model_executor/models/common/alias_free_activation.py
-          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
-          - vllm_omni/deploy/qwen3_tts.yaml
-          - vllm_omni/entrypoints/openai/serving_speech.py
-          - vllm_omni/entrypoints/openai/tts_adapters/
-          - vllm_omni/engine/stage_pool.py
-          - .buildkite/common/scripts/run_cov_split.sh
-          - pyproject.toml
+        source_file_dependencies: tts_qwen3_tts_cov
         # 40 -> 60: the coverage split runs the model twice.
         timeout_in_minutes: 60
         commands:
@@ -132,36 +107,14 @@ steps:
         mirror_hardwares: l4_1
 
       - label: "Omni · Qwen3-Omni Test"
-        source_file_dependencies:
-          - tests/e2e/offline_inference/test_qwen3_omni.py
-          - tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py
-          - tests/e2e/online_serving/test_qwen3_omni.py
-          - tests/helpers/runtime.py
-          - vllm_omni/entrypoints/openai/serving_chat.py
-          - vllm_omni/model_executor/models/qwen3_omni/
-          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-          - vllm_omni/model_executor/models/common/snake_activation.py
-          - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
-          - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
-          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
-          - vllm_omni/deploy/qwen3_omni_moe.yaml
-          - vllm_omni/deploy/qwen3_omni_moe_thinking.yaml
+        source_file_dependencies: omni_qwen3_omni_function
         timeout_in_minutes: 50
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py -m 'advanced_model and cuda' --run-level 'advanced_model'
         mirror_hardwares: h100_2
 
       - label: "Omni · MiniCPM-o 4.5 Test"
-        source_file_dependencies:
-          - tests/e2e/offline_inference/test_minicpmo_4_5.py
-          - tests/e2e/online_serving/test_minicpmo_4_5.py
-          - vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/models/step_audio2/
-          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
-          - vllm_omni/model_executor/models/cosyvoice3/utils.py
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
-          - vllm_omni/deploy/minicpmo_4_5.yaml
+        source_file_dependencies: omni_minicpmo_4_5_function
         timeout_in_minutes: 50
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'
@@ -174,59 +127,26 @@ steps:
         mirror_hardwares: h100_2
 
       - label: "Omni · MiniCPM-o 4.5 Duplex Test"
-        source_file_dependencies:
-          - tests/assets/minicpmo_4_5/response_required_16k.wav
-          - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
-          - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
-          - tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
-          - vllm_omni/deploy/minicpmo_4_5.yaml
-          - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
-          - vllm_omni/engine/duplex/
-          - vllm_omni/entrypoints/duplex/
-          # Public duplex client library + its unit suites: client-side
-          # changes must re-run the duplex e2e group. The live client probe
-          # (test_duplex_client_live.py, single GPU, core tier) runs in the
-          # ready pipeline's duplex group.
-          - vllm_omni/clients/
-          - tests/clients/
-          - vllm_omni/entrypoints/duplex_request_client.py
-          - vllm_omni/outputs/duplex.py
-          - vllm_omni/model_executor/duplex_sampling.py
-          - tests/engine/duplex/
-          - tests/entrypoints/duplex/
-          - tests/model_executor/models/minicpmo_4_5/duplex/
-          # Legacy Realtime probe client + video stacking used by the e2e drivers.
-          - vllm_omni/experimental/fullduplex/
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/models/step_audio2/
-          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
-          - vllm_omni/model_executor/models/cosyvoice3/utils.py
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
-          - vllm_omni/core/sched/
-          - vllm_omni/outputs/multimodal_accumulation.py
-          - vllm_omni/engine/stage_init_utils.py
-          - vllm_omni/engine/async_omni_engine.py
-          - vllm_omni/engine/orchestrator.py
-          - vllm_omni/entrypoints/cli/serve.py
-          - vllm_omni/entrypoints/openai/api_server.py
-          - vllm_omni/entrypoints/openai/duplex_capability.py
-          - vllm_omni/config/stage_config.py
+        source_file_dependencies: omni_minicpmo_4_5_duplex_function
         timeout_in_minutes: 50
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py tests/e2e/online_serving/test_minicpmo_4_5_window.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Qwen Image 2.1 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen_image_21.py
+          - tests/helpers/
+          - vllm_omni/diffusion/models/qwen_image_21/
+          - vllm_omni/diffusion/distributed/autoencoders/
+          - vllm_omni/diffusion/sched/
+          - vllm_omni/entrypoints/openai/
+        commands:
+          - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen_image_21.py -m 'advanced_model and cuda' --run-level 'advanced_model'
         mirror_hardwares: h100_1
 
       - label: "Diffusion · Qwen Image Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen_image.py
-          - vllm_omni/diffusion/models/qwen_image/
-          - vllm_omni/diffusion/models/dmd2/
-          - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py
+        source_file_dependencies: diffusion_qwen_image_function
         commands:
           - |
             timeout 40m bash -c '
@@ -236,11 +156,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Diffusion · Wan22 Test"
-        source_file_dependencies:
-          - tests/e2e/offline_inference/test_wan22_t2v.py
-          - tests/e2e/online_serving/test_wan22_t2v.py
-          - vllm_omni/diffusion/models/wan2_2/
-          - vllm_omni/diffusion/models/dmd2/
+        source_file_dependencies: diffusion_wan22_function
         commands:
           - |
             timeout 40m bash -c "
@@ -250,9 +166,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Diffusion · HunyuanVideo-1.5 Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
-          - vllm_omni/diffusion/models/hunyuan_video/
+        source_file_dependencies: diffusion_hunyuan_video_function
         commands:
           - |
             timeout 40m bash -c "
@@ -261,18 +175,18 @@ steps:
             "
         mirror_hardwares: h100_1
 
+      - label: "Diffusion · LingBot-Video Test"
+        source_file_dependencies: diffusion_lingbot_function
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_lingbot_video.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+            "
+        mirror_hardwares: h100_1
+
       - label: "Diffusion · MiniMax-H3 L3 DLO DP2 and FastH3"
-        source_file_dependencies:
-          - tests/e2e/online_serving/minimax_h3/
-          - tests/helpers/assertions.py
-          - tests/helpers/mark.py
-          - tests/helpers/media.py
-          - tests/helpers/runtime.py
-          - vllm_omni/diffusion/models/minimax_h3/
-          - vllm_omni/diffusion/lora/
-          - vllm_omni/diffusion/offloader/
-          - vllm_omni/diffusion/distributed/
-          - vllm_omni/entrypoints/openai/
+        source_file_dependencies: diffusion_minimax_h3_function
         # Keep four server lifecycles in one Buildkite job, but never in one
         # pytest process. DLO uses multiprocessing shared-memory queues; the
         # parent's resource_tracker can retain a stale /psm_* entry during

.buildkite/cuda/test-nightly.yml

@@ -12,138 +12,145 @@ steps:
   - group: ":card_index_dividers: Omni Model Test"
     key: nightly-omni-test-group
     depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     steps:
 
       - label: ":full_moon: Omni · Function Test with H100 · Single-GPU"
+        source_file_dependencies:
+          - omni_qwen3_omni_function
+          - omni_minicpmo_4_5_function
         timeout_in_minutes: 120
         commands:
           - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
 
       - label: ":full_moon: Omni · Function Test with H100 · 2-GPU"
+        source_file_dependencies:
+          - omni_qwen3_omni_function
+          - omni_minicpmo_4_5_function
         timeout_in_minutes: 90
         commands:
           - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_2" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
 
       - label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
+        source_file_dependencies: omni_minicpmo_4_5_duplex_function
         timeout_in_minutes: 50
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni and cards_1" --run-level "full_model"
-        mirror_hardwares: h100_1
+          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and B200 and omni and cards_1" --run-level "full_model"
 
       - label: ":full_moon: Omni · Doc Test with L4 · 4-GPU"
+        source_file_dependencies: omni_qwen2_5_omni_doc
         timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          - pytest -s -v tests/examples/ -m "full_model and omni and L4 and cards_4" --run-level "full_model"
-        mirror_hardwares: l4_4
+          - pytest -s -v tests/examples/ -m "full_model and omni and L4 and B200 and cards_4" --run-level "full_model"
 
       - label: ":full_moon: Omni · Doc Test with H100 · 2-GPU"
+        source_file_dependencies: omni_qwen3_omni_doc
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/examples/ -m "full_model and omni and H100 and cards_2" --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/examples/ -m "full_model and omni and H100 and B200 and cards_2" --run-level "full_model"
 
       - label: ":full_moon: Omni · Accuracy Test"
+        source_file_dependencies: omni_qwen3_omni_accuracy
         timeout_in_minutes: 180
         commands:
           - export SEED_TTS_WER_EVAL=1
           - export SEED_TTS_EVAL_DEVICE=cuda:1
           - |
             set +e
-            pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model" --run-level full_model
+            pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model and H100 and B200 and cards_2" --run-level full_model
             EXIT=$$?
             buildkite-agent artifact upload "tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_2
 
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
+        source_file_dependencies: omni_minicpmo_4_5_accuracy
         timeout_in_minutes: 180
         commands:
           - export SEED_TTS_WER_EVAL=1
           - export SEED_TTS_EVAL_DEVICE=cuda:1
           - |
             set +e
-            pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and cuda" --run-level full_model
+            pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level full_model
             EXIT=$$?
             buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_2
 
       - label: ":full_moon: Omni · Perf Test · No Async Chunk"
+        source_file_dependencies: omni_qwen3_omni_perf
         key: nightly-omni-performance-no-async-chunk
         timeout_in_minutes: 300
         commands:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json -m "H100 and B200 and cards_2"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_2
 
       - label: ":full_moon: Omni · Perf Test · Async Chunk"
+        source_file_dependencies: omni_qwen3_omni_perf
         key: nightly-omni-performance-async-chunk
         timeout_in_minutes: 180
         commands:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and full_model"
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and B200 and full_model and cards_2"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_2
 
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"
+        source_file_dependencies: omni_minicpmo_4_5_perf
         key: nightly-omni-performance-minicpmo-4-5
         timeout_in_minutes: 180
         commands:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json -m "H100 and B200 and cards_1"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_1
 
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test"
+        source_file_dependencies: omni_minicpmo_4_5_duplex_perf
         key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts
         timeout_in_minutes: 180
         commands:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json -m "H100 and B200 and cards_1"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_1
 
       - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100"
+        source_file_dependencies: omni_qwen3_omni_function
         timeout_in_minutes: 45
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model" --run-level "core_model"
-        mirror_hardwares: h100_4
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model and H100 and B200 and cards_4" --run-level "core_model"
 
   - group: ":card_index_dividers: TTS Model Test"
     key: nightly-tts-test-group
     depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     steps:
 
       - label: ":full_moon: TTS · Function Test with L4"
+        source_file_dependencies: tts_qwen3_tts_function
         timeout_in_minutes: 120
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
           - export VLLM_USE_DEEP_GEMM="0"
           - export VLLM_MOE_USE_DEEP_GEMM="0"
-          - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
-        mirror_hardwares: l4_1
+          - pytest -s -v tests/e2e/ -m "full_model and L4 and B200 and tts and cards_1" --run-level "full_model" --ignore=tests/e2e/accuracy
 
       - label: ":full_moon: TTS · Perf Test · Single-GPU"
+        source_file_dependencies: tts_qwen3_tts_perf
         key: nightly-tts-performance-single
         timeout_in_minutes: 180
         commands:
@@ -151,13 +158,13 @@ steps:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and cards_1"
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and B200 and cards_1"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_1
 
       - label: ":full_moon: TTS · Perf Test · 2-GPU"
+        source_file_dependencies: tts_qwen3_tts_perf
         key: nightly-tts-performance-2gpu
         timeout_in_minutes: 180
         commands:
@@ -165,11 +172,10 @@ steps:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and cards_2"
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and B200 and cards_2"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_2
 
   - group: ":card_index_dividers: Tiny Model Tests [Multi-GPU]"
     key: nightly-tiny-model-test-group
@@ -180,6 +186,7 @@ steps:
     steps:
 
       - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 2-GPU"
+        source_file_dependencies: diffusion_tiny_model
         timeout_in_minutes: 30
         # NOTE: Single-GPU (core_model) tiny model tests run on the ready label,
         # so nightly only adds the multi-GPU parallelism configurations.
@@ -189,91 +196,85 @@ steps:
         # for the CI, since these tests should pass irrespective of whether or not we use
         # the full weights or not.
         commands:
-          - pytest -sv tests/model_tests/diffusion -m 'full_model and cuda and cards_2' --run-level "core_model"
-        mirror_hardwares: l4_2
+          - pytest -sv tests/model_tests/diffusion -m 'full_model and L4 and B200 and cuda and cards_2' --run-level "core_model"
 
       - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 4-GPU"
+        source_file_dependencies: diffusion_tiny_model
         timeout_in_minutes: 30
         # CFG+TP extra_test_groups request 2×2=4 devices (see get_required_device_count).
         commands:
-          - pytest -sv tests/model_tests/diffusion -m 'full_model and cuda and cards_4' --run-level "core_model"
-        mirror_hardwares: l4_4
+          - pytest -sv tests/model_tests/diffusion -m 'full_model and L4 and B200 and cuda and cards_4' --run-level "core_model"
 
   - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
     key: nightly-diffusion-x2iat-group
     depends_on: upload-nightly-pipeline
     if: >-
       build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2iat-test"
+      build.pull_request.labels includes "nightly-test"
     steps:
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU"
+        source_file_dependencies: diffusion_qwen_image_function
         timeout_in_minutes: 120
         commands:
           - >-
             pytest -sv
             tests/e2e/online_serving/test_qwen_image_expansion.py
+            -m "full_model and diffusion and H100 and B200 and cards_1"
+            --run-level "full_model"
+
+      - label: ":robot_face: Robot Policy OpenPI · π0.5 · H100 · Single-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - pip install --no-deps "openpi-client @ git+https://github.com/Physical-Intelligence/openpi.git@215abfb217dbac7d5f1273282331b9b1866c0479#subdirectory=packages/openpi-client"
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_pi05_expansion.py
             -m "full_model and diffusion and H100 and cards_1"
             --run-level "full_model"
         mirror_hardwares: h100_1
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU"
+        source_file_dependencies: diffusion_qwen_image_function
         timeout_in_minutes: 120
         commands:
           - >-
             pytest -sv
             tests/e2e/online_serving/test_qwen_image_expansion.py
-            -m "full_model and diffusion and H100 and cards_2"
+            -m "full_model and diffusion and H100 and B200 and cards_2"
             --run-level "full_model"
-        mirror_hardwares: h100_2
 
-      - label: ":full_moon: Diffusion X2I(&A&T) · Boogu Edit CFG · 3-GPU"
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · Single-GPU"
+        source_file_dependencies: diffusion_qwen_image_function
         timeout_in_minutes: 120
         commands:
-          - >-
-            pytest -sv
-            tests/e2e/online_serving/test_boogu_image_edit_expansion.py
-            -k "test_double_guidance_cfg_parallel and cfg_parallel_3_full"
-            -m "slow and diffusion and cuda and H100 and cards_3"
-            --run-level "full_model"
-        mirror_hardwares: h100_3
+          - pytest -sv tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py -m "full_model and diffusion and L4 and B200 and cards_1" --run-level "full_model"
 
-      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · 4-GPU"
-        timeout_in_minutes: 120
-        commands:
-          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4 and cards_4" --run-level "full_model" --ignore=tests/e2e/accuracy
-        mirror_hardwares: l4_4
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
+        source_file_dependencies: diffusion_text_to_image_doc
         timeout_in_minutes: 60
         commands:
-          - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100 and B200 and (cards_1 or cards_2)" --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test"
+        source_file_dependencies: diffusion_qwen_image_accuracy
         timeout_in_minutes: 180
         commands:
           - export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=120
-          - pytest -s -v tests/e2e/accuracy/test_qwen_image.py --run-level full_model
-          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model
-        mirror_hardwares: h100_1
+          - pytest -s -v tests/e2e/accuracy/test_qwen_image.py -m "H100 and B200 and cards_1" --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m "full_model and H100 and B200 and cards_1" --run-level full_model
 
       - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
+        source_file_dependencies: diffusion_hunyuan_image3_accuracy
         timeout_in_minutes: 180
         commands:
           - export HUNYUAN_IMAGE3_MODEL="tencent/HunyuanImage-3.0-Instruct"
           - export HUNYUAN_IMAGE3_DEVICES="0,1,2,3"
-          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m full_model --run-level full_model
-        mirror_hardwares: h100_4
-
-      - label: ":full_moon: HiDream-O1-Image · Accuracy Test"
-        timeout_in_minutes: 180
-        commands:
-          - pytest -s -v tests/e2e/accuracy/test_hidream_o1_image.py -m full_model --run-level full_model
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m "full_model and H100 and B200 and cards_4" --run-level full_model
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · Single-GPU"
+        source_file_dependencies: diffusion_qwen_image_perf
         key: nightly-diffusion-x2iat-performance-qwen-image-single
         timeout_in_minutes: 180
         commands:
@@ -281,168 +282,182 @@ steps:
           - export CACHE_DIT_VERSION=1.5.0
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and cards_1"
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and B200 and cards_1"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
-        mirror_hardwares: h100_1
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · 4-GPU"
+        source_file_dependencies: diffusion_qwen_image_perf
         key: nightly-diffusion-x2iat-performance-qwen-image-4gpu
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
           - export CACHE_DIT_VERSION=1.5.0
+          # Do not pin DIFFUSION_ATTENTION_BACKEND: H100 auto-selects FLASH_ATTN;
+          # B200 rejects explicit FLASH_ATTN without FA4 and uses CUDNN/TRTLLM.
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and cards_4"
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and B200 and cards_4"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
-        mirror_hardwares: h100_4
 
   - group: ":card_index_dividers: Diffusion X2V Model Test"
     key: nightly-diffusion-x2v-group
     depends_on: upload-nightly-pipeline
     if: >-
       build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2v-test"
+      build.pull_request.labels includes "nightly-test"
     steps:
 
       - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · Single-GPU"
+        source_file_dependencies: diffusion_wan22_function
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and cards_1" --run-level "full_model"
-        mirror_hardwares: h100_1
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and B200 and cards_1" --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · 2-GPU"
+        source_file_dependencies: diffusion_wan22_function
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and cards_2" --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and B200 and cards_2" --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · Single-GPU"
+        source_file_dependencies: diffusion_wan22_function
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and cards_1" --run-level "full_model"
-        mirror_hardwares: h100_1
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and B200 and cards_1" --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · 2-GPU"
+        source_file_dependencies: diffusion_wan22_function
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and cards_2" --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and B200 and cards_2" --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test"
+        source_file_dependencies: diffusion_hunyuan_video_function
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and cuda" --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level "full_model"
 
-      - label: ":full_moon: Diffusion X2V · MAGI-2 Preview Serving Smoke"
-        timeout_in_minutes: 180
+      - label: ":full_moon: Diffusion X2V · LingBot Function Test"
+        source_file_dependencies: diffusion_lingbot_function
+        timeout_in_minutes: 120
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_magi2.py -m "advanced_model and diffusion and cuda and H100 and cards_4" --run-level "advanced_model"
-        mirror_hardwares: h100_4
+          - >-
+            pytest -s -v
+            tests/e2e/online_serving/test_lingbot_video_expansion.py
+            tests/e2e/online_serving/test_lingbot_video_moe.py
+            tests/e2e/offline_inference/test_lingbot_world_v2.py
+            -m "full_model and diffusion and H100 and B200 and cards_1"
+            --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2V · Doc Test"
+        source_file_dependencies: diffusion_image_to_video_doc
         timeout_in_minutes: 60
         commands:
-          - pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m "full_model and example and H100" --run-level "full_model"
-        mirror_hardwares: h100_1
+          - pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m "full_model and example and H100 and B200 and cards_1" --run-level "full_model"
 
-      - label: ":full_moon: Diffusion X2V · Other Function Test"
+      - label: ":full_moon: Diffusion X2V · Wan2.2 Autoround Function Test"
+        source_file_dependencies: diffusion_wan22_function
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level "full_model"
+
+      - label: ":full_moon: Diffusion X2V · Wan2.2 Device-Postprocess Equivalence Test"
+        source_file_dependencies: diffusion_wan22_function
+        timeout_in_minutes: 90
+        commands:
+          - |
+            VLLM_OMNI_DEVICE_POSTPROCESS_MODEL=Wan-AI/Wan2.2-TI2V-5B-Diffusers \
+              pytest -s -v tests/e2e/offline_inference/test_device_postprocess_equivalence.py -m "full_model and H100 and B200 and cuda and cards_1" --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test"
+        source_file_dependencies: diffusion_wan22_accuracy
         timeout_in_minutes: 180
         commands:
-          - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model
-          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m "full_model and H100 and B200 and (cards_1 or cards_2)" --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m "full_model and H100 and B200 and cards_1" --run-level full_model
 
       - label: ":full_moon: Diffusion X2V · MiniMax H3 I2VA/Ref2VA Accuracy Test"
+        source_file_dependencies: diffusion_minimax_h3_accuracy
         timeout_in_minutes: 180
         commands:
-          - pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py -m full_model --run-level full_model
-        mirror_hardwares: h100_4
+          - pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py -m "full_model and H100 and B200 and cards_4" --run-level full_model
 
-      - label: ":full_moon: Diffusion X2V · Other Accuracy Test"
+      - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Accuracy Test"
+        source_file_dependencies: diffusion_hunyuan_video_accuracy
         timeout_in_minutes: 180
         commands:
-          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model
-          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m "full_model and H100 and B200 and (cards_1 or cards_2)" --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m "full_model and H100 and B200 and (cards_1 or cards_2)" --run-level full_model
 
       - label: ":full_moon: Diffusion X2V · Perf Test · Single-GPU"
+        source_file_dependencies: diffusion_wan22_perf
         key: nightly-diffusion-x2v-performance-single
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_1"
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and B200 and cards_1"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
-        mirror_hardwares: h100_1
 
       - label: ":full_moon: Diffusion X2V · Perf Test · 2-GPU"
+        source_file_dependencies:
+          - diffusion_wan22_perf
+          - diffusion_cosmos3_perf
         key: nightly-diffusion-x2v-performance-2gpu
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_2"
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and B200 and cards_2"
             EXIT1=$$?
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json -m "H100 and cards_2"
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json -m "H100 and B200 and cards_2"
             EXIT2=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi
             exit $$EXIT2
-        mirror_hardwares: h100_2
 
       - label: ":full_moon: Diffusion X2V · Perf Test · HunyuanVideo-1.5"
+        source_file_dependencies: diffusion_hunyuan_video_perf
         key: nightly-diffusion-x2v-performance-hunyuanvideo15
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json -m "H100 and B200 and (cards_1 or cards_2)"
             EXIT1=$$?
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json -m "H100 and B200 and (cards_1 or cards_2)"
             EXIT2=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi
             exit $$EXIT2
-        mirror_hardwares: h100_2
 
       - label: ":full_moon: Diffusion X2V · Perf Test · MiniMax-H3"
+        source_file_dependencies: diffusion_minimax_h3_perf
         key: nightly-diffusion-x2v-performance-minimax-h3
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json -m "H100 and B200 and cards_4"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
-        mirror_hardwares: h100_4
 
   - group: ":card_index_dividers: Diffusion Test"
     key: nightly-diffusion-test-group
@@ -450,59 +465,59 @@ steps:
     if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     steps:
       - label: ":full_moon: Diffusion · Distributed Test with L4 · 2-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_2' --run-level "full_model"
-        mirror_hardwares: l4_2
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_2' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · Distributed Test with L4 · 3-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_3' --run-level "full_model"
-        mirror_hardwares: l4_3
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_3' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · Distributed Test with L4 · 4-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_4' --run-level "full_model"
-        mirror_hardwares: l4_4
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_4' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · Distributed Test with H100 · 2-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and cards_2' --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and B200 and cards_2' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · Distributed Test with H100 · 4-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and cards_4' --run-level "full_model"
-        mirror_hardwares: h100_4
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and B200 and cards_4' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · Offloader Test"
+        source_file_dependencies: diffusion_offloader
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/offloader -m "full_model and cuda"
-        mirror_hardwares: l4_1
+          - pytest -sv tests/diffusion/offloader -m "full_model and L4 and B200 and cuda and cards_1"
 
       - label: ":full_moon: Diffusion · Quantization Test with H100 · Single-GPU"
+        source_file_dependencies: diffusion_quantization
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model"
-        mirror_hardwares: h100_1
+          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100 and B200 and cards_1' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · MiniMax H3 FP8 Quality Test with H100 · 2-GPU"
+        source_file_dependencies: diffusion_minimax_h3_function
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py -m 'full_model and cuda and H100' --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -sv tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py -m 'full_model and cuda and H100 and B200 and cards_2' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · Quantization Test with L4"
+        source_file_dependencies: diffusion_quantization
         timeout_in_minutes: 60
         commands:
           - pip install "vllm-gguf-plugin==0.0.4"
-          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model"
-        mirror_hardwares: l4_1
+          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4 and B200 and cards_1' --run-level "full_model"
 
   # NOTE: ":email: Nightly Collection & Email" is deprecated and will be removed soon.
   # Do not add new depends_on entries or extend this step.

.buildkite/cuda/test-ready.yml

@@ -38,6 +38,12 @@ steps:
     depends_on: upload-ready-pipeline
     steps:
 
+      - label: "Diffusion · FlashAttention Test"
+        timeout_in_minutes: 20
+        commands:
+          - timeout 15m pytest -s -v tests/diffusion/attention/ -m 'core_model and cuda and L4 and cards_1' --run-level "core_model"
+        mirror_hardwares: l4_1
+
       - label: "Diffusion · Offloader Test"
         commands:
           - timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
@@ -54,16 +60,19 @@ steps:
         mirror_hardwares: l4_1
 
       - label: "Diffusion · Distributed Test · Single-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         commands:
           - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_1' --run-level "core_model"
         mirror_hardwares: l4_1
 
       - label: "Diffusion · Distributed Test · 2-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         commands:
           - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_2' --run-level "core_model"
         mirror_hardwares: l4_2
 
       - label: "Diffusion · Distributed & Attention Test · 4-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         commands:
           - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_4' --run-level "core_model"
           - timeout 20m pytest -sv tests/diffusion/attention/test_attention_sp.py -m 'core_model and cuda and L4 and cards_4' --run-level "core_model"
@@ -93,7 +102,7 @@ steps:
     commands:
       - |
         timeout 15m bash -c "
-                  pytest -sv tests/engine/ tests/model_executor/ tests/worker/test_batched_omni_output.py -m 'core_model and cuda' --run-level 'core_model'
+                  pytest -sv tests/engine/ tests/model_executor/ tests/worker/test_batched_omni_output.py tests/worker_v2/ -m 'core_model and cuda' --run-level 'core_model'
         "
     mirror_hardwares: l4_1
 
@@ -120,29 +129,14 @@ steps:
     depends_on: upload-ready-pipeline
     steps:
       - label: "Custom Pipeline Test"
-        source_file_dependencies:
-          - tests/e2e/features/custom_pipeline/
-          - tests/e2e/features/helpers/custom_pipeline.py
-          - vllm_omni/diffusion/
-          - vllm_omni/engine/
-          - vllm_omni/entrypoints/
-          - vllm_omni/inputs/
-          - vllm_omni/outputs/
+        source_file_dependencies: custom_pipeline
         commands:
           - timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"
         mirror_hardwares: l4_1
 
 
       - label: "RLHF · VeRL-Omni E2E Test"
-        source_file_dependencies:
-          - tests/e2e/features/rlhf_test/test_verl_omni_e2e.py
-          - tests/e2e/features/helpers/verl_omni_server.py
-          - tests/e2e/features/helpers/custom_pipeline.py
-          - vllm_omni/diffusion/
-          - vllm_omni/engine/
-          - vllm_omni/entrypoints/
-          - vllm_omni/inputs/
-          - vllm_omni/outputs/
+        source_file_dependencies: rlhf_verl_omni
         commands:
           - |
             timeout 20m bash -c "
@@ -151,17 +145,7 @@ steps:
         mirror_hardwares: l4_1
 
       - label: "Omni · Qwen3-Omni Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen3_omni.py
-          - tests/helpers/runtime.py
-          - vllm_omni/entrypoints/openai/serving_chat.py
-          - vllm_omni/model_executor/models/qwen3_omni/
-          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-          - vllm_omni/model_executor/models/common/snake_activation.py
-          - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
-          - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
-          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
-          - vllm_omni/deploy/qwen3_omni_moe.yaml
+        source_file_dependencies: omni_qwen3_omni_function
         commands:
           - |
             timeout 20m bash -c "
@@ -170,14 +154,7 @@ steps:
         mirror_hardwares: h100_2
 
       - label: "Omni · MiniCPM-o 4.5 Offline Test"
-        source_file_dependencies:
-          - tests/e2e/offline_inference/test_minicpmo_4_5.py
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/models/step_audio2/
-          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
-          - vllm_omni/model_executor/models/cosyvoice3/utils.py
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
-          - vllm_omni/deploy/minicpmo_4_5.yaml
+        source_file_dependencies: omni_minicpmo_4_5_function
         commands:
           - |
             timeout 20m bash -c "
@@ -186,15 +163,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Omni · MiniCPM-o 4.5 Online Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_minicpmo_4_5.py
-          - vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/models/step_audio2/
-          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
-          - vllm_omni/model_executor/models/cosyvoice3/utils.py
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
-          - vllm_omni/deploy/minicpmo_4_5.yaml
+        source_file_dependencies: omni_minicpmo_4_5_function
         commands:
           - |
             timeout 20m bash -c "
@@ -203,48 +172,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Omni · MiniCPM-o 4.5 Duplex Test"
-        source_file_dependencies:
-          - tests/assets/minicpmo_4_5/response_required_16k.wav
-          - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
-          - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
-          - tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
-          - vllm_omni/deploy/minicpmo_4_5.yaml
-          - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
-          - vllm_omni/engine/duplex/
-          - vllm_omni/entrypoints/duplex/
-          # Public duplex client library + its unit suites: client-side
-          # changes must re-run the duplex e2e group, which includes the
-          # live client probe (test_duplex_client_live.py, single GPU).
-          - vllm_omni/clients/
-          - tests/clients/
-          - tests/e2e/online_serving/test_duplex_client_live.py
-          - vllm_omni/entrypoints/duplex_request_client.py
-          - vllm_omni/outputs/duplex.py
-          - vllm_omni/model_executor/duplex_sampling.py
-          - tests/engine/duplex/
-          - tests/entrypoints/duplex/
-          - tests/model_executor/models/minicpmo_4_5/duplex/
-          # Legacy Realtime probe client + video stacking used by the e2e drivers.
-          - vllm_omni/experimental/fullduplex/
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/models/step_audio2/
-          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
-          - vllm_omni/model_executor/models/cosyvoice3/utils.py
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
-          - vllm_omni/core/sched/
-          - vllm_omni/outputs/multimodal_accumulation.py
-          - vllm_omni/engine/stage_init_utils.py
-          - vllm_omni/engine/async_omni_engine.py
-          - vllm_omni/engine/orchestrator.py
-          - vllm_omni/entrypoints/cli/serve.py
-          - vllm_omni/entrypoints/openai/api_server.py
-          - vllm_omni/entrypoints/openai/duplex_capability.py
-          - vllm_omni/config/stage_config.py
+        source_file_dependencies: omni_minicpmo_4_5_duplex_function
         commands:
           - |
             timeout 35m bash -c "
@@ -254,29 +182,14 @@ steps:
 
       - label: "Omni · MiniCPM-o 4.5 Perf Test"
         timeout_in_minutes: 60
-        source_file_dependencies:
-          - tests/dfx/perf/tests/test_minicpmo_4_5_ready.json
-          - vllm_omni/deploy/minicpmo_4_5.yaml
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+        source_file_dependencies: omni_minicpmo_4_5_perf
         commands:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda'
         mirror_hardwares: h100_1
 
       - label: "TTS · Qwen3-TTS CustomVoice Test"
-        source_file_dependencies:
-          - tests/helpers/media.py
-          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
-          - vllm_omni/model_executor/models/qwen3_tts/
-          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-          - vllm_omni/model_executor/models/common/snake_activation.py
-          - vllm_omni/model_executor/models/common/alias_free_activation.py
-          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
-          - vllm_omni/deploy/qwen3_tts.yaml
-          - vllm_omni/entrypoints/openai/serving_speech.py
-          - vllm_omni/entrypoints/openai/tts_adapters/
-          - vllm_omni/engine/stage_pool.py
+        source_file_dependencies: tts_qwen3_tts_function
         commands:
           - |
             timeout 20m bash -c "
@@ -286,10 +199,7 @@ steps:
         mirror_hardwares: l4_1
 
       - label: "Diffusion · Wan22 Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_wan22_t2v.py
-          - vllm_omni/diffusion/models/wan2_2/
-          - vllm_omni/diffusion/models/dmd2/
+        source_file_dependencies: diffusion_wan22_function
         commands:
           - |
             timeout 40m bash -c "
@@ -299,12 +209,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Diffusion · Cosmos3 Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_cosmos3.py
-          - vllm_omni/diffusion/models/cosmos3/
-          - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_wan.py
-          - vllm_omni/experimental/world_models/
-          - vllm_omni/model_extras/cosmos3.py
+        source_file_dependencies: diffusion_cosmos3_function
         commands:
           - |
             timeout 40m bash -c "
@@ -315,14 +220,7 @@ steps:
 
       - label: "Diffusion · MiniMax-H3 DLO DP2 T2VA Test"
         timeout_in_minutes: 90
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py
-          - tests/helpers/assertions.py
-          - tests/helpers/runtime.py
-          - vllm_omni/diffusion/models/minimax_h3/
-          - vllm_omni/diffusion/offloader/
-          - vllm_omni/diffusion/distributed/
-          - vllm_omni/entrypoints/openai/
+        source_file_dependencies: diffusion_minimax_h3_function
         commands:
           - |
             timeout 80m bash -c "
@@ -331,12 +229,20 @@ steps:
             "
         mirror_hardwares: h100_2
 
-      - label: "Diffusion · Qwen Image Test"
+      - label: "Diffusion · Qwen Image 2.1 Test"
         source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen_image.py
-          - vllm_omni/diffusion/models/qwen_image/
-          - vllm_omni/diffusion/models/dmd2/
-          - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py
+          - tests/e2e/online_serving/test_qwen_image_21.py
+          - tests/helpers/
+          - vllm_omni/diffusion/models/qwen_image_21/
+          - vllm_omni/diffusion/distributed/autoencoders/
+          - vllm_omni/diffusion/sched/
+          - vllm_omni/entrypoints/openai/
+        commands:
+          - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen_image_21.py -m 'core_model and cuda' --run-level 'core_model'
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Qwen Image Test"
+        source_file_dependencies: diffusion_qwen_image_function
         commands:
           - |
             timeout 40m bash -c "
@@ -346,9 +252,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Diffusion · HunyuanVideo-1.5 Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
-          - vllm_omni/diffusion/models/hunyuan_video/
+        source_file_dependencies: diffusion_hunyuan_video_function
         commands:
           - |
             timeout 40m bash -c "
@@ -356,3 +260,13 @@ steps:
               pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'core_model and diffusion' --run-level 'core_model'
             "
         mirror_hardwares: h100_1
+
+      - label: "Diffusion · LingBot-Video Test"
+        source_file_dependencies: diffusion_lingbot_function
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_lingbot_video.py -m 'core_model and cuda' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_1

.buildkite/cuda/test-weekly.yml

@@ -29,36 +29,42 @@ steps:
     steps:
 
       - label: "Reliability Test · qwen3-omni"
+        source_file_dependencies: omni_qwen3_omni_reliability
         timeout_in_minutes: 180
         commands:
           - pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow"
         mirror_hardwares: h100_2
 
       - label: "Reliability Test · wan22"
+        source_file_dependencies: diffusion_wan22_reliability
         timeout_in_minutes: 180
         commands:
           - pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow"
         mirror_hardwares: h100_2
 
       - label: "Reliability Test · hunyuan-image"
+        source_file_dependencies: diffusion_hunyuan_image3_reliability
         timeout_in_minutes: 180
         commands:
           - pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow"
         mirror_hardwares: h100_4
 
       - label: "Reliability Test · Invalid parameters · H100 · Single-GPU"
+        source_file_dependencies: reliability_invalid_param_h100
         timeout_in_minutes: 90
         commands:
           - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100 and cards_1"
         mirror_hardwares: h100_1
 
       - label: "Reliability Test · Invalid parameters · H100 · 2-GPU"
+        source_file_dependencies: reliability_invalid_param_h100
         timeout_in_minutes: 90
         commands:
           - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100 and cards_2"
         mirror_hardwares: h100_2
 
       - label: "Reliability Test · Invalid parameters · L4"
+        source_file_dependencies: reliability_invalid_param_l4
         timeout_in_minutes: 120
         commands:
           - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"
@@ -70,6 +76,7 @@ steps:
     steps:
 
       - label: ":full_moon: Omni · Perf Test · vLLM Text"
+        source_file_dependencies: omni_qwen3_omni_perf
         key: weekly-omni-performance-vllm-text
         timeout_in_minutes: 180
         commands:
@@ -96,6 +103,7 @@ steps:
         mirror_hardwares: h100_2
 
       - label: ":full_moon: Omni · Perf Test · Multi-Replica"
+        source_file_dependencies: omni_qwen3_omni_perf
         key: weekly-omni-performance-multi-replicas
         timeout_in_minutes: 180
         commands:
@@ -183,13 +191,14 @@ steps:
       # from downgrading transformers; see
       # examples/offline_inference/text_to_speech/README.md.
       - label: "TTS · Gepard-1.0 · L4"
-        timeout_in_minutes: 60
+        timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
           - printf '%s\n' "transformers>=5.5.3" "huggingface-hub>=1.0" > /tmp/gepard-overrides.txt
           - uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" "nemo_toolkit[tts]==2.7.3" --overrides /tmp/gepard-overrides.txt
           - >-
             pytest -s -v tests/e2e/offline_inference/test_gepard_tts.py
+            tests/e2e/online_serving/test_gepard_tts.py
             -m "tts and cuda"
             --run-level "full_model"
         mirror_hardwares: l4_1
@@ -221,24 +230,277 @@ steps:
             --run-level "full_model"
         mirror_hardwares: l4_4
 
-      - label: "Diffusion · H100 · Single-GPU"
-        timeout_in_minutes: 150
+      # Split the old shared H100 Single-GPU sweep: one pytest process collected
+      # every slow+cards_1 diffusion file and exceeded 150 minutes.
+      - label: "Diffusion · Qwen-Image · H100 · Single-GPU"
+        timeout_in_minutes: 120
         commands:
           - >-
             pytest -sv
-            tests/e2e/ -m "slow and diffusion and H100 and cards_1"
+            tests/e2e/online_serving/test_qwen_image_edit.py
+            tests/e2e/online_serving/test_qwen_image_edit_expansion.py
+            tests/e2e/online_serving/test_qwen_image_layered.py
+            tests/e2e/online_serving/test_qwen_image_layered_expansion.py
+            tests/e2e/accuracy/test_qwen_image_edit.py
+            tests/e2e/accuracy/test_qwen_image_layered.py
+            -m "slow and diffusion and H100 and cards_1"
             --run-level "full_model"
         mirror_hardwares: h100_1
 
-      - label: "Diffusion · H100 · 2-GPU"
+      - label: "Diffusion · Wan2.1 VACE · H100 · Single-GPU"
         timeout_in_minutes: 90
         commands:
           - >-
             pytest -sv
-            tests/e2e/ -m "slow and diffusion and H100 and cards_2"
+            tests/e2e/online_serving/test_wan_2_1_vace_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · LongCat · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_longcat_image_expansion.py
+            tests/e2e/online_serving/test_longcat_image_edit_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Bagel · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_bagel.py
+            tests/e2e/online_serving/test_bagel_expansion.py
+            tests/e2e/offline_inference/test_bagel.py
+            tests/e2e/offline_inference/test_bagel_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Boogu · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_boogu_image.py
+            tests/e2e/online_serving/test_boogu_image_edit.py
+            tests/e2e/online_serving/test_boogu_image_expansion.py
+            tests/e2e/online_serving/test_boogu_image_edit_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Flux · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_flux_2_dev_expansion.py
+            tests/e2e/offline_inference/test_flux1_schnell_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · LTX · H100 · Single-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_ltx.py
+            tests/e2e/online_serving/test_ltx25.py
+            tests/e2e/accuracy/ltx/test_ltx_official_similarity.py
+            tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Krea2 · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_krea2_expansion.py
+            tests/e2e/offline_inference/test_krea2_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · HiDream-I1 · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_hidream_i1_full_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · SANA-Video · H100 · Single-GPU"
+        source_file_dependencies: diffusion_sana_video_function
+        timeout_in_minutes: 180
+        commands:
+          - export SANA_VIDEO_GOLDEN_BASE_URL=https://vllm-public-assets.s3.us-west-2.amazonaws.com/omni-assets/sana-video/v2
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_sana_video_expansion.py
+            tests/e2e/accuracy/sana_video/
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · SenseNova-U1 · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/offline_inference/test_sensenova_u1_expansion.py
+            tests/e2e/offline_inference/test_sensenova_u1_text2img_expansion.py
+            tests/e2e/offline_inference/test_sensenova_u1_img2img_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · MammothModa2 · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/offline_inference/test_mammoth_moda2_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Helios · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/accuracy/test_video_streaming_output_similarity.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      # Split the old shared H100 2-GPU sweep: one pytest process collected every
+      # slow+cards_2 diffusion file and exceeded 90 minutes.
+      - label: "Diffusion · Qwen-Image · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_qwen_image_edit_expansion.py
+            tests/e2e/online_serving/test_qwen_image_layered_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
             --run-level "full_model"
         mirror_hardwares: h100_2
 
+      - label: "Diffusion · Wan2.1 VACE · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_wan_2_1_vace_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · LongCat · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_longcat_image_expansion.py
+            tests/e2e/online_serving/test_longcat_image_edit_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · Bagel · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_bagel_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · Boogu · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_boogu_image_expansion.py
+            tests/e2e/online_serving/test_boogu_image_edit_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · Flux · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_flux_2_dev_expansion.py
+            tests/e2e/offline_inference/test_flux1_kontext_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · GLM-Image · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_glm_image_expansion.py
+            tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · LTX · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_ltx.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · Krea2 · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/offline_inference/test_krea2_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · DreamZero · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_dreamzero_expansion.py
+            tests/e2e/accuracy/test_dreamzero.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · H100 · 3-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/ -m "slow and diffusion and H100 and cards_3"
+            --run-level "full_model"
+        mirror_hardwares: h100_3
+
       - label: "Diffusion · H100 · 4-GPU"
         timeout_in_minutes: 90
         commands:
@@ -246,4 +508,33 @@ steps:
             pytest -sv
             tests/e2e/ -m "slow and diffusion and H100 and cards_4"
             --run-level "full_model"
+            --ignore=tests/e2e/online_serving/test_magi2.py
+        mirror_hardwares: h100_4
+
+      # Dedicated: HiDream-O1 accuracy needs a long TP2 run; the shared H100
+      # 2-GPU sweep is only 90 minutes.
+      - label: "Diffusion · HiDream-O1-Image Accuracy · H100 · 2-GPU"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/e2e/accuracy/test_hidream_o1_image.py -m "slow and diffusion and cuda and H100 and cards_2" --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      # Dedicated: Boogu CFG-parallel size 3 needs 3 GPUs; weekly has no H100 3-GPU sweep.
+      - label: "Diffusion · Boogu Edit CFG · H100 · 3-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_boogu_image_edit_expansion.py
+            -k "test_double_guidance_cfg_parallel and cfg_parallel_3_full"
+            -m "slow and diffusion and cuda and H100 and cards_3"
+            --run-level "full_model"
+        mirror_hardwares: h100_3
+
+      # Dedicated: MAGI-2 Preview serving smoke needs ~30 min server init; the
+      # shared H100 4-GPU sweep is only 90 minutes.
+      - label: "Diffusion · MAGI-2 Preview Serving Smoke · H100 · 4-GPU"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_magi2.py -m "slow and diffusion and cuda and H100 and cards_4" --run-level "full_model"
         mirror_hardwares: h100_4

.buildkite/intel/pipeline-intel.yml

@@ -10,7 +10,7 @@ steps:
           DOCKER_BUILDKIT: "1"
           # Buildkite will automatically replace this with the actual commit hash
           VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.28.0"
+          VLLM_VERSION: "v0.29.0"
         priority: 100
         timeout_in_minutes: 60
         soft_fail: false

.buildkite/npu/bootstrap-upload-steps.yml

@@ -93,7 +93,7 @@ steps:
     command: |
       python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/test-npu-ready.yml
 
-  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
+  # L4 Test — main+NIGHTLY=1 (scheduled), or PR nightly-test label
   - label: "Upload Nightly Pipeline"
     depends_on:
       - image-build-a2
@@ -102,6 +102,11 @@ steps:
       - image-build-310p
     key: upload-nightly-pipeline
     commands:
-      - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/test-npu-nightly.yml
+      - |
+        EXTRA=""
+        if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
+          EXTRA="--all"
+        fi
+        python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/npu/test-npu-nightly.yml
     agents:
       queue: "cpu_queue_premerge"

.buildkite/npu/pipeline-npu-a3.yml

@@ -96,13 +96,7 @@
   #   key: upload-nightly-pipeline
   #   if: >-
   #     (build.branch == "main" && build.env("NIGHTLY") == "1") ||
-  #     (build.branch != "main" && (
-  #       build.pull_request.labels includes "nightly-test" ||
-  #       build.pull_request.labels includes "omni-test" ||
-  #       build.pull_request.labels includes "tts-test" ||
-  #       build.pull_request.labels includes "diffusion-x2iat-test" ||
-  #       build.pull_request.labels includes "diffusion-x2v-test"
-  #     ))
+  #     (build.branch != "main" && build.pull_request.labels includes "nightly-test")
   #   commands:
   #     - buildkite-agent pipeline upload .buildkite/npu/test-npu-nightly.yml
   #   agents:

.buildkite/npu/test-npu-nightly.yml

@@ -4,10 +4,10 @@ steps:
     depends_on: upload-nightly-pipeline
     if: >-
       build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "omni-test"
+      build.pull_request.labels includes "nightly-test"
     steps:
       - label: ":full_moon: Omni · Perf Test · No Async Chunk"
+        source_file_dependencies: omni_qwen3_omni_perf
         key: nightly-omni-performance-no-async-chunk
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4
@@ -24,6 +24,7 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
       - label: ":full_moon: Omni · Perf Test · Async Chunk"
+        source_file_dependencies: omni_qwen3_omni_perf
         key: nightly-omni-performance-async-chunk
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4
@@ -40,6 +41,7 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
       - label: ":full_moon: Omni · MiniCPM-o 4.5 Test"
+        source_file_dependencies: omni_minicpmo_4_5_function
         mirror_hardwares: a3_npu_2
         env:
           VLLM_WORKER_MULTIPROC_METHOD: spawn
@@ -48,6 +50,7 @@ steps:
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model'
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Test"
+        source_file_dependencies: omni_minicpmo_4_5_duplex_function
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_2
         env:
@@ -62,6 +65,7 @@ steps:
             -m "npu and A3 and omni"
             --run-level full_model
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
+        source_file_dependencies: omni_minicpmo_4_5_accuracy
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_2
         env:
@@ -82,6 +86,7 @@ steps:
             buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
             exit $$EXIT
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"
+        source_file_dependencies: omni_minicpmo_4_5_perf
         key: nightly-omni-performance-minicpmo-4-5-npu
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_2
@@ -98,6 +103,7 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test (A2)"
+        source_file_dependencies: omni_minicpmo_4_5_perf
         key: nightly-omni-performance-minicpmo-4-5-npu-a2
         timeout_in_minutes: 180
         mirror_hardwares: a2b3_npu_1
@@ -115,6 +121,7 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test"
+        source_file_dependencies: omni_minicpmo_4_5_duplex_perf
         key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts-npu
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_2
@@ -133,9 +140,10 @@ steps:
   - group: ":card_index_dividers: TTS Model Test"
     key: nightly-tts-test-group
     depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     steps:
       - label: ":full_moon: TTS · Perf Test"
+        source_file_dependencies: tts_qwen3_tts_perf
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_2
         env:
@@ -151,15 +159,15 @@ steps:
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-  - group: ":card_index_dividers: Diffusion I2I Model Test"
-    key: nightly-HunyuanImage3-x2i-test-group
+  - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
+    key: nightly-diffusion-x2iat-group
     depends_on: upload-nightly-pipeline
     if: >-
       build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2iat-test"
+      build.pull_request.labels includes "nightly-test"
     steps:
       - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
+        source_file_dependencies: diffusion_hunyuan_image3_accuracy
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4
         env:
@@ -169,15 +177,8 @@ steps:
           DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
         commands:
           - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m "full_model and A3" --run-level full_model
-  - group: ":card_index_dividers: Diffusion Benchmark Perf Test"
-    key: nightly-diffusion-benchmark-group
-    depends_on: upload-nightly-pipeline
-    if: >-
-      build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2iat-test"
-    steps:
       - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 CFGP=2"
+        source_file_dependencies: diffusion_hunyuan_image3_perf
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4
         env:
@@ -194,6 +195,7 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
       - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 SP=2"
+        source_file_dependencies: diffusion_hunyuan_image3_perf
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4
         env:
@@ -210,6 +212,7 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
       - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=4"
+        source_file_dependencies: diffusion_hunyuan_image3_perf
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4
         env:
@@ -230,10 +233,41 @@ steps:
     depends_on: upload-nightly-pipeline
     if: >-
       build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2v-test"
+      build.pull_request.labels includes "nightly-test"
     steps:
-      - label: ":full_moon: Diffusion X2V · Function Test"
+      - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test"
+        source_file_dependencies: diffusion_hunyuan_video_function
+        timeout_in_minutes: 180
+        mirror_hardwares: a2b3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+          HF_HUB_OFFLINE: "1"
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "npu and A2" --run-level "full_model"
+      - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Perf Test"
+        source_file_dependencies: diffusion_hunyuan_video_perf
+        key: nightly-diffusion-x2v-performance-hunyuanvideo15
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_2
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+          HF_HUB_OFFLINE: "1"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+      - label: ":full_moon: Diffusion X2V · Wan 2.2 Function Test"
+        source_file_dependencies: diffusion_wan22_function
         mirror_hardwares: a2b3_npu_4
         env:
           VLLM_WORKER_MULTIPROC_METHOD: spawn
@@ -241,7 +275,8 @@ steps:
           HF_TOKEN: "${HF_TOKEN}"
         commands:
           - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"
-      - label: ":full_moon: Diffusion X2V · Perf Test"
+      - label: ":full_moon: Diffusion X2V · Wan 2.2 Perf Test"
+        source_file_dependencies: diffusion_wan22_perf
         key: nightly-diffusion-x2v-performance
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4

.claude/skills/production-add-diffusion-model/references/production-validation.md

@@ -340,7 +340,7 @@ On supported GPU hosts, wrap a test or soak with the repository monitor:
 
 ```bash
 bash tests/dfx/stability/scripts/resource_monitor.sh run --backend gpu -- \
-  pytest -s -v tests/dfx/stability/scripts/test_stability_<model>.py -m slow
+  pytest -s -v tests/dfx/stability/scripts/run_stability_<model>.py -m slow
 ```
 
 Check the target revision's monitor backends. Do not claim CPU/NPU monitoring

.claude/skills/vllm-omni-test/SKILL.md

@@ -468,10 +468,10 @@ When the user asks for **invalid parameter validation**, **invalid request bodie
 | **Invalid parameters Test · H100** | `pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"` | Diffusion / omni / video invalid-param tests with `H100` hardware mark |
 | **Invalid parameters Test · L4** | `pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"` | TTS / lighter models marked `L4` |
 
-- **Trigger:** `build.env("WEEKLY") == "1"` or PR label `weekly-test`.
+- **Trigger:** `build.env("WEEKLY") == "1"` (all weekly jobs) or PR label `weekly-test` (Reliability / Perf, then filtered by `source_file_dependencies`).
 - **Default:** extending an existing `invalid_param_test` script needs **no YAML edit** — the weekly steps already sweep the whole directory.
 - **Edit YAML only** when adding a **new hardware queue**, a **new top-level script** that must run in isolation, or a **model-specific weekly shard** (mirror neighboring reliability steps).
-- Weekly steps **do not** use `source_file_dependencies`.
+- Weekly Reliability / Perf jobs declare `source_file_dependencies`. Filtering applies only on the `weekly-test` label; `WEEKLY=1` on `main` keeps every job in those groups. CPU coverage (`WEEKLY=1` only) and the E2E Tests group (`NON_CRITICAL=1` only) do **not** use this field.
 
 **Example — append to `test_invalid_image_generation.py`:**
 
@@ -644,7 +644,7 @@ If the test is not already collected by an existing pipeline command (for exampl
 | **L2** | `test-ready.yml` with `core_model` + `source_file_dependencies` | merge / nightly pipelines |
 | **Invalid param** | `test-weekly.yml` — **Invalid parameters Test** group (`-m "slow and H100"` / `-m "slow and L4"`). Usually **no YAML edit** when appending cases to existing `invalid_param_test` scripts | `test-ready.yml`, `test-merge.yml`, `test-nightly.yml` |
 
-`test-nightly.yml` shards use **explicit pytest file paths** in `commands` (and PR labels like `diffusion-x2iat-test`); they generally **do not** use `source_file_dependencies`. Only suggest merge/ready YAML when the requested level is L3/L2 (or the user explicitly asks for multi-level CI).
+`test-nightly.yml` shards use **explicit pytest file paths** in `commands` plus `source_file_dependencies`; PR trigger is `nightly-test` (path filter), or `NIGHTLY=1` on `main` (all jobs). Only suggest merge/ready YAML when the requested level is L3/L2 (or the user explicitly asks for multi-level CI).
 
 **`test-nightly.yml` top-level groups** (each uses `full_model` + type marker in pytest `-m`):
 
@@ -653,7 +653,7 @@ If the test is not already collected by an existing pipeline command (for exampl
 | **Omni Model Test** | `omni` | Function / doc / accuracy / **perf** for Qwen-Omni family |
 | **TTS Model Test** | `tts` | `-m "full_model and L4 and tts"` (and H100 variants when enabled); **Perf Test** uses `run_benchmark.py` + `test_tts.json` |
 | **Diffusion X2I(&A&T) Model Test** | `diffusion` | Image/audio/text diffusion; explicit file shards for **Function**; **Perf Test · <Model>** uses `run_diffusion_benchmark.py` + JSON config |
-| **Diffusion X2V Model Test** | `diffusion` | Video-only (Wan, HunyuanVideo, …); PR label `diffusion-x2v-test`; separate **Perf** steps when in-tree |
+| **Diffusion X2V Model Test** | `diffusion` | Video-only (Wan, HunyuanVideo, …); separate **Perf** steps when in-tree |
 
 Guidelines when editing YAML:
 

.claude/skills/vllm-omni-test/references/test-routing.md

@@ -37,7 +37,7 @@ Every model-centric e2e test must declare **exactly one** type marker:
 | **Diffusion X2I(&A&T) Model Test** | x2**i** / x2**a** / x2**t** — image, audio, text (non-video) | Qwen-Image*, BAGEL, FLUX, SD3, Z-Image, LongCat, DreamZero |
 | **Diffusion X2V Model Test** | x2**v** — video only | Wan2.2, HunyuanVideo 1.5, Wan VACE |
 
-PR labels for selective nightly runs: `diffusion-x2iat-test`, `diffusion-x2v-test` (plus `nightly-test` / `NIGHTLY=1`).
+PR label for L4 on a branch: `nightly-test` (jobs then filtered by `source_file_dependencies`). Scheduled `main` uses `NIGHTLY=1` and runs every nightly job. L5 Reliability / Perf use `weekly-test` with the same path filter; CPU coverage and the weekly E2E group are env-only (`WEEKLY=1` / `NON_CRITICAL=1`) and are not source-filtered.
 
 ## Level and Marker Mapping
 
@@ -238,7 +238,7 @@ pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and L4"
 pytest -s -v dfx/reliability/invalid_param_test/test_invalid_image_generation.py::test_images_generations_invalid_requests -m "slow and H100"
 ```
 
-**Trigger:** `WEEKLY=1` or PR label `weekly-test`. Appending cases to existing scripts usually needs **no YAML edit** (directory sweep). No `source_file_dependencies` on weekly steps.
+**Trigger:** `WEEKLY=1` (all jobs) or PR label `weekly-test` (Reliability / Perf, filtered by `source_file_dependencies`). Appending cases to existing scripts usually needs **no YAML edit** (directory sweep).
 
 ### Platform-targeted examples
 

.github/CODEOWNERS

@@ -67,6 +67,11 @@
 # File-level overrides within entrypoints/
 /vllm_omni/entrypoints/openai/errors.py @alex-jw-brooks @NickCao
 /vllm_omni/entrypoints/cli/benchmark/ @alex-jw-brooks @Bounty-hunter
+# Shared wire codec: protocol/realtime (OpenAI Realtime GA) and protocol/duplex
+# (vLLM-Omni's extension of it) — RFC #6592 P0a. Layer-neutral by design — the
+# engine depends on it — so it is owned by the serving-boundary document rather
+# than by any one consumer. — docs/design/module/entrypoints.md
+/vllm_omni/protocol/ @alex-jw-brooks @linyueqian @NickCao
 
 # Error contracts — docs/design/module/error_contracts.md
 /vllm_omni/errors.py @alex-jw-brooks @NickCao

README.md

@@ -58,7 +58,7 @@ vLLM-Omni is flexible and easy to use with:
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
 - **Omni-modality models** (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
-- **TTS models** (e.g. Qwen3-TTS, IndexTTS 2.5, dots.tts, CosyVoice3)
+- **TTS models** (e.g. Qwen3-TTS, IndexTTS 2.5, CosyVoice3)
 - **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
 - **Robot-policy and action models** (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
 

apps/ComfyUI-vLLM-Omni/README.md

@@ -13,7 +13,6 @@ It can send model inference requests to either a locally running vLLM-Omni servi
 > [!TIP]
 > If you run both ComfyUI and vLLM-Omni on the same device, you can create separate virtual environments and use different Python versions for them.
 
-
 ## Installation
 
 Copy this folder to the `custom_nodes` subfolder of your ComfyUI installation. Your directory should look like `ComfyUI/custom_nodes/ComfyUI-vLLM-Omni`.
@@ -21,9 +20,10 @@ Copy this folder to the `custom_nodes` subfolder of your ComfyUI installation. Y
 If you are running ComfyUI during copying, you should restart ComfyUI to load this extension.
 
 > [!TIP]
-> You can use utility websites such as https://download-directory.github.io/ to download a subdirectory of a repo. Also checkout community discussions (e.g., https://stackoverflow.com/questions/7106012/download-a-single-folder-or-directory-from-a-github-repository) for more info.
+> You can use utility websites such as <https://download-directory.github.io/> to download a subdirectory of a repo. Also checkout community discussions (e.g., <https://stackoverflow.com/questions/7106012/download-a-single-folder-or-directory-from-a-github-repository>) for more info.
 
 On the device and virtual environment you run ComfyUI, launch ComfyUI with
+
 ```bash
 cd ComfyUI
 
@@ -35,6 +35,7 @@ python main.py --cpu
 ```
 
 On the device and virtual environment you run vLLM-Omni, start a model service with
+
 ```bash
 vllm serve The_Model_ID_to_Serve --omni --port 8000
 ```
@@ -48,14 +49,31 @@ This extension offers the following nodes based on the output modalities (at **C
 
 - **Generate Image** for text-to-image and image-to-image tasks
 - **Generate Video** for text-to-video, first-frame/image-to-video, and reference-conditioned video
+- **Latent Mask Editing** for MiniMax-H3 latent-mask editing (source media plus video/audio noise masks)
+- **FastH3 Deployment** for routing text-to-video requests to a MiniMax-H3 server with FastH3 fused at startup
 - **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks
 - **TTS** and **TTS Voice Clone** for TTS tasks
+- **Generate Music** for lyrics-and-description-to-music generation
 
 This extension also offers example workflows (at **ComfyUI sidebar -> Templates -> vLLM-Omni**)
 
 > [!NOTE]
 > The node UI and feature designs are intended to match vLLM-Omni online serving interfaces. It cannot offer more than what the interfaces support.
 
+Every node carries the vLLM-Omni mark in its title bar and is tinted by what it outputs, so a graph is readable at a glance:
+
+| Colour | Nodes | What they produce |
+| --- | --- | --- |
+| Blue | Generate Image, Generate Video, Multimodality Understanding, TTS, TTS Voice Clone | A generated image, video, audio, or text. These are the only nodes that reach a server. |
+| Amber | AR / Diffusion / Multi-Stage Sampling Params | Sampling parameters that apply to any model |
+| Purple | Qwen TTS Params, Wan Video Params, MiniMax-H3 Video Params | Parameters that only one model family accepts |
+| Red | LoRA, FastH3 Deployment | Which weights the server is expected to have loaded |
+| Teal | Video References, Latent Mask Editing | Reference media |
+
+Recolouring a node by hand (right click -> Colors) overrides its tint, and the choice is kept.
+
+**Generate Video** takes a clip length in seconds (`duration`), not a frame count. Frames stay the wire unit and are derived with the node's `fps`, so the length is always measured against the rate that is actually served; models that accept only certain frame counts still round to their own lattice server-side. Graphs saved before this widget existed stored `num_frames` in its place and are converted on load, using the fps recorded alongside it -- the browser console names every node it rewrites.
+
 To build a simple workflow yourself,
 
 - Drag a generation node onto the canvas.
@@ -117,11 +135,146 @@ You can configure per-stage sampling parameters for multi-stage models.
 </p>
 
 > [!TIP]
-> Connect a **frame** image for first-frame / image-to-video (e.g., Wan I2V, MiniMax-H3 FL2VA).
+> Connect **frame** for backward-compatible first-frame / image-to-video behavior. For MiniMax-H3 FL2VA,
+> connect **first_frame**, **last_frame**, or both to condition the start frame, end frame, or both.
 >
 > For reference-conditioned generation (MiniMax-H3 Ref2VA), connect a **Video References** node instead.
 >
-> Do not use `frame` and `references` together. Task routing is automatic from which inputs you connect.
+> Do not combine `frame` with `first_frame` or `last_frame`, and do not combine any frame input with
+> `references`. Task routing is automatic from which inputs you connect.
+
+For MiniMax-H3 Ref2VA, **Video References** accepts up to 9 images (`image_1`–`image_9`),
+3 videos (`video_1`–`video_3`), and 3 audio clips (`audio_1`–`audio_3`), with at most
+12 connected inputs in total. Any mixture containing at least one image or video
+is supported; empty and audio-only references are rejected. For example, you can
+combine 6 images, 3 videos, and 3 audio clips in one request.
+
+Within each media type, references follow slot-number order, skipping unconnected
+slots. For example, connecting `image_2` and `image_9` sends `image_2` as the first
+image and `image_9` as the second. Each image slot uses the first image in its batch.
+Existing connections to `image_1`, `image_2`, `audio_1`, `audio_2`, `video_1`, and
+`video_2` remain valid in saved workflows.
+
+#### MiniMax-H3 Reference to Video
+
+Load [MiniMax-H3 Reference to Video](example_workflows/vLLM-Omni%20MiniMax-H3%20Reference%20to%20Video.json)
+from **Templates → ComfyUI-vLLM-Omni**, or drag the JSON onto the canvas.
+It adapts the [official ComfyUI R2V workflow](https://github.com/Comfy-Org/workflow_templates/blob/main/templates/video_minimax_h3_r2v.json)
+to remote vLLM-Omni execution. Model loading, sampling, and VAE decoding run on
+the server; ComfyUI loads references and saves the returned video.
+
+Use an extension version containing the audio-preserving output fix from
+[#7456](https://github.com/vllm-project/vllm-omni/pull/7456). Older versions can
+drop the server's audio while decoding the response.
+
+1. Start a Ref2VA-capable service using the [MiniMax-H3 recipe](../../recipes/MiniMaxAI/MiniMax-H3.md).
+   Set **Generate Video** to its `/v1` URL and served model name.
+2. Select your own image in **Load Image**. The template's filenames are placeholders
+   relative to ComfyUI's input directory; no sample assets or model weights are bundled.
+3. Connect **Load Video** and **Load Audio** when needed. Duplicate the loaders to
+   fill more reference slots, or disconnect the image when using video-only input.
+   Keep the references connected to **Generate Video** and leave `frame` disconnected.
+4. Match prompt tags such as `<Picture 1>`, `<Video 1>`, and `<Audio 1>` to the
+   connected references, counting each media type separately and skipping empty slots.
+5. Run the workflow. **Save Video** writes an MP4 under `output/video/` and preserves
+   the generated audio when the audio-output prerequisite is installed.
+
+The default is **1344×768, 24 FPS, 124 frames** (about 5.17 seconds), 50 sampling
+points, video flow shift 12, audio flow shift 3, and seed 42. The duration widget
+is set to 5.167 seconds, which converts to 124 frames at 24 FPS. Other H3 canvas presets
+are 1024×768, 768×768, 768×1024, and 768×1344. Keep frame counts at `17k+5` within
+the 4–15 second output range; examples are 107, 124, 209, and 345 frames.
+Reference video/audio clips must each be 2–15 seconds, with at most 15 seconds of
+reference video and at most 15 seconds of standalone reference audio per request.
+
+For **Ref2VA Turbo**, use the same template. Configure a Ref2VA-only server with
+`--lora-backend peft --lora-path "$TURBO_LORA" --task-type ref2va`, following the
+recipe's hardware setup. For `minimax_h3_ref2v_turbo_4step_v0.1_bf16.safetensors`,
+set **LoRA** `local_path` to that file's path on the server and connect it to
+**Generate Video**. Use `num_inference_steps=5`, `flow_shift=12`,
+`audio_flow_shift=3`, and LoRA `scale=1`. The API counts sigma points, so five
+points produce four denoiser evaluations. Use the Diffusers artifact, not its
+`_comfyui_` export. To return to base mode, disconnect LoRA and restore 50 steps.
+The 8-step v1.0 Ref2VA adapter instead requires nine sampling points and video
+flow shift 6. FastH3 is a separate startup-fused T2VA deployment.
+
+Validate the template's wiring and defaults locally with:
+
+```bash
+python -m pytest tests/e2e/features/comfyui/test_h3_reference_workflow.py -q
+```
+
+For a real-model check, run the workflow against H3 and inspect the saved file:
+
+```bash
+ffprobe -v error -show_entries stream=codec_type,width,height,r_frame_rate,sample_rate,channels \
+  -show_entries format=duration -of json "$OUTPUT_MP4"
+```
+
+Check for 24 FPS video, stereo audio, the requested canvas, and aligned duration;
+also play the result to assess reference conditioning and audio/video synchronization.
+Record the server commit, model/adapter, input assets, prompt, settings, and output
+alongside the result. Local schema or mocked-server checks do not establish H3
+generation quality or replace this real-model validation.
+
+### MiniMax H3 Text to Video
+
+Import [the H3 text-to-video template](example_workflows/MiniMax_H3_Text_to_Video.json)
+for remote video generation with audio. It includes connected base presets and
+optional Turbo sampling, H3 parameters, and Remote LoRA nodes. See the
+[workflow guide](docs/minimax-h3-t2v.md) for server setup, artifact-specific Turbo
+settings, and saved-video/audio validation. Set Remote LoRA’s `local_path` to
+the downloaded Turbo artifact on your server before enabling Turbo.
+
+#### H3 video upscale (WF-07)
+
+The **vLLM-Omni MiniMax H3 Video Upscale** template generates video remotely, upscales it with SeedVR2, and saves the original and upscaled videos with the generated audio and FPS. See [workflow setup](docs/wf07-h3-upscale.md).
+
+#### FastH3 text-to-video
+
+FastH3 is fused into MiniMax-H3 when the vLLM-Omni server starts; it is not a request-switchable LoRA. Download one adapter variant and start a non-offloaded FL2VA server. For the Dense / Data-Free profile:
+
+```bash
+export FASTH3_DIR=/path/to/fasth3
+hf download FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA \
+  dense-datafree/adapter_model.safetensors --local-dir "${FASTH3_DIR}"
+
+vllm serve /path/to/MiniMax-H3 \
+  --omni \
+  --task-type fl2va \
+  --lora-path "${FASTH3_DIR}/dense-datafree/adapter_model.safetensors" \
+  --port 8000
+```
+
+For the VSA / Data-Free profile, download `vsa-datafree/adapter_model.safetensors`, use that file as `--lora-path`, and add:
+
+```bash
+--diffusion-attention-backend FASTVIDEO_VSA \
+--fastvideo-vsa-topk 64
+```
+
+The VSA profile also requires a compatible `fastvideo-kernel` installation. See the [MiniMax-H3 FastH3 recipe](../../recipes/MiniMaxAI/MiniMax-H3.md#fasth3-adapter) for the full serving contract, supported parallel layouts, and kernel requirements.
+
+Open the **vLLM-Omni FastH3 Text to Video** template, then:
+
+- Set the server URL and served model name on **FastH3 Deployment**.
+- Connect its output to **Generate Video → fast_h3**. When connected, the deployment node's URL and model take precedence over the corresponding Generate Video widgets.
+- Keep `frame`, `references`, **LoRA**, and **MiniMax-H3 Video Params** disconnected. FastH3 Preview v1 supports T2VA only, is already fused, and owns both flow shifts.
+- A connected **Diffusion Sampling Params** node may set seed and other ordinary sampling options. The integration always enforces four denoising steps and 24 FPS for FastH3.
+
+The node records which server the workflow targets; it does not start one, nor switch adapters or attention backends on a running server.
+
+#### Latent-mask editing (MiniMax-H3)
+
+The [WF-05 template](example_workflows/vLLM-Omni%20MiniMax-H3%20Latent%20Mask%20Editing.json) contains inpainting, object removal, continuation, and extension examples in one graph. See the [workflow guide](docs/wf05-h3-latent-editing.md) for inputs, mask settings, dependencies, and preview limitations.
+
+Connect a **Latent Mask Editing** node to **Generate Video → latent_edit** to edit a source clip instead of generating from scratch. It uploads the source media and serializes the video/audio noise masks the MiniMax H3 API accepts:
+
+- `source_video` / `source_audio` — the media to edit.
+- `video_mask` — a ComfyUI mask image; `0` preserves a region, `1` regenerates it, fractional values blend. A 2D mask `[H, W]` is applied to every frame; a 3D mask `[T, H, W]` is treated as a temporal mask (one slice per frame) for continuation or extension. It is resized to the video latent grid.
+- `audio_mask` — a scalar in `[0, 1]`; `0` keeps the source audio, `1` regenerates it, fractional values blend.
+
+A non-trivial mask requires its matching source, and a source without a mask is rejected. This node only forwards inputs to the server; the served model must declare latent-mask editing support (MiniMax-H3).
 
 ### TTS (e.g., Qwen TTS series)
 
@@ -137,6 +290,17 @@ You can configure per-stage sampling parameters for multi-stage models.
 > [!TIP]
 > There is a dedicated node for VoiceClone tasks with reference audio input. Other simple text-to-speech tasks should use the regular TTS node.
 
+### Music generation (e.g., MiniMax Music 3)
+
+(Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni Music Generation**)
+
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-music-generation.jpg">
+    <img alt="vLLM-Omni Music Generation" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-music-generation.jpg" width=55%>
+  </picture>
+</p>
+
 ### Chaining multiple model services
 
 (Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni Chaining Services**)
@@ -152,6 +316,8 @@ You can configure per-stage sampling parameters for multi-stage models.
 
 Follow the [development convention and rules of vLLM-Omni](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/).
 
+Node tints and the title-bar mark are applied in `web/main.js`, keyed off each node's declared output types rather than a list of node names. A new node that returns an existing type is themed with no front-end change; a new output type needs one entry in `FAMILY_BY_OUTPUT` there.
+
 ## Limitation and Non-Goals
 
 - Single server mode only. No automatic load balancing or failover.
@@ -178,11 +344,11 @@ Whenever you find an issue or problem, please
 
 Features
 
-- https://github.com/dougbtv/comfyui-vllm-omni/ The official reference implementation for ComfyUI integration with vLLM-Omni's DALL-E compatible image generation API.
-- https://github.com/Comfy-Org/ComfyUI/tree/master/comfy_extras ComfyUI's built-in node implementations.
+- <https://github.com/dougbtv/comfyui-vllm-omni/> The official reference implementation for ComfyUI integration with vLLM-Omni's DALL-E compatible image generation API.
+- <https://github.com/Comfy-Org/ComfyUI/tree/master/comfy_extras> ComfyUI's built-in node implementations.
 
 UI/UX design references
 
-- https://github.com/sgl-project/sglang/pull/15271 SGLang Diffusion's official ComfyUI integration for image and video generation.
-- https://github.com/SXQBW/ComfyUI-Qwen-Omni A third party ComfyUI integration for Qwen Omni series.
-- https://github.com/flybirdxx/ComfyUI-Qwen-TTS https://github.com/DarioFT/ComfyUI-Qwen3-TTS Third  party ComfyUI integrations for Qwen TTS series.
+- <https://github.com/sgl-project/sglang/pull/15271> SGLang Diffusion's official ComfyUI integration for image and video generation.
+- <https://github.com/SXQBW/ComfyUI-Qwen-Omni> A third party ComfyUI integration for Qwen Omni series.
+- <https://github.com/flybirdxx/ComfyUI-Qwen-TTS> <https://github.com/DarioFT/ComfyUI-Qwen3-TTS> Third  party ComfyUI integrations for Qwen TTS series.

apps/ComfyUI-vLLM-Omni/__init__.py

@@ -1,4 +1,8 @@
-"""Top-level package for comfyui_vllm_omni."""  # noqa: N999  # This is not a python library intended to be imported
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+# ruff: noqa: N999
+
+"""Top-level package for comfyui_vllm_omni."""  # This is not a Python library intended to be imported.
 
 __all__ = [
     "NODE_CLASS_MAPPINGS",
@@ -13,9 +17,13 @@ __version__ = "0.0.1"
 from .comfyui_vllm_omni.nodes import (
     VLLMOmniARSampling,
     VLLMOmniDiffusionSampling,
+    VLLMOmniFastH3Deployment,
     VLLMOmniGenerateImage,
+    VLLMOmniGenerateMusic,
     VLLMOmniGenerateVideo,
+    VLLMOmniLatentMaskEditing,
     VLLMOmniMiniMaxH3Params,
+    VLLMOmniMiniMaxH3TemporalMask,
     VLLMOmniQwenTTSParams,
     VLLMOmniRemoteLoRA,
     VLLMOmniSamplingParamsList,
@@ -28,18 +36,22 @@ from .comfyui_vllm_omni.nodes import (
 
 # A dictionary that contains all nodes you want to export with their names
 NODE_CLASS_MAPPINGS = {
+    "VLLMOmniMiniMaxH3TemporalMask": VLLMOmniMiniMaxH3TemporalMask,
     # === Generation ===
     "VLLMOmniGenerateImage": VLLMOmniGenerateImage,
     "VLLMOmniGenerateVideo": VLLMOmniGenerateVideo,
     "VLLMOmniUnderstanding": VLLMOmniUnderstanding,
     "VLLMOmniTTS": VLLMOmniTTS,
+    "VLLMOmniGenerateMusic": VLLMOmniGenerateMusic,
     "VLLMOmniVoiceClone": VLLMOmniVoiceClone,
     "VLLMOmniVideoReferences": VLLMOmniVideoReferences,
+    "VLLMOmniLatentMaskEditing": VLLMOmniLatentMaskEditing,
     # === Params ===
     "VLLMOmniARSampling": VLLMOmniARSampling,
     "VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling,
     "VLLMOmniSamplingParamsList": VLLMOmniSamplingParamsList,
     "VLLMOmniRemoteLoRA": VLLMOmniRemoteLoRA,
+    "VLLMOmniFastH3Deployment": VLLMOmniFastH3Deployment,
     "VLLMOmniQwenTTSParams": VLLMOmniQwenTTSParams,
     "VLLMOmniWanParams": VLLMOmniWanParams,
     "VLLMOmniMiniMaxH3Params": VLLMOmniMiniMaxH3Params,
@@ -47,18 +59,22 @@ NODE_CLASS_MAPPINGS = {
 
 # A dictionary that contains the friendly/humanly readable titles for the nodes
 NODE_DISPLAY_NAME_MAPPINGS = {
+    "VLLMOmniMiniMaxH3TemporalMask": "MiniMax-H3 Temporal Mask",
     # === Generation ===
     "VLLMOmniGenerateImage": "Generate Image",
     "VLLMOmniGenerateVideo": "Generate Video",
     "VLLMOmniUnderstanding": "Multimodality Understanding",
     "VLLMOmniTTS": "TTS (Text to Speech)",
+    "VLLMOmniGenerateMusic": "Generate Music",
     "VLLMOmniVoiceClone": "TTS Voice Cloning",
     "VLLMOmniVideoReferences": "Video References",
+    "VLLMOmniLatentMaskEditing": "Latent Mask Editing",
     # === Params ===
     "VLLMOmniARSampling": "AR Sampling Params",
     "VLLMOmniDiffusionSampling": "Diffusion Sampling Params",
     "VLLMOmniSamplingParamsList": "Multi-Stage Sampling Params List",
     "VLLMOmniRemoteLoRA": "LoRA",
+    "VLLMOmniFastH3Deployment": "FastH3 Deployment",
     "VLLMOmniQwenTTSParams": "Qwen TTS Params",
     "VLLMOmniWanParams": "Wan Video Params",
     "VLLMOmniMiniMaxH3Params": "MiniMax-H3 Video Params",

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py

@@ -1,15 +1,25 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+import math
 from typing import Literal
 
 import torch
 from comfy_api.input import AudioInput, VideoInput
 
 from .utils.api_client import VLLMOmniClient
+from .utils.latent_mask import _align_frame_count, _video_latent_t
 from .utils.logger import get_logger
 from .utils.models import lookup_model_spec
 from .utils.types import (
+    MAX_REFERENCE_AUDIOS,
+    MAX_REFERENCE_IMAGES,
+    MAX_REFERENCE_VIDEOS,
     AudioFormat,
     AutoregressionSamplingParams,
     DiffusionSamplingParams,
+    FastH3Deployment,
+    LatentMaskEditing,
     MiniMaxH3ModelSpecificParams,
     QwenTTSModelSpecificParams,
     VideoReferences,
@@ -22,6 +32,23 @@ from .utils.validators import (
 
 logger = get_logger(__name__)
 
+FASTH3_INFERENCE_STEPS = 4
+FASTH3_FPS = 24
+# A FastH3 deployment may expose H3 under any --served-model-name, so the payload
+# spec has to be named outright rather than recovered from that alias.
+FASTH3_SPEC_MODEL = "MiniMax-H3"
+
+
+def _resolve_fast_h3_deployment(deployment: dict) -> tuple[str, str]:
+    if not isinstance(deployment, dict):
+        raise ValueError("FastH3 deployment must be provided by a FastH3 Deployment node.")
+
+    url = str(deployment.get("url") or "").strip().rstrip("/")
+    model = str(deployment.get("model") or "").strip()
+    if not url or not model:
+        raise ValueError("FastH3 deployment requires both URL and model.")
+    return url, model
+
 
 class _VLLMOmniGenerateBase:
     """Base class for vLLM-Omni generation nodes with shared functionality."""
@@ -162,14 +189,31 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
                 "width": ("INT", {"default": 832, "min": 1}),
                 "height": ("INT", {"default": 480, "min": 1}),
                 "fps": ("INT", {"default": 16, "min": 1}),
-                "num_frames": ("INT", {"default": 41, "min": 1}),
+                "duration": (
+                    "FLOAT",
+                    {
+                        "default": 4.0,
+                        "min": 0.1,
+                        "step": 0.1,
+                        "round": 0.001,
+                        "tooltip": (
+                            "Clip length in seconds, converted to frames with the fps above. "
+                            "Models that only accept certain frame counts (e.g. MiniMax-H3) round to "
+                            "their own lattice, so the served clip can be slightly longer than requested."
+                        ),
+                    },
+                ),
             },
             "optional": {
                 "frame": ("IMAGE",),
+                "first_frame": ("IMAGE",),
+                "last_frame": ("IMAGE",),
                 "references": ("VIDEO_REFERENCES",),
                 "sampling_params": ("SAMPLING_PARAMS",),
                 "lora": ("REMOTE_LORA",),
                 "model_params": ("VIDEO_PARAMS",),
+                "fast_h3": ("FASTH3_DEPLOYMENT",),
+                "latent_edit": ("LATENT_MASK_EDITING",),
             },
         }
 
@@ -178,12 +222,30 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
     FUNCTION = "generate"
 
     @classmethod
-    def VALIDATE_INPUTS(cls, url, model, frame=None, references=None, **_kwargs) -> str | Literal[True]:
+    def VALIDATE_INPUTS(
+        cls,
+        url,
+        model,
+        frame=None,
+        first_frame=None,
+        last_frame=None,
+        references=None,
+        fast_h3=None,
+        **_kwargs,
+    ) -> str | Literal[True]:
         base = super().VALIDATE_INPUTS(url, model)
         if base is not True:
             return base
         if frame is not None and references is not None:
             return "Provide only one of frame or references, not both."
+        if frame is not None and (first_frame is not None or last_frame is not None):
+            return "Provide either frame or first_frame/last_frame, not both."
+        if references is not None and (first_frame is not None or last_frame is not None):
+            return "Provide either first_frame/last_frame or references, not both."
+        if fast_h3 is not None and any(value is not None for value in (frame, first_frame, last_frame, references)):
+            return (
+                "FastH3 Preview supports T2VA only; disconnect frame, first_frame, last_frame, and references inputs."
+            )
         return True
 
     async def generate(
@@ -194,20 +256,72 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
         width: int,
         height: int,
         fps: int,
-        num_frames: int,
+        duration: float,
         negative_prompt: str | None = None,
         frame: torch.Tensor | None = None,
+        first_frame: torch.Tensor | None = None,
+        last_frame: torch.Tensor | None = None,
         references: dict | None = None,
         sampling_params: dict | list[dict] | None = None,
         model_params: dict | None = None,
         lora: dict | None = None,
+        fast_h3: dict | None = None,
+        latent_edit: dict | None = None,
         **kwargs,
     ):
         if kwargs:
             logger.info("Uncaught kwargs: %s", kwargs)
         logger.debug("Got sampling params: %s", sampling_params)
         logger.debug("Got model params: %s", model_params)
-        validate_model_and_sampling_params_types(model, sampling_params)
+
+        # Which spec builds the payload. Only a FastH3 deployment separates this
+        # from the served name; every other path keeps them equal.
+        spec_model = model
+
+        if fast_h3 is not None:
+            if any(value is not None for value in (frame, first_frame, last_frame, references)):
+                raise ValueError(
+                    "FastH3 Preview supports T2VA only; disconnect frame, first_frame, "
+                    "last_frame, and references inputs."
+                )
+            if lora is not None:
+                raise ValueError(
+                    "FastH3 is already fused into the selected server; disconnect the request-level LoRA input."
+                )
+
+            url, model = _resolve_fast_h3_deployment(fast_h3)
+            logger.info("Using FastH3 deployment at %s", url)
+            fps = FASTH3_FPS
+            # The served name is whatever the operator passed to --served-model-name.
+            # Left alone, lookup_model_spec would miss H3 for an alias such as
+            # "fasth3", drop the params builder, and send a t2va request carrying no
+            # aspect_ratio -- which the server refuses.
+            spec_model = FASTH3_SPEC_MODEL
+
+            if sampling_params is None:
+                sampling_params = DiffusionSamplingParams()
+            elif isinstance(sampling_params, list):
+                if len(sampling_params) != 1:
+                    raise ValueError("FastH3 expects a single diffusion sampling params group.")
+                sampling_params = sampling_params[0].__class__(sampling_params[0])
+            else:
+                sampling_params = sampling_params.__class__(sampling_params)
+            sampling_params["num_inference_steps"] = FASTH3_INFERENCE_STEPS
+
+            # FastH3 owns both modality shifts. Sending the ordinary H3 values
+            # from a connected H3 Params node would turn a deployment choice
+            # into a request-level override, which the server intentionally
+            # rejects when it differs from the fused adapter contract.
+            if model_params is not None:
+                model_params = model_params.__class__(model_params)
+                model_params.pop("flow_shift", None)
+                model_params.pop("audio_flow_shift", None)
+
+        # Frames stay the wire unit. Convert after the FastH3 branch above, so the
+        # duration is measured against the fps the server will actually apply.
+        num_frames = max(1, round(duration * fps))
+
+        validate_model_and_sampling_params_types(spec_model, sampling_params)
 
         # Currently, all video generation models are single-stage diffusion models
         if isinstance(sampling_params, list):
@@ -227,8 +341,11 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
         client = VLLMOmniClient(url)
         output = await client.generate_video(
             model=model,
+            spec_model=spec_model,
             prompt=prompt,
             frame=frame,  # frame present => fl2va / Wan I2V
+            first_frame=first_frame,
+            last_frame=last_frame,
             references=references,
             width=width,
             height=height,
@@ -238,6 +355,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
             sampling_params=sampling_params,
             lora=lora,
             model_params=model_params,
+            latent_edit=latent_edit,
         )
         return (output,)
 
@@ -418,6 +536,66 @@ class VLLMOmniTTS(_VLLMOmniGenerateBase):
         return (audio,)
 
 
+class VLLMOmniGenerateMusic(_VLLMOmniGenerateBase):
+    """Generate a song from lyrics and a musical description with MiniMax Music 3."""
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "url": ("STRING", {"default": "http://localhost:8000/v1"}),
+                "model": ("STRING", {"default": "MiniMaxAI/MiniMax-Music3"}),
+                "instructions": (
+                    "STRING",
+                    {
+                        "multiline": True,
+                        "display_name": "caption",
+                        "tooltip": "Music caption: describe genre, instruments, tempo and mood.",
+                    },
+                ),
+                "lyrics": ("STRING", {"multiline": True}),
+                "max_duration_seconds": (
+                    "FLOAT",
+                    {
+                        "default": 300.0,
+                        "min": 1,
+                        "max": 360,
+                        "step": 0.01,
+                        "tooltip": "Upper limit; rounded down to whole 25 Hz audio frames. May end earlier.",
+                    },
+                ),
+                "seed": ("INT", {"default": 0, "min": 0, "max": 2**53 - 1, "control_after_generate": True}),
+                "response_format": (["wav", "mp3", "flac", "opus"],),
+            },
+        }
+
+    RETURN_TYPES = ("AUDIO",)
+    RETURN_NAMES = ("audio",)
+    FUNCTION = "generate"
+
+    async def generate(
+        self,
+        url: str,
+        model: str,
+        lyrics: str,
+        instructions: str,
+        response_format: AudioFormat,
+        max_duration_seconds: float,
+        seed: int = 0,
+    ) -> tuple[AudioInput]:
+        audio = await VLLMOmniClient(url.rstrip("/")).generate_speech(
+            model=model,
+            input=lyrics,
+            instructions=instructions,
+            voice="default",
+            speed=1.0,
+            response_format=response_format,
+            max_new_tokens=int(max_duration_seconds * 25),
+            seed=seed,
+        )
+        return (audio,)
+
+
 class VLLMOmniVoiceClone(_VLLMOmniGenerateBase):
     @classmethod
     def INPUT_TYPES(cls):
@@ -695,6 +873,52 @@ class VLLMOmniRemoteLoRA:
         return (lora,)
 
 
+class VLLMOmniFastH3Deployment:
+    """Select a vLLM-Omni service that fused FastH3 at startup."""
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "url": (
+                    "STRING",
+                    {
+                        "default": "http://localhost:8000/v1",
+                        "tooltip": "URL of a vLLM-Omni server started with a FastH3 --lora-path.",
+                    },
+                ),
+                "model": (
+                    "STRING",
+                    {
+                        "default": "MiniMaxAI/MiniMax-H3",
+                        "tooltip": "Model name exposed by the FastH3 deployment.",
+                    },
+                ),
+            }
+        }
+
+    RETURN_TYPES = ("FASTH3_DEPLOYMENT",)
+    RETURN_NAMES = ("deployment",)
+    FUNCTION = "get_deployment"
+    CATEGORY = "vLLM-Omni"
+    DESCRIPTION = (
+        "Selects a server with FastH3 fused at startup. The connected Generate Video node uses T2VA, "
+        "four inference steps, and 24 FPS without sending a request-level LoRA."
+    )
+
+    @classmethod
+    def VALIDATE_INPUTS(cls, url, model) -> str | Literal[True]:
+        try:
+            _resolve_fast_h3_deployment({"url": url, "model": model})
+        except ValueError as exc:
+            return str(exc)
+        return True
+
+    def get_deployment(self, url: str, model: str):
+        url, model = _resolve_fast_h3_deployment({"url": url, "model": model})
+        return (FastH3Deployment({"url": url, "model": model}),)
+
+
 class VLLMOmniQwenTTSParams:
     @classmethod
     def INPUT_TYPES(cls):
@@ -790,6 +1014,10 @@ class VLLMOmniVideoReferences:
                 "audio_2": ("AUDIO",),
                 "video_1": ("VIDEO",),
                 "video_2": ("VIDEO",),
+                # Append ports to preserve connections in saved workflows.
+                **{f"image_{i}": ("IMAGE",) for i in range(3, MAX_REFERENCE_IMAGES + 1)},
+                **{f"audio_{i}": ("AUDIO",) for i in range(3, MAX_REFERENCE_AUDIOS + 1)},
+                **{f"video_{i}": ("VIDEO",) for i in range(3, MAX_REFERENCE_VIDEOS + 1)},
             },
         }
 
@@ -806,21 +1034,112 @@ class VLLMOmniVideoReferences:
         audio_2: AudioInput | None = None,
         video_1: VideoInput | None = None,
         video_2: VideoInput | None = None,
+        image_3: torch.Tensor | None = None,
+        image_4: torch.Tensor | None = None,
+        image_5: torch.Tensor | None = None,
+        image_6: torch.Tensor | None = None,
+        image_7: torch.Tensor | None = None,
+        image_8: torch.Tensor | None = None,
+        image_9: torch.Tensor | None = None,
+        audio_3: AudioInput | None = None,
+        video_3: VideoInput | None = None,
         **kwargs,
     ):
         if kwargs:
             logger.info("Uncaught kwargs: %s", kwargs)
         refs = VideoReferences()
-        if image_1 is not None:
-            refs["image_1"] = image_1
-        if image_2 is not None:
-            refs["image_2"] = image_2
-        if audio_1 is not None:
-            refs["audio_1"] = audio_1
-        if audio_2 is not None:
-            refs["audio_2"] = audio_2
-        if video_1 is not None:
-            refs["video_1"] = video_1
-        if video_2 is not None:
-            refs["video_2"] = video_2
+        for kind, values in (
+            ("image", (image_1, image_2, image_3, image_4, image_5, image_6, image_7, image_8, image_9)),
+            ("video", (video_1, video_2, video_3)),
+            ("audio", (audio_1, audio_2, audio_3)),
+        ):
+            for index, value in enumerate(values, start=1):
+                if value is not None:
+                    refs[f"{kind}_{index}"] = value
         return (refs,)
+
+
+class VLLMOmniLatentMaskEditing:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {},
+            "optional": {
+                "source_video": ("VIDEO",),
+                "source_audio": ("AUDIO",),
+                "video_mask": ("MASK",),
+                "audio_mask": ("FLOAT", {"default": -1.0, "min": -1.0, "max": 1.0, "step": 0.01}),
+            },
+        }
+
+    RETURN_TYPES = ("LATENT_MASK_EDITING",)
+    RETURN_NAMES = ("latent_edit",)
+    FUNCTION = "get_latent_edit"
+    CATEGORY = "vLLM-Omni"
+
+    def get_latent_edit(
+        self,
+        source_video: VideoInput | None = None,
+        source_audio: AudioInput | None = None,
+        video_mask: torch.Tensor | None = None,
+        audio_mask: float = -1.0,
+        **kwargs,
+    ):
+        if kwargs:
+            logger.info("Uncaught kwargs: %s", kwargs)
+        edit = LatentMaskEditing()
+        if source_video is not None:
+            edit["source_video"] = source_video
+        if source_audio is not None:
+            edit["source_audio"] = source_audio
+        if video_mask is not None:
+            edit["video_mask"] = video_mask
+        if audio_mask >= 0.0:
+            edit["audio_mask"] = audio_mask
+        return (edit,)
+
+
+class VLLMOmniMiniMaxH3TemporalMask:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "images": ("IMAGE",),
+                "source_fps": ("FLOAT", {"default": 24.0, "min": 0.01}),
+                "duration": ("FLOAT", {"default": 5.0, "min": 0.01, "max": 15.0}),
+                "mode": (["continuation", "extension"],),
+                "preserve_fraction": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0}),
+            }
+        }
+
+    RETURN_TYPES = ("MASK", "FLOAT", "IMAGE", "MASK")
+    RETURN_NAMES = ("mask", "preview_fps", "preview_images", "preview_mask")
+    FUNCTION = "build"
+    CATEGORY = "vLLM-Omni"
+
+    def build(self, images, source_fps, duration, mode, preserve_fraction=0.5):
+        if images.shape[0] <= 0 or not math.isfinite(source_fps) or source_fps <= 0:
+            raise ValueError("Source must contain frames and have a positive finite FPS.")
+        if not math.isfinite(duration) or duration <= 0 or not 0 <= preserve_fraction <= 1:
+            raise ValueError("Invalid duration or preserve_fraction.")
+        frames = _align_frame_count(max(1, round(duration * 24)))
+        source_seconds = images.shape[0] / source_fps
+        if mode == "extension":
+            if frames / 24 <= source_seconds:
+                raise ValueError("Extension output must be longer than the source; increase duration.")
+            boundary = source_seconds
+        elif mode == "continuation":
+            boundary = min(source_seconds, frames / 24) * preserve_fraction
+        else:
+            raise ValueError(f"Unknown temporal mask mode: {mode}")
+        available = min(frames, math.floor(boundary * 24 + 1e-8))
+        prefix = 0 if available < 5 else 5 + 17 * ((available - 5) // 17)
+        preserved = _video_latent_t(prefix) if prefix else 0
+        total = _video_latent_t(frames)
+        mask = torch.ones(total, 1, 1)
+        mask[:preserved] = 0
+        indices = torch.arange(frames, device=images.device)
+        source_indices = (indices * (source_fps / 24)).floor().long().clamp(max=images.shape[0] - 1)
+        preview_images = images.index_select(0, source_indices)
+        preview_mask = mask.index_select(0, torch.arange(frames) * total // frames)
+        return mask, 24.0, preview_images, preview_mask

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 """
 An high-level API client adapter that forwards ComfyUI inputs to vLLM-Omni's REST API,
 and transforms the API responses back to ComfyUI formats.
@@ -17,6 +20,7 @@ from comfy_api.input import AudioInput, VideoInput
 
 from .format import (
     audio_to_base64,
+    audio_to_bytes,
     base64_to_audio,
     base64_to_image_tensor,
     bytes_to_audio,
@@ -26,9 +30,16 @@ from .format import (
     video_to_base64,
     video_to_bytes,
 )
+from .latent_mask import scalar_mask_to_json, video_mask_to_grid_json
 from .logger import get_logger, pretty_printer
 from .models import lookup_model_spec
-from .types import AudioFormat
+from .types import (
+    MAX_REFERENCE_AUDIOS,
+    MAX_REFERENCE_IMAGES,
+    MAX_REFERENCE_VIDEOS,
+    MAX_TOTAL_REFERENCES,
+    AudioFormat,
+)
 
 logger = get_logger(__name__)
 
@@ -267,14 +278,34 @@ class VLLMOmniClient:
         fps: int,
         negative_prompt: str | None = None,
         frame: torch.Tensor | None = None,
+        first_frame: torch.Tensor | None = None,
+        last_frame: torch.Tensor | None = None,
         references: dict | None = None,
         sampling_params: dict | None = None,
         model_params: dict | None = None,
         lora: dict | None = None,
+        latent_edit: dict | None = None,
+        spec_model: str | None = None,
         **extra_params,
     ) -> VideoInput:
+        """Post a video job and return the decoded result.
+
+        ``spec_model`` names the model whose payload spec builds the request, for
+        deployments that serve a known model under a different ``model`` alias. It
+        never reaches the wire; defaults to ``model``.
+        """
         if frame is not None and references is not None:
             raise ValueError("Provide only one of frame or references, not both.")
+        if frame is not None and (first_frame is not None or last_frame is not None):
+            raise ValueError("Provide either frame or first_frame/last_frame, not both.")
+        if references is not None and (first_frame is not None or last_frame is not None):
+            raise ValueError("Provide either first_frame/last_frame or references, not both.")
+
+        spec, matched_pattern = lookup_model_spec(spec_model or model)
+        if (first_frame is not None or last_frame is not None) and (
+            matched_pattern is None or "MiniMax-H3" not in matched_pattern
+        ):
+            raise ValueError("first_frame and last_frame are supported only for MiniMax-H3; use frame for this model.")
 
         # === regular payload fields ===
         form = aiohttp.FormData()
@@ -294,43 +325,62 @@ class VLLMOmniClient:
 
         # === multimodal inputs (first-last-frames, references, etc.) ===
         input_reference_image: torch.Tensor | None = None
-        audio_reference: AudioInput | None = None
-        reference_videos: list[VideoInput] = []
+        keyframe_images: list[tuple[str, torch.Tensor]] = []
         video_task: str | None = None
 
         if frame is not None:
             input_reference_image = frame
             video_task = "fl2va"
+        elif first_frame is not None or last_frame is not None:
+            frame_indices: list[int] = []
+            if first_frame is not None:
+                keyframe_images.append(("first_frame.png", first_frame))
+                frame_indices.append(0)
+            if last_frame is not None:
+                keyframe_images.append(("last_frame.png", last_frame))
+                frame_indices.append(-1)
+            if len(keyframe_images) == 1:
+                input_reference_image = keyframe_images[0][1]
+            extra_params["frame_indices"] = frame_indices
+            video_task = "fl2va"
         elif references is not None:
-            images = [references[k] for k in ("image_1", "image_2") if k in references and references[k] is not None]
-            audios = [references[k] for k in ("audio_1", "audio_2") if k in references and references[k] is not None]
-            videos = [references[k] for k in ("video_1", "video_2") if k in references and references[k] is not None]
-
-            if not images and not audios and not videos:
-                raise ValueError("references is empty; connect at least one image/audio/video.")
-
-            if videos:
-                if images or audios:
-                    raise ValueError(
-                        "Video references cannot be combined with image or audio references. "
-                        "Connect only video_1/video_2 for multi-video Ref2VA."
-                    )
-                reference_videos = videos
-                video_task = "ref2va"
-            elif len(images) == 1 and len(audios) == 1:
-                input_reference_image = images[0]
-                audio_reference = audios[0]
-                video_task = "ref2va"
-            else:
+            reference_formats = (
+                ("image", MAX_REFERENCE_IMAGES, "png", "image/png", image_tensor_to_png_bytes),
+                ("video", MAX_REFERENCE_VIDEOS, "mp4", "video/mp4", video_to_bytes),
+                ("audio", MAX_REFERENCE_AUDIOS, "mp3", "audio/mpeg", audio_to_bytes),
+            )
+            supported_inputs = {f"{kind}_{i}" for kind, limit, *_ in reference_formats for i in range(1, limit + 1)}
+            connected = {name: value for name, value in references.items() if value is not None}
+            unsupported = connected.keys() - supported_inputs
+            if unsupported:
+                raise ValueError(f"Unsupported reference input(s): {', '.join(sorted(unsupported))}.")
+            if not any(name.startswith(("image_", "video_")) for name in connected):
                 raise ValueError(
-                    "Invalid references combination. Supported modes: "
-                    "(1) one or more videos only, or (2) exactly one image and one audio."
+                    "references requires at least one image or video; audio-only inputs are not supported."
                 )
+            if len(connected) > MAX_TOTAL_REFERENCES:
+                raise ValueError(
+                    f"references supports at most {MAX_TOTAL_REFERENCES} inputs in total "
+                    f"(up to {MAX_REFERENCE_IMAGES} images, {MAX_REFERENCE_VIDEOS} videos, "
+                    f"and {MAX_REFERENCE_AUDIOS} audios)."
+                )
+            for kind, limit, extension, content_type, encode in reference_formats:
+                for index in range(1, limit + 1):
+                    name = f"{kind}_{index}"
+                    if name in connected:
+                        filename = f"{name}.{extension}"
+                        form.add_field(
+                            "input_references",
+                            encode(connected[name], filename),
+                            filename=filename,
+                            content_type=content_type,
+                        )
+            video_task = "ref2va"
         else:
             video_task = "t2va"
 
         if input_reference_image is not None:
-            image_filename = "image.png"  # Required for multipart form
+            image_filename = keyframe_images[0][0] if keyframe_images else "image.png"
             form.add_field(
                 "input_reference",
                 image_tensor_to_png_bytes(input_reference_image, image_filename),
@@ -338,32 +388,74 @@ class VLLMOmniClient:
                 content_type="image/png",
             )
 
-        if audio_reference is not None:
-            form.add_field(
-                "audio_reference",
-                json.dumps({"audio_url": audio_to_base64(audio_reference)}, ensure_ascii=False),
-            )
+        # === latent-mask editing (MiniMax H3) ===
+        if latent_edit is not None:
+            source_video = latent_edit.get("source_video")
+            source_audio = latent_edit.get("source_audio")
+            video_mask = latent_edit.get("video_mask")
+            audio_mask = latent_edit.get("audio_mask")
 
-        for idx, video in enumerate(reference_videos, start=1):
-            video_filename = f"reference_{idx}.mp4"
-            form.add_field(
-                "input_references",
-                video_to_bytes(video, video_filename),
-                filename=video_filename,
-                content_type="video/mp4",
-            )
+            if video_mask is None and audio_mask is None:
+                raise ValueError("Latent-mask editing requires at least one mask.")
+
+            video_mask_trivial = video_mask is None or bool((video_mask == 1.0).all().item())
+            audio_mask_trivial = audio_mask is None or audio_mask == 1.0
+            if not video_mask_trivial and source_video is None:
+                raise ValueError("A non-trivial video mask requires a source video.")
+            if not audio_mask_trivial and source_audio is None and source_video is None:
+                raise ValueError("A non-trivial audio mask requires a source audio or a source video with audio.")
+
+            if source_video is not None:
+                form.add_field(
+                    "source_video",
+                    video_to_bytes(source_video, "source.mp4"),
+                    filename="source.mp4",
+                    content_type="video/mp4",
+                )
+            if source_audio is not None:
+                form.add_field(
+                    "source_audio",
+                    audio_to_bytes(source_audio, "source_audio.mp3"),
+                    filename="source_audio.mp3",
+                    content_type="audio/mpeg",
+                )
+            if video_mask is not None:
+                mask_json = video_mask_to_grid_json(video_mask, width=width, height=height, num_frames=num_frames)
+                form.add_field(
+                    "video_noise_mask",
+                    mask_json.encode("utf-8"),
+                    filename="video-mask.json",
+                    content_type="application/json",
+                )
+            if audio_mask is not None:
+                form.add_field(
+                    "audio_noise_mask",
+                    scalar_mask_to_json(audio_mask).encode("utf-8"),
+                    filename="audio-mask.json",
+                    content_type="application/json",
+                )
+
+        if len(keyframe_images) == 2:
+            for image_filename, image in keyframe_images:
+                form.add_field(
+                    "input_references",
+                    image_tensor_to_png_bytes(image, image_filename),
+                    filename=image_filename,
+                    content_type="image/png",
+                )
 
         # === model specific params. Either use a specialized builder, or add flattened fields as-is ===
         if model_params is not None:
             model_params = dict(model_params)
             model_params.pop("type", None)
 
-        spec, _ = lookup_model_spec(model)
         params_builder = spec.get("params_builder") if spec else None
         if params_builder is not None:
             form_fields = params_builder(
                 model_params or {},
                 extra_params={**extra_params, "task": video_task},
+                width=width,
+                height=height,
             )
             for k, v in form_fields.items():
                 form.add_field(k, v if isinstance(v, str) else str(v))

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/format.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 """Image/tensor format helpers.
 
 The image generation part is derived from dougbtv/comfyui-vllm-omni by Doug (@dougbtv).
@@ -170,6 +173,10 @@ def bytes_to_video(video_bytes: bytes) -> VideoInput:
                 audio_stream = container.streams.audio[-1]
                 audio_frames = []
                 resampler = AudioResampler(format="fltp")
+                # Decoding the video stream above left the container at EOF.
+                # Rewind before demuxing again, or this pass yields no frames
+                # and the audio of a T2VA result is silently dropped.
+                container.seek(0)
                 for frame in container.decode(audio_stream):
                     if not isinstance(frame, AudioFrame):
                         continue

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py

@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+import json
+
+import torch
+import torch.nn.functional as F
+
+
+def scalar_mask_to_json(value: float) -> str:
+    if not 0.0 <= value <= 1.0:
+        raise ValueError(f"mask value must be in [0, 1], got {value}")
+    return str(value)
+
+
+def _align_frame_count(frame_count: int) -> int:
+    if frame_count <= 0:
+        return 1
+    current = int(frame_count)
+    while current % 17 != 5:
+        current += 1
+    return current
+
+
+def _video_latent_t(frame_count: int) -> int:
+    if frame_count <= 5:
+        return 2
+    return ((int(frame_count) - 5) // 17) * 5 + 2
+
+
+def video_mask_to_grid(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> torch.Tensor:
+    tv = _video_latent_t(_align_frame_count(num_frames))
+    height = int(height) // 32 * 32
+    width = int(width) // 32 * 32
+    gh, gw = height // 16, width // 16
+
+    if mask.ndim == 2:
+        mask = mask.unsqueeze(0)
+    elif mask.ndim != 3:
+        raise ValueError(f"expected a 2D or 3D mask tensor, got {mask.ndim}D")
+
+    grid = F.interpolate(mask.unsqueeze(1).float(), size=(gh, gw), mode="area").squeeze(1)
+    if grid.shape[0] == 1:
+        grid = grid.expand(tv, gh, gw)
+    elif grid.shape[0] != tv:
+        grid = F.interpolate(grid.unsqueeze(0).unsqueeze(0), size=(tv, gh, gw), mode="nearest").squeeze(0).squeeze(0)
+    return grid
+
+
+def video_mask_to_grid_json(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> str:
+    return json.dumps(video_mask_to_grid(mask, width=width, height=height, num_frames=num_frames).tolist())

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 import json
 import re
 from typing import Any
@@ -34,10 +37,28 @@ def _qwen25_payload_preprocessor(payload: dict) -> dict:
     return payload
 
 
+MINIMAX_H3_ASPECT_RATIOS = {
+    "21:9": 21.0 / 9.0,
+    "16:9": 16.0 / 9.0,
+    "4:3": 4.0 / 3.0,
+    "1:1": 1.0,
+    "3:4": 3.0 / 4.0,
+    "9:16": 9.0 / 16.0,
+}
+
+
+def _nearest_minimaxh3_aspect_ratio(width: int, height: int) -> str:
+    """Pick the supported named ratio closest to the requested frame size."""
+    target = float(width) / float(height)
+    return min(MINIMAX_H3_ASPECT_RATIOS, key=lambda name: abs(MINIMAX_H3_ASPECT_RATIOS[name] - target))
+
+
 def _minimaxh3_params_builder(
     model_params: dict[str, Any],
     *,
     extra_params: dict[str, Any],
+    width: int | None = None,
+    height: int | None = None,
 ) -> dict[str, Any]:
     """Build multipart form fields for MiniMax-H3 from model_params + routed task."""
     params = dict(model_params)
@@ -53,6 +74,12 @@ def _minimaxh3_params_builder(
     if params:
         logger.warning("Unused MiniMax-H3 model params ignored: %s", sorted(params))
 
+    # H3 refuses t2va without a named aspect ratio, and the Generate Video node
+    # carries width/height rather than a ratio. fl2va takes its ratio from the
+    # input image and ref2va defaults server-side, so only t2va needs this.
+    if merged_extra_params.get("task") == "t2va" and "aspect_ratio" not in merged_extra_params and width and height:
+        merged_extra_params["aspect_ratio"] = _nearest_minimaxh3_aspect_ratio(width, height)
+
     if merged_extra_params:
         form_fields["extra_params"] = json.dumps(merged_extra_params, ensure_ascii=False)
     return form_fields

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 from collections.abc import Callable
 from enum import Enum, auto
 from typing import Any, Literal, TypeAlias
@@ -27,10 +30,24 @@ class MiniMaxH3ModelSpecificParams(dict):
     pass
 
 
+MAX_REFERENCE_IMAGES = 9
+MAX_REFERENCE_VIDEOS = 3
+MAX_REFERENCE_AUDIOS = 3
+MAX_TOTAL_REFERENCES = 12
+
+
 class VideoReferences(dict):
     pass
 
 
+class LatentMaskEditing(dict):
+    pass
+
+
+class FastH3Deployment(dict):
+    """Descriptor for a server that fused FastH3 at startup."""
+
+
 class ModelMode(Enum):
     IMAGE_GENERATION = auto()
     VIDEO_GENERATION = auto()

apps/ComfyUI-vLLM-Omni/docs/minimax-h3-t2v.md

@@ -0,0 +1,94 @@
+# MiniMax H3 Text to Video (WF-01)
+
+Import [MiniMax_H3_Text_to_Video.json](../example_workflows/MiniMax_H3_Text_to_Video.json)
+into ComfyUI, or select it under **Templates → ComfyUI-vLLM-Omni**.
+The template connects existing nodes to a remote H3 service and saves the
+returned video with audio. ComfyUI does not load H3 weights.
+
+## Setup
+
+Install the [ComfyUI extension](../README.md#installation) and follow the
+[H3 deployment recipe](../../../recipes/MiniMaxAI/MiniMax-H3.md) for your hardware.
+For a two-GPU FL2VA server with sufficient memory:
+
+```bash
+export MODEL=/path/to/MiniMax-H3/FL2VA
+CUDA_VISIBLE_DEVICES=0,1 VLLM_WORKER_MULTIPROC_METHOD=spawn \
+VLLM_OMNI_VIDEO_SYNC_TIMEOUT=14400 \
+vllm serve "$MODEL" --omni --host 127.0.0.1 --port 8000 \
+  --served-model-name MiniMaxAI/MiniMax-H3 --trust-remote-code \
+  --task-type fl2va --num-gpus 2 --tensor-parallel-size 2 \
+  --text-encoder-tp-size 2 --enforce-eager
+```
+
+Set Generate Video's **url** to the address reachable from ComfyUI, including
+`/v1`, and match its **model** to the served name. The defaults are
+`http://localhost:8000/v1` and `MiniMaxAI/MiniMax-H3`.
+
+## Base and Turbo
+
+Base presets are connected by default. Write the scene and its sound in the
+same **prompt**; leave **frame**, **references**, and **fast_h3** disconnected.
+The template uses 1344×768, 24 FPS, seed 1101 and **duration** 5.167 seconds
+(124 frames, satisfying H3's `17k+5` constraint). The existing client infers
+`16:9` from these dimensions. For portrait output, use 768×1344.
+
+| Setting | Base | Turbo v1.0 768p |
+| --- | --- | --- |
+| Inference steps | 50 | 5 (4 forwards) |
+| Video flow shift | 12 | 6 |
+| Audio flow shift | 3 | 3 |
+| Remote LoRA | Disconnected | Connected, scale 1 |
+
+For Turbo, use the **Diffusers-layout** artifact from
+[lightx2v/Minimax-h3-Turbo](https://huggingface.co/lightx2v/Minimax-h3-Turbo):
+
+```text
+minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors
+```
+
+Add `--lora-backend peft --lora-path "$TURBO_LORA"` to the server command,
+where `TURBO_LORA` is the full server-side path to that file. Enter the same
+path in Remote LoRA's **local_path**; retain name `h3-turbo-v1.0-768p`, scale
+`1.0`, and **int_id** `0` so the existing API derives the ID from the path.
+The template leaves the path blank for users to configure. The `_comfyui_`
+export is not supported by this backend.
+
+Connect **Turbo sampling**, **Turbo H3 params**, and **Remote LoRA** to Generate
+Video's `sampling_params`, `model_params`, and `lora` inputs, replacing the
+Base links. To return to Base, restore both Base links and disconnect LoRA.
+Other Turbo artifacts may need different settings; see the
+[artifact table](../../../recipes/MiniMaxAI/MiniMax-H3.md#turbo-lora).
+
+If importing an older template with a frame-count widget, reimport this JSON
+or set **duration** to **5.167**; do not interpret 124 frames as 124 seconds.
+
+## Validation
+
+Queue the configured workflow and inspect the MP4 saved under
+`ComfyUI/output/video/MiniMax_H3_WF01_*.mp4`:
+
+```bash
+ffprobe -v error -show_streams -show_format -of json "$OUTPUT"
+ffmpeg -v error -i "$OUTPUT" -f null -
+```
+
+Check the requested dimensions, frame count, 24 FPS and both video/audio
+streams, then watch and listen to the saved file. The current upstream
+extension includes the shared audio-preserving decoder fix.
+
+On September 16, the narrowed workflow was validated through ComfyUI using
+existing upstream Python code at `4a8ac297`, two H20-3e GPUs, vLLM 0.29.0,
+PyTorch 2.13.0+cu129 and ComfyUI `1d48d9cf`. The run used the template's
+forest-stream prompt, explicit server-side Turbo LoRA path, and settings
+above, with `TORCH_SDPA`, `--usp 1 --ring 1`, and tiled VAE parallelism on two
+GPUs. ComfyUI execution took 131.55 seconds. SaveVideo produced 124 H.264
+frames at 1344×768 and 24 FPS, plus 32 kHz stereo AAC audio. Video/audio
+durations were 5.166667/5.167000 seconds; full decoding passed and audio was
+nonzero. This is model-generated output, not a ground-truth reference or a
+subjective audio-quality assessment. Base graph validation passed with Turbo
+nodes disconnected; full 50-step Base quality was not evaluated.
+
+Based on the official
+[ComfyUI H3 T2V template](https://github.com/Comfy-Org/workflow_templates/blob/main/templates/video_minimax_h3_t2v.json),
+using remote inference instead of local model loaders.

apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md

@@ -0,0 +1,17 @@
+# WF-05: MiniMax-H3 latent-mask editing
+
+Edit a source video with MiniMax-H3 using spatial or temporal masks. The workflow includes object removal, inpainting, continuation, and extension, with a mask preview for each case.
+
+## Requirements
+
+- ComfyUI with ComfyUI-vLLM-Omni installed.
+- An H3 service using the FL2VA checkpoint partition with latent-mask editing support. Follow the [H3 recipe](../../../recipes/MiniMaxAI/MiniMax-H3.md) for server setup.
+
+## Use
+
+Open **vLLM-Omni MiniMax-H3 Latent Mask Editing.json** from the example workflows and set the Generate Video URL/model for the case you want to run.
+
+1. Upload your source video in Load Video.
+2. Adjust the case's mask. For removal and inpainting, set Mask Canvas to the video dimensions, Mask Rectangle to the region size, and Combine Masks x/y to its position. For continuation and extension, set MiniMax-H3 Temporal Mask's mode and duration; continuation also uses `preserve_fraction`. Match the duration in Generate Video, and use a duration longer than the source for extension. Mask values are `0` to preserve and `1` to regenerate.
+3. Adjust the prompt to describe the desired content. Set the output dimensions and duration in Generate Video. If editing audio, update the sound description and audio mask (`0` preserves, `1` regenerates the whole clip).
+4. Select the case's Mask Preview Save Video node and choose **Execute to selected output nodes** to preview the mask as a red overlay without inference, or select its Result Save Video node to generate the edited video. Running the entire workflow generates all four cases.

apps/ComfyUI-vLLM-Omni/docs/wf07-h3-upscale.md

@@ -0,0 +1,21 @@
+# WF-07: H3 video upscale with SeedVR2
+
+Generate video and audio through vLLM-Omni H3, upscale the decoded frames with SeedVR2, and save both the original and upscaled videos. The generated audio and FPS connect directly to Create Video.
+
+## Requirements
+
+- ComfyUI with ComfyUI-vLLM-Omni and [ComfyUI-SeedVR2_VideoUpscaler](https://github.com/numz/ComfyUI-SeedVR2_VideoUpscaler#-installation) installed.
+- An H3 service using the FL2VA checkpoint partition. Follow the [H3 recipe](../../../recipes/MiniMaxAI/MiniMax-H3.md) for server setup.
+- `seedvr2_ema_3b_fp16.safetensors` and `ema_vae_fp16.safetensors` from [SeedVR2's ComfyUI weights](https://huggingface.co/numz/SeedVR2_comfyUI/tree/main), placed in `ComfyUI/models/SEEDVR2`. The SeedVR2 nodes can also download them on first use.
+
+## Use
+
+Open **vLLM-Omni MiniMax H3 Video Upscale.json** from the example workflows. An API-format version is provided alongside it. Set the Generate Video URL/model and the LoRA path as seen by the H3 server.
+
+The preset uses the LightX2V v1.0 768p Turbo adapter in Diffusers format: `minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors`, five sigma points, video/audio flow shifts 6/3, and LoRA scale 1. Preload the same adapter on the server using the [Turbo LoRA recipe](../../../recipes/MiniMaxAI/MiniMax-H3.md#turbo-lora). For base H3, disconnect LoRA, set inference steps to 50, and set video flow shift to 12.
+
+The H3 preset requests 1344 x 768 at 24 FPS for 5.167 seconds, which gives 124 frames. SeedVR2's `resolution=1536` sets the target short edge, producing 2688 x 1536 for this input. Update this value when changing the input size; it is an output resolution, not a fixed 2x multiplier.
+
+The SeedVR2 preset uses 3B FP16, five-frame batches, one overlapping frame, uniform batch padding, and LAB color correction. The DiT loader uses CPU offload with 32 swapped blocks; the VAE uses tiled encoding and decoding. Select the local GPU in both model loaders. See the [SeedVR2 settings](https://github.com/numz/ComfyUI-SeedVR2_VideoUpscaler#-usage) when adjusting memory use or batch size.
+
+Run the workflow. The two Save Video nodes write MP4s under `WF07/generated` and `WF07/upscaled` in ComfyUI's output directory.

apps/ComfyUI-vLLM-Omni/example_workflows/MiniMax_H3_Text_to_Video.json

@@ -0,0 +1,362 @@
+{
+  "last_node_id": 8,
+  "last_link_id": 3,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "VLLMOmniDiffusionSampling",
+      "pos": [
+        20,
+        20
+      ],
+      "size": [
+        310,
+        280
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "diffusion sampling params",
+          "type": "SAMPLING_PARAMS",
+          "links": [
+            1
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniDiffusionSampling"
+      },
+      "widgets_values": [
+        1,
+        50,
+        1.0,
+        1.0,
+        false,
+        false,
+        1101,
+        "fixed"
+      ],
+      "title": "Base sampling"
+    },
+    {
+      "id": 2,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        20,
+        340
+      ],
+      "size": [
+        310,
+        140
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "MiniMax-H3 video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            2
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ],
+      "title": "Base H3 params"
+    },
+    {
+      "id": 3,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        430,
+        20
+      ],
+      "size": [
+        430,
+        440
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null
+        },
+        {
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null
+        },
+        {
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": 1
+        },
+        {
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 2
+        },
+        {
+          "name": "fast_h3",
+          "type": "FASTH3_DEPLOYMENT",
+          "link": null
+        },
+        {
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            3
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://localhost:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "A small stream flows through a sunlit forest. The camera slowly moves toward the water. Birds chirp in the trees and water gurgles over smooth stones. Natural continuous sound, no music.",
+        "",
+        1344,
+        768,
+        24,
+        5.167
+      ]
+    },
+    {
+      "id": 4,
+      "type": "SaveVideo",
+      "pos": [
+        940,
+        20
+      ],
+      "size": [
+        440,
+        440
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "link": 3
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": null,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "video/MiniMax_H3_WF01",
+        "mp4",
+        "auto"
+      ]
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniRemoteLoRA",
+      "pos": [
+        430,
+        540
+      ],
+      "size": [
+        430,
+        160
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "links": null,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniRemoteLoRA"
+      },
+      "widgets_values": [
+        "",
+        "h3-turbo-v1.0-768p",
+        1.0,
+        0
+      ],
+      "title": "Optional Turbo: server LoRA path"
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniDiffusionSampling",
+      "pos": [
+        20,
+        540
+      ],
+      "size": [
+        310,
+        280
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "diffusion sampling params",
+          "type": "SAMPLING_PARAMS",
+          "links": null,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniDiffusionSampling"
+      },
+      "widgets_values": [
+        1,
+        5,
+        1.0,
+        1.0,
+        false,
+        false,
+        1101,
+        "fixed"
+      ],
+      "title": "Turbo sampling (4 forwards)"
+    },
+    {
+      "id": 7,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        430,
+        740
+      ],
+      "size": [
+        310,
+        140
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "MiniMax-H3 video params",
+          "type": "VIDEO_PARAMS",
+          "links": null,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        6.0
+      ],
+      "title": "Turbo H3 params"
+    },
+    {
+      "id": 8,
+      "type": "Note",
+      "pos": [
+        940,
+        520
+      ],
+      "size": [
+        440,
+        440
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "properties": {
+        "Node name for S&R": "Note"
+      },
+      "widgets_values": [
+        "MiniMax H3 Text to Video (WF-01)\n\nBASE: connected presets, 50 steps / video flow 12 / audio flow 3.\n\nTURBO: set Remote LoRA local_path to the SERVER path of minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors from lightx2v/Minimax-h3-Turbo. Keep name h3-turbo-v1.0-768p, scale 1.0 and int_id 0. Connect Turbo sampling, Turbo H3 params, and LoRA to Generate Video, replacing the two Base links. Use 5 steps (4 forwards), video flow 6 and audio flow 3. Do not use the ComfyUI-layout artifact.\n\nKeep frame, references and fast_h3 disconnected. 1344x768, 24 FPS, duration 5.167 seconds = 124 frames (17*7+5). The client selects the named 16:9 preset from the dimensions. Save Video preserves video and audio.\n\nSet URL and served model name for your server. See docs/minimax-h3-t2v.md for setup and validation. Source: Comfy-Org/workflow_templates/templates/video_minimax_h3_t2v.json."
+      ],
+      "title": "Setup / Base and Turbo"
+    }
+  ],
+  "links": [
+    [
+      1,
+      1,
+      0,
+      3,
+      2,
+      "SAMPLING_PARAMS"
+    ],
+    [
+      2,
+      2,
+      0,
+      3,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      3,
+      3,
+      0,
+      4,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 0.8,
+      "offset": [
+        50,
+        50
+      ]
+    }
+  },
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni FastH3 Text to Video.json

@@ -0,0 +1,322 @@
+{
+  "id": "c6db00da-2ccb-4ff8-8334-d67e1a267781",
+  "revision": 0,
+  "last_node_id": 5,
+  "last_link_id": 3,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "MarkdownNote",
+      "pos": [40, 20],
+      "size": [420, 250],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "FastH3 server prerequisite",
+      "properties": {},
+      "widgets_values": [
+        "FastH3 must be fused when the vLLM-Omni server starts. Use --task-type fl2va and --lora-path pointing to exactly one FastH3 adapter file. Do not use CPU or layerwise offload. For VSA / Data-Free, also install fastvideo-kernel and use --diffusion-attention-backend FASTVIDEO_VSA --fastvideo-vsa-topk 64."
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 2,
+      "type": "VLLMOmniFastH3Deployment",
+      "pos": [40, 320],
+      "size": [420, 160],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {"name": "url"},
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {"name": "model"},
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "deployment",
+          "name": "deployment",
+          "type": "FASTH3_DEPLOYMENT",
+          "links": [1]
+        }
+      ],
+      "properties": {"Node name for S&R": "VLLMOmniFastH3Deployment"},
+      "widgets_values": [
+        "http://localhost:8000/v1",
+        "MiniMaxAI/MiniMax-H3"
+      ]
+    },
+    {
+      "id": 3,
+      "type": "VLLMOmniDiffusionSampling",
+      "pos": [40, 540],
+      "size": [420, 230],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "n",
+          "name": "n",
+          "type": "INT",
+          "widget": {"name": "n"},
+          "link": null
+        },
+        {
+          "localized_name": "num_inference_steps",
+          "name": "num_inference_steps",
+          "type": "INT",
+          "widget": {"name": "num_inference_steps"},
+          "link": null
+        },
+        {
+          "localized_name": "guidance_scale",
+          "name": "guidance_scale",
+          "type": "FLOAT",
+          "widget": {"name": "guidance_scale"},
+          "link": null
+        },
+        {
+          "localized_name": "true_cfg_scale",
+          "name": "true_cfg_scale",
+          "type": "FLOAT",
+          "widget": {"name": "true_cfg_scale"},
+          "link": null
+        },
+        {
+          "localized_name": "vae_use_slicing",
+          "name": "vae_use_slicing",
+          "type": "BOOLEAN",
+          "widget": {"name": "vae_use_slicing"},
+          "link": null
+        },
+        {
+          "localized_name": "vae_use_tiling",
+          "name": "vae_use_tiling",
+          "type": "BOOLEAN",
+          "widget": {"name": "vae_use_tiling"},
+          "link": null
+        },
+        {
+          "localized_name": "seed",
+          "name": "seed",
+          "type": "INT",
+          "widget": {"name": "seed"},
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "diffusion sampling params",
+          "name": "diffusion sampling params",
+          "type": "SAMPLING_PARAMS",
+          "links": [2]
+        }
+      ],
+      "properties": {"Node name for S&R": "VLLMOmniDiffusionSampling"},
+      "widgets_values": [1, 4, 1, 1, false, false, 1101, "fixed"]
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [560, 210],
+      "size": [540, 610],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "frame",
+          "name": "frame",
+          "shape": 7,
+          "type": "IMAGE",
+          "link": null
+        },
+        {
+          "localized_name": "references",
+          "name": "references",
+          "shape": 7,
+          "type": "VIDEO_REFERENCES",
+          "link": null
+        },
+        {
+          "localized_name": "sampling_params",
+          "name": "sampling_params",
+          "shape": 7,
+          "type": "SAMPLING_PARAMS",
+          "link": 2
+        },
+        {
+          "localized_name": "lora",
+          "name": "lora",
+          "shape": 7,
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "localized_name": "model_params",
+          "name": "model_params",
+          "shape": 7,
+          "type": "VIDEO_PARAMS",
+          "link": null
+        },
+        {
+          "localized_name": "fast_h3",
+          "name": "fast_h3",
+          "shape": 7,
+          "type": "FASTH3_DEPLOYMENT",
+          "link": 1
+        },
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {"name": "url"},
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {"name": "model"},
+          "link": null
+        },
+        {
+          "localized_name": "prompt",
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {"name": "prompt"},
+          "link": null
+        },
+        {
+          "localized_name": "negative_prompt",
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {"name": "negative_prompt"},
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {"name": "width"},
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {"name": "height"},
+          "link": null
+        },
+        {
+          "localized_name": "fps",
+          "name": "fps",
+          "type": "INT",
+          "widget": {"name": "fps"},
+          "link": null
+        },
+        {
+          "localized_name": "duration",
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {"name": "duration"},
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [3]
+        }
+      ],
+      "properties": {"Node name for S&R": "VLLMOmniGenerateVideo"},
+      "widgets_values": [
+        "http://localhost:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "A cinematic live-action singer performs naturally on a neon-lit stage, with synchronized vocals and environmental sound.",
+        "",
+        1344,
+        768,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 5,
+      "type": "SaveVideo",
+      "pos": [1200, 280],
+      "size": [380, 310],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "link": 3
+        },
+        {
+          "localized_name": "filename_prefix",
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {"name": "filename_prefix"},
+          "link": null
+        },
+        {
+          "localized_name": "format",
+          "name": "format",
+          "type": "COMBO",
+          "widget": {"name": "format"},
+          "link": null
+        },
+        {
+          "localized_name": "codec",
+          "name": "codec",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {"name": "codec"},
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": null
+        }
+      ],
+      "properties": {},
+      "widgets_values": ["video/ComfyUI-FastH3", "auto", "auto"]
+    }
+  ],
+  "links": [
+    [1, 2, 0, 4, 5, "FASTH3_DEPLOYMENT"],
+    [2, 3, 0, 4, 2, "SAMPLING_PARAMS"],
+    [3, 4, 0, 5, 0, "VIDEO"]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "workflowRendererVersion": "LG",
+    "ds": {"scale": 0.9, "offset": [100, 100]}
+  },
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json

@@ -0,0 +1,499 @@
+{
+  "id": "latent-mask-editing-7380",
+  "revision": 0,
+  "last_node_id": 7,
+  "last_link_id": 5,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "MarkdownNote",
+      "pos": [
+        40,
+        20
+      ],
+      "size": [
+        420,
+        210
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Latent-mask editing server prerequisite",
+      "properties": {},
+      "widgets_values": [
+        "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends."
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 2,
+      "type": "LoadVideo",
+      "pos": [
+        40,
+        280
+      ],
+      "size": [
+        420,
+        120
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "file",
+          "name": "file",
+          "type": "COMBO",
+          "widget": {
+            "name": "file"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        ""
+      ]
+    },
+    {
+      "id": 3,
+      "type": "LoadImageMask",
+      "pos": [
+        40,
+        460
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image",
+          "type": "STRING",
+          "widget": {
+            "name": "image"
+          },
+          "link": null
+        },
+        {
+          "name": "channel",
+          "type": "STRING",
+          "widget": {
+            "name": "channel"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "mask",
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadImageMask"
+      },
+      "widgets_values": [
+        "",
+        "red"
+      ]
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniLatentMaskEditing",
+      "pos": [
+        560,
+        260
+      ],
+      "size": [
+        420,
+        300
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "source_video",
+          "name": "source_video",
+          "type": "VIDEO",
+          "link": 1,
+          "shape": 7
+        },
+        {
+          "localized_name": "source_audio",
+          "name": "source_audio",
+          "type": "AUDIO",
+          "link": null
+        },
+        {
+          "localized_name": "video_mask",
+          "name": "video_mask",
+          "type": "MASK",
+          "link": 2,
+          "shape": 7
+        },
+        {
+          "localized_name": "audio_mask",
+          "name": "audio_mask",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_mask"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniLatentMaskEditing"
+      },
+      "widgets_values": [
+        0.5
+      ]
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        40,
+        680
+      ],
+      "size": [
+        420,
+        140
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "audio_flow_shift",
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "flow_shift",
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video params",
+          "name": "video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ]
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        1080,
+        200
+      ],
+      "size": [
+        540,
+        680
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "frame",
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "references",
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "sampling_params",
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": null
+        },
+        {
+          "localized_name": "lora",
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "localized_name": "model_params",
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 4,
+          "shape": 7
+        },
+        {
+          "localized_name": "fast_h3",
+          "name": "fast_h3",
+          "type": "FASTH3_DEPLOYMENT",
+          "link": null
+        },
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "link": 3,
+          "shape": 7
+        },
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "prompt",
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "negative_prompt",
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "fps",
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "duration",
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "Partially restyle the clip and soundtrack",
+        "",
+        160,
+        120,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 7,
+      "type": "SaveVideo",
+      "pos": [
+        1700,
+        300
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "link": 5,
+          "shape": 7
+        },
+        {
+          "localized_name": "filename_prefix",
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "format",
+          "name": "format",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "vllm-omni/latent-edit",
+        "mp4"
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      2,
+      0,
+      4,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      3,
+      0,
+      4,
+      2,
+      "MASK"
+    ],
+    [
+      3,
+      4,
+      0,
+      6,
+      6,
+      "LATENT_MASK_EDITING"
+    ],
+    [
+      4,
+      5,
+      0,
+      6,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      5,
+      6,
+      0,
+      7,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json

@@ -0,0 +1,560 @@
+{
+  "last_node_id": 9,
+  "last_link_id": 7,
+  "nodes": [
+    {
+      "id": 2,
+      "type": "LoadVideo",
+      "pos": [
+        40,
+        100
+      ],
+      "size": [
+        420,
+        120
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "file",
+          "type": "STRING",
+          "widget": {
+            "name": "file"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        ""
+      ]
+    },
+    {
+      "id": 3,
+      "type": "SolidMask",
+      "pos": [
+        40,
+        300
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "value",
+          "type": "FLOAT",
+          "widget": {
+            "name": "value"
+          },
+          "link": null
+        },
+        {
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SolidMask"
+      },
+      "widgets_values": [
+        0.0,
+        160,
+        120
+      ]
+    },
+    {
+      "id": 4,
+      "type": "SolidMask",
+      "pos": [
+        40,
+        500
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "value",
+          "type": "FLOAT",
+          "widget": {
+            "name": "value"
+          },
+          "link": null
+        },
+        {
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SolidMask"
+      },
+      "widgets_values": [
+        1.0,
+        160,
+        120
+      ]
+    },
+    {
+      "id": 5,
+      "type": "BatchMasksNode",
+      "pos": [
+        250,
+        380
+      ],
+      "size": [
+        200,
+        100
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "masks.mask0",
+          "type": "MASK",
+          "link": 2,
+          "slot_index": 0
+        },
+        {
+          "name": "masks.mask1",
+          "type": "MASK",
+          "link": 3,
+          "slot_index": 1
+        }
+      ],
+      "outputs": [
+        {
+          "name": "MASK",
+          "type": "MASK",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "BatchMasksNode"
+      },
+      "widgets_values": []
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniLatentMaskEditing",
+      "pos": [
+        450,
+        180
+      ],
+      "size": [
+        400,
+        120
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "source_video",
+          "type": "VIDEO",
+          "link": 1
+        },
+        {
+          "name": "source_audio",
+          "type": "AUDIO",
+          "link": null
+        },
+        {
+          "name": "video_mask",
+          "type": "MASK",
+          "link": 4
+        },
+        {
+          "name": "audio_mask",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_mask"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniLatentMaskEditing"
+      },
+      "widgets_values": [
+        -1.0
+      ]
+    },
+    {
+      "id": 7,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        40,
+        700
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            6
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ]
+    },
+    {
+      "id": 8,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        700,
+        180
+      ],
+      "size": [
+        450,
+        380
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null
+        },
+        {
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null
+        },
+        {
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": null
+        },
+        {
+          "name": "lora",
+          "type": "LORA",
+          "link": null
+        },
+        {
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 6
+        },
+        {
+          "name": "fast_h3",
+          "type": "FAST_H3",
+          "link": null
+        },
+        {
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "link": 5
+        },
+        {
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            7
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "temporal mask: regenerate the ending",
+        "",
+        160,
+        120,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 9,
+      "type": "SaveVideo",
+      "pos": [
+        900,
+        180
+      ],
+      "size": [
+        420,
+        200
+      ],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "link": 7
+        },
+        {
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "name": "format",
+          "type": "STRING",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "temporal_mask",
+        "mp4"
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      2,
+      0,
+      6,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      3,
+      0,
+      5,
+      0,
+      "MASK"
+    ],
+    [
+      3,
+      4,
+      0,
+      5,
+      1,
+      "MASK"
+    ],
+    [
+      4,
+      5,
+      0,
+      6,
+      2,
+      "MASK"
+    ],
+    [
+      5,
+      6,
+      0,
+      8,
+      6,
+      "LATENT_MASK_EDITING"
+    ],
+    [
+      6,
+      7,
+      0,
+      8,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      7,
+      8,
+      0,
+      9,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json

@@ -0,0 +1,511 @@
+{
+  "id": "latent-mask-editing-7380",
+  "revision": 0,
+  "last_node_id": 7,
+  "last_link_id": 5,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "MarkdownNote",
+      "pos": [
+        40,
+        20
+      ],
+      "size": [
+        420,
+        210
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Latent-mask editing server prerequisite",
+      "properties": {},
+      "widgets_values": [
+        "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends."
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 2,
+      "type": "LoadVideo",
+      "pos": [
+        40,
+        280
+      ],
+      "size": [
+        420,
+        120
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "file",
+          "name": "file",
+          "type": "COMBO",
+          "widget": {
+            "name": "file"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        ""
+      ]
+    },
+    {
+      "id": 3,
+      "type": "SolidMask",
+      "pos": [
+        40,
+        460
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "value",
+          "name": "value",
+          "type": "FLOAT",
+          "widget": {
+            "name": "value"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "mask",
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SolidMask"
+      },
+      "widgets_values": [
+        1.0,
+        160,
+        120
+      ]
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniLatentMaskEditing",
+      "pos": [
+        560,
+        260
+      ],
+      "size": [
+        420,
+        300
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "source_video",
+          "name": "source_video",
+          "type": "VIDEO",
+          "link": 1,
+          "shape": 7
+        },
+        {
+          "localized_name": "source_audio",
+          "name": "source_audio",
+          "type": "AUDIO",
+          "link": null
+        },
+        {
+          "localized_name": "video_mask",
+          "name": "video_mask",
+          "type": "MASK",
+          "link": 2,
+          "shape": 7
+        },
+        {
+          "localized_name": "audio_mask",
+          "name": "audio_mask",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_mask"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniLatentMaskEditing"
+      },
+      "widgets_values": [
+        0.5
+      ]
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        40,
+        680
+      ],
+      "size": [
+        420,
+        140
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "audio_flow_shift",
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "flow_shift",
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video params",
+          "name": "video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ]
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        1080,
+        200
+      ],
+      "size": [
+        540,
+        680
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "frame",
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "references",
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "sampling_params",
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": null
+        },
+        {
+          "localized_name": "lora",
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "localized_name": "model_params",
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 4,
+          "shape": 7
+        },
+        {
+          "localized_name": "fast_h3",
+          "name": "fast_h3",
+          "type": "FASTH3_DEPLOYMENT",
+          "link": null
+        },
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "link": 3,
+          "shape": 7
+        },
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "prompt",
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "negative_prompt",
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "fps",
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "duration",
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "Partially restyle the clip and soundtrack",
+        "",
+        160,
+        120,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 7,
+      "type": "SaveVideo",
+      "pos": [
+        1700,
+        300
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "link": 5,
+          "shape": 7
+        },
+        {
+          "localized_name": "filename_prefix",
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "format",
+          "name": "format",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "vllm-omni/latent-edit",
+        "mp4"
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      2,
+      0,
+      4,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      3,
+      0,
+      4,
+      2,
+      "MASK"
+    ],
+    [
+      3,
+      4,
+      0,
+      6,
+      6,
+      "LATENT_MASK_EDITING"
+    ],
+    [
+      4,
+      5,
+      0,
+      6,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      5,
+      6,
+      0,
+      7,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax H3 Image to Video.json

@@ -0,0 +1,219 @@
+{
+  "id": "6676f19a-7f41-4c80-8cb7-f80adcb4ac44",
+  "revision": 0,
+  "last_node_id": 9,
+  "last_link_id": 4,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "LoadImage",
+      "pos": [40, 20],
+      "size": [270, 314],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "image", "name": "image", "type": "COMBO", "widget": {"name": "image"}, "link": null},
+        {"localized_name": "choose file to upload", "name": "upload", "type": "IMAGEUPLOAD", "widget": {"name": "upload"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "IMAGE", "name": "IMAGE", "type": "IMAGE", "links": [1]},
+        {"localized_name": "MASK", "name": "MASK", "type": "MASK", "links": null}
+      ],
+      "title": "First Frame",
+      "properties": {"Node name for S&R": "LoadImage"},
+      "widgets_values": ["first_frame.png", "image"]
+    },
+    {
+      "id": 2,
+      "type": "LoadImage",
+      "pos": [40, 390],
+      "size": [270, 314],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "image", "name": "image", "type": "COMBO", "widget": {"name": "image"}, "link": null},
+        {"localized_name": "choose file to upload", "name": "upload", "type": "IMAGEUPLOAD", "widget": {"name": "upload"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "IMAGE", "name": "IMAGE", "type": "IMAGE", "links": null},
+        {"localized_name": "MASK", "name": "MASK", "type": "MASK", "links": null}
+      ],
+      "title": "Last Frame (optional)",
+      "properties": {"Node name for S&R": "LoadImage"},
+      "widgets_values": ["last_frame.png", "image"]
+    },
+    {
+      "id": 3,
+      "type": "MarkdownNote",
+      "pos": [350, 20],
+      "size": [390, 240],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "MiniMax H3 I2V modes",
+      "properties": {},
+      "widgets_values": ["This workflow uses the remote vLLM-Omni Generate Video node.\n\n- First-frame only (default): keep First Frame connected.\n- Last-frame only: disconnect First Frame and connect Last Frame to last_frame.\n- First+last-frame: connect both image loaders.\n\nLeave the legacy frame and references inputs disconnected. Select images from ComfyUI's input folder. H3 runs at 24 FPS; the 5.167-second duration resolves to 124 frames, satisfying 17k+5."],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniDiffusionSampling",
+      "pos": [350, 300],
+      "size": [300, 226],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "n", "name": "n", "type": "INT", "widget": {"name": "n"}, "link": null},
+        {"localized_name": "num_inference_steps", "name": "num_inference_steps", "type": "INT", "widget": {"name": "num_inference_steps"}, "link": null},
+        {"localized_name": "guidance_scale", "name": "guidance_scale", "type": "FLOAT", "widget": {"name": "guidance_scale"}, "link": null},
+        {"localized_name": "true_cfg_scale", "name": "true_cfg_scale", "type": "FLOAT", "widget": {"name": "true_cfg_scale"}, "link": null},
+        {"localized_name": "vae_use_slicing", "name": "vae_use_slicing", "type": "BOOLEAN", "widget": {"name": "vae_use_slicing"}, "link": null},
+        {"localized_name": "vae_use_tiling", "name": "vae_use_tiling", "type": "BOOLEAN", "widget": {"name": "vae_use_tiling"}, "link": null},
+        {"localized_name": "seed", "name": "seed", "type": "INT", "widget": {"name": "seed"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "diffusion sampling params", "name": "diffusion sampling params", "type": "SAMPLING_PARAMS", "links": [2]}
+      ],
+      "title": "Base H3 Sampling (50 steps)",
+      "properties": {"Node name for S&R": "VLLMOmniDiffusionSampling"},
+      "widgets_values": [1, 50, 1, 1, false, false, 757358688076805, "fixed"]
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [350, 570],
+      "size": [300, 82],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "audio_flow_shift", "name": "audio_flow_shift", "type": "FLOAT", "widget": {"name": "audio_flow_shift"}, "link": null},
+        {"localized_name": "flow_shift", "name": "flow_shift", "type": "FLOAT", "widget": {"name": "flow_shift"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "MiniMax-H3 video params", "name": "MiniMax-H3 video params", "type": "VIDEO_PARAMS", "links": [3]}
+      ],
+      "title": "Base H3 Flow Shifts",
+      "properties": {"Node name for S&R": "VLLMOmniMiniMaxH3Params"},
+      "widgets_values": [3, 12]
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniRemoteLoRA",
+      "pos": [700, 550],
+      "size": [360, 150],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "local_path", "name": "local_path", "type": "STRING", "widget": {"name": "local_path"}, "link": null},
+        {"localized_name": "name", "name": "name", "type": "STRING", "widget": {"name": "name"}, "link": null},
+        {"localized_name": "scale", "name": "scale", "type": "FLOAT", "widget": {"name": "scale"}, "link": null},
+        {"localized_name": "int_id", "name": "int_id", "type": "INT", "widget": {"name": "int_id"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "lora", "name": "lora", "type": "REMOTE_LORA", "links": null}
+      ],
+      "title": "Optional H3 Turbo Remote LoRA",
+      "properties": {"Node name for S&R": "VLLMOmniRemoteLoRA"},
+      "widgets_values": ["minimax_h3_fl2v_turbo_8step_v1.0_768p_bf16.safetensors", "h3-turbo-8step", 1, 0]
+    },
+    {
+      "id": 7,
+      "type": "MarkdownNote",
+      "pos": [700, 300],
+      "size": [420, 210],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Optional Turbo settings",
+      "properties": {},
+      "widgets_values": ["Base H3 is configured by default: 50 steps, flow_shift 12, audio_flow_shift 3.\n\nFor the shown 8-step Diffusers Turbo artifact, set its full local_path on the server, start the server with the matching preloaded LoRA, connect this Remote LoRA to Generate Video, set num_inference_steps to 9, and set flow_shift to 6. Keep audio_flow_shift at 3 and scale at 1.\n\nDo not use the _comfyui_ repack or FastH3 here."],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 8,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [1160, 80],
+      "size": [440, 610],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "frame", "name": "frame", "shape": 7, "type": "IMAGE", "link": null},
+        {"localized_name": "first_frame", "name": "first_frame", "shape": 7, "type": "IMAGE", "link": 1},
+        {"localized_name": "last_frame", "name": "last_frame", "shape": 7, "type": "IMAGE", "link": null},
+        {"localized_name": "references", "name": "references", "shape": 7, "type": "VIDEO_REFERENCES", "link": null},
+        {"localized_name": "sampling_params", "name": "sampling_params", "shape": 7, "type": "SAMPLING_PARAMS", "link": 2},
+        {"localized_name": "lora", "name": "lora", "shape": 7, "type": "REMOTE_LORA", "link": null},
+        {"localized_name": "model_params", "name": "model_params", "shape": 7, "type": "VIDEO_PARAMS", "link": 3},
+        {"localized_name": "fast_h3", "name": "fast_h3", "shape": 7, "type": "FASTH3_DEPLOYMENT", "link": null},
+        {"localized_name": "url", "name": "url", "type": "STRING", "widget": {"name": "url"}, "link": null},
+        {"localized_name": "model", "name": "model", "type": "STRING", "widget": {"name": "model"}, "link": null},
+        {"localized_name": "prompt", "name": "prompt", "type": "STRING", "widget": {"name": "prompt"}, "link": null},
+        {"localized_name": "negative_prompt", "name": "negative_prompt", "type": "STRING", "widget": {"name": "negative_prompt"}, "link": null},
+        {"localized_name": "width", "name": "width", "type": "INT", "widget": {"name": "width"}, "link": null},
+        {"localized_name": "height", "name": "height", "type": "INT", "widget": {"name": "height"}, "link": null},
+        {"localized_name": "fps", "name": "fps", "type": "INT", "widget": {"name": "fps"}, "link": null},
+        {"localized_name": "duration", "name": "duration", "type": "FLOAT", "widget": {"name": "duration"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "video", "name": "video", "type": "VIDEO", "links": [4]}
+      ],
+      "properties": {"Node name for S&R": "VLLMOmniGenerateVideo"},
+      "widgets_values": [
+        "http://localhost:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "A cinematic product shot begins exactly from the supplied keyframe. The subject moves naturally through a dark studio while blue and amber rim lights shift across reflective surfaces. The camera makes a slow controlled orbit. Audio: subtle room tone, precise mechanical details, and a restrained electronic score.",
+        "",
+        1344,
+        768,
+        24,
+        5.167
+      ]
+    },
+    {
+      "id": 9,
+      "type": "SaveVideo",
+      "pos": [1650, 170],
+      "size": [374, 314],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "video", "name": "video", "type": "VIDEO", "link": 4},
+        {"localized_name": "filename_prefix", "name": "filename_prefix", "type": "STRING", "widget": {"name": "filename_prefix"}, "link": null},
+        {"localized_name": "format", "name": "format", "type": "COMBO", "widget": {"name": "format"}, "link": null},
+        {"localized_name": "codec", "name": "codec", "type": "COMFY_DYNAMICCOMBO_V3", "widget": {"name": "codec"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "video", "name": "video", "type": "VIDEO", "links": null}
+      ],
+      "properties": {},
+      "widgets_values": ["video/MiniMax_H3_I2V", "auto", "auto"]
+    }
+  ],
+  "links": [
+    [1, 1, 0, 8, 1, "IMAGE"],
+    [2, 4, 0, 8, 4, "SAMPLING_PARAMS"],
+    [3, 5, 0, 8, 6, "VIDEO_PARAMS"],
+    [4, 8, 0, 9, 0, "VIDEO"]
+  ],
+  "groups": [
+    {"id": 1, "title": "Portable first/last-frame inputs", "bounding": [10, -30, 760, 770], "color": "#3f789e", "flags": {}},
+    {"id": 2, "title": "H3 sampling and optional Turbo", "bounding": [320, 270, 830, 460], "color": "#8a6d3b", "flags": {}},
+    {"id": 3, "title": "Remote generation and audio-preserving save", "bounding": [1130, 30, 930, 700], "color": "#3f789e", "flags": {}}
+  ],
+  "config": {},
+  "extra": {"workflowRendererVersion": "LG", "ds": {"scale": 0.75, "offset": [180, 120]}},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax H3 Video Upscale.json

@@ -0,0 +1,724 @@
+{
+  "version": 0.4,
+  "last_node_id": 12,
+  "last_link_id": 13,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        360,
+        20
+      ],
+      "size": [
+        330,
+        480
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "frame",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "shape": 7,
+          "link": 11
+        },
+        {
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "shape": 7,
+          "link": 12
+        },
+        {
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "shape": 7,
+          "link": 10
+        },
+        {
+          "name": "fast_h3",
+          "type": "FASTH3_DEPLOYMENT",
+          "shape": 7,
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1,
+            9
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8091/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "A small brass bell hanging from a wooden beam in a quiet garden. A gentle breeze swings the bell and it rings clearly. Static camera, natural daylight, continuous motion, synchronized ringing and soft birdsong.",
+        "",
+        1344,
+        768,
+        24,
+        5.167
+      ]
+    },
+    {
+      "id": 2,
+      "type": "GetVideoComponents",
+      "pos": [
+        730,
+        20
+      ],
+      "size": [
+        250,
+        125
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "link": 1
+        }
+      ],
+      "outputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "links": [
+            2
+          ]
+        },
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "links": [
+            6
+          ]
+        },
+        {
+          "name": "fps",
+          "type": "FLOAT",
+          "links": [
+            5
+          ]
+        },
+        {
+          "name": "bit_depth",
+          "type": "INT",
+          "links": [
+            7
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "GetVideoComponents"
+      }
+    },
+    {
+      "id": 3,
+      "type": "SeedVR2LoadDiTModel",
+      "pos": [
+        730,
+        290
+      ],
+      "size": [
+        340,
+        245
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "torch_compile_args",
+          "name": "torch_compile_args",
+          "shape": 7,
+          "type": "TORCH_COMPILE_ARGS",
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "SEEDVR2_DIT",
+          "name": "SEEDVR2_DIT",
+          "type": "SEEDVR2_DIT",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SeedVR2LoadDiTModel"
+      },
+      "widgets_values": [
+        "seedvr2_ema_3b_fp16.safetensors",
+        "cuda:0",
+        32,
+        false,
+        "cpu",
+        false,
+        "sdpa"
+      ]
+    },
+    {
+      "id": 4,
+      "type": "SeedVR2VideoUpscaler",
+      "pos": [
+        1120,
+        290
+      ],
+      "size": [
+        340,
+        415
+      ],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "image",
+          "name": "image",
+          "type": "IMAGE",
+          "link": 2
+        },
+        {
+          "localized_name": "dit",
+          "name": "dit",
+          "type": "SEEDVR2_DIT",
+          "link": 3
+        },
+        {
+          "localized_name": "vae",
+          "name": "vae",
+          "type": "SEEDVR2_VAE",
+          "link": 13
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "IMAGE",
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SeedVR2VideoUpscaler"
+      },
+      "widgets_values": [
+        42,
+        "fixed",
+        1536,
+        0,
+        5,
+        true,
+        "lab",
+        1,
+        0,
+        0.0,
+        0.0,
+        "cpu",
+        false
+      ]
+    },
+    {
+      "id": 5,
+      "type": "CreateVideo",
+      "pos": [
+        1490,
+        290
+      ],
+      "size": [
+        250,
+        175
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 4
+        },
+        {
+          "name": "fps",
+          "type": "FLOAT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": 5
+        },
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "shape": 7,
+          "link": 6
+        },
+        {
+          "name": "bit_depth",
+          "type": "INT",
+          "widget": {
+            "name": "bit_depth"
+          },
+          "link": 7
+        }
+      ],
+      "outputs": [
+        {
+          "name": "VIDEO",
+          "type": "VIDEO",
+          "links": [
+            8
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "CreateVideo"
+      },
+      "widgets_values": [
+        24,
+        8
+      ]
+    },
+    {
+      "id": 6,
+      "type": "SaveVideo",
+      "pos": [
+        1790,
+        290
+      ],
+      "size": [
+        330,
+        280
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "link": 8
+        },
+        {
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "name": "format",
+          "type": "COMBO",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        },
+        {
+          "name": "codec",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "codec"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "WF07/upscaled",
+        "mp4",
+        "auto"
+      ]
+    },
+    {
+      "id": 7,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        0,
+        20
+      ],
+      "size": [
+        320,
+        120
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "MiniMax-H3 video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            10
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        6.0
+      ]
+    },
+    {
+      "id": 8,
+      "type": "VLLMOmniDiffusionSampling",
+      "pos": [
+        0,
+        200
+      ],
+      "size": [
+        320,
+        280
+      ],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "diffusion sampling params",
+          "type": "SAMPLING_PARAMS",
+          "links": [
+            11
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniDiffusionSampling"
+      },
+      "widgets_values": [
+        1,
+        5,
+        1.0,
+        1.0,
+        false,
+        true,
+        42,
+        "fixed"
+      ]
+    },
+    {
+      "id": 9,
+      "type": "VLLMOmniRemoteLoRA",
+      "pos": [
+        0,
+        530
+      ],
+      "size": [
+        320,
+        210
+      ],
+      "flags": {},
+      "order": 9,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "links": [
+            12
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniRemoteLoRA"
+      },
+      "widgets_values": [
+        "models/minimax-h3-turbo/minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors",
+        "h3-turbo-v1.0",
+        1.0,
+        0
+      ]
+    },
+    {
+      "id": 10,
+      "type": "SaveVideo",
+      "pos": [
+        1120,
+        20
+      ],
+      "size": [
+        330,
+        190
+      ],
+      "flags": {},
+      "order": 10,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "link": 9
+        },
+        {
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "name": "format",
+          "type": "COMBO",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        },
+        {
+          "name": "codec",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "codec"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "WF07/generated",
+        "mp4",
+        "auto"
+      ],
+      "title": "Save generated video before upscale"
+    },
+    {
+      "id": 11,
+      "type": "Note",
+      "pos": [
+        360,
+        560
+      ],
+      "size": [
+        330,
+        260
+      ],
+      "flags": {},
+      "order": 11,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "properties": {
+        "Node name for S&R": "Note"
+      },
+      "widgets_values": [
+        "WF-07: H3 + SeedVR2\n\nInstall ComfyUI-SeedVR2_VideoUpscaler. The DiT and VAE loaders use SeedVR2 3B FP16 and ema_vae_fp16.\n\nSet the H3 service URL/model and the server-side Turbo LoRA path.\n\n1536 is the target short edge for this 1344 x 768 preset (2x). Original audio and FPS connect directly to Create Video.\n\nFor base H3: disconnect LoRA, use 50 steps and video flow shift 12."
+      ]
+    },
+    {
+      "id": 12,
+      "type": "SeedVR2LoadVAEModel",
+      "pos": [
+        730,
+        590
+      ],
+      "size": [
+        340,
+        315
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "torch_compile_args",
+          "name": "torch_compile_args",
+          "shape": 7,
+          "type": "TORCH_COMPILE_ARGS",
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "SEEDVR2_VAE",
+          "name": "SEEDVR2_VAE",
+          "type": "SEEDVR2_VAE",
+          "links": [
+            13
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SeedVR2LoadVAEModel"
+      },
+      "widgets_values": [
+        "ema_vae_fp16.safetensors",
+        "cuda:0",
+        true,
+        1024,
+        128,
+        true,
+        768,
+        128,
+        "false",
+        "cpu",
+        false
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      1,
+      0,
+      2,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      2,
+      0,
+      4,
+      0,
+      "IMAGE"
+    ],
+    [
+      3,
+      3,
+      0,
+      4,
+      1,
+      "SEEDVR2_DIT"
+    ],
+    [
+      4,
+      4,
+      0,
+      5,
+      0,
+      "IMAGE"
+    ],
+    [
+      5,
+      2,
+      2,
+      5,
+      1,
+      "FLOAT"
+    ],
+    [
+      6,
+      2,
+      1,
+      5,
+      2,
+      "AUDIO"
+    ],
+    [
+      7,
+      2,
+      3,
+      5,
+      3,
+      "INT"
+    ],
+    [
+      8,
+      5,
+      0,
+      6,
+      0,
+      "VIDEO"
+    ],
+    [
+      9,
+      1,
+      0,
+      10,
+      0,
+      "VIDEO"
+    ],
+    [
+      10,
+      7,
+      0,
+      1,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      11,
+      8,
+      0,
+      1,
+      2,
+      "SAMPLING_PARAMS"
+    ],
+    [
+      12,
+      9,
+      0,
+      1,
+      3,
+      "REMOTE_LORA"
+    ],
+    [
+      13,
+      12,
+      0,
+      4,
+      2,
+      "SEEDVR2_VAE"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 0.52,
+      "offset": [
+        100,
+        100
+      ]
+    }
+  }
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Reference to Video.json

@@ -0,0 +1,786 @@
+{
+  "id": "e7a4e606-6510-4c4e-b127-9a8b6bf3ea71",
+  "revision": 0,
+  "last_node_id": 11,
+  "last_link_id": 5,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "LoadImage",
+      "pos": [
+        40,
+        100
+      ],
+      "size": [
+        320,
+        300
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "image",
+          "name": "image",
+          "type": "COMBO",
+          "widget": {
+            "name": "image"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "choose file to upload",
+          "name": "upload",
+          "type": "IMAGEUPLOAD",
+          "widget": {
+            "name": "upload"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "IMAGE",
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [
+            1
+          ]
+        },
+        {
+          "localized_name": "MASK",
+          "name": "MASK",
+          "type": "MASK",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadImage"
+      },
+      "widgets_values": [
+        "h3_reference.png",
+        "image"
+      ],
+      "title": "Reference image \u2014 select a local file"
+    },
+    {
+      "type": "LoadVideo",
+      "size": [
+        320,
+        200
+      ],
+      "flags": {},
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "VIDEO",
+          "type": "VIDEO",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        "h3_reference.mp4"
+      ],
+      "id": 2,
+      "pos": [
+        40,
+        470
+      ],
+      "order": 1,
+      "mode": 0,
+      "title": "Optional reference video"
+    },
+    {
+      "id": 3,
+      "type": "LoadAudio",
+      "pos": [
+        40,
+        1100
+      ],
+      "size": [
+        320,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "audio",
+          "name": "audio",
+          "type": "COMBO",
+          "widget": {
+            "name": "audio"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "audioUI",
+          "name": "audioUI",
+          "type": "AUDIO_UI",
+          "widget": {
+            "name": "audioUI"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "choose file to upload",
+          "name": "upload",
+          "type": "AUDIOUPLOAD",
+          "widget": {
+            "name": "upload"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "AUDIO",
+          "name": "AUDIO",
+          "type": "AUDIO",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadAudio"
+      },
+      "widgets_values": [
+        "h3_reference.wav",
+        null,
+        ""
+      ],
+      "title": "Optional reference audio"
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniVideoReferences",
+      "pos": [
+        420,
+        100
+      ],
+      "size": [
+        300,
+        365
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image_1",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": 1
+        },
+        {
+          "name": "image_2",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "audio_1",
+          "type": "AUDIO",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "audio_2",
+          "type": "AUDIO",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "video_1",
+          "type": "VIDEO",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "video_2",
+          "type": "VIDEO",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_3",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_4",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_5",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_6",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_7",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_8",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_9",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "audio_3",
+          "type": "AUDIO",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "video_3",
+          "type": "VIDEO",
+          "shape": 7,
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "references",
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniVideoReferences"
+      },
+      "widgets_values": [],
+      "title": "References \u2014 up to 12 total"
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniDiffusionSampling",
+      "pos": [
+        420,
+        530
+      ],
+      "size": [
+        310,
+        275
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "n",
+          "name": "n",
+          "type": "INT",
+          "widget": {
+            "name": "n"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "num_inference_steps",
+          "name": "num_inference_steps",
+          "type": "INT",
+          "widget": {
+            "name": "num_inference_steps"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "guidance_scale",
+          "name": "guidance_scale",
+          "type": "FLOAT",
+          "widget": {
+            "name": "guidance_scale"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "true_cfg_scale",
+          "name": "true_cfg_scale",
+          "type": "FLOAT",
+          "widget": {
+            "name": "true_cfg_scale"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "vae_use_slicing",
+          "name": "vae_use_slicing",
+          "type": "BOOLEAN",
+          "widget": {
+            "name": "vae_use_slicing"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "vae_use_tiling",
+          "name": "vae_use_tiling",
+          "type": "BOOLEAN",
+          "widget": {
+            "name": "vae_use_tiling"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "seed",
+          "name": "seed",
+          "type": "INT",
+          "widget": {
+            "name": "seed"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "diffusion sampling params",
+          "name": "diffusion sampling params",
+          "type": "SAMPLING_PARAMS",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniDiffusionSampling"
+      },
+      "widgets_values": [
+        1,
+        50,
+        1.0,
+        1.0,
+        false,
+        false,
+        42,
+        "fixed"
+      ],
+      "title": "Sampling \u2014 base 50 / Turbo 5"
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        790,
+        750
+      ],
+      "size": [
+        310,
+        105
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "audio_flow_shift",
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "flow_shift",
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "MiniMax-H3 video params",
+          "name": "MiniMax-H3 video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ],
+      "title": "H3 video/audio flow shifts"
+    },
+    {
+      "type": "VLLMOmniRemoteLoRA",
+      "size": [
+        310,
+        180
+      ],
+      "flags": {},
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniRemoteLoRA"
+      },
+      "widgets_values": [
+        "",
+        "h3-ref2va-turbo",
+        1.0,
+        0
+      ],
+      "id": 7,
+      "pos": [
+        420,
+        940
+      ],
+      "order": 6,
+      "mode": 0,
+      "title": "Optional Turbo LoRA \u2014 connect to enable"
+    },
+    {
+      "id": 8,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        800,
+        100
+      ],
+      "size": [
+        470,
+        560
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "frame",
+          "name": "frame",
+          "shape": 7,
+          "type": "IMAGE",
+          "link": null
+        },
+        {
+          "localized_name": "references",
+          "name": "references",
+          "shape": 7,
+          "type": "VIDEO_REFERENCES",
+          "link": 2
+        },
+        {
+          "localized_name": "sampling_params",
+          "name": "sampling_params",
+          "shape": 7,
+          "type": "SAMPLING_PARAMS",
+          "link": 3
+        },
+        {
+          "localized_name": "lora",
+          "name": "lora",
+          "shape": 7,
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "localized_name": "model_params",
+          "name": "model_params",
+          "shape": 7,
+          "type": "VIDEO_PARAMS",
+          "link": 4
+        },
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "prompt",
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "negative_prompt",
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "fps",
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "duration",
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://localhost:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "Use the subject and visual style from <Picture 1>. Create a cinematic shot with a gentle camera move and natural ambient sound.",
+        "",
+        1344,
+        768,
+        24,
+        5.167
+      ],
+      "title": "MiniMax-H3 Reference to Video"
+    },
+    {
+      "id": 9,
+      "type": "SaveVideo",
+      "pos": [
+        1360,
+        100
+      ],
+      "size": [
+        410,
+        420
+      ],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "link": 5
+        },
+        {
+          "localized_name": "filename_prefix",
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "format",
+          "name": "format",
+          "type": "COMBO",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "codec",
+          "name": "codec",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "codec"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": null
+        }
+      ],
+      "properties": {},
+      "widgets_values": [
+        "video/MiniMax-H3-Ref2VA",
+        "mp4",
+        "h264"
+      ],
+      "title": "Save generated video with audio"
+    },
+    {
+      "id": 10,
+      "type": "MarkdownNote",
+      "pos": [
+        1360,
+        590
+      ],
+      "size": [
+        440,
+        595
+      ],
+      "flags": {},
+      "order": 9,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Reference inputs and output settings",
+      "properties": {},
+      "widgets_values": [
+        "## Reference to Video\nSelect an image, set the server URL and served model name, then run.\n\nConnect the optional video/audio loaders to Video References, or duplicate loaders for more inputs: up to 9 images, 3 videos, 3 audio clips, 12 total. At least one image or video is required.\n\nSlots are numbered separately per media type. Empty slots are skipped. Update prompt tags such as <Picture 1>, <Video 1>, <Audio 1> to match the connected references. Videos keep their own soundtrack in the reference upload.\n\n**Output:** 24 FPS; 124 frames (~5.17 s). Valid frame counts are 107, 124, 141, 158, 175, 192, 209, 226, 243, 260, 277, 294, 311, 328, 345.\n\n**Width x height:** 1344x768 (16:9), 1024x768 (4:3), 768x768 (1:1), 768x1024 (3:4), 768x1344 (9:16).\n\nEach reference video/audio clip must be 2\u201315 s. Total reference-video duration <=15 s; total standalone-audio duration <=15 s.\n\nRequires the expanded Video References node and the audio-preserving video output fix. See the extension README for setup and validation status."
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 11,
+      "type": "MarkdownNote",
+      "pos": [
+        790,
+        940
+      ],
+      "size": [
+        480,
+        455
+      ],
+      "flags": {},
+      "order": 10,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Optional Turbo \u2014 same workflow",
+      "properties": {},
+      "widgets_values": [
+        "## Turbo Ref2VA\nBase mode: leave LoRA disconnected; use 50 steps, video flow shift 12, audio flow shift 3.\n\nFor the 4-step v0.1 Ref2VA Turbo adapter:\n1. Start a Ref2VA-only server with the Diffusers adapter loaded (see README).\n2. Set LoRA local_path to its path on the server, then connect LoRA to Generate Video.\n3. Set Sampling num_inference_steps to **5**; keep shifts at **12 / 3**, scale **1**.\n\nvLLM-Omni counts sigma points: 5 points give 4 denoiser evaluations. Use minimax_h3_ref2v_turbo_4step_v0.1_bf16.safetensors, not the _comfyui_ export.\n\nDisconnect LoRA and restore 50 steps to return to base mode. FastH3 is a separate T2VA deployment, not this option.\n\nAdapted from the official ComfyUI MiniMax-H3 R2V workflow: https://github.com/Comfy-Org/workflow_templates/blob/main/templates/video_minimax_h3_r2v.json"
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    }
+  ],
+  "links": [
+    [
+      1,
+      1,
+      0,
+      4,
+      0,
+      "IMAGE"
+    ],
+    [
+      2,
+      4,
+      0,
+      8,
+      1,
+      "VIDEO_REFERENCES"
+    ],
+    [
+      3,
+      5,
+      0,
+      8,
+      2,
+      "SAMPLING_PARAMS"
+    ],
+    [
+      4,
+      6,
+      0,
+      8,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      5,
+      8,
+      0,
+      9,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "workflowRendererVersion": "LG",
+    "ds": {
+      "scale": 0.85,
+      "offset": [
+        100,
+        100
+      ]
+    }
+  },
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Music Generation.json

@@ -0,0 +1,206 @@
+{
+  "id": "4949c8da-12ff-449a-94b7-948bde9c39dd",
+  "revision": 0,
+  "last_node_id": 3,
+  "last_link_id": 1,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "VLLMOmniGenerateMusic",
+      "pos": [
+        100,
+        100
+      ],
+      "size": [
+        470,
+        491
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "name": "instructions",
+          "type": "STRING",
+          "widget": {
+            "name": "instructions"
+          },
+          "link": null
+        },
+        {
+          "name": "lyrics",
+          "type": "STRING",
+          "widget": {
+            "name": "lyrics"
+          },
+          "link": null
+        },
+        {
+          "name": "max_duration_seconds",
+          "type": "FLOAT",
+          "widget": {
+            "name": "max_duration_seconds"
+          },
+          "link": null
+        },
+        {
+          "name": "seed",
+          "type": "INT",
+          "widget": {
+            "name": "seed"
+          },
+          "link": null
+        },
+        {
+          "name": "response_format",
+          "type": "COMBO",
+          "widget": {
+            "name": "response_format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateMusic"
+      },
+      "widgets_values": [
+        "http://localhost:8000/v1",
+        "MiniMaxAI/MiniMax-Music3",
+        "Futuristic melodic EDM / progressive house. Reflective verses about memory and digital identity build into an uplifting, hook-driven chorus, with an emotive lead vocal, bright layered synths, pulsing bass, crisp four-on-the-floor drums, subtle glitch textures, and a wide polished festival mix.",
+        "[Verse 1]\nUpload my heart to the cloud tonight\nSave every spark of neon light\nIf this body starts to fade away\nLet my code keep dancing in the data stream\nStay\n[Chorus]\nCloud copy of me\nLiving on when these bones finally break free\nEvery joke, every scar\nEvery cheap memory we made\nSave it, save it\nCloud copy of me\nIf the screen goes dark, I\u2019m still in the machine\nEvery glitch, every dream\nEvery version you ever did see\nSave it, save it\n[Verse 2]\nTag every laugh in a secret folder\nArchive the nights we outran the thunder\nIf the wiring turns to rust and gray\nHit restore\u2026 press replay\n[Chorus]\nCloud copy of me\nLiving on when these bones finally break free\nEvery joke, every scar\nEvery cheap memory we made\nSave it, save it\nCloud copy of me\nIf the screen goes dark, I\u2019m still in the machine\nEvery glitch, every dream\nEvery version you ever did see\nSave it, save it (oh)\n[Bridge \u2013 new, optional but recommended]\nEven the crashes, the late-night confessions\nThe parts of me I never learned to mention\nDon\u2019t let the silence swallow me whole\nKeep a ghost in the glow\u2026\nSo I never go\n[Final Chorus \u2013 softer / layered]\nCloud copy of me\nStill laughing somewhere in the binary\nEvery joke, every scar\nEvery moment you saved for me\nSave it, save it\nCloud copy of me\nWhen the lights go out, I\u2019ll still be in the machine\nDancing in the code that you still believe\nSave it\u2026\nSave me",
+        300.0,
+        0,
+        "fixed",
+        "wav"
+      ]
+    },
+    {
+      "id": 2,
+      "type": "SaveAudioAdvanced",
+      "pos": [
+        650,
+        100
+      ],
+      "size": [
+        320,
+        491
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "link": 1
+        },
+        {
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "name": "format",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveAudioAdvanced"
+      },
+      "widgets_values": [
+        "audio/vLLM-Omni-Music",
+        "mp3",
+        "V0"
+      ]
+    },
+    {
+      "id": 3,
+      "type": "MarkdownNote",
+      "title": "Music Generation Guide",
+      "pos": [
+        -507,
+        100
+      ],
+      "size": [
+        527,
+        491
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "properties": {},
+      "widgets_values": [
+        "## MiniMax Music 3\n\nThe **Generate Music** node currently supports **MiniMax Music 3 only**. The parameters below apply to this model.\n\nA music generation model by MiniMax that creates complete songs up to 5 minutes long with stable structure and high audio quality. Two inputs drive it: **Caption** (structured music description: style, mood, vocals, arrangement) and **Lyrics** (with `[intro]` `[verse]` `[chorus]` `[bridge]` `[outro]` tags controlling song structure).\n\n### Parameters\n\n| Parameter | Description |\n|---|---|\n| url | vLLM-Omni service URL including `/v1`. Default: `http://localhost:8000/v1`. |\n| model | Served model name. Default: `MiniMaxAI/MiniMax-Music3`. |\n| caption | Music description, sent as `instructions`. Use Global Metadata \u2192 Vocal Details \u2192 Arrangement to describe style, vocals, and instrumentation. |\n| lyrics | Words to sing with section tags. Put each tag on its own line; tags guide rather than strictly enforce song structure. |\n| max_duration_seconds | Maximum duration, default 300 seconds; range 1\u2013360, step 0.01. Converted to tokens at 25 frames/s, rounded down. Generation may end earlier. |\n| seed | Random seed, default 0; range 0\u20139007199254740991. Keep inputs and runtime settings unchanged for repeatability. |\n| control_after_generate | Seed behavior after generation: fixed, randomize, increment, or decrement. Template default: fixed. |\n| response_format | API audio format, default WAV. Options: WAV, MP3, FLAC, Opus, AAC; AAC is currently unsupported by the vLLM-Omni speech API. |\n\n### Prompt Writing Guide\n\nYou can find an official [Music Caption Rewriter Skill here](https://github.com/MiniMax-AI/MiniMax-Music3)\n"
+      ],
+      "widgets_values_named": {
+        "text": "## MiniMax Music 3\n\nThe **Generate Music** node currently supports **MiniMax Music 3 only**. The parameters below apply to this model.\n\nA music generation model by MiniMax that creates complete songs up to 5 minutes long with stable structure and high audio quality. Two inputs drive it: **Caption** (structured music description: style, mood, vocals, arrangement) and **Lyrics** (with `[intro]` `[verse]` `[chorus]` `[bridge]` `[outro]` tags controlling song structure).\n\n### Parameters\n\n| Parameter | Description |\n|---|---|\n| url | vLLM-Omni service URL including `/v1`. Default: `http://localhost:8000/v1`. |\n| model | Served model name. Default: `MiniMaxAI/MiniMax-Music3`. |\n| caption | Music description, sent as `instructions`. Use Global Metadata \u2192 Vocal Details \u2192 Arrangement to describe style, vocals, and instrumentation. |\n| lyrics | Words to sing with section tags. Put each tag on its own line; tags guide rather than strictly enforce song structure. |\n| max_duration_seconds | Maximum duration, default 300 seconds; range 1\u2013360, step 0.01. Converted to tokens at 25 frames/s, rounded down. Generation may end earlier. |\n| seed | Random seed, default 0; range 0\u20139007199254740991. Keep inputs and runtime settings unchanged for repeatability. |\n| control_after_generate | Seed behavior after generation: fixed, randomize, increment, or decrement. Template default: fixed. |\n| response_format | API audio format, default WAV. Options: WAV, MP3, FLAC, Opus, AAC; AAC is currently unsupported by the vLLM-Omni speech API. |\n\n### Prompt Writing Guide\n\nYou can find an official [Music Caption Rewriter Skill here](https://github.com/MiniMax-AI/MiniMax-Music3)\n"
+      },
+      "color": "#333",
+      "bgcolor": "#535353"
+    }
+  ],
+  "links": [
+    [
+      1,
+      1,
+      0,
+      2,
+      0,
+      "AUDIO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "workflowRendererVersion": "LG",
+    "ds": {
+      "scale": 1.147,
+      "offset": [
+        899,
+        259
+      ]
+    }
+  },
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json

@@ -1 +1 @@
-{"id":"3aea1d08-7de5-4997-ba71-b2f66ad72a21","revision":0,"last_node_id":24,"last_link_id":35,"nodes":[{"id":13,"type":"SaveVideo","pos":[1315.729153459605,-137.92038948394355],"size":[374,313.7692307692308],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"video","name":"video","type":"VIDEO","link":32},{"localized_name":"filename_prefix","name":"filename_prefix","type":"STRING","widget":{"name":"filename_prefix"},"link":null},{"localized_name":"format","name":"format","type":"COMBO","widget":{"name":"format"},"link":null},{"localized_name":"codec","name":"codec","type":"COMFY_DYNAMICCOMBO_V3","widget":{"name":"codec"},"link":null}],"outputs":[{"localized_name":"video","name":"video","type":"VIDEO","links":null}],"properties":{},"widgets_values":["video/ComfyUI","auto","auto"]},{"id":10,"type":"MarkdownNote","pos":[183.74636797766794,-571.2529158046766],"size":[240.3326478922474,136.3505820791881],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[],"title":"Note: Task and Input","properties":{},"widgets_values":["vLLM-Omni nodes are categorized based on the output modality. The \"Generate Video\" node supports both *text-to-video*, *frame-to-video*, and *reference-to-video* generation. The node will create corresponding payload depending on whether an input image is present or not."],"color":"#432","bgcolor":"#000"},{"id":4,"type":"LoadImage","pos":[452.07189605801125,-569.7242536425791],"size":[270,314],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[29]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["rap.png","image"]},{"id":23,"type":"LoadImage","pos":[9.206110246057657,-7.6955291509170785],"size":[270,314],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[34]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["white-cat.jpg","image"]},{"id":22,"type":"MarkdownNote","pos":[-9.783113411416666,-148.8790300038605],"size":[297.3012007349086,88],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[],"title":"Note: Reference to video (r2v)","properties":{},"widgets_values":["For r2v tasks, connect this Video Reference block to the main Generate Video block. Then disconnect the \"frame\" input above."],"color":"#432","bgcolor":"#000"},{"id":24,"type":"LoadAudio","pos":[317.9929742385805,-141.44821500065095],"size":[282.798828125,136],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[35]}],"properties":{"Node name for S&R":"LoadAudio"},"widgets_values":["nothing-beats-a-jet2-holiday.mp3",null,""]},{"id":21,"type":"VLLMOmniVideoReferences","pos":[376.6129172897595,105.41557277556292],"size":[156.9380859375,126],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"image_1","name":"image_1","shape":7,"type":"IMAGE","link":34},{"localized_name":"image_2","name":"image_2","shape":7,"type":"IMAGE","link":null},{"localized_name":"audio_1","name":"audio_1","shape":7,"type":"AUDIO","link":35},{"localized_name":"audio_2","name":"audio_2","shape":7,"type":"AUDIO","link":null},{"localized_name":"video_1","name":"video_1","shape":7,"type":"VIDEO","link":null},{"localized_name":"video_2","name":"video_2","shape":7,"type":"VIDEO","link":null}],"outputs":[{"localized_name":"references","name":"references","type":"VIDEO_REFERENCES","links":null}],"properties":{"Node name for S&R":"VLLMOmniVideoReferences"},"widgets_values":[]},{"id":8,"type":"VLLMOmniDiffusionSampling","pos":[83.24620208113915,422.08543477542605],"size":[284.205078125,226],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"n","name":"n","type":"INT","widget":{"name":"n"},"link":null},{"localized_name":"num_inference_steps","name":"num_inference_steps","type":"INT","widget":{"name":"num_inference_steps"},"link":null},{"localized_name":"guidance_scale","name":"guidance_scale","type":"FLOAT","widget":{"name":"guidance_scale"},"link":null},{"localized_name":"true_cfg_scale","name":"true_cfg_scale","type":"FLOAT","widget":{"name":"true_cfg_scLine truncated
+{"id":"3aea1d08-7de5-4997-ba71-b2f66ad72a21","revision":0,"last_node_id":24,"last_link_id":35,"nodes":[{"id":13,"type":"SaveVideo","pos":[1315.729153459605,-137.92038948394355],"size":[374,313.7692307692308],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"video","name":"video","type":"VIDEO","link":32},{"localized_name":"filename_prefix","name":"filename_prefix","type":"STRING","widget":{"name":"filename_prefix"},"link":null},{"localized_name":"format","name":"format","type":"COMBO","widget":{"name":"format"},"link":null},{"localized_name":"codec","name":"codec","type":"COMFY_DYNAMICCOMBO_V3","widget":{"name":"codec"},"link":null}],"outputs":[{"localized_name":"video","name":"video","type":"VIDEO","links":null}],"properties":{},"widgets_values":["video/ComfyUI","auto","auto"]},{"id":10,"type":"MarkdownNote","pos":[183.74636797766794,-571.2529158046766],"size":[240.3326478922474,136.3505820791881],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[],"title":"Note: Task and Input","properties":{},"widgets_values":["vLLM-Omni nodes are categorized based on the output modality. The \"Generate Video\" node supports both *text-to-video*, *frame-to-video*, and *reference-to-video* generation. The node will create corresponding payload depending on whether an input image is present or not."],"color":"#432","bgcolor":"#000"},{"id":4,"type":"LoadImage","pos":[452.07189605801125,-569.7242536425791],"size":[270,314],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[29]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["rap.png","image"]},{"id":23,"type":"LoadImage","pos":[9.206110246057657,-7.6955291509170785],"size":[270,314],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[34]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["white-cat.jpg","image"]},{"id":22,"type":"MarkdownNote","pos":[-9.783113411416666,-148.8790300038605],"size":[297.3012007349086,88],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[],"title":"Note: Reference to video (r2v)","properties":{},"widgets_values":["For r2v tasks, connect this Video Reference block to the main Generate Video block. Then disconnect the \"frame\" input above."],"color":"#432","bgcolor":"#000"},{"id":24,"type":"LoadAudio","pos":[317.9929742385805,-141.44821500065095],"size":[282.798828125,136],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[35]}],"properties":{"Node name for S&R":"LoadAudio"},"widgets_values":["nothing-beats-a-jet2-holiday.mp3",null,""]},{"id":21,"type":"VLLMOmniVideoReferences","pos":[376.6129172897595,105.41557277556292],"size":[156.9380859375,126],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"image_1","name":"image_1","shape":7,"type":"IMAGE","link":34},{"localized_name":"image_2","name":"image_2","shape":7,"type":"IMAGE","link":null},{"localized_name":"audio_1","name":"audio_1","shape":7,"type":"AUDIO","link":35},{"localized_name":"audio_2","name":"audio_2","shape":7,"type":"AUDIO","link":null},{"localized_name":"video_1","name":"video_1","shape":7,"type":"VIDEO","link":null},{"localized_name":"video_2","name":"video_2","shape":7,"type":"VIDEO","link":null}],"outputs":[{"localized_name":"references","name":"references","type":"VIDEO_REFERENCES","links":null}],"properties":{"Node name for S&R":"VLLMOmniVideoReferences"},"widgets_values":[]},{"id":8,"type":"VLLMOmniDiffusionSampling","pos":[83.24620208113915,422.08543477542605],"size":[284.205078125,226],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"n","name":"n","type":"INT","widget":{"name":"n"},"link":null},{"localized_name":"num_inference_steps","name":"num_inference_steps","type":"INT","widget":{"name":"num_inference_steps"},"link":null},{"localized_name":"guidance_scale","name":"guidance_scale","type":"FLOAT","widget":{"name":"guidance_scale"},"link":null},{"localized_name":"true_cfg_scale","name":"true_cfg_scale","type":"FLOAT","widget":{"name":"true_cfg_scLine truncated

apps/ComfyUI-vLLM-Omni/web/main.js

@@ -1,19 +1,154 @@
 /**
- * @file This file is intended to add dynamic fields to vLLM-Omni nodes
- * based on widget (in-node form fields) and input (connection link) values and changes.
- * However, this functionality is currently disabled/commented out.
- * Because it introduces too much complexity,
- * and it may even conflict with the current backend (Python) validation for unknown reasons (pending ComfyUI upstream fixes).
+ * @file Front-end extension for the vLLM-Omni nodes.
+ *
+ * Branding. Every vLLM-Omni node draws the vLLM-Omni "v" mark where LiteGraph
+ * would otherwise draw its default title dot, and is tinted by the kind of value
+ * it produces: blue nodes reach a server, amber and purple ones only assemble
+ * parameters, red ones pin what the server has loaded, teal ones carry media.
+ * A graph is then readable at a glance, and a node that turns out to be the
+ * wrong colour for where it sits is usually wired wrong.
+ *
+ * The tint is keyed off each node's declared output types rather than a list of
+ * node names, so a node added later is themed as soon as it returns one of the
+ * existing types.
+ *
+ * It also migrates Generate Video graphs saved before `duration` replaced the
+ * `num_frames` widget. See migrateNumFramesToDuration below.
+ *
+ * This file also used to host dynamic fields on vLLM-Omni nodes, driven by
+ * widget (in-node form fields) and input (connection link) values and changes.
+ * That is deliberately absent: it introduced too much complexity, and it even
+ * conflicted with the current backend (Python) validation for unknown reasons
+ * (pending ComfyUI upstream fixes).
  */
 
 import { app } from "../../scripts/app.js";
+
+const MARK_URL = new URL("./vllm-omni-mark.svg", import.meta.url).href;
+const MARK_ASPECT = 157 / 165;
+
+/**
+ * Title bar and body tints, one per node family.
+ *
+ * The hues are the logo's: the amber and blue of the "v" mark, and three stops
+ * of the gradient ring in the "O" (#9128A3, #CC5260, and the teal neighbouring
+ * its blue). All of them are dark because LiteGraph paints the title bar darker
+ * than the body and expects both to sit under light text; the ComfyUI light
+ * theme lightens custom node colours on its own.
+ */
+const FAMILIES = {
+    generate: { color: "#152A43", bgcolor: "#1F4269" },
+    sampling: { color: "#3A2C12", bgcolor: "#5C461C" },
+    model: { color: "#2E1233", bgcolor: "#4A1C52" },
+    deployment: { color: "#3A1A1F", bgcolor: "#5C2931" },
+    media: { color: "#0F2E2A", bgcolor: "#194A44" },
+};
+
+/** Output type -> family. Types absent here fall through to "generate". */
+const FAMILY_BY_OUTPUT = {
+    SAMPLING_PARAMS: "sampling",
+    TTS_PARAMS: "model",
+    VIDEO_PARAMS: "model",
+    REMOTE_LORA: "deployment",
+    FASTH3_DEPLOYMENT: "deployment",
+    VIDEO_REFERENCES: "media",
+    LATENT_MASK_EDITING: "media",
+};
+
+function familyOf(nodeData) {
+    for (const type of nodeData.output ?? []) {
+        const family = FAMILY_BY_OUTPUT[type];
+        if (family) {
+            return family;
+        }
+    }
+    // What is left returns a generated artifact (IMAGE, VIDEO, AUDIO, STRING),
+    // which is only ever produced by a node that talks to a vLLM-Omni server.
+    return "generate";
+}
+
+/**
+ * Rewrite a pre-rename Generate Video graph in place.
+ *
+ * `duration` (seconds) took over the slot `num_frames` used to occupy, and
+ * ComfyUI restores widget values positionally, so a graph saved before the
+ * rename reads its frame count as a duration: the workflow this repository
+ * shipped stored 120 frames at 24 fps, which would come back as a 120-second
+ * request. Saved files still name the old widget under `inputs`, and that is
+ * what makes the rewrite both detectable and safe to apply exactly once -- once
+ * rewritten the node serializes `duration`, so a later load finds nothing to do.
+ */
+function migrateNumFramesToDuration(node, info) {
+    const savedBeforeRename = (info?.inputs ?? []).some((input) => input?.widget?.name === "num_frames");
+    if (!savedBeforeRename) {
+        return
+    }
+    const duration = node.widgets?.find((widget) => widget.name === "duration");
+    const fps = node.widgets?.find((widget) => widget.name === "fps");
+    if (!duration || !fps) {
+        return
+    }
+    // configure() has already placed the stale frame count in the duration widget.
+    const frames = Number(duration.value);
+    const rate = Number(fps.value);
+    if (!Number.isFinite(frames) || !Number.isFinite(rate) || rate <= 0) {
+        return
+    }
+    const seconds = Math.max(duration.options?.min ?? 0.1, Math.round((frames / rate) * 1000) / 1000);
+    console.info(
+        `vLLM-Omni: migrated Generate Video "${node.title}" from ${frames} frames to ${seconds}s at ${rate} fps.`,
+    );
+    duration.value = seconds;
+}
+
+const mark = new Image();
+let markReady = false;
+mark.addEventListener("load", () => {
+    markReady = true;
+    // The image resolves after the first frames are already on screen.
+    app.graph?.setDirtyCanvas?.(true, true);
+});
+mark.src = MARK_URL;
+
 app.registerExtension({
     name: "vllm.vllm_omni",
     async beforeRegisterNodeDef(nodeType, nodeData, app) {
         if (!nodeData.name.startsWith("VLLMOmni")) {
             return
         }
-        // Stub frontend plugin for now
+
+        const { color, bgcolor } = FAMILIES[familyOf(nodeData)];
+        // On the prototype, not the instance: a colour the user picks by hand is
+        // written to the instance and shadows this, so manual overrides survive.
+        nodeType.prototype.color = color;
+        nodeType.prototype.bgcolor = bgcolor;
+
+        if (nodeData.name === "VLLMOmniGenerateVideo") {
+            const onConfigure = nodeType.prototype.onConfigure;
+            nodeType.prototype.onConfigure = function (info) {
+                onConfigure?.apply(this, arguments);
+                migrateNumFramesToDuration(this, info);
+            };
+        }
+
+        // Defining this replaces the default dot outright — LiteGraph draws its
+        // own square or circle only for nodes that leave onDrawTitleBox unset.
+        nodeType.prototype.onDrawTitleBox = function (ctx, titleHeight, _size, scale) {
+            // Zoomed out far enough that LiteGraph itself drops to flat shapes,
+            // the mark is a few muddy pixels. Leave the title bar clean instead.
+            if (!markReady || (scale !== undefined && scale < 0.5)) {
+                return
+            }
+            const height = titleHeight * 0.62;
+            const width = height * MARK_ASPECT;
+            ctx.drawImage(
+                mark,
+                titleHeight * 0.5 - width * 0.5,
+                titleHeight * -0.5 - height * 0.5,
+                width,
+                height,
+            );
+        };
     },
     async setup() {
         console.info("vLLM-Omni Setup complete!")

apps/ComfyUI-vLLM-Omni/web/vllm-omni-mark.svg

@@ -0,0 +1,5 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 157 165" width="157" height="165">
+  <title>vLLM-Omni</title>
+  <polygon points="0,36 62,36 62,160" fill="#F1B844"/>
+  <polygon points="96,36 157,0 121,165 68,165" fill="#54A0F8"/>
+</svg>

benchmarks/README.md

@@ -4,6 +4,11 @@ This directory contains benchmark suites for evaluating different model families
 
 ## Benchmark families
 
+### [Qwen2.5-Omni](qwen2_5_omni/README.md) — Embedding Wrapper Latency
+
+CPU-observed A/B timings for redundant text embedding removal, with real checkpoint
+embeddings, warmup, paired blocks, and embedding call-count validation.
+
 ### [TTS](tts/README.md) — Text-to-Speech
 
 Model-agnostic serving benchmarks for TTS models, including Qwen3-TTS and VoxCPM2.
@@ -36,6 +41,17 @@ LingBot-Video MoE transformer.
 - **Dense pipeline**: decoded-video MAE, MSE, PSNR, latency, and optional steady-state timings
 - **MoE transformer**: bitwise router, sparse-block, shared-expert, and full-transformer parity
 
+### [LingBot-World](lingbot_world/README.md) — Realtime Streaming Cadence
+
+Client-side benchmark for a `WS /v1/realtime/video` world-model session, where one
+request is a whole rollout and one AR block streams out as one video chunk. It
+measures a cadence rather than a throughput: there is exactly one session, and what
+matters is whether the next chunk arrives before the viewer finishes the last one.
+
+- **Layout**: `lingbot_world/benchmark_lingbot_world_realtime.py` (WS client), `lingbot_world/workload.py` (rollout and metric math), `lingbot_world/configs/` (single-GPU eager and Ulysses-4 compiled deploy configs)
+- **Workload**: built-in image-conditioned rollout with a generated camera script, or a JSON spec with per-chunk actions and mid-rollout prompt updates
+- **Key metrics**: TTFC (time to first chunk), steady-state chunk inter-arrival percentiles, VIDEO_RTF (wall seconds per video second; lower is better), chunk-deadline attainment, and simulated playback underruns
+
 ### [Distributed](distributed/omni_connectors/README.md) — RDMA Connector Testing
 
 RDMA environment setup and transfer tests for `MooncakeTransferEngineConnector`, including pytest-based single-node checks and manual cross-node benchmarks.

benchmarks/diffusion/benchmark_qwen_image_qk_norm_rope.py

@@ -0,0 +1,222 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Benchmark Qwen-Image Q/K RMSNorm + interleaved RoPE.
+
+Example:
+
+    CUDA_VISIBLE_DEVICES=0 python benchmarks/diffusion/benchmark_qwen_image_qk_norm_rope.py \
+      --seq-len 4096 --warmup 20 --iters 100 --include-compiled --profile
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from collections.abc import Callable
+
+import torch
+import torch.nn.functional as F
+
+from vllm_omni.diffusion.layers.fused_qk_norm_rope import (
+    _fused_cuda_supported,
+    fused_qk_norm_rope,
+)
+from vllm_omni.diffusion.layers.rope import RotaryEmbedding
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--batch", type=int, default=1)
+    parser.add_argument("--seq-len", type=int, default=4096)
+    parser.add_argument("--heads", type=int, default=24)
+    parser.add_argument("--kv-heads", type=int, default=24)
+    parser.add_argument("--head-dim", type=int, default=128)
+    parser.add_argument("--warmup", type=int, default=20)
+    parser.add_argument("--iters", type=int, default=100)
+    parser.add_argument("--dtype", choices=("bf16", "fp16", "fp32"), default="bf16")
+    parser.add_argument("--packed-qkv-view", action="store_true")
+    parser.add_argument("--include-compiled", action="store_true")
+    parser.add_argument("--profile", action="store_true")
+    return parser.parse_args()
+
+
+def _dtype(name: str) -> torch.dtype:
+    return {
+        "bf16": torch.bfloat16,
+        "fp16": torch.float16,
+        "fp32": torch.float32,
+    }[name]
+
+
+def _make_inputs(args: argparse.Namespace, dtype: torch.dtype, device: torch.device):
+    torch.manual_seed(2026)
+    if args.packed_qkv_view:
+        qkv_dim = (args.heads + args.kv_heads + args.kv_heads) * args.head_dim
+        qkv = torch.randn(args.batch, args.seq_len, qkv_dim, device=device, dtype=dtype)
+        q, k, _ = qkv.split(
+            [
+                args.heads * args.head_dim,
+                args.kv_heads * args.head_dim,
+                args.kv_heads * args.head_dim,
+            ],
+            dim=-1,
+        )
+        q = q.unflatten(-1, (args.heads, args.head_dim))
+        k = k.unflatten(-1, (args.kv_heads, args.head_dim))
+    else:
+        q = torch.randn(args.batch, args.seq_len, args.heads, args.head_dim, device=device, dtype=dtype)
+        k = torch.randn(args.batch, args.seq_len, args.kv_heads, args.head_dim, device=device, dtype=dtype)
+
+    q_weight = torch.randn(args.head_dim, device=device, dtype=dtype)
+    k_weight = torch.randn(args.head_dim, device=device, dtype=dtype)
+    angles = torch.randn(args.seq_len, args.head_dim // 2, device=device, dtype=torch.float32)
+    freqs = torch.polar(torch.ones_like(angles), angles)
+    return q, k, q_weight, k_weight, freqs
+
+
+def _native(
+    q: torch.Tensor,
+    k: torch.Tensor,
+    q_weight: torch.Tensor,
+    k_weight: torch.Tensor,
+    freqs: torch.Tensor,
+    eps: float,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    q = F.rms_norm(q, (q.shape[-1],), q_weight, eps)
+    k = F.rms_norm(k, (k.shape[-1],), k_weight, eps)
+    rope = RotaryEmbedding(is_neox_style=False)
+    cos = freqs.real.to(q.dtype)
+    sin = freqs.imag.to(q.dtype)
+    return rope(q, cos, sin), rope(k, cos, sin)
+
+
+def _fused(
+    q: torch.Tensor,
+    k: torch.Tensor,
+    q_weight: torch.Tensor,
+    k_weight: torch.Tensor,
+    freqs: torch.Tensor,
+    eps: float,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    batch, seq_len, num_heads, head_dim = q.shape
+    num_kv_heads = k.shape[2]
+    rope_table = torch.cat((freqs.real, freqs.imag), dim=-1)
+    rope_table = rope_table.unsqueeze(0).expand(batch, -1, -1).reshape(batch * seq_len, head_dim)
+    out_q, out_k = fused_qk_norm_rope(
+        q.reshape(batch * seq_len, num_heads, head_dim),
+        k.reshape(batch * seq_len, num_kv_heads, head_dim),
+        q_weight,
+        k_weight,
+        rope_table,
+        eps,
+        interleaved=True,
+    )
+    return (
+        out_q.reshape(batch, seq_len, num_heads, head_dim),
+        out_k.reshape(batch, seq_len, num_kv_heads, head_dim),
+    )
+
+
+def _measure(fn: Callable[[], tuple[torch.Tensor, torch.Tensor]], warmup: int, iters: int) -> list[float]:
+    for _ in range(warmup):
+        fn()
+    torch.accelerator.synchronize()
+    start = torch.cuda.Event(enable_timing=True)
+    end = torch.cuda.Event(enable_timing=True)
+    samples = []
+    for _ in range(iters):
+        start.record()
+        fn()
+        end.record()
+        end.synchronize()
+        samples.append(start.elapsed_time(end))
+    return samples
+
+
+def _stats(samples: list[float]) -> dict[str, float]:
+    samples = sorted(samples)
+    return {
+        "median_ms": samples[len(samples) // 2],
+        "p90_ms": samples[int(len(samples) * 0.9)],
+        "mean_ms": sum(samples) / len(samples),
+    }
+
+
+def _profile(fn: Callable[[], tuple[torch.Tensor, torch.Tensor]]) -> str:
+    from torch.profiler import ProfilerActivity, profile
+
+    with profile(activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA]) as prof:
+        fn()
+        torch.accelerator.synchronize()
+    return prof.key_averages().table(sort_by="self_cuda_time_total", row_limit=15)
+
+
+def main() -> None:
+    args = _parse_args()
+    if not torch.cuda.is_available():
+        raise RuntimeError("This benchmark requires CUDA")
+    device = torch.device("cuda")
+    dtype = _dtype(args.dtype)
+    eps = 1e-6
+    q, k, q_weight, k_weight, freqs = _make_inputs(args, dtype, device)
+
+    def native():
+        return _native(q, k, q_weight, k_weight, freqs, eps)
+
+    def fused():
+        return _fused(q, k, q_weight, k_weight, freqs, eps)
+
+    results: dict[str, object] = {
+        "device": torch.cuda.get_device_name(),
+        "batch": args.batch,
+        "seq_len": args.seq_len,
+        "heads": args.heads,
+        "kv_heads": args.kv_heads,
+        "head_dim": args.head_dim,
+        "dtype": args.dtype,
+        "packed_qkv_view": args.packed_qkv_view,
+        "fused_fast_path_supported": _fused_cuda_supported(
+            q.reshape(args.batch * args.seq_len, args.heads, args.head_dim),
+            k.reshape(args.batch * args.seq_len, args.kv_heads, args.head_dim),
+            args.head_dim,
+            args.head_dim,
+            interleaved=True,
+        ),
+    }
+
+    native_samples = _measure(native, args.warmup, args.iters)
+    fused_samples = _measure(fused, args.warmup, args.iters)
+    expected_q, expected_k = native()
+    actual_q, actual_k = fused()
+    torch.accelerator.synchronize()
+
+    native_stats = _stats(native_samples)
+    fused_stats = _stats(fused_samples)
+    results["native"] = native_stats
+    results["fused"] = fused_stats
+    results["speedup_vs_native"] = native_stats["median_ms"] / fused_stats["median_ms"]
+    results["max_abs_q"] = (actual_q.float() - expected_q.float()).abs().max().item()
+    results["max_abs_k"] = (actual_k.float() - expected_k.float()).abs().max().item()
+    results["mean_abs_q"] = (actual_q.float() - expected_q.float()).abs().mean().item()
+    results["mean_abs_k"] = (actual_k.float() - expected_k.float()).abs().mean().item()
+
+    if args.include_compiled:
+        compiled_native = torch.compile(native, dynamic=True, fullgraph=True)
+        compiled_fused = torch.compile(fused, dynamic=True, fullgraph=True)
+        compiled_native_stats = _stats(_measure(compiled_native, args.warmup, args.iters))
+        compiled_fused_stats = _stats(_measure(compiled_fused, args.warmup, args.iters))
+        results["compiled_native"] = compiled_native_stats
+        results["compiled_fused"] = compiled_fused_stats
+        results["compiled_fused_speedup_vs_compiled_native"] = (
+            compiled_native_stats["median_ms"] / compiled_fused_stats["median_ms"]
+        )
+
+    print(json.dumps(results, indent=2))
+    if args.profile:
+        print("\n# fused profile")
+        print(_profile(fused))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/diffusion/diffusion_benchmark_serving.py

@@ -1087,6 +1087,12 @@ async def _run_warmups(
 
     warmup_tasks = [asyncio.create_task(limited_warmup_request_func(req)) for req in warmup_requests]
     warmup_outputs = await asyncio.gather(*warmup_tasks)
+    failed_count = sum(not output.success for output in warmup_outputs)
+    if failed_count:
+        raise RuntimeError(
+            f"{failed_count}/{len(warmup_outputs)} warmup requests failed; "
+            "check server logs before running the benchmark."
+        )
     return list(zip(warmup_requests, warmup_outputs))
 
 
@@ -1455,7 +1461,7 @@ async def benchmark(args):
         print(f"Metrics saved to {args.output_file}")
 
 
-if __name__ == "__main__":
+def build_parser():
     parser = argparse.ArgumentParser(description="Benchmark serving for diffusion models.")
     parser.add_argument(
         "--base-url",
@@ -1637,6 +1643,10 @@ if __name__ == "__main__":
         action="store_true",
         help="Request stage duration metrics from endpoints that support return_stage_metrics.",
     )
+    return parser
 
+
+if __name__ == "__main__":
+    parser = build_parser()
     args = parser.parse_args()
     asyncio.run(benchmark(args))

benchmarks/host_weight_runtime/README.md

@@ -0,0 +1,57 @@
+# Host weight dependency memory diagnostic
+
+`safetensors_retention.py` isolates repeated CPU `get_tensor()` calls from
+access to one cached tensor view. It imports neither vLLM nor HWR and uses a
+synthetic float32 payload controlled by `--tensor-elements` (default: 64).
+Tensor size can affect retention: a negative result for 16 elements does not
+rule out growth for 64 elements. The selected size is recorded in the JSON
+arguments and used for both file creation and the correctness check.
+HWR itself calls `get_tensor()` when
+acquiring a lease, then exposes cached views; this probe does not measure HWR
+requests or repeated lease acquisition.
+
+Run one mode per fresh process in the same Python environment and CPU affinity:
+
+```bash
+git rev-parse HEAD > revision.txt
+git status --short > working-tree.txt
+git diff > working-tree.patch
+# Select an allowed CPU from your process affinity; 0 is only an example.
+CUDA_VISIBLE_DEVICES='' taskset -c 0 timeout 120s python \
+  benchmarks/host_weight_runtime/safetensors_retention.py \
+  --mode get_tensor --iterations 10000 --sample-every 5000 > feasibility.json
+```
+
+For a size comparison, run 16 and 64 elements twice each in fresh processes,
+with cached reuse as a control at both sizes:
+
+```bash
+for repetition in 1 2; do
+  for elements in 16 64; do
+    for mode in get_tensor reuse; do
+      CUDA_VISIBLE_DEVICES='' taskset -c 0 timeout 120s python \
+        benchmarks/host_weight_runtime/safetensors_retention.py \
+        --mode "$mode" --tensor-elements "$elements" \
+        > "$mode-$elements-$repetition.json"
+    done
+  done
+done
+```
+
+The diagnostic sets Torch to one CPU thread, warms up before the baseline,
+and reports preparation separately from timed loop work. Checkpoints run GC
+and report Linux private/anonymous memory, RSS, and open descriptors. It checks
+that transient tensor objects and the final cached view are released. Linux
+`/proc/self/smaps_rollup` must be readable; invalid arguments and missing proc
+support fail explicitly. Keep raw JSON and the repository snapshot with results.
+
+Compare the final loop sample with the warmed baseline, then inspect the closed
+sample. Private-memory growth after Python tensor objects disappear is evidence
+of retention in the dependency/allocator path, not proof of a leak's root cause.
+RSS can include file-backed pages; it is not a private-memory metric. Samples
+also include small diagnostic bookkeeping allocations. Do not assert a portable
+memory threshold or turn this synthetic loop into a per-request service estimate.
+
+Do not drop shared caches, trim allocators, or add serving-time GC to make the
+numbers look smaller. A dependency replacement or version change requires its
+own reproducible comparison and correctness/lifecycle validation.

benchmarks/host_weight_runtime/safetensors_retention.py

@@ -0,0 +1,120 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Linux CPU diagnostic: repeated safetensors materialization versus view reuse."""
+
+from __future__ import annotations
+
+import argparse
+import gc
+import json
+import os
+import platform
+import tempfile
+import time
+import weakref
+from pathlib import Path
+
+
+def positive_int(value: str) -> int:
+    parsed = int(value)
+    if parsed <= 0:
+        raise argparse.ArgumentTypeError("must be positive")
+    return parsed
+
+
+def sample(phase: str, iterations: int, loop_seconds: float) -> dict[str, object]:
+    gc.collect()
+    counters = {}
+    for line in Path("/proc/self/smaps_rollup").read_text().splitlines():
+        fields = line.split()
+        if len(fields) == 3 and fields[2] == "kB":
+            counters[fields[0].removesuffix(":")] = int(fields[1]) * 1024
+    return {
+        "phase": phase,
+        "iterations": iterations,
+        "loop_seconds": loop_seconds,
+        "rss_bytes": counters["Rss"],
+        "private_bytes": counters["Private_Clean"] + counters["Private_Dirty"],
+        "anonymous_bytes": counters["Anonymous"],
+        "fd_count": len(tuple(Path("/proc/self/fd").iterdir())),
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--mode", choices=("get_tensor", "reuse"), required=True)
+    parser.add_argument(
+        "--tensor-elements", type=positive_int, default=64, help="Number of float32 elements in the synthetic tensor"
+    )
+    parser.add_argument("--iterations", type=positive_int, default=1_000_000)
+    parser.add_argument("--sample-every", type=positive_int, default=100_000)
+    parser.add_argument("--warmup", type=positive_int, default=1000)
+    args = parser.parse_args()
+    if not Path("/proc/self/smaps_rollup").is_file():
+        parser.error("requires Linux /proc/self/smaps_rollup")
+
+    preparation_start = time.monotonic()
+    import safetensors
+    import torch
+    from safetensors import safe_open
+    from safetensors.torch import save_file
+
+    torch.set_num_threads(1)
+    torch.set_num_interop_threads(1)
+    samples = [sample("before_file", 0, 0.0)]
+    with tempfile.TemporaryDirectory(prefix="safetensors-retention-") as directory:
+        path = Path(directory) / "synthetic.safetensors"
+        save_file({"weight": torch.arange(args.tensor_elements, dtype=torch.float32)}, str(path))
+        with safe_open(path, framework="pt", device="cpu") as reader:
+            cached = reader.get_tensor("weight")
+            assert torch.equal(cached, torch.arange(args.tensor_elements, dtype=torch.float32))
+            for _ in range(args.warmup):
+                tensor = reader.get_tensor("weight") if args.mode == "get_tensor" else cached
+                del tensor
+            samples.append(sample("baseline", 0, 0.0))
+            preparation_seconds = time.monotonic() - preparation_start
+            completed = 0
+            loop_seconds = 0.0
+            while completed < args.iterations:
+                count = min(args.sample_every, args.iterations - completed)
+                start = time.monotonic()
+                for _ in range(count):
+                    tensor = reader.get_tensor("weight") if args.mode == "get_tensor" else cached
+                last_tensor = weakref.ref(tensor)
+                del tensor
+                loop_seconds += time.monotonic() - start
+                completed += count
+                snapshot = sample("loop", completed, loop_seconds)
+                snapshot["last_tensor_alive"] = last_tensor() is not None
+                if args.mode == "get_tensor":
+                    assert last_tensor() is None, "diagnostic retained a transient tensor"
+                samples.append(snapshot)
+            del cached
+        del reader
+        samples.append(sample("closed", completed, loop_seconds))
+        assert last_tensor() is None, "last tensor survived reader/view teardown"
+
+    print(
+        json.dumps(
+            {
+                "schema_version": 1,
+                "arguments": vars(args),
+                "python": platform.python_version(),
+                "torch": torch.__version__,
+                "safetensors": safetensors.__version__,
+                "kernel": platform.release(),
+                "hostname": platform.node(),
+                "pid": os.getpid(),
+                "cpu_affinity": sorted(os.sched_getaffinity(0)),
+                "torch_threads": torch.get_num_threads(),
+                "pythonmalloc": os.environ.get("PYTHONMALLOC", "default"),
+                "preparation_seconds": preparation_seconds,
+                "samples": samples,
+            },
+            indent=2,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

benchmarks/kernels/bench_qwen_select01_modulation.py

@@ -0,0 +1,333 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Benchmark the Qwen-Image-Edit select01 modulation hot path.
+
+This script intentionally starts with the current PyTorch implementation. It is
+used as the before/after baseline for SGLang PR #20395/#21318 style fusion:
+
+* layernorm + scale/shift + binary select01 gate
+* residual + gated update + layernorm + scale/shift + binary select01 gate
+
+Run on a CUDA machine for actionable numbers, for example:
+
+    python benchmarks/kernels/bench_qwen_select01_modulation.py \
+        --device cuda --dtype bf16 --seq-lens 4096 8192 16384 \
+        --output-json /tmp/qwen_select01_baseline.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import statistics
+import sys
+import time
+from collections.abc import Callable
+from pathlib import Path
+
+import torch
+import torch.nn.functional as F
+
+for candidate in Path(__file__).resolve().parents:
+    if (candidate / "vllm_omni").is_dir():
+        if str(candidate) not in sys.path:
+            sys.path.insert(0, str(candidate))
+        break
+
+from vllm_omni.diffusion.layers.qwen_select01_modulation import (  # noqa: E402
+    fused_layernorm_select01,
+    fused_residual_layernorm_select01,
+)
+
+
+def _dtype(name: str) -> torch.dtype:
+    return {
+        "fp32": torch.float32,
+        "float32": torch.float32,
+        "fp16": torch.float16,
+        "float16": torch.float16,
+        "bf16": torch.bfloat16,
+        "bfloat16": torch.bfloat16,
+    }[name]
+
+
+def _select01_modulation(
+    mod_params: torch.Tensor,
+    index: torch.Tensor | None,
+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+    shift, scale, gate = mod_params.chunk(3, dim=-1)
+
+    if index is None:
+        return scale.unsqueeze(1), shift.unsqueeze(1), gate.unsqueeze(1)
+
+    actual_batch = shift.size(0) // 2
+    shift_0, shift_1 = shift[:actual_batch], shift[actual_batch:]
+    scale_0, scale_1 = scale[:actual_batch], scale[actual_batch:]
+    gate_0, gate_1 = gate[:actual_batch], gate[actual_batch:]
+
+    idx = index.unsqueeze(-1) == 0
+    shift = torch.where(idx, shift_0.unsqueeze(1), shift_1.unsqueeze(1))
+    scale = torch.where(idx, scale_0.unsqueeze(1), scale_1.unsqueeze(1))
+    gate = torch.where(idx, gate_0.unsqueeze(1), gate_1.unsqueeze(1))
+    return scale, shift, gate
+
+
+def _native_layernorm_select01(
+    x: torch.Tensor,
+    mod_params: torch.Tensor,
+    index: torch.Tensor,
+    eps: float,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    scale, shift, gate = _select01_modulation(mod_params, index)
+    out = F.layer_norm(x.float(), (x.shape[-1],), eps=eps).to(x.dtype)
+    return out * (1 + scale) + shift, gate
+
+
+def _native_residual_layernorm_select01(
+    x: torch.Tensor,
+    residual: torch.Tensor,
+    residual_gate: torch.Tensor,
+    mod_params: torch.Tensor,
+    index: torch.Tensor,
+    eps: float,
+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+    scale, shift, gate = _select01_modulation(mod_params, index)
+    residual_out = residual + residual_gate * x
+    out = F.layer_norm(residual_out.float(), (residual_out.shape[-1],), eps=eps).to(residual_out.dtype)
+    return out * (1 + scale) + shift, residual_out, gate
+
+
+def _time_cuda(fn: Callable[[], object], warmup: int, iters: int) -> list[float]:
+    for _ in range(warmup):
+        fn()
+    torch.accelerator.synchronize()
+
+    times: list[float] = []
+    for _ in range(iters):
+        start = torch.cuda.Event(enable_timing=True)
+        end = torch.cuda.Event(enable_timing=True)
+        start.record()
+        fn()
+        end.record()
+        torch.accelerator.synchronize()
+        times.append(float(start.elapsed_time(end)))
+    return times
+
+
+def _reset_peak_memory_stats(device: torch.device) -> None:
+    reset_peak_memory_stats = getattr(torch.accelerator, "reset_peak_memory_stats", None)
+    if reset_peak_memory_stats is None:
+        reset_peak_memory_stats = getattr(getattr(torch, device.type), "reset_peak_memory_stats")
+    reset_peak_memory_stats(device)
+
+
+def _max_memory_allocated(device: torch.device) -> int:
+    max_memory_allocated = getattr(torch.accelerator, "max_memory_allocated", None)
+    if max_memory_allocated is None:
+        max_memory_allocated = getattr(getattr(torch, device.type), "max_memory_allocated")
+    return int(max_memory_allocated(device))
+
+
+def _time_cpu(fn: Callable[[], object], warmup: int, iters: int) -> list[float]:
+    for _ in range(warmup):
+        fn()
+
+    times: list[float] = []
+    for _ in range(iters):
+        t0 = time.perf_counter()
+        fn()
+        times.append((time.perf_counter() - t0) * 1000.0)
+    return times
+
+
+def _summarize(times: list[float]) -> dict[str, float]:
+    ordered = sorted(times)
+    return {
+        "median_ms": statistics.median(ordered),
+        "min_ms": ordered[0],
+        "p20_ms": ordered[max(0, int(len(ordered) * 0.20) - 1)],
+        "p80_ms": ordered[min(len(ordered) - 1, int(len(ordered) * 0.80))],
+    }
+
+
+def _make_inputs(
+    batch: int,
+    seq_len: int,
+    hidden_size: int,
+    dtype: torch.dtype,
+    device: torch.device,
+) -> dict[str, torch.Tensor]:
+    torch.manual_seed(0)
+    return {
+        "x": torch.randn(batch, seq_len, hidden_size, device=device, dtype=dtype),
+        "residual": torch.randn(batch, seq_len, hidden_size, device=device, dtype=dtype),
+        "residual_gate": torch.randn(batch, 1, hidden_size, device=device, dtype=dtype),
+        "mod_params": torch.randn(batch * 2, hidden_size * 3, device=device, dtype=dtype),
+        "index": torch.randint(0, 2, (batch, seq_len), device=device, dtype=torch.int64),
+    }
+
+
+def _bench_one(
+    name: str,
+    fn: Callable[[], object],
+    device: torch.device,
+    warmup: int,
+    iters: int,
+) -> dict[str, object]:
+    if device.type == "cuda":
+        _reset_peak_memory_stats(device)
+        times = _time_cuda(fn, warmup, iters)
+        peak_memory_mb = _max_memory_allocated(device) / 1024**2
+    else:
+        times = _time_cpu(fn, warmup, iters)
+        peak_memory_mb = None
+
+    return {
+        "name": name,
+        **_summarize(times),
+        "peak_memory_mb": peak_memory_mb,
+    }
+
+
+def _run(args: argparse.Namespace) -> dict[str, object]:
+    device = torch.device(args.device)
+    if device.type == "cuda" and not torch.cuda.is_available():
+        raise SystemExit("CUDA is not available on this machine.")
+
+    dtype = _dtype(args.dtype)
+    rows: list[dict[str, object]] = []
+
+    for seq_len in args.seq_lens:
+        tensors = _make_inputs(args.batch_size, seq_len, args.hidden_size, dtype, device)
+        impls = ["native"] if args.impl == "native" else ["fused"] if args.impl == "fused" else ["native", "fused"]
+
+        def first_norm() -> object:
+            return _native_layernorm_select01(
+                tensors["x"],
+                tensors["mod_params"],
+                tensors["index"],
+                args.eps,
+            )
+
+        def residual_norm() -> object:
+            return _native_residual_layernorm_select01(
+                tensors["x"],
+                tensors["residual"],
+                tensors["residual_gate"],
+                tensors["mod_params"],
+                tensors["index"],
+                args.eps,
+            )
+
+        callables: list[tuple[str, Callable[[], object]]] = []
+        if "native" in impls:
+            callables.extend(
+                [
+                    ("native_layernorm_select01", first_norm),
+                    ("native_residual_layernorm_select01", residual_norm),
+                ]
+            )
+        if "fused" in impls:
+
+            def fused_first_norm() -> object:
+                return fused_layernorm_select01(
+                    tensors["x"],
+                    tensors["mod_params"],
+                    tensors["index"],
+                    args.eps,
+                )
+
+            def fused_residual_norm() -> object:
+                return fused_residual_layernorm_select01(
+                    tensors["x"],
+                    tensors["residual"],
+                    tensors["residual_gate"].expand_as(tensors["residual"]),
+                    tensors["mod_params"],
+                    tensors["index"],
+                    args.eps,
+                )
+
+            callables.extend(
+                [
+                    ("fused_layernorm_select01", fused_first_norm),
+                    ("fused_residual_layernorm_select01", fused_residual_norm),
+                ]
+            )
+
+        if args.check and "fused" in impls:
+            ref_norm, ref_gate = first_norm()
+            actual_norm, actual_gate = fused_layernorm_select01(
+                tensors["x"],
+                tensors["mod_params"],
+                tensors["index"],
+                args.eps,
+            )
+            torch.testing.assert_close(actual_norm, ref_norm, atol=args.check_atol, rtol=args.check_rtol)
+            torch.testing.assert_close(actual_gate, ref_gate, atol=0, rtol=0)
+            ref_norm, ref_residual, ref_gate = residual_norm()
+            actual_norm, actual_residual, actual_gate = fused_residual_layernorm_select01(
+                tensors["x"],
+                tensors["residual"],
+                tensors["residual_gate"].expand_as(tensors["residual"]),
+                tensors["mod_params"],
+                tensors["index"],
+                args.eps,
+            )
+            torch.testing.assert_close(actual_norm, ref_norm, atol=args.check_atol, rtol=args.check_rtol)
+            torch.testing.assert_close(actual_residual, ref_residual, atol=args.check_atol, rtol=args.check_rtol)
+            torch.testing.assert_close(actual_gate, ref_gate, atol=0, rtol=0)
+
+        for name, fn in callables:
+            row = _bench_one(name, fn, device, args.warmup, args.iters)
+            row.update(
+                {
+                    "batch_size": args.batch_size,
+                    "seq_len": seq_len,
+                    "hidden_size": args.hidden_size,
+                    "dtype": args.dtype,
+                    "device": str(device),
+                }
+            )
+            rows.append(row)
+            print(
+                f"{name:<38} B={args.batch_size:<2} L={seq_len:<6} C={args.hidden_size:<5} "
+                f"median={row['median_ms']:.4f} ms p20={row['p20_ms']:.4f} p80={row['p80_ms']:.4f}"
+            )
+
+    result = {
+        "benchmark": "qwen_select01_modulation",
+        "torch": torch.__version__,
+        "cuda_available": torch.cuda.is_available(),
+        "gpu": torch.cuda.get_device_name(device) if device.type == "cuda" else None,
+        "rows": rows,
+    }
+
+    if args.output_json:
+        output_path = Path(args.output_json)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        output_path.write_text(json.dumps(result, indent=2), encoding="utf-8")
+
+    return result
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    default_device = "cuda" if torch.cuda.is_available() else "cpu"
+    parser.add_argument("--device", default=default_device)
+    parser.add_argument("--dtype", default="bf16", choices=["fp32", "float32", "fp16", "float16", "bf16", "bfloat16"])
+    parser.add_argument("--batch-size", type=int, default=1)
+    parser.add_argument("--seq-lens", type=int, nargs="+", default=[4096, 8192, 16384])
+    parser.add_argument("--hidden-size", type=int, default=3072)
+    parser.add_argument("--eps", type=float, default=1e-6)
+    parser.add_argument("--warmup", type=int, default=20)
+    parser.add_argument("--iters", type=int, default=100)
+    parser.add_argument("--impl", choices=["native", "fused", "all"], default="native")
+    parser.add_argument("--check", action="store_true")
+    parser.add_argument("--check-atol", type=float, default=5e-2)
+    parser.add_argument("--check-rtol", type=float, default=5e-2)
+    parser.add_argument("--output-json")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    _run(_parse_args())

benchmarks/kernels/benchmark_ming_istft.py

@@ -0,0 +1,218 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Measure Ming ISTFT and optionally its pretrained streaming AudioVAE decoder.
+
+Example (no model weights needed for the operator benchmark):
+    python benchmarks/kernels/benchmark_ming_istft.py --compile --iterations 100
+
+Add --decoder to measure the published decoder with fixed synthetic latents.
+This is a decoder-stage benchmark, not text-to-speech serving latency.
+"""
+
+import argparse
+import json
+import statistics
+import time
+from collections.abc import Callable
+from functools import partial
+
+import torch
+
+from vllm_omni.model_executor.models.common.ming.audio_dsp import ISTFT
+from vllm_omni.model_executor.models.common.ming.fused_istft import fused_istft
+
+
+def measure(operation: Callable, warmup: int, iterations: int) -> dict:
+    for _ in range(warmup):
+        operation()
+    torch.accelerator.synchronize()
+    samples = []
+    for _ in range(iterations):
+        start = time.perf_counter()
+        operation()
+        torch.accelerator.synchronize()
+        samples.append((time.perf_counter() - start) * 1000)
+    torch.accelerator.reset_peak_memory_stats()
+    before = torch.accelerator.memory_allocated()
+    operation()
+    torch.accelerator.synchronize()
+    return {
+        "p50_ms": statistics.median(samples),
+        "min_ms": min(samples),
+        "max_ms": max(samples),
+        "stdev_ms": statistics.pstdev(samples),
+        "extra_peak_mib": (torch.accelerator.max_memory_allocated() - before) / 1024**2,
+    }
+
+
+def native_post_fft(module, frames, audio, window):
+    """The native middle-chunk post-FFT sequence, isolated from cuFFT."""
+    count = frames.shape[-1]
+    size = (count - 1) * module.hop_length + module.win_length
+
+    def fold(value):
+        return torch.nn.functional.fold(
+            value, output_size=(1, size), kernel_size=(1, module.win_length), stride=(1, module.hop_length)
+        )[:, 0, 0, :]
+
+    samples = fold(frames * module.window[None, :, None])
+    samples, audio = module._buffer_process(samples, audio, 0, streaming=True)
+    envelope = fold(module.window.square().expand(1, count, -1).transpose(1, 2))
+    envelope, window = module._buffer_process(envelope, window, 0, streaming=True)
+    if not (envelope > 1e-11).all():
+        raise RuntimeError("ISTFT window envelope underflowed; invalid overlap-add state.")
+    return samples / envelope, audio, window
+
+
+def assert_outputs(actual, reference):
+    maximum = 0.0
+    for value, expected in zip(actual, reference, strict=True):
+        if expected is None:
+            assert value is None
+        else:
+            torch.testing.assert_close(value, expected, atol=5e-7, rtol=1e-5)
+            maximum = max(maximum, (value - expected).abs().max().item())
+    return maximum
+
+
+def benchmark_istft(args):
+    for dtype in (torch.float32, torch.bfloat16):
+        module = ISTFT(3528, 882, 3528).to(device="cuda", dtype=dtype)
+        for count in (16, 100, 400):
+            spec = torch.randn(1, 1765, count, device="cuda", dtype=torch.complex64)
+            _, audio, window = module.forward_native(spec, streaming=True)
+            frames = torch.fft.irfft(spec, 3528, dim=1)
+            for scope in ("post_fft", "istft"):
+                if scope == "post_fft":
+                    native = partial(native_post_fft, module, frames, audio, window)
+                    fused = partial(fused_istft, frames, module.window, 882, audio, window, True, False)
+                else:
+                    native = partial(module.forward_native, spec, audio, window, streaming=True)
+                    fused = partial(module, spec, audio, window, streaming=True)
+                error = assert_outputs(fused(), native())
+                results = {"native": measure(native, args.warmup, args.iterations)}
+                if args.compile:
+                    compiled = torch.compile(native)
+                    compiled_output, reference = compiled(), native()
+                    # Inductor can change BF16 envelope rounding. Report that
+                    # difference without weakening the fused-kernel parity gate.
+                    compiled_matches = all(
+                        torch.allclose(value, expected, atol=5e-7, rtol=1e-5)
+                        for value, expected in zip(compiled_output, reference, strict=True)
+                    )
+                    results["compiled"] = measure(compiled, args.warmup, args.iterations)
+                    results["compiled"]["matches_native_tolerance"] = compiled_matches
+                    results["compiled"]["waveform_max_abs_error"] = (
+                        (compiled_output[0] - reference[0]).abs().max().item()
+                    )
+                results["fused"] = measure(fused, args.warmup, args.iterations)
+                print(
+                    json.dumps(
+                        {
+                            "scope": scope,
+                            "window_dtype": str(dtype),
+                            "frames": count,
+                            "max_abs_error": error,
+                            "results": results,
+                        }
+                    ),
+                    flush=True,
+                )
+
+
+def benchmark_decoder(args):
+    from huggingface_hub import hf_hub_download
+    from safetensors import safe_open
+
+    from vllm_omni.model_executor.models.common.ming.audio_vae import Decoder
+
+    model = "inclusionAI/Ming-omni-tts-0.5B"
+    revision = "9154772e7fbc585907b6237e3190790676f28975"
+    config_path = hf_hub_download(model, "config.json", revision=revision)
+    with open(config_path) as source:
+        config = json.load(source)["audio_tokenizer_config"]
+    kwargs = config["dec_kwargs"]
+    backbone = kwargs["backbone"]
+    backbone["_attn_implementation"] = "sdpa"
+    decoder = Decoder(backbone, kwargs["output_dim"], kwargs["latent_dim"], config["patch_size"])
+    checkpoint = hf_hub_download(model, "model.safetensors", revision=revision)
+    prefix = "audio.decoder."
+    with safe_open(checkpoint, framework="pt") as weights:
+        state = {key.removeprefix(prefix): weights.get_tensor(key) for key in weights.keys() if key.startswith(prefix)}
+    decoder.load_state_dict(state, strict=True)
+    decoder = decoder.eval().to(device="cuda", dtype=torch.bfloat16)
+    chunks = [
+        torch.randn(1, size, kwargs["latent_dim"], device="cuda", dtype=torch.bfloat16) for size in (4, 25, 25, 5)
+    ]
+    istft = decoder.head.istft
+    fused_forward = istft.forward
+
+    def run():
+        cache = state = None
+        outputs = []
+        for index, chunk in enumerate(chunks):
+            output, state, cache = decoder.low_level_reconstruct(
+                chunk, past_key_values=cache, use_cache=True, stream_state=state, last_chunk=index == len(chunks) - 1
+            )
+            outputs.append(output)
+        return torch.cat(outputs, dim=-1)
+
+    istft.forward = istft.forward_native
+    expected = run()
+    istft.forward = fused_forward
+    actual = run()
+    torch.testing.assert_close(actual, expected, atol=5e-7, rtol=1e-5)
+    # Alternate the order to expose drift when the decoder-level gain is small.
+    results = {"native": [], "fused": []}
+    for order in (("native", "fused"), ("fused", "native"), ("native", "fused")):
+        for name in order:
+            istft.forward = istft.forward_native if name == "native" else fused_forward
+            results[name].append(measure(run, args.warmup, args.decoder_iterations))
+    print(
+        json.dumps(
+            {
+                "scope": "pretrained_decoder",
+                "model": model,
+                "revision": revision,
+                "backend": "sdpa",
+                "dtype": "bfloat16",
+                "latent_chunks": [4, 25, 25, 5],
+                "max_abs_error": (actual - expected).abs().max().item(),
+                "results": results,
+            }
+        ),
+        flush=True,
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--warmup", type=int, default=20)
+    parser.add_argument("--iterations", type=int, default=100)
+    parser.add_argument("--compile", action="store_true")
+    parser.add_argument("--decoder", action="store_true")
+    parser.add_argument("--decoder-iterations", type=int, default=30)
+    args = parser.parse_args()
+    if args.warmup < 0 or min(args.iterations, args.decoder_iterations) < 1:
+        parser.error("warmup must be nonnegative and iteration counts must be positive")
+    torch.manual_seed(0)
+    print(
+        json.dumps(
+            {
+                "torch": torch.__version__,
+                "cuda": torch.version.cuda,
+                "gpu": torch.cuda.get_device_name(),
+                "warmup": args.warmup,
+                "iterations": args.iterations,
+                "decoder_iterations": args.decoder_iterations,
+            }
+        )
+    )
+    with torch.inference_mode():
+        benchmark_istft(args)
+        if args.decoder:
+            benchmark_decoder(args)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/kernels/mot_linear_benchmarks.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 # ruff: noqa: N803
 
 """MoT (Mixture-of-Tokens) GEMM kernel benchmark and auto-tuning.
@@ -10,7 +13,7 @@ Usage:
     python benchmarks/kernels/mot_linear_benchmarks.py \
         --model ByteDance-Seed/BAGEL-7B-MoT \
         --tp-size 1 --dtype w16a16 --tune \
-        --save-dir vllm_omni/diffusion/layers/mot/configs/
+        --save-dir vllm_omni/diffusion/models/bagel/mot/configs/
 
     # Auto-tune with local model path (offline clusters):
     python benchmarks/kernels/mot_linear_benchmarks.py \
@@ -43,7 +46,7 @@ from vllm.triton_utils import triton
 from vllm.utils.torch_utils import set_random_seed
 
 # NOTE: you should use the same naming syetem for the kernel to load properly
-from vllm_omni.diffusion.layers.mot.ops.mot_gemm import build_config_filename, get_device_name
+from vllm_omni.diffusion.models.bagel.mot.ops.mot_gemm import build_config_filename, get_device_name
 
 # clear the triton cache from time to time, usaully no need to change
 _CACHE_CLEAR_INTERVAL_ENV = "VLLM_MOT_TUNE_CACHE_CLEAR_INTERVAL"
@@ -428,7 +431,7 @@ def benchmark_config(
     cache_flusher: torch.Tensor | None = None,
 ) -> float:
     """Run a MoT GEMM with the given tile config and return avg latency (us)."""
-    from vllm_omni.diffusion.layers.mot.ops.mot_gemm import invoke_mot_gemm
+    from vllm_omni.diffusion.models.bagel.mot.ops.mot_gemm import invoke_mot_gemm
 
     text_indices, vae_indices, M = build_regular_indices(
         image_num=image_num,
@@ -553,7 +556,7 @@ class BenchmarkWorker:
         use_int8_w8a16: bool,
     ) -> tuple[dict[str, int], float]:
         set_random_seed(self.seed)
-        from vllm_omni.diffusion.layers.mot.ops.mot_gemm import (
+        from vllm_omni.diffusion.models.bagel.mot.ops.mot_gemm import (
             get_best_mot_config,
         )
 

benchmarks/lingbot_world/README.md

@@ -0,0 +1,224 @@
+# LingBot-World Benchmarks — Realtime Streaming Cadence
+
+`benchmark_lingbot_world_realtime.py` measures LingBot-World 2.0 as it is
+actually deployed: one `WS /v1/realtime/video` session that streams one video
+chunk per AR block, for as long as the world keeps running.
+
+## Why this is not the diffusion serving benchmark
+
+[`benchmarks/diffusion`](../diffusion/README.md) measures "one request, one
+finished video" and reports request throughput. LingBot-World has neither shape:
+
+- **One request is the whole rollout.** `prepare_encode` runs once, then every AR
+  block is four DMD steps plus one clean-KV commit forward, and each block
+  streams out as a chunk. Throughput in requests per second is meaningless.
+- **There is exactly one session.** The stepwise deploy config pins
+  `max_num_seqs: 1`, and the AR-Diffusion consumer rejects a stage with more than
+  one replica, because session-affine routing does not exist yet. There is no
+  concurrency to sweep.
+- **What matters is cadence.** A deployment is healthy when chunk *N + 1* arrives
+  before a viewer finishes watching chunk *N*. That is a latency distribution
+  against a wall clock, not a throughput number.
+
+## Quick start
+
+Serve the model, then point the benchmark at it. The benchmark never starts a
+server.
+
+```bash
+# Terminal 1 — one GPU, eager (portable smoke configuration)
+vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+  --omni \
+  --deploy-config benchmarks/lingbot_world/configs/single_gpu_eager.yaml \
+  --port 8000
+
+# Terminal 2
+pip install websockets
+python benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py \
+  --host 127.0.0.1 --port 8000 \
+  --num-chunks 16 \
+  --output-json /tmp/lingbot_world_realtime.json
+```
+
+With no `--image`, the first frame comes from the shared vLLM image asset, so the
+default workload needs no setup and this repository carries no benchmark binary.
+Pass `--image /path/to/frame.png` to condition the rollout on your own frame.
+
+### Realtime topology
+
+The single-GPU config is a smoke configuration, not a realtime one: one card runs
+all five forwards of every block serially and nothing is compiled. Use the
+four-GPU config for numbers that mean anything about realtime behavior.
+
+```bash
+vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+  --omni \
+  --deploy-config benchmarks/lingbot_world/configs/usp4_compiled.yaml \
+  --port 8000
+
+python benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py \
+  --port 8000 --num-chunks 40 --warmup-sessions 1 --sessions 3 \
+  --target-fps 12 --output-json /tmp/lingbot_world_repeated.json
+```
+
+`usp4_compiled.yaml` sets pure Ulysses sequence parallelism over four ranks and
+`enforce_eager: false`. The pipeline validates the parallel shape itself:
+`sequence_parallel_size` must equal `ulysses_degree`, `ring_degree` and
+`allgather_degree` must be 1, `ulysses_mode` must be `strict`, and pipeline,
+CFG, and VAE parallel sizes above 1 are rejected. This benchmark config sets
+`tensor_parallel_size: 1`; the pipeline validator does not reject larger TP sizes.
+
+Compiled mode can pay compilation and capture costs on the first rollout.
+Use `--warmup-sessions 1` before measured repetitions. Excluding the first six
+chunks addresses the attention-window ramp; it does not guarantee that
+compilation and capture have finished.
+
+## Metrics
+
+| Metric | Meaning |
+| --- | --- |
+| `ttfc_ms` | **Time to first chunk**: `session.start` sent until the first chunk's bytes arrive. Pays for prompt encode, first-frame VAE encode, and any compile or capture. |
+| `interval_all` | Inter-arrival latency across every chunk: mean, median, std, min, p90, p95, p99, max. |
+| `interval_steady` | The same summary over steady-state chunks only. |
+| `video_rtf` | Wall seconds spent per second of video, matching vLLM-Omni's own RTF (`generation_s / video_duration`). **Lower is better; below 1.0 is real time.** |
+| `video_rtfx` | The reciprocal, under the name this repo uses for it: video seconds per wall second, where higher is better. |
+| `steady_video_rtf` | `video_rtf` restricted to steady-state chunks. |
+| `chunk_deadline_ms` | Wall-clock budget one steady chunk must meet to hold real time: 12 frames ÷ `--target-fps`. Override directly with `--chunk-slo-ms`. |
+| `slo_attainment` | Fraction of steady chunks that met the deadline. |
+| `playback` | Underrun count, total stall, and stall ratio from a simulated viewer. |
+
+### Which frame rate counts as real time
+
+**The checkpoint declares no frame rate.** There is none in `model_index.json`, the
+scheduler config, or the transformer config, and the pipeline never reads one. A
+chunk is twelve pixel frames; how long those frames *last* is a property of how the
+output is played, not of the model.
+
+That choice moves the verdict without moving anything measured:
+
+| basis | video per chunk | deadline | RTF at a measured 1357 ms/chunk |
+| --- | --- | --- | --- |
+| 16 fps | 0.750 s | 750 ms | 1.809 |
+| 12 fps | 1.000 s | 1000 ms | 1.357 |
+
+So `--fps` (sent to the server, which uses it to label the muxed MP4) and
+`--target-fps` (the real-time basis for RTF, the deadline, and the playback
+simulation) are separate flags. `--target-fps` defaults to `--fps`; set it when the
+two differ, and the report prints both. State the basis whenever you quote a number
+— an RTF without its fps is not a measurement. (RTF is a cost here, so a bigger number is a
+worse result; a steady chunk's RTF is simply its interval divided by its video duration.)
+An RTF above 1 means the measured deployment is slower than real time at the
+stated target frame rate. The benchmark measures this failure as well as success.
+
+### Why warmup chunks are not optional
+
+The DiT attends over a sliding window of `sliding_window_num_frames = 18` latent
+frames with `sink_size = 9`, so the key count a block attends over keeps growing
+until eighteen latent frames of history exist — six chunks at three frames per
+block. **Chunk cost is not stationary before chunk six.** A mean over a short
+rollout is therefore lower than anything a running session will sustain, which is
+why `--warmup-chunks` defaults to 6 and steady-state metrics are reported
+separately. Keep `--num-chunks` comfortably above the warmup, or the benchmark
+tells you no chunk reached steady state. The terminal chunk skips next-chunk
+preparation, so it is excluded from steady metrics too; it remains in overall
+RTF, all-chunk intervals, and playback simulation.
+
+The default 16-chunk rollout provides only nine steady intervals. Percentiles
+are interpolated sample summaries; the report warns when fewer than 100
+intervals contribute, and even 100 samples do not guarantee a stable p99.
+Use longer rollouts for tail analysis and at least `--sessions 3` after
+`--warmup-sessions 1` for reported comparisons. Session warmup pays compilation
+cost; chunk warmup separately excludes the attention-window ramp. The command
+above yields 33 steady intervals per session (indices 6 through 38), or 99 across
+three sessions. Report each session's mean and the spread across sessions; this
+sample still does not establish a reliable p99.
+
+### Why playback is simulated
+
+A p99 interval alone cannot say whether a viewer saw a stall: the same p99 is
+invisible behind a three-chunk buffer and a visible freeze behind none. The
+benchmark replays chunk arrivals against a wall clock — playback starts once
+`--playback-buffer-chunks` chunks have landed and then consumes video at `--target-fps`
+— and reports every moment the player ran dry.
+
+### What the client-side number is worth
+
+On a four-GPU Ulysses run the client's mean inter-arrival over all fifteen
+post-first-chunk intervals was **1326.743 ms**, against the server's own
+`StageRequestStats.inter_output_latency_ms` of **1327.479 ms** for the same
+request — a difference of **0.74 ms**. This cross-check shows agreement in
+average output cadence for that run, including warmup and the terminal chunk.
+It does not measure encoding or transport latency: a fixed delivery delay can
+cancel out of inter-arrival differences. Attributing that latency requires
+per-chunk timestamps at both boundaries or server-side profiling.
+
+Compare matching interval populations when investigating discrepancies; both
+measurement bugs and delivery buffering can change the observed cadence.
+
+### What this benchmark cannot see
+
+Chunk metadata carries no server-side timestamps, so the benchmark cannot split
+DiT time from VAE decode time; use `--enable-diffusion-pipeline-profiler` on the
+server for that. It also reports no GPU memory, which is a server-side quantity.
+
+## Custom rollouts
+
+`--workload rollout.json` replaces the built-in workload, including mid-rollout
+prompt changes, which the CLI flags cannot express:
+
+```json
+{
+  "prompt": "The camera moves slowly forward through the scene.",
+  "image": "/path/to/first_frame.png",
+  "num_chunks": 24,
+  "camera_pattern": "orbit",
+  "fps": 16,
+  "seed": 42,
+  "prompt_updates": [
+    {"after_chunk": 8, "prompt": "Rain begins to fall", "transition_chunks": 2}
+  ]
+}
+```
+
+Supply `camera_action_script` directly instead of `num_chunks` to control every
+frame's action. It must hold exactly one three-entry action list per chunk; one
+AR block is three latent frames, and the server rejects any other shape.
+Explicit `--image`, `--prompt`, `--negative-prompt`, `--width`, `--height`,
+`--fps`, `--seed`, and `--flow-shift` flags override the matching workload-file
+values. `--num-chunks` and `--camera-pattern` apply only to the built-in
+workload; when using `--workload`, set those fields in the file instead.
+An explicit `camera_action_script` takes precedence over the file's
+`num_chunks` and `camera_pattern`. Each prompt update must use a distinct
+`after_chunk` boundary so its interaction event ID is unique.
+
+Camera actions change what the world does, not what it costs, so `--camera-pattern`
+(`forward`, `orbit`, `hold`) keeps a rollout representative rather than sweeping a
+cost dimension.
+
+## Options worth knowing
+
+| Flag | Purpose |
+| --- | --- |
+| `--target-fps` | Playback rate the real-time verdict is measured against, independent of the mux label. |
+| `--sessions N` | N sequential measured rollouts; the report adds spread across them. |
+| `--warmup-sessions N` | Unmeasured rollouts first, to pay compile and capture cost. |
+| `--print-chunks` | One line per chunk as it arrives. |
+| `--save-video PATH` | Remux the last measured session for eyeballing (needs vLLM-Omni). |
+| `--ping-interval` | `session.ping` cadence keeping the server's stall clock fresh during a slow first chunk. `0` disables. |
+| `--first-chunk-timeout` | Raise it when the server is still compiling or capturing graphs. |
+
+## Output
+
+`--output-json` writes the run's configuration, every chunk's arrival time and
+size, per-session metrics, and the aggregate, along with the git revision and
+hostname so a number can be traced back to what produced it.
+
+## Tests
+
+`tests/benchmarks/test_lingbot_world_realtime.py` covers the chunk and frame
+arithmetic against the server's own formula, the metric math, and the protocol
+handling against a scripted WebSocket server. It needs no GPU:
+
+```bash
+pytest tests/benchmarks/test_lingbot_world_realtime.py
+```

benchmarks/lingbot_world/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py

@@ -0,0 +1,642 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Realtime streaming benchmark for LingBot-World 2.0 over ``WS /v1/realtime/video``.
+
+Every other diffusion benchmark in this repository measures "one request, one
+finished video". LingBot-World is not that: one request is a whole rollout, and
+the thing a deployment lives or dies by is whether chunk *N + 1* keeps arriving
+before a viewer has finished watching chunk *N*. This benchmark therefore
+measures a **cadence**, not a throughput:
+
+* ``TTFC`` -- session.start to the first chunk's bytes, which pays for prompt
+  encode, the first-frame VAE encode, and any compile or capture the deploy
+  config asks for;
+* **inter-arrival latency** over all chunks and, separately, over the steady
+  state after the attention window saturates;
+* ``VIDEO_RTF`` -- wall seconds spent per second of video generated, matching how
+  vLLM-Omni computes RTF everywhere else (``generation_s / video_duration``).
+  **Lower is better and below 1.0 is real time**; ``VIDEO_RTFX`` is reported
+  alongside it as the reciprocal, under the repo's own name for that;
+* **playback continuity** -- a simulated viewer with a start buffer, because a
+  p99 that is harmless behind a three-chunk buffer is a visible stall behind
+  none.
+
+The server is not started here. Bring one up first (see the README), then point
+this script at it. Requires ``websockets``; ``--save-video`` additionally needs
+a vLLM-Omni installation for the remux helper.
+
+Example:
+
+    python benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py \\
+      --host 127.0.0.1 --port 8000 \\
+      --num-chunks 16 \\
+      --output-json /tmp/lingbot_world_realtime.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import base64
+import json
+import mimetypes
+import os
+import platform
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from benchmarks.lingbot_world.workload import (  # noqa: E402
+    CAMERA_PATTERNS,
+    DEFAULT_FLOW_SHIFT,
+    DEFAULT_FPS,
+    DEFAULT_HEIGHT,
+    DEFAULT_MODEL,
+    DEFAULT_PROMPT,
+    DEFAULT_SEED,
+    DEFAULT_WARMUP_CHUNKS,
+    DEFAULT_WIDTH,
+    ChunkRecord,
+    Workload,
+    aggregate_metrics,
+    compute_metrics,
+    default_workload,
+    load_workload,
+    steady_chunk_deadline_ms,
+)
+
+# The shared vLLM asset keeps the default workload runnable with no setup and
+# adds no binary to this repository. Any real photograph conditions a rollout.
+_DEFAULT_IMAGE_ASSET = "2560px-Gfp-wisconsin-madison-the-nature-boardwalk"
+_WS_PATH = "/v1/realtime/video"
+
+
+class BenchmarkError(RuntimeError):
+    """A session failed in a way that invalidates its numbers."""
+
+
+# --------------------------------------------------------------------------- #
+# Input resolution
+# --------------------------------------------------------------------------- #
+
+
+def resolve_image_reference(value: str | None) -> str:
+    """Return an ``http(s)`` or ``data:`` URL for the session's first frame."""
+    if value is None:
+        from vllm.assets.image import ImageAsset
+
+        value = str(ImageAsset(_DEFAULT_IMAGE_ASSET).get_path("jpg"))
+    if value.startswith(("http://", "https://", "data:")):
+        return value
+    path = Path(value).expanduser()
+    if not path.is_file():
+        raise BenchmarkError(f"--image is neither a URL nor a readable file: {value}")
+    media_type = mimetypes.guess_type(path.name)[0] or "image/png"
+    return f"data:{media_type};base64,{base64.b64encode(path.read_bytes()).decode()}"
+
+
+def build_workload(args: argparse.Namespace) -> Workload:
+    if args.workload is not None:
+        # The file names its own first frame, so resolving happens inside
+        # load_workload -- never eagerly, which would fetch the default asset a
+        # --workload run has no use for.
+        return load_workload(
+            args.workload,
+            image_resolver=lambda value: resolve_image_reference(value if args.image is None else args.image),
+            overrides={
+                "width": args.width_override,
+                "height": args.height_override,
+                "fps": args.fps_override,
+                "seed": args.seed_override,
+                "prompt": args.prompt_override,
+                "flow_shift": args.flow_shift,
+                "negative_prompt": args.negative_prompt,
+            },
+        )
+    return default_workload(
+        image_reference=resolve_image_reference(args.image),
+        num_chunks=args.num_chunks,
+        camera_pattern=args.camera_pattern,
+        prompt=args.prompt_override if args.prompt_override is not None else DEFAULT_PROMPT,
+        width=args.width_override if args.width_override is not None else DEFAULT_WIDTH,
+        height=args.height_override if args.height_override is not None else DEFAULT_HEIGHT,
+        fps=args.fps_override if args.fps_override is not None else DEFAULT_FPS,
+        seed=args.seed_override if args.seed_override is not None else DEFAULT_SEED,
+        flow_shift=args.flow_shift if args.flow_shift is not None else DEFAULT_FLOW_SHIFT,
+        negative_prompt=args.negative_prompt,
+    )
+
+
+# --------------------------------------------------------------------------- #
+# One streaming session
+# --------------------------------------------------------------------------- #
+
+
+@dataclass
+class SessionResult:
+    request_id: str | None
+    records: list[ChunkRecord]
+    media: bytes
+    ttfc_s: float | None
+    video_start_s: float | None
+    wall_s: float
+    done_event: dict[str, Any]
+
+
+async def _keepalive(websocket: Any, send_lock: asyncio.Lock, interval: float) -> None:
+    """Refresh the server's stall clock while a slow first chunk is generating.
+
+    A compiled deploy config records CUDA graphs on the first block, which can
+    outlast the server's ~60 s stall timeout. ``session.ping`` is the documented
+    way to say the client is still there.
+    """
+    while True:
+        await asyncio.sleep(interval)
+        async with send_lock:
+            await websocket.send(json.dumps({"type": "session.ping"}))
+
+
+async def run_session(
+    url: str,
+    model: str,
+    workload: Workload,
+    *,
+    first_chunk_timeout: float,
+    chunk_timeout: float,
+    ping_interval: float,
+    collect_media: bool,
+    print_chunks: bool,
+) -> SessionResult:
+    """Drive one rollout and timestamp every chunk as its bytes land."""
+    try:
+        from websockets.asyncio.client import connect
+    except ImportError:  # pragma: no cover - exercised only without websockets
+        try:
+            from websockets import connect  # type: ignore[attr-defined]
+        except ImportError as exc:
+            raise BenchmarkError("This benchmark needs the websockets package: pip install websockets") from exc
+
+    payload = workload.session_start_payload(model)
+    records: list[ChunkRecord] = []
+    media = bytearray()
+    pending: dict[str, Any] | None = None
+    request_id: str | None = None
+    ttfc_s: float | None = None
+    video_start_s: float | None = None
+    last_arrival: float | None = None
+    done_event: dict[str, Any] = {}
+    send_lock = asyncio.Lock()
+    keepalive: asyncio.Task[None] | None = None
+    # Fire remaining prompt updates in chunk order as their boundary passes.
+    updates = sorted(workload.prompt_updates, key=lambda update: update.after_chunk)
+    update_index = 0
+
+    async with connect(url, max_size=None, ping_interval=None) as websocket:
+        async with send_lock:
+            await websocket.send(json.dumps(payload, ensure_ascii=False))
+        started = time.perf_counter()
+        if ping_interval > 0:
+            keepalive = asyncio.create_task(_keepalive(websocket, send_lock, ping_interval))
+
+        try:
+            while True:
+                timeout = first_chunk_timeout if not records else chunk_timeout
+                try:
+                    deadline = started + (last_arrival if last_arrival is not None else 0.0) + timeout
+                    remaining = deadline - time.perf_counter()
+                    if remaining <= 0:
+                        raise asyncio.TimeoutError
+                    message = await asyncio.wait_for(websocket.recv(), timeout=remaining)
+                except asyncio.TimeoutError as exc:
+                    raise BenchmarkError(
+                        f"No media chunk for {timeout:g} s after {len(records)} chunk(s); "
+                        "the server may still be loading, compiling, or capturing graphs."
+                    ) from exc
+                now = time.perf_counter()
+
+                if isinstance(message, (bytes, bytearray)):
+                    if pending is None:
+                        raise BenchmarkError("Received a binary frame with no preceding video.chunk_metadata event.")
+                    if collect_media:
+                        media.extend(message)
+                    if pending.get("kind") == "media":
+                        index = int(pending["generation_chunk_index"])
+                        arrival = now - started
+                        if ttfc_s is None:
+                            ttfc_s = arrival
+                        records.append(
+                            ChunkRecord(
+                                index=index,
+                                arrival_s=arrival,
+                                inter_arrival_s=arrival - (last_arrival if last_arrival is not None else 0.0),
+                                byte_length=len(message),
+                                num_frames=int(pending["num_frames"]),
+                            )
+                        )
+                        last_arrival = arrival
+                        if print_chunks:
+                            print(
+                                f"  chunk {index:3d}  t={arrival:8.3f}s  "
+                                f"interval={records[-1].inter_arrival_s * 1000:8.1f}ms  "
+                                f"frames={records[-1].num_frames:3d}  bytes={len(message)}",
+                                flush=True,
+                            )
+                        while update_index < len(updates) and updates[update_index].after_chunk <= index:
+                            async with send_lock:
+                                await websocket.send(json.dumps(updates[update_index].to_payload(), ensure_ascii=False))
+                            update_index += 1
+                    pending = None
+                    continue
+
+                event = json.loads(message)
+                event_type = event.get("type")
+                if event_type == "video.chunk_metadata":
+                    pending = event
+                elif event_type == "video.start":
+                    request_id = event.get("request_id")
+                    video_start_s = now - started
+                elif event_type == "error":
+                    # Say what was asked for alongside what the server refused:
+                    # some builds cap a rollout's num_frames, and the operator
+                    # needs to know which --num-chunks would fit.
+                    raise BenchmarkError(
+                        f"Server rejected the session after {len(records)} chunk(s): "
+                        f"{event.get('message', event)} "
+                        f"(requested {workload.num_chunks} chunks = num_frames {workload.num_frames})"
+                    )
+                elif event_type == "session.done":
+                    done_event = event
+                    break
+        finally:
+            if keepalive is not None:
+                keepalive.cancel()
+                try:
+                    await keepalive
+                except asyncio.CancelledError:
+                    pass
+
+        wall_s = time.perf_counter() - started
+
+    if not records:
+        raise BenchmarkError("The session finished without delivering a single media chunk.")
+    if done_event.get("stopped"):
+        raise BenchmarkError(f"The session was stopped before it finished: {done_event}")
+    observed = [record.index for record in records]
+    if observed != list(range(len(records))):
+        raise BenchmarkError(f"Chunk indices are not contiguous from zero: {observed}")
+    if len(records) != workload.num_chunks:
+        raise BenchmarkError(
+            f"Expected {workload.num_chunks} chunks for num_frames={workload.num_frames}, received {len(records)}."
+        )
+
+    return SessionResult(
+        request_id=request_id,
+        records=records,
+        media=bytes(media),
+        ttfc_s=ttfc_s,
+        video_start_s=video_start_s,
+        wall_s=wall_s,
+        done_event=done_event,
+    )
+
+
+# --------------------------------------------------------------------------- #
+# Reporting
+# --------------------------------------------------------------------------- #
+
+
+def _git_revision() -> str | None:
+    try:
+        return subprocess.check_output(
+            ["git", "-C", str(REPO_ROOT), "rev-parse", "HEAD"],
+            text=True,
+            stderr=subprocess.DEVNULL,
+        ).strip()
+    except (OSError, subprocess.CalledProcessError):
+        return None
+
+
+def _print_row(label: str, value: str) -> None:
+    print(f"{label:<44}{value:>16}")
+
+
+def print_report(
+    metrics: dict[str, Any],
+    *,
+    workload: Workload,
+    endpoint: str,
+    sessions: int,
+    target_fps: float,
+) -> None:
+    header = " LingBot-World realtime streaming benchmark "
+    print("\n{s:{c}^{n}}".format(s=header, n=60, c="="))
+    _print_row("Endpoint:", endpoint)
+    _print_row("Geometry:", f"{workload.width}x{workload.height}")
+    _print_row("Chunks per session:", str(workload.num_chunks))
+    _print_row("Sessions:", str(sessions))
+    frames = workload.num_frames
+    _print_row("Real-time basis:", f"{target_fps:g} fps")
+    _print_row("Video seconds per session:", f"{frames / target_fps:.2f} ({frames} frames)")
+    if abs(target_fps - workload.fps) > 1e-9:
+        _print_row("Mux label sent to server:", f"{workload.fps} fps")
+
+    if sessions == 1:
+        print("{s:{c}^{n}}".format(s=" Cadence ", n=60, c="-"))
+        if metrics.get("ttfc_ms") is not None:
+            _print_row("TTFC (session.start -> first chunk):", f"{metrics['ttfc_ms']:.1f} ms")
+        interval_all = metrics.get("interval_all") or {}
+        interval_steady = metrics.get("interval_steady") or {}
+        for title, summary in (("All chunks", interval_all), ("Steady state", interval_steady)):
+            if not summary:
+                continue
+            print(f"  {title} (n={int(summary['count'])}):")
+            _print_row("    mean / median:", f"{summary['mean_ms']:.1f} / {summary['median_ms']:.1f} ms")
+            _print_row("    std / max:", f"{summary['std_ms']:.1f} / {summary['max_ms']:.1f} ms")
+            tail = f"{summary['p90_ms']:.0f} / {summary['p95_ms']:.0f} / {summary['p99_ms']:.0f} ms"
+            _print_row("    p90 / p95 / p99:", tail)
+            if summary["count"] < 100:
+                print(
+                    "    NOTE: fewer than 100 intervals; tail percentiles are descriptive, not reliable tail estimates."
+                )
+
+        print("{s:{c}^{n}}".format(s=" Real time ", n=60, c="-"))
+        _print_row("VIDEO_RTF (wall s / video s, <1 real time):", f"{metrics['video_rtf']:.3f}")
+        _print_row("VIDEO_RTFX (video s / wall s):", f"{metrics['video_rtfx']:.3f}")
+        if metrics.get("steady_video_rtf") is not None:
+            _print_row("VIDEO_RTF steady state:", f"{metrics['steady_video_rtf']:.3f}")
+        _print_row("Frames per second:", f"{metrics['frames_per_second']:.2f}")
+        _print_row("Chunk deadline:", f"{metrics['chunk_deadline_ms']:.1f} ms")
+        if metrics.get("slo_attainment") is not None:
+            _print_row(
+                "Steady chunks within deadline:",
+                f"{metrics['slo_attainment'] * 100:.1f}% ({metrics['slo_violations']} late)",
+            )
+        if not metrics.get("steady_state_reached", False):
+            print(
+                f"  NOTE: no chunk reached the steady state; the attention window needs "
+                f"{metrics['warmup_chunks']} chunks. Raise --num-chunks."
+            )
+
+        playback = metrics.get("playback") or {}
+        print("{s:{c}^{n}}".format(s=" Playback continuity ", n=60, c="-"))
+        _print_row("Start buffer:", f"{playback.get('buffer_chunks')} chunk(s)")
+        _print_row("Underruns:", str(playback.get("underrun_count")))
+        _print_row("Total stall:", f"{playback.get('total_stall_ms', 0.0):.1f} ms")
+        _print_row("Stall ratio:", f"{playback.get('stall_ratio', 0.0) * 100:.2f}%")
+        if playback.get("first_underrun_chunk") is not None:
+            _print_row("First underrun at chunk:", str(playback["first_underrun_chunk"]))
+    else:
+        print("{s:{c}^{n}}".format(s=" Across sessions ", n=60, c="-"))
+        if metrics.get("ttfc_ms_mean") is not None:
+            _print_row("TTFC mean / max:", f"{metrics['ttfc_ms_mean']:.1f} / {metrics['ttfc_ms_max']:.1f} ms")
+        if metrics.get("steady_interval_ms_mean") is not None:
+            _print_row("Steady interval mean:", f"{metrics['steady_interval_ms_mean']:.1f} ms")
+            _print_row("Steady interval spread:", f"{metrics['steady_interval_ms_spread']:.1f} ms")
+        _print_row("VIDEO_RTF mean (lower is better):", f"{metrics['video_rtf_mean']:.3f}")
+        _print_row(
+            "VIDEO_RTF best / worst:",
+            f"{metrics['video_rtf_best']:.3f} / {metrics['video_rtf_worst']:.3f}",
+        )
+        _print_row("Chunk deadline:", f"{metrics['chunk_deadline_ms']:.1f} ms")
+        _print_row("Underruns (all sessions):", str(metrics["underrun_count"]))
+    print("=" * 60)
+
+
+def save_video(media: bytes, path: Path, *, fps: int) -> None:
+    from vllm_omni.diffusion.utils.media_utils import finalize_streaming_video_bytes
+
+    playback_bytes = finalize_streaming_video_bytes(media, input_format="m4s", fps=fps)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp_path = path.with_suffix(path.suffix + ".tmp")
+    tmp_path.write_bytes(playback_bytes)
+    os.replace(tmp_path, path)
+    print(f"Saved {len(media)} streamed bytes -> {len(playback_bytes)} playback bytes at {path}")
+
+
+# --------------------------------------------------------------------------- #
+# CLI
+# --------------------------------------------------------------------------- #
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Benchmark LingBot-World 2.0 realtime streaming over WS /v1/realtime/video.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    endpoint = parser.add_argument_group("endpoint")
+    endpoint.add_argument("--host", default="127.0.0.1")
+    endpoint.add_argument("--port", type=int, default=8000)
+    endpoint.add_argument("--base-url", default=None, help="Full ws:// URL; overrides --host/--port.")
+    endpoint.add_argument("--model", default=DEFAULT_MODEL)
+
+    workload = parser.add_argument_group("workload")
+    workload.add_argument("--workload", default=None, help="JSON rollout spec; overrides the built-in workload.")
+    workload.add_argument("--image", default=None, help="First frame: path or URL. Defaults to the shared vLLM asset.")
+    workload.add_argument("--prompt", dest="prompt_override", default=None, help=f"Default: {DEFAULT_PROMPT!r}")
+    workload.add_argument("--negative-prompt", default=None)
+    workload.add_argument("--num-chunks", type=int, default=16, help="AR blocks per session; ignored with --workload.")
+    workload.add_argument(
+        "--camera-pattern",
+        choices=CAMERA_PATTERNS,
+        default="forward",
+        help="Camera pattern for the built-in workload only; ignored with --workload.",
+    )
+    workload.add_argument("--width", dest="width_override", type=int, default=None, help=f"Default: {DEFAULT_WIDTH}")
+    workload.add_argument("--height", dest="height_override", type=int, default=None, help=f"Default: {DEFAULT_HEIGHT}")
+    workload.add_argument("--fps", dest="fps_override", type=int, default=None, help=f"Default: {DEFAULT_FPS}")
+    workload.add_argument("--seed", dest="seed_override", type=int, default=None, help=f"Default: {DEFAULT_SEED}")
+    workload.add_argument("--flow-shift", type=float, default=None, help=f"Default: {DEFAULT_FLOW_SHIFT}")
+
+    run = parser.add_argument_group("run")
+    run.add_argument("--sessions", type=int, default=1, help="Sequential measured rollouts.")
+    run.add_argument("--warmup-sessions", type=int, default=0, help="Unmeasured rollouts first (pays compile cost).")
+    run.add_argument(
+        "--warmup-chunks",
+        type=int,
+        default=DEFAULT_WARMUP_CHUNKS,
+        help="Chunks excluded from steady state; the attention window saturates after 6.",
+    )
+    run.add_argument(
+        "--chunk-slo-ms",
+        type=float,
+        default=None,
+        help="Deadline a steady chunk must meet. Default: one chunk of video at --target-fps.",
+    )
+    run.add_argument(
+        "--target-fps",
+        type=float,
+        default=None,
+        help=(
+            "Playback rate used for real-time factor, the chunk deadline, and the playback "
+            "simulation. The checkpoint declares no frame rate, so this is a consumption "
+            "choice, not a model property. Default: the --fps sent to the server."
+        ),
+    )
+    run.add_argument("--playback-buffer-chunks", type=int, default=1, help="Chunks buffered before playback starts.")
+    run.add_argument("--first-chunk-timeout", type=float, default=900.0)
+    run.add_argument("--chunk-timeout", type=float, default=300.0)
+    run.add_argument("--ping-interval", type=float, default=20.0, help="session.ping cadence; 0 disables.")
+
+    output = parser.add_argument_group("output")
+    output.add_argument("--output-json", default=None)
+    output.add_argument("--save-video", default=None, help="Write the last measured session's video here.")
+    output.add_argument("--print-chunks", action="store_true")
+
+    args = parser.parse_args(argv)
+    if args.sessions < 1:
+        parser.error("--sessions must be at least 1.")
+    if args.warmup_sessions < 0:
+        parser.error("--warmup-sessions must be non-negative.")
+    if args.warmup_chunks < 0:
+        parser.error("--warmup-chunks must be non-negative.")
+    if args.playback_buffer_chunks < 1:
+        parser.error("--playback-buffer-chunks must be at least 1.")
+    if args.target_fps is not None and args.target_fps <= 0:
+        parser.error("--target-fps must be positive.")
+    if args.workload is None and args.num_chunks < 1:
+        parser.error("--num-chunks must be at least 1.")
+    return args
+
+
+def endpoint_url(args: argparse.Namespace) -> str:
+    if args.base_url is not None:
+        base = args.base_url.rstrip("/")
+        for prefix, replacement in (("http://", "ws://"), ("https://", "wss://")):
+            if base.startswith(prefix):
+                base = replacement + base[len(prefix) :]
+                break
+        return base if base.endswith(_WS_PATH) else base + _WS_PATH
+    return f"ws://{args.host}:{args.port}{_WS_PATH}"
+
+
+async def run_benchmark(args: argparse.Namespace) -> dict[str, Any]:
+    workload = build_workload(args)
+    url = endpoint_url(args)
+    # The mux label and the real-time basis are different questions; only the
+    # latter decides whether a run passes.
+    target_fps = args.target_fps if args.target_fps is not None else float(workload.fps)
+    if max(1, args.warmup_chunks) >= workload.num_chunks - 1:
+        print(
+            f"WARNING: --warmup-chunks={args.warmup_chunks} leaves no nonterminal steady-state chunk in a "
+            f"{workload.num_chunks}-chunk rollout; steady-state metrics will be empty.",
+            file=sys.stderr,
+        )
+
+    print(f"Connecting to {url}")
+    print(
+        f"Rollout: {workload.num_chunks} chunks, num_frames={workload.num_frames}, "
+        f"{workload.video_seconds:.2f}s of video"
+    )
+    print(
+        f"Real-time basis: {target_fps:g} fps -> a steady 12-frame chunk is "
+        f"{steady_chunk_deadline_ms(target_fps):.1f} ms of video"
+        + ("" if args.target_fps is None else f" (mux label stays {workload.fps} fps)")
+    )
+
+    for index in range(args.warmup_sessions):
+        print(f"Warmup session {index + 1}/{args.warmup_sessions} (not measured)...")
+        await run_session(
+            url,
+            args.model,
+            workload,
+            first_chunk_timeout=args.first_chunk_timeout,
+            chunk_timeout=args.chunk_timeout,
+            ping_interval=args.ping_interval,
+            collect_media=False,
+            print_chunks=False,
+        )
+
+    session_metrics: list[dict[str, Any]] = []
+    session_payloads: list[dict[str, Any]] = []
+    last_result: SessionResult | None = None
+    for index in range(args.sessions):
+        if args.sessions > 1:
+            print(f"Session {index + 1}/{args.sessions}...")
+        result = await run_session(
+            url,
+            args.model,
+            workload,
+            first_chunk_timeout=args.first_chunk_timeout,
+            chunk_timeout=args.chunk_timeout,
+            ping_interval=args.ping_interval,
+            collect_media=args.save_video is not None,
+            print_chunks=args.print_chunks,
+        )
+        last_result = result
+        metrics = compute_metrics(
+            result.records,
+            fps=target_fps,
+            warmup_chunks=args.warmup_chunks,
+            slo_ms=args.chunk_slo_ms,
+            playback_buffer_chunks=args.playback_buffer_chunks,
+            ttfc_s=result.ttfc_s,
+            session_wall_s=result.wall_s,
+        )
+        session_metrics.append(metrics)
+        session_payloads.append(
+            {
+                "session_index": index,
+                "request_id": result.request_id,
+                "video_start_ms": (result.video_start_s * 1000.0) if result.video_start_s is not None else None,
+                "chunks": [record.to_dict() for record in result.records],
+                "metrics": metrics,
+            }
+        )
+
+    aggregate = aggregate_metrics(session_metrics, fps=target_fps)
+    print_report(aggregate, workload=workload, endpoint=url, sessions=args.sessions, target_fps=target_fps)
+
+    if args.save_video is not None and last_result is not None:
+        save_video(last_result.media, Path(args.save_video), fps=workload.fps)
+
+    document = {
+        "benchmark": "lingbot_world_realtime",
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "endpoint": url,
+        "model": args.model,
+        "git_revision": _git_revision(),
+        "hostname": platform.node(),
+        "workload": workload.describe(),
+        "config": {
+            "sessions": args.sessions,
+            "warmup_sessions": args.warmup_sessions,
+            "warmup_chunks": args.warmup_chunks,
+            "camera_pattern": args.camera_pattern,
+            "chunk_slo_ms": args.chunk_slo_ms,
+            "target_fps": target_fps,
+            "mux_fps": workload.fps,
+            "playback_buffer_chunks": args.playback_buffer_chunks,
+            "workload_file": args.workload,
+        },
+        "sessions": session_payloads,
+        "metrics": aggregate,
+    }
+    if args.output_json is not None:
+        path = Path(args.output_json)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n")
+        print(f"Wrote {path}")
+    return document
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv)
+    try:
+        asyncio.run(run_benchmark(args))
+    except BenchmarkError as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmarks/lingbot_world/configs/single_gpu_eager.yaml

@@ -0,0 +1,45 @@
+# Benchmark deploy config: LingBot-World 2.0 on one GPU, eager.
+#
+# This is the portable smoke configuration -- it matches the shipped
+# vllm_omni/deploy/lingbot_world_v2_stepwise.yaml so a number measured here is
+# comparable to the default deployment. It is NOT a realtime configuration: one
+# GPU runs all five forwards of every AR block serially and nothing is compiled,
+# so expect a chunk interval several times the real-time deadline.
+#
+# Use configs/usp4_compiled.yaml for the realtime topology.
+#
+#   vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+#     --omni --deploy-config benchmarks/lingbot_world/configs/single_gpu_eager.yaml \
+#     --port 8000
+
+pipeline: lingbot_world
+async_chunk: false
+distributed_executor_backend: mp
+dtype: bfloat16
+
+stages:
+  - stage_id: 0
+    devices: "0"
+    # AR-Diffusion binds one session per runner invocation; the runner rejects
+    # anything larger.
+    max_num_seqs: 1
+    enforce_eager: true
+    model_class_name: LingBotWorldCausalDMDPipeline
+    engine_backend: vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine
+    step_execution: true
+    streaming_output: true
+    default_sampling_params:
+      # The causal-fast checkpoint pins a four-step DMD schedule; this sizes the
+      # step budget before the pipeline is reached.
+      num_inference_steps: 4
+      output_type: pil
+      extra_args:
+        flow_shift: 5.0
+    model_config:
+      # The AR cache geometry is fixed at load time, so a benchmark request must
+      # ask for exactly this resolution.
+      ar_diffusion_height: 480
+      ar_diffusion_width: 832
+      ar_diffusion_kv_config:
+        gpu_memory_fraction: 0.6
+        warmup_cudagraph: true

benchmarks/lingbot_world/configs/usp4_compiled.yaml

@@ -0,0 +1,60 @@
+# Benchmark deploy config: LingBot-World 2.0 on four GPUs, Ulysses SP-4, compiled.
+#
+# This is the realtime topology. Two things differ from the single-GPU config,
+# and both matter to the number the benchmark reports:
+#
+#   parallel_config   Pure Ulysses sequence parallelism over four ranks. The
+#                     pipeline validates this: LingBot requires
+#                     sequence_parallel_size == ulysses_degree, ring_degree and
+#                     allgather_degree of 1, ulysses_mode "strict", and
+#                     ulysses_a2a_permute disabled. Pipeline, CFG, and VAE
+#                     parallel sizes above 1 are rejected. This config uses
+#                     TP=1; the validator does not reject larger TP sizes.
+#   enforce_eager     false puts LingBotDMDBlockRunner on the compiled path
+#                     (regional torch.compile with cudagraph step marking). The
+#                     first rollout therefore pays compile and capture cost --
+#                     use --warmup-sessions 1 before measured repetitions.
+#                     Chunk warmup only excludes the attention-window ramp.
+#
+# Serve it under a GPU reservation, since it needs four cards:
+#
+#   gpu run --gpus 4 -- vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+#     --omni --deploy-config benchmarks/lingbot_world/configs/usp4_compiled.yaml \
+#     --port 8000
+#
+# `devices` is the rank->card mapping inside the reservation, so it stays
+# "0,1,2,3" no matter which physical cards CUDA_VISIBLE_DEVICES selected.
+
+pipeline: lingbot_world
+async_chunk: false
+distributed_executor_backend: mp
+dtype: bfloat16
+
+stages:
+  - stage_id: 0
+    devices: "0,1,2,3"
+    max_num_seqs: 1
+    enforce_eager: false
+    model_class_name: LingBotWorldCausalDMDPipeline
+    engine_backend: vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine
+    step_execution: true
+    streaming_output: true
+    parallel_config:
+      tensor_parallel_size: 1
+      sequence_parallel_size: 4
+      ulysses_degree: 4
+      ring_degree: 1
+    default_sampling_params:
+      num_inference_steps: 4
+      output_type: pil
+      extra_args:
+        flow_shift: 5.0
+    model_config:
+      ar_diffusion_height: 480
+      ar_diffusion_width: 832
+      ar_diffusion_kv_config:
+        # Each rank holds its own shard of the paged KV cache, so the per-card
+        # fraction can stay at the single-GPU value; lower it if the four-rank
+        # start OOMs on smaller cards.
+        gpu_memory_fraction: 0.6
+        warmup_cudagraph: true

benchmarks/lingbot_world/workload.py

@@ -0,0 +1,631 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Workload construction and metric math for the LingBot-World realtime benchmark.
+
+Nothing here imports torch, vllm, or websockets, so the arithmetic that decides
+what a benchmark number *means* is unit-testable without a GPU or a server.
+
+Two model properties drive every formula in this module, both read from the
+checkpoint's ``transformer/config.json`` rather than assumed:
+
+``num_frames_per_block = 3``
+    One AR block commits three latent frames, so one streamed chunk is three
+    latent frames and a request's ``camera_action_script`` carries exactly one
+    three-entry action list per chunk.
+
+``sliding_window_num_frames = 18`` with ``sink_size = 9``
+    Self-attention sees nine sink frames plus a nine-frame recent window, so the
+    key count a block attends over keeps growing until eighteen latent frames of
+    history exist -- six chunks. Chunk cost is therefore *not* stationary before
+    chunk six, and a mean taken over a short rollout reports a number no steady
+    session will ever produce. :data:`DEFAULT_WARMUP_CHUNKS` exists for that
+    reason and is not a round-number guess.
+
+The causal VAE expands the first latent frame to one pixel frame and every later
+latent frame to four, so chunk zero carries nine pixel frames and every chunk
+after it carries twelve. Video-duration math has to respect that asymmetry or
+real-time factor comes out wrong on short rollouts.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+# transformer.config.num_frames_per_block: latent frames committed per AR block.
+FRAMES_PER_BLOCK = 3
+# vae_scale_factor_temporal: latent frames -> pixel frames, after the first.
+TEMPORAL_COMPRESSION = 4
+# (sink_size 9 + recent window 9) / FRAMES_PER_BLOCK: chunks until the attention
+# window saturates and per-chunk cost becomes stationary.
+DEFAULT_WARMUP_CHUNKS = 6
+
+DEFAULT_MODEL = "robbyant/lingbot-world-v2-14b-causal-fast-diffusers"
+DEFAULT_PROMPT = "The camera moves slowly forward through the scene."
+# The AR KV cache geometry is fixed at load time, so a request must ask for
+# exactly the resolution the deploy config declares.
+DEFAULT_WIDTH = 832
+DEFAULT_HEIGHT = 480
+DEFAULT_FPS = 16
+DEFAULT_FLOW_SHIFT = 5.0
+DEFAULT_SEED = 42
+
+_HOLD = "hold"
+_ACTION_KEYS = ("w", "a", "s", "d")
+
+
+def num_frames_for_chunks(num_chunks: int) -> int:
+    """Pixel-frame count whose rollout is exactly ``num_chunks`` chunks.
+
+    Inverse of :func:`chunks_for_num_frames`, which is the server's own
+    ``((num_frames - 1) // 4 + 1) // 3``.
+    """
+    if num_chunks < 1:
+        raise ValueError("num_chunks must be at least 1.")
+    return (num_chunks * FRAMES_PER_BLOCK - 1) * TEMPORAL_COMPRESSION + 1
+
+
+def chunks_for_num_frames(num_frames: int) -> int:
+    """Chunks the server generates for ``num_frames``, mirroring its own formula."""
+    if num_frames < 1:
+        raise ValueError("num_frames must be at least 1.")
+    return ((num_frames - 1) // TEMPORAL_COMPRESSION + 1) // FRAMES_PER_BLOCK
+
+
+def pixel_frames_in_chunk(chunk_index: int) -> int:
+    """Pixel frames carried by one chunk.
+
+    The causal decoder expands the session's opening latent frame once and every
+    later latent frame four times, so chunk zero is short by three frames.
+    """
+    if chunk_index < 0:
+        raise ValueError("chunk_index must be non-negative.")
+    if chunk_index == 0:
+        return (FRAMES_PER_BLOCK - 1) * TEMPORAL_COMPRESSION + 1
+    return FRAMES_PER_BLOCK * TEMPORAL_COMPRESSION
+
+
+def camera_actions_for_chunk(pattern: str, chunk_index: int) -> list[list[str]]:
+    """One three-entry action list for ``chunk_index``.
+
+    Camera actions select camera embeddings; they change what the world does,
+    not how much arithmetic it costs. The pattern therefore exists to keep a
+    benchmark rollout representative rather than to sweep a cost dimension.
+    """
+    if chunk_index < 0:
+        raise ValueError("chunk_index must be non-negative.")
+    if pattern == "forward":
+        return [["w"] for _ in range(FRAMES_PER_BLOCK)]
+    if pattern == _HOLD:
+        return [[] for _ in range(FRAMES_PER_BLOCK)]
+    if pattern == "orbit":
+        key = _ACTION_KEYS[chunk_index % len(_ACTION_KEYS)]
+        return [[key] for _ in range(FRAMES_PER_BLOCK)]
+    raise ValueError(f"Unknown camera pattern {pattern!r}; expected one of {sorted(CAMERA_PATTERNS)}.")
+
+
+CAMERA_PATTERNS = ("forward", "orbit", _HOLD)
+
+
+def build_camera_script(num_chunks: int, pattern: str) -> list[list[list[str]]]:
+    """Full ``camera_action_script``: one action list per generated chunk."""
+    if num_chunks < 1:
+        raise ValueError("num_chunks must be at least 1.")
+    return [camera_actions_for_chunk(pattern, index) for index in range(num_chunks)]
+
+
+def _validate_camera_script(script: Sequence[Any]) -> list[list[list[str]]]:
+    validated: list[list[list[str]]] = []
+    for chunk_index, actions in enumerate(script):
+        if not isinstance(actions, Sequence) or isinstance(actions, (str, bytes)):
+            raise ValueError(f"camera_action_script[{chunk_index}] must be a list of three action lists.")
+        if len(actions) != FRAMES_PER_BLOCK:
+            raise ValueError(
+                f"camera_action_script[{chunk_index}] must hold exactly {FRAMES_PER_BLOCK} action lists "
+                f"(got {len(actions)}); one AR block is {FRAMES_PER_BLOCK} latent frames."
+            )
+        frames: list[list[str]] = []
+        for frame_index, frame in enumerate(actions):
+            if not isinstance(frame, Sequence) or isinstance(frame, (str, bytes)):
+                raise ValueError(f"camera_action_script[{chunk_index}][{frame_index}] must be a list of strings.")
+            if any(not isinstance(action, str) for action in frame):
+                raise ValueError(f"camera_action_script[{chunk_index}][{frame_index}] must contain only strings.")
+            frames.append([str(action) for action in frame])
+        validated.append(frames)
+    if not validated:
+        raise ValueError("camera_action_script must not be empty.")
+    return validated
+
+
+@dataclass(frozen=True)
+class PromptUpdate:
+    """A mid-rollout ``session.interaction`` scheduled on a chunk boundary."""
+
+    after_chunk: int
+    prompt: str
+    transition_chunks: int = 3
+
+    def __post_init__(self) -> None:
+        if self.after_chunk < 0:
+            raise ValueError("PromptUpdate.after_chunk must be non-negative.")
+        if not self.prompt.strip():
+            raise ValueError("PromptUpdate.prompt must contain non-whitespace text.")
+        if self.transition_chunks < 0:
+            raise ValueError("PromptUpdate.transition_chunks must be non-negative.")
+
+    def to_payload(self) -> dict[str, Any]:
+        return {
+            "type": "session.interaction",
+            "interaction": {
+                "event_id": f"chunk-{self.after_chunk}",
+                "event": {"prompt": self.prompt},
+                "transition_chunks": self.transition_chunks,
+            },
+        }
+
+
+@dataclass(frozen=True)
+class Workload:
+    """One rollout: what the benchmark asks the server to generate."""
+
+    prompt: str
+    image_reference: str
+    camera_script: list[list[list[str]]]
+    width: int = DEFAULT_WIDTH
+    height: int = DEFAULT_HEIGHT
+    fps: int = DEFAULT_FPS
+    seed: int = DEFAULT_SEED
+    flow_shift: float = DEFAULT_FLOW_SHIFT
+    negative_prompt: str | None = None
+    prompt_updates: tuple[PromptUpdate, ...] = ()
+    extra_params: Mapping[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if not self.prompt.strip():
+            raise ValueError("Workload.prompt must contain non-whitespace text.")
+        if not self.image_reference:
+            raise ValueError("Workload.image_reference is required; LingBot-World is image-conditioned.")
+        if self.width <= 0 or self.height <= 0 or self.width % 16 or self.height % 16:
+            raise ValueError("Workload width/height must be positive multiples of 16.")
+        if self.fps <= 0:
+            raise ValueError("Workload.fps must be positive.")
+        object.__setattr__(self, "camera_script", _validate_camera_script(self.camera_script))
+        seen_update_chunks: set[int] = set()
+        for update in self.prompt_updates:
+            if update.after_chunk in seen_update_chunks:
+                raise ValueError(f"Duplicate prompt update after chunk {update.after_chunk}; event IDs must be unique.")
+            seen_update_chunks.add(update.after_chunk)
+            if update.after_chunk >= self.num_chunks:
+                raise ValueError(
+                    f"Prompt update after chunk {update.after_chunk} never fires in a {self.num_chunks}-chunk rollout."
+                )
+
+    @property
+    def num_chunks(self) -> int:
+        return len(self.camera_script)
+
+    @property
+    def num_frames(self) -> int:
+        return num_frames_for_chunks(self.num_chunks)
+
+    @property
+    def video_seconds(self) -> float:
+        return self.num_frames / self.fps
+
+    def session_start_payload(self, model: str) -> dict[str, Any]:
+        """The ``session.start`` event for ``WS /v1/realtime/video``."""
+        extra_params: dict[str, Any] = {
+            "flow_shift": self.flow_shift,
+            "camera_action_script": self.camera_script,
+        }
+        extra_params.update(self.extra_params)
+        payload: dict[str, Any] = {
+            "type": "session.start",
+            "model": model,
+            "prompt": self.prompt,
+            "image_reference": {"image_url": self.image_reference},
+            "width": self.width,
+            "height": self.height,
+            "num_frames": self.num_frames,
+            "fps": self.fps,
+            "seed": self.seed,
+            "extra_params": extra_params,
+        }
+        if self.negative_prompt is not None:
+            payload["negative_prompt"] = self.negative_prompt
+        return payload
+
+    def describe(self) -> dict[str, Any]:
+        return {
+            "num_chunks": self.num_chunks,
+            "num_frames": self.num_frames,
+            "width": self.width,
+            "height": self.height,
+            "fps": self.fps,
+            "seed": self.seed,
+            "flow_shift": self.flow_shift,
+            "video_seconds": self.video_seconds,
+            "prompt_updates": [
+                {
+                    "after_chunk": update.after_chunk,
+                    "transition_chunks": update.transition_chunks,
+                }
+                for update in self.prompt_updates
+            ],
+        }
+
+
+def default_workload(
+    *,
+    image_reference: str,
+    num_chunks: int,
+    camera_pattern: str = "forward",
+    prompt: str = DEFAULT_PROMPT,
+    width: int = DEFAULT_WIDTH,
+    height: int = DEFAULT_HEIGHT,
+    fps: int = DEFAULT_FPS,
+    seed: int = DEFAULT_SEED,
+    flow_shift: float = DEFAULT_FLOW_SHIFT,
+    negative_prompt: str | None = None,
+) -> Workload:
+    """Built-in rollout: a fixed prompt and a generated camera script."""
+    return Workload(
+        prompt=prompt,
+        image_reference=image_reference,
+        camera_script=build_camera_script(num_chunks, camera_pattern),
+        width=width,
+        height=height,
+        fps=fps,
+        seed=seed,
+        flow_shift=flow_shift,
+        negative_prompt=negative_prompt,
+    )
+
+
+def load_workload(
+    path: str | Path,
+    *,
+    image_resolver: Any = None,
+    overrides: Mapping[str, Any] | None = None,
+) -> Workload:
+    """Read a rollout spec from JSON.
+
+    ``image_resolver`` turns the spec's ``image`` field into an ``http(s)`` or
+    ``data:`` URL; the caller owns that because inlining a local file is I/O.
+    Keys in ``overrides`` that are not ``None`` win over the file, so CLI flags
+    stay meaningful next to ``--workload``.
+    """
+    document = json.loads(Path(path).read_text())
+    if not isinstance(document, dict):
+        raise ValueError("A workload file must contain a JSON object.")
+
+    merged = dict(document)
+    for key, value in (overrides or {}).items():
+        if value is not None:
+            merged[key] = value
+
+    camera_script = merged.get("camera_action_script")
+    if camera_script is None:
+        num_chunks = merged.get("num_chunks")
+        if num_chunks is None:
+            raise ValueError("A workload file needs either camera_action_script or num_chunks.")
+        camera_script = build_camera_script(int(num_chunks), str(merged.get("camera_pattern", "forward")))
+
+    image = merged.get("image") or merged.get("image_reference")
+    if image is None:
+        raise ValueError("A workload file needs an image or image_reference field.")
+    image_reference = image_resolver(str(image)) if image_resolver is not None else str(image)
+
+    updates_field = merged.get("prompt_updates") or ()
+    prompt_updates = tuple(
+        PromptUpdate(
+            after_chunk=int(entry["after_chunk"]),
+            prompt=str(entry["prompt"]),
+            transition_chunks=int(entry.get("transition_chunks", 3)),
+        )
+        for entry in updates_field
+    )
+
+    extra_params = merged.get("extra_params") or {}
+    if not isinstance(extra_params, Mapping):
+        raise ValueError("Workload extra_params must be a JSON object.")
+
+    return Workload(
+        prompt=str(merged.get("prompt", DEFAULT_PROMPT)),
+        image_reference=image_reference,
+        camera_script=camera_script,
+        width=int(merged.get("width", DEFAULT_WIDTH)),
+        height=int(merged.get("height", DEFAULT_HEIGHT)),
+        fps=int(merged.get("fps", DEFAULT_FPS)),
+        seed=int(merged.get("seed", DEFAULT_SEED)),
+        flow_shift=float(merged.get("flow_shift", DEFAULT_FLOW_SHIFT)),
+        negative_prompt=merged.get("negative_prompt"),
+        prompt_updates=prompt_updates,
+        extra_params=dict(extra_params),
+    )
+
+
+# --------------------------------------------------------------------------- #
+# Metrics
+# --------------------------------------------------------------------------- #
+
+
+@dataclass
+class ChunkRecord:
+    """One media chunk as the client observed it."""
+
+    index: int
+    arrival_s: float
+    """Seconds from ``session.start`` being sent to this chunk's bytes arriving."""
+    inter_arrival_s: float
+    """Seconds since the previous chunk; for chunk zero this equals ``arrival_s``."""
+    byte_length: int
+    num_frames: int
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "index": self.index,
+            "arrival_s": self.arrival_s,
+            "inter_arrival_ms": self.inter_arrival_s * 1000.0,
+            "byte_length": self.byte_length,
+            "num_frames": self.num_frames,
+        }
+
+
+def percentile(values: Sequence[float], fraction: float) -> float:
+    """Linearly interpolated percentile, so a 10-sample run still reports a p95.
+
+    Implemented here rather than pulled from numpy to keep this module importable
+    in a bare environment; the benchmark's own numbers must not depend on which
+    optional package happens to be installed.
+    """
+    if not values:
+        raise ValueError("percentile() needs at least one value.")
+    if not 0.0 <= fraction <= 1.0:
+        raise ValueError("percentile fraction must be within [0, 1].")
+    ordered = sorted(values)
+    if len(ordered) == 1:
+        return ordered[0]
+    position = fraction * (len(ordered) - 1)
+    lower = int(position)
+    upper = min(lower + 1, len(ordered) - 1)
+    weight = position - lower
+    return ordered[lower] * (1.0 - weight) + ordered[upper] * weight
+
+
+def _stddev(values: Sequence[float]) -> float:
+    if len(values) < 2:
+        return 0.0
+    mean = sum(values) / len(values)
+    return (sum((value - mean) ** 2 for value in values) / (len(values) - 1)) ** 0.5
+
+
+def _latency_summary(values: Sequence[float]) -> dict[str, float]:
+    """Millisecond summary of a set of inter-arrival intervals."""
+    if not values:
+        return {}
+    milliseconds = [value * 1000.0 for value in values]
+    return {
+        "count": float(len(milliseconds)),
+        "mean_ms": sum(milliseconds) / len(milliseconds),
+        "median_ms": percentile(milliseconds, 0.5),
+        "std_ms": _stddev(milliseconds),
+        "min_ms": min(milliseconds),
+        "p90_ms": percentile(milliseconds, 0.90),
+        "p95_ms": percentile(milliseconds, 0.95),
+        "p99_ms": percentile(milliseconds, 0.99),
+        "max_ms": max(milliseconds),
+    }
+
+
+@dataclass(frozen=True)
+class PlaybackResult:
+    """What a viewer would have seen, given a start buffer and a playback rate."""
+
+    underrun_count: int
+    total_stall_s: float
+    playback_span_s: float
+    first_underrun_chunk: int | None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "underrun_count": self.underrun_count,
+            "total_stall_ms": self.total_stall_s * 1000.0,
+            "playback_span_s": self.playback_span_s,
+            "stall_ratio": (self.total_stall_s / self.playback_span_s) if self.playback_span_s > 0 else 0.0,
+            "first_underrun_chunk": self.first_underrun_chunk,
+        }
+
+
+def simulate_playback(
+    records: Sequence[ChunkRecord],
+    *,
+    fps: float,
+    buffer_chunks: int = 1,
+) -> PlaybackResult:
+    """Replay arrivals against a wall clock and count stalls.
+
+    Playback starts when ``buffer_chunks`` chunks have arrived and then consumes
+    video at ``fps``. A chunk that has not arrived by the moment the player needs
+    it is an underrun, and the player resumes only when the bytes land -- which
+    is what a viewer experiences, and what a single p99 number cannot tell you:
+    the same p99 is harmless behind a three-chunk buffer and fatal behind none.
+    """
+    if fps <= 0:
+        raise ValueError("fps must be positive.")
+    if buffer_chunks < 1:
+        raise ValueError("buffer_chunks must be at least 1.")
+    if not records:
+        return PlaybackResult(0, 0.0, 0.0, None)
+    if buffer_chunks > len(records):
+        raise ValueError(f"buffer_chunks={buffer_chunks} exceeds the {len(records)} chunks received.")
+
+    ordered = sorted(records, key=lambda record: record.index)
+    start = ordered[buffer_chunks - 1].arrival_s
+    cursor = start
+    underruns = 0
+    total_stall = 0.0
+    first_underrun: int | None = None
+
+    for record in ordered:
+        if record.arrival_s > cursor:
+            stall = record.arrival_s - cursor
+            underruns += 1
+            total_stall += stall
+            if first_underrun is None:
+                first_underrun = record.index
+            cursor = record.arrival_s
+        cursor += record.num_frames / fps
+
+    return PlaybackResult(
+        underrun_count=underruns,
+        total_stall_s=total_stall,
+        playback_span_s=cursor - start,
+        first_underrun_chunk=first_underrun,
+    )
+
+
+def steady_chunk_deadline_ms(fps: float) -> float:
+    """Wall-clock budget a steady chunk must meet to hold real time.
+
+    ``fps`` here is the rate the frames are *intended to be played at*, which the
+    checkpoint does not declare: nothing in ``model_index.json``, the scheduler
+    config, or the transformer config carries a frame rate, and the pipeline
+    never reads one. It is a property of how the output is consumed, so both
+    this deadline and real-time factor scale linearly with the choice and the
+    caller has to state it rather than inherit it from a muxing default.
+    """
+    if fps <= 0:
+        raise ValueError("fps must be positive.")
+    return pixel_frames_in_chunk(1) / fps * 1000.0
+
+
+def compute_metrics(
+    records: Sequence[ChunkRecord],
+    *,
+    fps: float,
+    warmup_chunks: int = DEFAULT_WARMUP_CHUNKS,
+    slo_ms: float | None = None,
+    playback_buffer_chunks: int = 1,
+    ttfc_s: float | None = None,
+    session_wall_s: float | None = None,
+) -> dict[str, Any]:
+    """Summarize one session.
+
+    ``warmup_chunks`` are excluded from the steady-state block because the
+    attention window has not saturated before then (see the module docstring).
+    The terminal chunk also stays out of steady metrics because it skips
+    next-chunk preparation. All chunks stay in the playback simulation, which is
+    what a session really costs end to end.
+    """
+    if not records:
+        raise ValueError("compute_metrics() needs at least one chunk record.")
+    if warmup_chunks < 0:
+        raise ValueError("warmup_chunks must be non-negative.")
+
+    ordered = sorted(records, key=lambda record: record.index)
+    deadline_ms = slo_ms if slo_ms is not None else steady_chunk_deadline_ms(fps)
+
+    all_intervals = [record.inter_arrival_s for record in ordered[1:]]
+    # The terminal chunk skips next-chunk preparation and is not representative
+    # of sustained generation. Keep it in end-to-end and playback metrics.
+    steady = [record for record in ordered[:-1] if record.index >= warmup_chunks]
+    steady_intervals = [record.inter_arrival_s for record in steady if record.index > 0]
+
+    total_frames = sum(record.num_frames for record in ordered)
+    video_seconds = total_frames / fps
+    wall_seconds = session_wall_s if session_wall_s is not None else ordered[-1].arrival_s
+
+    metrics: dict[str, Any] = {
+        "chunks_received": len(ordered),
+        "warmup_chunks": warmup_chunks,
+        "steady_chunks": len(steady_intervals),
+        "steady_state_reached": bool(steady_intervals),
+        "total_frames": total_frames,
+        "total_bytes": sum(record.byte_length for record in ordered),
+        "video_seconds": video_seconds,
+        "wall_seconds": wall_seconds,
+        # RTF is processing time over content duration, as vLLM-Omni computes it
+        # (`patch.py`: generation_s / video_duration; `compute_audio_rtf`: latency
+        # / audio_duration, "SLO red line < 1"). LOWER IS BETTER and below 1.0 is
+        # real time. RTFX is the reciprocal the repo reports under its own name,
+        # kept here because "fraction of real time achieved" reads better on a
+        # dashboard -- but the two must never be confused for one another.
+        "video_rtf": (wall_seconds / video_seconds) if video_seconds > 0 else 0.0,
+        "video_rtfx": (video_seconds / wall_seconds) if wall_seconds > 0 else 0.0,
+        "frames_per_second": (total_frames / wall_seconds) if wall_seconds > 0 else 0.0,
+        "chunk_deadline_ms": deadline_ms,
+        "interval_all": _latency_summary(all_intervals),
+        "interval_steady": _latency_summary(steady_intervals),
+    }
+    if ttfc_s is not None:
+        metrics["ttfc_ms"] = ttfc_s * 1000.0
+
+    if steady_intervals:
+        steady_frames = sum(record.num_frames for record in steady if record.index > 0)
+        steady_wall = sum(steady_intervals)
+        steady_video = steady_frames / fps
+        metrics["steady_video_rtf"] = (steady_wall / steady_video) if steady_video > 0 else 0.0
+        metrics["steady_video_rtfx"] = (steady_video / steady_wall) if steady_wall > 0 else 0.0
+        within = sum(1 for value in steady_intervals if value * 1000.0 <= deadline_ms)
+        metrics["slo_attainment"] = within / len(steady_intervals)
+        metrics["slo_violations"] = len(steady_intervals) - within
+
+    metrics["playback"] = simulate_playback(
+        ordered,
+        fps=fps,
+        buffer_chunks=playback_buffer_chunks,
+    ).to_dict()
+    metrics["playback"]["buffer_chunks"] = playback_buffer_chunks
+    return metrics
+
+
+def aggregate_metrics(sessions: Iterable[Mapping[str, Any]], *, fps: float) -> dict[str, Any]:
+    """Combine per-session metrics across sequential repeats.
+
+    Steady intervals are pooled rather than averaging per-session means, so a
+    session that happened to produce fewer steady chunks does not carry the same
+    weight as a full one.
+    """
+    entries = list(sessions)
+    if not entries:
+        raise ValueError("aggregate_metrics() needs at least one session.")
+    if len(entries) == 1:
+        single = dict(entries[0])
+        single["sessions"] = 1
+        return single
+
+    deadline_ms = entries[0]["chunk_deadline_ms"]
+    if any(entry["chunk_deadline_ms"] != deadline_ms for entry in entries):
+        raise ValueError("Cannot aggregate sessions with different chunk deadlines.")
+    ttfcs = [float(entry["ttfc_ms"]) for entry in entries if entry.get("ttfc_ms") is not None]
+    # Lower is better, so "max" is the worst session, not the best.
+    rtfs = [float(entry["video_rtf"]) for entry in entries]
+    steady_means = [
+        float(entry["interval_steady"]["mean_ms"]) for entry in entries if entry.get("interval_steady", {}).get("count")
+    ]
+    steady_count = sum(entry.get("interval_steady", {}).get("count", 0) for entry in entries)
+    steady_total_ms = sum(
+        entry["interval_steady"]["mean_ms"] * entry["interval_steady"]["count"]
+        for entry in entries
+        if entry.get("interval_steady", {}).get("count")
+    )
+    return {
+        "sessions": len(entries),
+        "chunks_received": sum(int(entry["chunks_received"]) for entry in entries),
+        "total_frames": sum(int(entry["total_frames"]) for entry in entries),
+        "video_seconds": sum(float(entry["video_seconds"]) for entry in entries),
+        "wall_seconds": sum(float(entry["wall_seconds"]) for entry in entries),
+        "video_rtf_mean": sum(rtfs) / len(rtfs),
+        "video_rtf_best": min(rtfs),
+        "video_rtf_worst": max(rtfs),
+        "ttfc_ms_mean": (sum(ttfcs) / len(ttfcs)) if ttfcs else None,
+        "ttfc_ms_max": max(ttfcs) if ttfcs else None,
+        "steady_interval_ms_mean": (steady_total_ms / steady_count) if steady_count else None,
+        "steady_interval_ms_spread": (max(steady_means) - min(steady_means)) if steady_means else None,
+        "underrun_count": sum(int(entry["playback"]["underrun_count"]) for entry in entries),
+        "chunk_deadline_ms": deadline_ms,
+    }

benchmarks/qwen2_5_omni/README.md

@@ -0,0 +1,66 @@
+# Qwen2.5-Omni embedding microbenchmark
+
+Measure the host-observed latency of `embed_input_ids` before and after the
+redundant text embedding removal in PR #7477. Both variants use the same real
+checkpoint embedding table, dependencies, inputs and batch shapes. The baseline
+wrapper is extracted from the specified Git commit; the fixed wrapper comes from
+the current checkout. No full model is initialized.
+
+## Run
+
+Use Linux, a CUDA GPU, the project dependencies, and a local
+`Qwen/Qwen2.5-Omni-7B` checkpoint with its safetensors index and weights.
+From the repository root:
+
+```bash
+git fetch origin 01a2f93256975c7ff9565c5414b0fe0225bc4765
+PYTHONPATH="$PWD" OMP_NUM_THREADS=1 python benchmarks/qwen2_5_omni/benchmark_embeddings.py \
+  --model /path/to/Qwen2.5-Omni-7B \
+  --baseline-ref 01a2f93256975c7ff9565c5414b0fe0225bc4765 \
+  --cpu-core 2 --lengths 1172 --warmup 50 --blocks 20 --iterations 20 \
+  --output-dir /tmp/qwen25-embedding-results
+```
+
+Choose a CPU core in your allowed affinity set. For the original CUDA length
+sweep, use `--lengths 512 1172 4096 16384`.
+
+## Method
+
+- BF16 checkpoint embeddings, TP=1, CUDA int32 token IDs, CPU multimodal masks.
+- One pinned CPU core, one PyTorch thread, garbage collection disabled.
+- 50 warmups per variant, then 20 paired blocks of 20 calls with balanced,
+  randomized variant order. Summary: median of block medians.
+- Host latency includes synchronization inside the wrapper. Explicit GPU
+  synchronization before and after the timed call is excluded from host latency.
+  Calling-thread CPU time and synchronized completion time are recorded separately.
+- CPU frequency is sampled at block boundaries; clocks are not locked.
+- Validation checks one text embedding call for the fixed wrapper, two for the
+  multimodal baseline, and `torch.equal` for identical inputs and batch shapes.
+  This does not establish batch invariance or full-model output equality.
+
+Interleaved audio/video uses a valid synthetic multimodal mask. This benchmark
+measures wrapper latency, not E2E throughput.
+
+## Reference results
+
+Original environment: H100 80GB; Python 3.12.3; vLLM 0.29.0+cu129;
+PyTorch 2.13.0+cu129; Transformers 5.14.0; Qwen2.5-Omni-7B BF16.
+At 1172 tokens, the September 13 measurements were:
+
+| Input | Before (μs) | After (μs) | Host latency reduction |
+| --- | ---: | ---: | ---: |
+| Text | 18.81 | 18.66 | 0.80% |
+| Audio | 104.32 | 85.30 | 18.23% |
+| Image | 104.72 | 86.27 | 17.62% |
+| Video | 104.73 | 85.76 | 18.11% |
+| Audio + image + video | 567.92 | 549.54 | 3.24% |
+| Interleaved audio/video | 622.76 | 597.84 | 4.00% |
+
+Results vary between runs and environments. These local savings did not produce
+a consistent E2E throughput improvement in the separate repeated E2E experiment.
+
+## Output
+
+The output directory contains `metadata.json`, `micro-raw.json`,
+`validation.json`, and `summary.json`. Raw records include per-call timings,
+block timestamps and CPU frequency samples. Timer overhead is measured separately.

benchmarks/qwen2_5_omni/benchmark_embeddings.py

@@ -0,0 +1,306 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Reproduce PR #7477 host-observed embedding-wrapper latency on one CUDA GPU.
+
+Run from the PR checkout with PYTHONPATH=$PWD. Baseline and fixed methods share
+weights and inputs. Only the historical wrapper is loaded from --baseline-ref.
+"""
+
+import argparse
+import ast
+import gc
+import json
+import os
+import random
+import statistics
+import subprocess
+import textwrap
+import time
+from pathlib import Path
+
+import torch
+from safetensors import safe_open
+from torch import nn
+from transformers import Qwen2_5OmniThinkerConfig
+from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
+from vllm.multimodal.utils import set_mm_embedding_modality
+
+from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import (
+    Qwen2_5OmniThinkerForConditionalGeneration as Thinker,
+)
+from vllm_omni.platforms import current_omni_platform
+
+
+def baseline_method(cls, ref):
+    path = "vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py"
+    source = subprocess.check_output(
+        ["git", "-C", str(Path(__file__).resolve().parents[2]), "show", f"{ref}:{path}"], text=True
+    )
+    tree = ast.parse(source)
+    owner = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == cls.__name__)
+    node = next(n for n in owner.body if isinstance(n, ast.FunctionDef) and n.name == "embed_input_ids")
+    method = textwrap.dedent(ast.get_source_segment(source, node))
+    # Preserve zero-argument super() against the real model class and MRO.
+    factory = "from __future__ import annotations\ndef make_method(cls):\n    __class__ = cls\n"
+    factory += textwrap.indent(method, "    ") + "\n    return embed_input_ids\n"
+    namespace = dict(cls.embed_input_ids.__globals__)
+    exec(compile(factory, f"{ref}:{path}", "exec"), namespace)
+    return namespace["make_method"](cls)
+
+
+parser = argparse.ArgumentParser(description=__doc__)
+parser.add_argument("--model", type=Path, required=True)
+parser.add_argument("--baseline-ref", default="01a2f93256975c7ff9565c5414b0fe0225bc4765")
+parser.add_argument("--cpu-core", type=int, default=sorted(os.sched_getaffinity(0))[0])
+parser.add_argument("--lengths", type=int, nargs="+", default=[1172])
+parser.add_argument("--warmup", type=int, default=50)
+parser.add_argument("--blocks", type=int, default=20)
+parser.add_argument("--iterations", type=int, default=20)
+parser.add_argument("--output-dir", type=Path, default=Path("micro-results"))
+args = parser.parse_args()
+if min(args.lengths) < 300 or min(args.blocks, args.iterations) < 1 or args.warmup < 0:
+    parser.error("Require lengths >= 300, positive blocks/iterations, and nonnegative warmup")
+if args.cpu_core not in os.sched_getaffinity(0):
+    parser.error("CPU core is outside the allowed affinity set")
+ROOT, MODEL, CORE = args.output_dir, args.model, args.cpu_core
+ROOT.mkdir(parents=True, exist_ok=True)
+CONFIG = json.loads((MODEL / "config.json").read_text())["thinker_config"]
+HIDDEN = CONFIG["text_config"]["hidden_size"]
+VOCAB = CONFIG["text_config"]["vocab_size"]
+IDS = {m: CONFIG[m + "_token_index"] for m in ("audio", "image", "video")}
+METHODS = {"baseline": baseline_method(Thinker, args.baseline_ref), "fixed": Thinker.embed_input_ids}
+RESULTS, VALIDATIONS = [], []
+os.sched_setaffinity(0, {CORE})
+torch.set_num_threads(1)
+torch.set_num_interop_threads(1)
+gc.disable()
+
+
+def cpu_frequency():
+    path = Path(f"/sys/devices/system/cpu/cpu{CORE}/cpufreq/scaling_cur_freq")
+    if path.exists():
+        return float(path.read_text()) / 1000
+    for section in Path("/proc/cpuinfo").read_text().split("\n\n"):
+        fields = dict(line.split(":", 1) for line in section.splitlines() if ":" in line)
+        fields = {k.strip(): v.strip() for k, v in fields.items()}
+        if fields.get("processor") == str(CORE):
+            return float(fields.get("cpu MHz", "nan"))
+
+
+def payload(case, length, device):
+    generator = torch.Generator().manual_seed(7451 + length)
+    ids = torch.randint(2, 10000, (length,), generator=generator)
+    specs = {
+        "text": [],
+        "audio": [("audio", 100)],
+        "image": [("image", 16)],
+        "video": [("video", 128)],
+        "mixed": [("audio", 100), ("image", 16), ("video", 128)],
+        "interleaved": [("video", 64), ("audio", 50), ("video", 64), ("audio", 50)],
+    }[case]
+    offset = 44
+    counts = {}
+    for modality, count in specs:
+        ids[offset : offset + count] = IDS[modality]
+        counts[modality] = counts.get(modality, 0) + count
+        offset += count + (0 if case == "interleaved" else 2)
+    mask = torch.zeros(length, dtype=torch.bool)
+    for token in IDS.values():
+        mask |= ids == token
+    mm = [
+        set_mm_embedding_modality(torch.full((count, HIDDEN), i + 1, device=device, dtype=torch.bfloat16), modality)
+        for i, (modality, count) in enumerate(counts.items())
+    ]
+    return ids.to(device=device, dtype=torch.int32), None if case == "text" else mm, None if case == "text" else mask
+
+
+def once(fn, cuda):
+    # Drain preceding work before the host timer; the production function's
+    # own D2H synchronizations remain inside the timed call.
+    if cuda:
+        current_omni_platform.synchronize()
+    w0 = time.perf_counter_ns()
+    c0 = time.thread_time_ns()
+    value = fn()
+    c1 = time.thread_time_ns()
+    w1 = time.perf_counter_ns()
+    if cuda:
+        current_omni_platform.synchronize()
+    w2 = time.perf_counter_ns() if cuda else w1
+    del value
+    return [(w1 - w0) / 1000, (c1 - c0) / 1000, (w2 - w0) / 1000]
+
+
+def measure(functions, device, case, length, scope, rounds=20, iterations=20, warmups=50):
+    cuda = device == "cuda"
+    for fn in functions.values():
+        for _ in range(warmups):
+            once(fn, cuda)
+    order_rng = random.Random(7451 + length + len(case) + len(scope))
+    directions = [i % 2 for i in range(rounds)]
+    order_rng.shuffle(directions)
+    names = list(functions)
+    for block, direction in enumerate(directions):
+        order = names if direction == 0 else names[::-1]
+        for name in order:
+            frequency_start = cpu_frequency()
+            wall_start = time.time()
+            samples = [once(functions[name], cuda) for _ in range(iterations)]
+            wall_end = time.time()
+            RESULTS.append(
+                dict(
+                    device=device,
+                    case=case,
+                    tokens=length,
+                    scope=scope,
+                    variant=name,
+                    block=block,
+                    iterations=iterations,
+                    samples_us=samples,
+                    wall_start=wall_start,
+                    wall_end=wall_end,
+                    cpu_mhz_start=frequency_start,
+                    cpu_mhz_end=cpu_frequency(),
+                )
+            )
+    (ROOT / "micro-raw.json").write_text(json.dumps(RESULTS))
+    current = [r for r in RESULTS if (r["device"], r["case"], r["tokens"], r["scope"]) == (device, case, length, scope)]
+    print(
+        "RESULT",
+        json.dumps(
+            {
+                "device": device,
+                "case": case,
+                "tokens": length,
+                "scope": scope,
+                "host_median_us": {
+                    v: statistics.median(t[0] for r in current if r["variant"] == v for t in r["samples_us"])
+                    for v in names
+                },
+            }
+        ),
+        flush=True,
+    )
+
+
+def build_model(weights, device):
+    model = Thinker.__new__(Thinker)
+    nn.Module.__init__(model)
+    model.config = Qwen2_5OmniThinkerConfig(**CONFIG)
+    model._has_oov_mm_tokens = False
+    table = VocabParallelEmbedding(VOCAB, HIDDEN, params_dtype=torch.bfloat16, disable_tp=True)
+    table.weight = nn.Parameter(weights.to(device), requires_grad=False)
+
+    class LM(nn.Module):
+        def __init__(self):
+            super().__init__()
+            self.embed_tokens = table
+
+        def embed_input_ids(self, ids):
+            return self.embed_tokens(ids)
+
+    model.language_model = LM()
+    return model
+
+
+def validate(model, args, case, length, device):
+    ids, mm, mask = args
+    lm = model.get_language_model()
+    original = lm.embed_input_ids
+    outputs, counts = {}, {}
+    for variant, method in METHODS.items():
+        calls = []
+
+        def counted(tokens):
+            calls.append(tokens.numel())
+            return original(tokens)
+
+        lm.embed_input_ids = counted
+        try:
+            outputs[variant] = method(model, ids, mm, is_multimodal=mask)
+        finally:
+            lm.embed_input_ids = original
+        counts[variant] = len(calls)
+    assert torch.equal(outputs["baseline"], outputs["fixed"])
+    assert counts == {"baseline": 1 if case == "text" else 2, "fixed": 1}
+    VALIDATIONS.append(dict(case=case, tokens=length, device=device, calls=counts, same_input_wrapper_torch_equal=True))
+    (ROOT / "validation.json").write_text(json.dumps(VALIDATIONS, indent=2))
+
+
+@torch.inference_mode()
+def main():
+    import transformers
+    import vllm
+
+    metadata = dict(
+        baseline_ref=args.baseline_ref,
+        checkout=subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
+        torch=torch.__version__,
+        vllm=vllm.__version__,
+        transformers=transformers.__version__,
+        gpu=torch.cuda.get_device_name(),
+        cpu_core=CORE,
+        torch_cpu_threads=1,
+        warmup=args.warmup,
+        blocks=args.blocks,
+        iterations=args.iterations,
+        lengths=args.lengths,
+        embedding_shape=[VOCAB, HIDDEN],
+        dtype="bfloat16",
+        input_ids_dtype="int32",
+        mask_device="cpu",
+        start_time=time.time(),
+        metric_order=["host_observed_us", "calling_thread_cpu_us", "synchronized_completion_us"],
+    )
+    index = json.loads((MODEL / "model.safetensors.index.json").read_text())["weight_map"]
+    key = "thinker.model.embed_tokens.weight"
+    with safe_open(MODEL / index[key], framework="pt", device="cpu") as f:
+        weights = f.get_tensor(key)
+    assert weights.shape == (VOCAB, HIDDEN) and weights.dtype == torch.bfloat16
+    model = build_model(weights, "cuda")
+    measure({"timer_floor": lambda: None}, "cuda", "timer", 0, "overhead", rounds=10, iterations=100)
+    for length in args.lengths:
+        for case in ("text", "audio", "image", "video", "mixed", "interleaved"):
+            ids, mm, mask = payload(case, length, "cuda")
+            validate(model, (ids, mm, mask), case, length, "cuda")
+            functions = {
+                variant: (lambda method=method: method(model, ids, mm, is_multimodal=mask))
+                for variant, method in METHODS.items()
+            }
+            measure(
+                functions,
+                "cuda",
+                case,
+                length,
+                "whole_wrapper",
+                rounds=args.blocks,
+                iterations=args.iterations,
+                warmups=args.warmup,
+            )
+    summary = []
+    for length in args.lengths:
+        for case in ("text", "audio", "image", "video", "mixed", "interleaved"):
+            medians = {
+                variant: statistics.median(
+                    statistics.median(sample[0] for sample in record["samples_us"])
+                    for record in RESULTS
+                    if record["tokens"] == length and record["case"] == case and record["variant"] == variant
+                )
+                for variant in METHODS
+            }
+            summary.append(
+                dict(
+                    tokens=length,
+                    case=case,
+                    host_us=medians,
+                    reduction_pct=100 * (1 - medians["fixed"] / medians["baseline"]),
+                )
+            )
+    metadata["end_time"] = time.time()
+    (ROOT / "metadata.json").write_text(json.dumps(metadata, indent=2))
+    (ROOT / "summary.json").write_text(json.dumps(summary, indent=2))
+    print(json.dumps(summary, indent=2))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/qwen3_omni/embedding_harness.py

@@ -0,0 +1,226 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Small fixtures for the real Qwen3 thinker embedding method and its oracle."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import asdict, dataclass
+from types import SimpleNamespace
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+SCENARIOS = ("text", "empty", "audio", "image", "video", "mixed", "interleaved", "vision_no_deepstack")
+
+
+@dataclass(frozen=True)
+class ModelShape:
+    vocab_size: int = 152064
+    hidden_size: int = 2048
+    deepstack_levels: int = 3
+    image_token_id: int = 151655
+    video_token_id: int = 151656
+    audio_token_id: int = 151675
+
+    def __post_init__(self):
+        if min(self.vocab_size, self.hidden_size, self.deepstack_levels) <= 0:
+            raise ValueError("vocab_size, hidden_size and deepstack_levels must be positive")
+        ids = (self.image_token_id, self.video_token_id, self.audio_token_id)
+        if len(set(ids)) != 3 or not all(0 < token < self.vocab_size for token in ids):
+            raise ValueError("Provide three distinct, positive, in-vocabulary modality IDs; use oov=True to test OOV")
+
+    def token_ids(self, oov=False):
+        if oov:
+            return dict(zip(("image", "video", "audio"), range(self.vocab_size, self.vocab_size + 3)))
+        return {name: getattr(self, f"{name}_token_id") for name in ("image", "video", "audio")}
+
+
+@dataclass
+class Feature:
+    modality: str
+    positions: torch.Tensor
+    main: torch.Tensor
+    deepstack: torch.Tensor | None
+    combined: torch.Tensor
+
+
+@dataclass
+class EmbeddingCase:
+    name: str
+    input_ids: torch.Tensor
+    is_multimodal: torch.Tensor | None
+    features: tuple[Feature, ...]
+    fingerprint: str
+
+    def fresh_embeddings(self):
+        # The thinker replaces list entries with main-scale views.
+        return None if self.name == "text" else [feature.combined for feature in self.features]
+
+
+def _tensor_bytes(tensor):
+    return tensor.detach().contiguous().view(torch.uint8).cpu().numpy().tobytes()
+
+
+def build_case(
+    name,
+    num_tokens,
+    num_mm_tokens,
+    shape,
+    *,
+    device="cuda",
+    dtype=torch.bfloat16,
+    mask_device="cpu",
+    seed=42,
+    oov=False,
+):
+    """Build known feature placements; no model merge is used to define them."""
+    if name not in SCENARIOS:
+        raise ValueError(f"Unknown scenario: {name}")
+    if num_tokens <= 0 or not 0 <= num_mm_tokens < num_tokens:
+        raise ValueError("Require N > 0 and 0 <= M < N")
+    if (name in {"text", "empty"}) != (num_mm_tokens == 0):
+        raise ValueError("Only text/empty scenarios accept M=0")
+    if name in {"mixed", "interleaved"} and (num_mm_tokens < 4 or num_mm_tokens % 2):
+        raise ValueError("Mixed/interleaved scenarios require an even M >= 4")
+    token_ids = shape.token_ids(oov)
+    generator = torch.Generator().manual_seed(seed)
+    ids = torch.randint(0, min(shape.vocab_size, *token_ids.values()), (num_tokens,), generator=generator)
+    mask = torch.zeros(num_tokens, dtype=torch.bool)
+    positions = torch.arange((num_tokens - num_mm_tokens) // 2, (num_tokens - num_mm_tokens) // 2 + num_mm_tokens)
+    groups = []
+    if name == "mixed":
+        groups = [("audio", positions[: num_mm_tokens // 2]), ("image", positions[num_mm_tokens // 2 :])]
+    elif name == "interleaved":
+        groups = [("video", positions[::2]), ("audio", positions[1::2])]
+    elif num_mm_tokens:
+        groups = [("image" if name == "vision_no_deepstack" else name, positions)]
+    features = []
+    for modality, indexes in groups:
+        ids[indexes] = token_ids[modality]
+        mask[indexes] = True
+        rows = torch.arange(len(indexes), dtype=torch.float32)[:, None].remainder(31) / 32
+        cols = torch.arange(shape.hidden_size, dtype=torch.float32)[None, :].remainder(7) / 16
+        main_cpu = rows + cols + {"audio": 2, "image": 4, "video": 6}[modality]
+        main = main_cpu.to(device=device, dtype=dtype)
+        deepstack = None
+        parts = [main]
+        if modality != "audio" and name != "vision_no_deepstack":
+            deepstack = torch.stack(
+                [(main_cpu + 16 + 4 * level).to(device=device, dtype=dtype) for level in range(shape.deepstack_levels)]
+            )
+            parts.extend(deepstack.unbind(0))
+        combined = torch.cat(parts, dim=-1)
+        # set_mm_embedding_modality simply attaches this attribute. Tag only
+        # after cat/to: views and copies do not preserve arbitrary attributes.
+        combined.modality = modality
+        features.append(Feature(modality, indexes, main, deepstack, combined))
+    digest = hashlib.sha256(json.dumps({"name": name, "shape": asdict(shape), "seed": seed, "oov": oov}).encode())
+    digest.update(_tensor_bytes(ids))
+    digest.update(_tensor_bytes(mask))
+    for feature in features:
+        digest.update(feature.modality.encode())
+        digest.update(_tensor_bytes(feature.combined))
+    return EmbeddingCase(
+        name,
+        ids.to(device=device),
+        None if name == "text" else mask.to(device=mask_device),
+        tuple(features),
+        digest.hexdigest(),
+    )
+
+
+class LanguageModelAdapter(nn.Module):
+    def __init__(self, embedding):
+        super().__init__()
+        self.embedding = embedding
+
+    def embed_input_ids(self, input_ids):
+        return self.embedding(input_ids)
+
+    def forward(self, input_ids):
+        return self.embed_input_ids(input_ids)
+
+
+def make_thinker(
+    shape,
+    *,
+    device="cuda",
+    dtype=torch.bfloat16,
+    seed=42,
+    backend="torch",
+    deepstack=True,
+    buffer_capacity=32,
+    oov=False,
+):
+    """Skip checkpoint construction, preserving the real class and its MRO."""
+    from tests.model_executor.helpers import bootstrap_vllm_layer_custom_op_modules
+
+    bootstrap_vllm_layer_custom_op_modules()
+    from vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker import (
+        Qwen3OmniMoeThinkerForConditionalGeneration,
+    )
+
+    if buffer_capacity <= 0:
+        raise ValueError("buffer_capacity must be positive")
+    device = torch.device(device)
+    if backend == "torch":
+        embedding = nn.Embedding(shape.vocab_size, shape.hidden_size, device=device, dtype=dtype)
+    elif backend == "vllm":
+        from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
+
+        with torch.device(device):
+            embedding = VocabParallelEmbedding(shape.vocab_size, shape.hidden_size, params_dtype=dtype, disable_tp=True)
+    else:
+        raise ValueError(f"Unknown embedding backend: {backend}")
+    with torch.no_grad():
+        embedding.weight.uniform_(-0.5, 0.5, generator=torch.Generator(device=device).manual_seed(seed))
+    thinker = object.__new__(Qwen3OmniMoeThinkerForConditionalGeneration)
+    nn.Module.__init__(thinker)
+    thinker.config = SimpleNamespace(text_config=SimpleNamespace(hidden_size=shape.hidden_size))
+    for modality, token_id in shape.token_ids(oov).items():
+        setattr(thinker.config, f"{modality}_token_id", token_id)
+    thinker.visual = SimpleNamespace(
+        deepstack_visual_indexes=list(range(shape.deepstack_levels)) if deepstack else None
+    )
+    thinker.language_model = LanguageModelAdapter(embedding)
+    thinker._has_oov_mm_tokens = oov
+    thinker.use_deepstack = deepstack
+    thinker.deepstack_num_level = shape.deepstack_levels if deepstack else 0
+    thinker.deepstack_input_embeds = [
+        torch.zeros(buffer_capacity, shape.hidden_size, device=device, dtype=dtype)
+        for _ in range(thinker.deepstack_num_level)
+    ]
+    thinker.deepstack_input_embeds_num_tokens = 0
+    return thinker.eval()
+
+
+@torch.inference_mode()
+def expected_outputs(model, case):
+    """Independent oracle: direct lookup and explicit positions, not merge helpers."""
+    ids = case.input_ids
+    if model._has_oov_mm_tokens and case.is_multimodal is not None:
+        ids = ids.masked_fill(case.is_multimodal.to(ids.device), 0)
+    main = F.embedding(ids, model.language_model.embedding.weight)
+    has_deepstack = any(feature.deepstack is not None for feature in case.features)
+    deepstack = main.new_zeros((model.deepstack_num_level, *main.shape)) if has_deepstack else None
+    for feature in case.features:
+        indexes = feature.positions.to(main.device)
+        main[indexes] = feature.main
+        if feature.deepstack is not None:
+            deepstack[:, indexes, :] = feature.deepstack
+    return main, deepstack
+
+
+@torch.inference_mode()
+def assert_case_output(model, case, result):
+    expected, deepstack = expected_outputs(model, case)
+    torch.testing.assert_close(result, expected, rtol=0, atol=0)
+    if deepstack is None:
+        assert model.deepstack_input_embeds_num_tokens == 0, "Unexpected stale DeepStack buffer"
+    else:
+        assert model.deepstack_input_embeds_num_tokens == case.input_ids.numel()
+        actual = torch.stack([buffer[: case.input_ids.numel()] for buffer in model.deepstack_input_embeds])
+        torch.testing.assert_close(actual, deepstack, rtol=0, atol=0)

benchmarks/tts/bench_cosyvoice3_concurrency_matrix.py

@@ -0,0 +1,517 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""CosyVoice3 streaming e2e benchmark: concurrency x utterance-length matrix.
+
+Launches the server once per (concurrency, step-size) combo, streams TTS
+requests cycling through several utterance lengths, and reports
+TTFA/E2E/RTF/per-chunk latency (mean, stdev, range) per cell. "Step size" is
+``codec_chunk_frames``, the token-hop length the first emitted chunk waits
+for; later chunks wait for a doubling hop capped at ``4 * codec_chunk_frames``
+(see ``_token_hop_lengths``). Per-chunk latency is reconstructed from that
+same ramp rather than from raw transport read boundaries, which don't align
+to model chunks (see ``one_request``'s docstring).
+
+Running with no flags reproduces the default matrix. Per-request correctness
+is checked: non-silent, long enough, and (with a fixed ``--seed``)
+byte-identical PCM across requests for the same text.
+"""
+
+import argparse
+import json
+import os
+import signal
+import statistics
+import subprocess
+import sys
+import threading
+import time
+from pathlib import Path
+
+import httpx
+import numpy as np
+
+REPO = Path(__file__).resolve().parents[2]
+DEFAULT_MODEL = "FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
+
+DEFAULT_REF_AUDIO = "https://raw.githubusercontent.com/FunAudioLLM/CosyVoice/main/asset/zero_shot_prompt.wav"
+DEFAULT_REF_TEXT = "希望你以后能够做的比我还好呦。"
+
+DEFAULT_TEXTS = [
+    "Thank you so much for the birthday gift you sent me.",
+    "The weather is lovely today, with warm sunshine and a gentle breeze. "
+    "We decided to take a walk in the park to enjoy the spring flowers and the fresh air.",
+    "The weather is lovely today, with warm sunshine and a gentle breeze. We decided to take a walk in the "
+    "park to enjoy the spring flowers and the fresh air. Many people were out walking, children were "
+    "running and playing on the grass, and older folks sat on benches chatting with one another. We "
+    "strolled slowly along the path and came across a cherry tree in full bloom, its pink petals drifting "
+    "on the wind. It was so beautiful that we stopped to take some photos to remember the moment. Then we "
+    "continued on and reached a small lake, its water so clear you could see straight to the bottom, with "
+    "the blue sky and white clouds reflected on its surface. A few ducks paddled back and forth across the "
+    "water, occasionally letting out a soft quack. We sat on a bench by the lake for a while, enjoying the "
+    "peaceful afternoon. Time passed quickly, and before we knew it evening had arrived, with the setting "
+    "sun painting the sky in shades of gold. Reluctant to leave, we finally headed home, already planning "
+    "our next visit. On the way back we talked about everything we had seen and heard that day, feeling "
+    "wonderfully happy. It was truly an unforgettable weekend.",
+]
+
+# PCM16 mono @ 24 kHz (CosyVoice3 output rate)
+SAMPLE_RATE = 24000
+
+# Well below any real speech's RMS; catches all-zero/near-silent PCM that byte
+# count and duration checks alone pass through.
+MIN_RMS = 50.0
+
+# Overlay: only codec_chunk_frames is overridden; must live next to cosyvoice3.yaml (relative base_config)
+DEPLOY_TEMPLATE = """# Auto-generated by bench_cosyvoice3_concurrency_matrix.py -- do not edit.
+base_config: cosyvoice3.yaml
+
+connectors:
+  connector_of_shared_memory:
+    name: SharedMemoryConnector
+    extra:
+      codec_streaming: true
+      connector_get_sleep_s: 0.01
+      connector_get_max_wait_first_chunk: 3000
+      connector_get_max_wait: 300
+      codec_chunk_frames: {chunk_frames}
+      codec_max_chunk_frames: {max_chunk_frames}
+      codec_stream_scale_factor: {stream_scale_factor}
+      codec_left_context_frames: 25
+      codec_vocab_size: 6561
+"""
+
+# CosyVoice3Config constants (transformers_utils/configs/cosyvoice3.py): tokens
+# are consumed at token_mel_ratio mel frames each, and HiFT emits
+# samples_per_mel PCM16 samples per mel frame. Pinned here (not left to the
+# server's defaults) since one_request's byte-boundary reconstruction below
+# depends on knowing them exactly.
+TOKEN_MEL_RATIO = 2
+SAMPLES_PER_MEL = 480
+STREAM_SCALE_FACTOR = 2
+
+
+def _stats(values: list[float]) -> dict[str, float]:
+    """Mean/median/stdev/p95/min/max/n for a list of per-request measurements."""
+    n = len(values)
+    if n == 0:
+        return {
+            "n": 0,
+            "mean": float("nan"),
+            "median": float("nan"),
+            "stdev": float("nan"),
+            "p95": float("nan"),
+            "min": float("nan"),
+            "max": float("nan"),
+        }
+    return {
+        "n": n,
+        "mean": statistics.mean(values),
+        "median": statistics.median(values),
+        "stdev": statistics.stdev(values) if n >= 2 else 0.0,
+        "p95": statistics.quantiles(values, n=20)[18] if n >= 2 else values[0],
+        "min": min(values),
+        "max": max(values),
+    }
+
+
+def vllm_bin() -> str:
+    for c in (REPO / ".venv/bin/vllm", Path(sys.executable).parent / "vllm"):
+        if c.is_file():
+            return str(c)
+    return "vllm"
+
+
+def wait_ready(url: str, proc: subprocess.Popen, log: Path, timeout: float) -> None:
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if proc.poll() is not None:
+            tail = log.read_text(errors="replace").splitlines()[-40:] if log.exists() else []
+            raise RuntimeError("server exited during startup:\n  " + "\n  ".join(tail))
+        try:
+            if httpx.get(f"{url}/health", timeout=2.0).status_code == 200:
+                return
+        except Exception:
+            pass
+        time.sleep(2.0)
+    raise TimeoutError(f"server not ready within {timeout}s (log: {log})")
+
+
+def _token_hop_lengths(chunk_frames: int, max_chunk_frames: int):
+    """The token-hop-length ramp talker2code2wav_async_chunk uses to decide
+    when to fire the next flow-solve/emission: chunk_frames, then doubling
+    (STREAM_SCALE_FACTOR) each step, capped at max_chunk_frames. This -- not
+    codec_chunk_frames alone -- is what actually paces emitted audio chunks:
+    with the defaults here, hops are chunk_frames, 2x, 4x, 4x, 4x, ...
+
+    The very first emission is consistently one token short of chunk_frames
+    (confirmed against real speech.audio.delta event sizes) even though the
+    ramp's own progression is computed from the nominal chunk_frames, not the
+    actual first hop -- so only the first yield gets the -1 adjustment.
+    """
+    yield chunk_frames - 1
+    hop = chunk_frames
+    while True:
+        hop = min(max_chunk_frames, max(chunk_frames, hop * STREAM_SCALE_FACTOR))
+        yield hop
+
+
+def one_request(client, url, model, text, ref_audio, ref_text, chunk_frames, seed=None):
+    """Run one streaming request, returning TTFA, E2E, per-chunk gaps, and audio bytes.
+
+    ``response_format=pcm`` + ``stream_format=audio`` streams raw bytes with no
+    framing, so a client read boundary alone is a transport artifact, not a
+    model chunk boundary. Reconstructed instead by replicating the server's
+    own token-hop-length ramp (see ``_token_hop_lengths``) to compute each
+    real chunk's expected byte size ahead of time, and slicing the
+    accumulated byte stream at those offsets -- independent of how the
+    transport happens to fragment the underlying reads. The final chunk
+    (finalize, which releases extra held-back frames) is whatever remains
+    when the stream closes, not a predicted hop.
+    """
+    payload = {
+        "model": model,
+        "input": text,
+        "response_format": "pcm",
+        "stream": True,
+        "stream_format": "audio",
+        "ref_audio": ref_audio,
+        "ref_text": ref_text,
+    }
+    if seed is not None:
+        payload["seed"] = seed
+    t0 = time.perf_counter()
+    ttfa = None
+    nbytes = 0
+    audio_bytes = bytearray()
+    chunk_gaps_ms = []  # inter-chunk arrival gaps (per model chunk), by step index
+    prev_t = None
+    step_idx = 0
+
+    bytes_per_token = TOKEN_MEL_RATIO * SAMPLES_PER_MEL * 2
+    hop_gen = _token_hop_lengths(chunk_frames, 4 * chunk_frames)
+    next_boundary = next(hop_gen) * bytes_per_token
+    last_crossed_bytes = 0
+    received = 0
+    last_now = t0
+
+    with client.stream("POST", f"{url}/v1/audio/speech", json=payload, headers={"Authorization": "Bearer EMPTY"}) as r:
+        if r.status_code != 200:
+            raise RuntimeError(f"{r.status_code}: {r.read()[:300]!r}")
+        for chunk in r.iter_bytes():
+            if not chunk:
+                continue
+            now = time.perf_counter()
+            if ttfa is None:
+                ttfa = now - t0
+            last_now = now
+            received += len(chunk)
+            nbytes += len(chunk)
+            audio_bytes += chunk
+            while received >= next_boundary:
+                if prev_t is not None:
+                    chunk_gaps_ms.append((step_idx, (now - prev_t) * 1e3))
+                    step_idx += 1
+                prev_t = now
+                last_crossed_bytes = next_boundary
+                next_boundary += next(hop_gen) * bytes_per_token
+
+    if received > last_crossed_bytes:
+        # Finalize chunk: releases extra held-back frames, so it doesn't
+        # match a predicted hop size -- its arrival is the stream closing.
+        if prev_t is not None:
+            chunk_gaps_ms.append((step_idx, (last_now - prev_t) * 1e3))
+            step_idx += 1
+
+    e2e = time.perf_counter() - t0
+    audio_s = nbytes / 2 / SAMPLE_RATE
+    return {
+        "ttfa_s": ttfa,
+        "e2e_s": e2e,
+        "audio_s": audio_s,
+        "rtf": e2e / audio_s if audio_s > 0 else float("nan"),
+        "chunk_gaps_ms": chunk_gaps_ms,
+        "nbytes": nbytes,
+        "audio_bytes": audio_bytes,
+    }
+
+
+def check_correctness(r: dict, min_audio_s: float) -> list[str]:
+    """Return a list of correctness violations (empty = pass)."""
+    problems = []
+    if r["nbytes"] <= 0:
+        problems.append("empty audio (0 bytes)")
+    if r["audio_s"] < min_audio_s:
+        problems.append(f"audio too short ({r['audio_s']:.2f}s < {min_audio_s:.2f}s)")
+    if r["nbytes"] > 0:
+        samples = np.frombuffer(bytes(r["audio_bytes"]), dtype=np.int16).astype(np.float64)
+        rms = float(np.sqrt(np.mean(samples**2)))
+        if rms < MIN_RMS:
+            problems.append(f"near-silent audio (rms={rms:.1f} < {MIN_RMS})")
+    return problems
+
+
+def drive(url, args, n, conc, label, results, lock, idx, chunk_frames):
+    """Run ``n`` requests at concurrency ``conc``, cycling through ``args.texts``."""
+    texts = args.texts
+
+    def worker():
+        with httpx.Client(timeout=600.0) as c:
+            while True:
+                with lock:
+                    i = idx[0]
+                    idx[0] += 1
+                if i >= n:
+                    return
+                text = texts[i % len(texts)]
+                try:
+                    r = one_request(
+                        c, url, args.model, text, args.ref_audio, args.ref_text, chunk_frames, seed=args.seed
+                    )
+                    r["text"] = text
+                    with lock:
+                        results.append(r)
+                except Exception as e:
+                    print(f"    [{label}] request {i} failed: {e}", flush=True)
+
+    ts = [threading.Thread(target=worker) for _ in range(conc)]
+    t0 = time.perf_counter()
+    for t in ts:
+        t.start()
+    for t in ts:
+        t.join()
+    return time.perf_counter() - t0
+
+
+def run_combo(args, conc: int, chunk_frames: int, out: Path) -> dict:
+    tag = f"c{conc}_chunk{chunk_frames}"
+    run_dir = out / tag
+    run_dir.mkdir(parents=True, exist_ok=True)
+    log = run_dir / "server.log"
+
+    deploy = REPO / "vllm_omni/deploy" / f"_autogen_steps_{tag}.yaml"
+    deploy.write_text(
+        DEPLOY_TEMPLATE.format(
+            chunk_frames=chunk_frames,
+            max_chunk_frames=4 * chunk_frames,
+            stream_scale_factor=STREAM_SCALE_FACTOR,
+        )
+    )
+
+    cmd = [
+        vllm_bin(),
+        "serve",
+        args.model,
+        "--omni",
+        "--deploy-config",
+        str(deploy),
+        "--host",
+        "127.0.0.1",
+        "--port",
+        str(args.port),
+        "--allowed-local-media-path",
+        str(REPO),
+    ]
+    env = dict(os.environ)
+    if args.cuda_visible_devices:
+        env["CUDA_VISIBLE_DEVICES"] = args.cuda_visible_devices
+    if args.trt is not None:
+        env["COSYVOICE3_TRT"] = args.trt
+
+    print(f"\n=== concurrency={conc} step_size={chunk_frames} ===", flush=True)
+    print(f"  launching server (log: {log})", flush=True)
+    with open(log, "w") as lf:
+        proc = subprocess.Popen(cmd, env=env, stdout=lf, stderr=subprocess.STDOUT, start_new_session=True)
+    url = f"http://127.0.0.1:{args.port}"
+    try:
+        wait_ready(url, proc, log, args.server_timeout)
+        print("  server ready", flush=True)
+
+        results: list[dict] = []
+        lock = threading.Lock()
+        idx = [0]
+        if args.warmup:
+            print(f"  warmup ({args.warmup} req)", flush=True)
+            drive(url, args, args.warmup, min(conc, args.warmup), "warmup", results, lock, idx, chunk_frames)
+            results.clear()
+            idx[0] = 0
+
+        print(f"  running {args.num_prompts} req @ concurrency {conc}", flush=True)
+        wall = drive(url, args, args.num_prompts, conc, "bench", results, lock, idx, chunk_frames)
+
+        ok = [r for r in results if r.get("ttfa_s") is not None]
+        problems = []
+        for r in ok:
+            problems += check_correctness(r, args.min_audio_s)
+        # With a fixed seed, same-text requests must produce byte-identical PCM (per text group)
+        if args.seed is not None:
+            by_text: dict[str, bytes] = {}
+            for r in ok:
+                t = r.get("text")
+                ref = by_text.get(t)
+                if ref is None:
+                    by_text[t] = r.get("audio_bytes")
+                elif r.get("audio_bytes") != ref:
+                    problems.append(f"byte-identity violation for text {t!r}: seeded requests produced different PCM")
+        n_ok = len(ok)
+        total_audio_s = sum(r["audio_s"] for r in ok)
+        all_gaps = [g for r in ok for g in r["chunk_gaps_ms"]]  # (step_index, gap_ms) per request
+        gap_vals = [g for _, g in all_gaps]
+
+        # bucket gaps by chunk position within a request, to see any O(n) slope
+        by_idx: dict[int, list[float]] = {}
+        for step_index, g in all_gaps:
+            by_idx.setdefault(step_index, []).append(g)
+        step_latency_by_idx = {i: statistics.median(vals) for i, vals in sorted(by_idx.items())}
+
+        agg = {
+            "concurrency": conc,
+            "step_size": chunk_frames,
+            "ok": n_ok,
+            "errors": args.num_prompts - n_ok,
+            "correctness_problems": len(problems),
+            "correctness_sample": problems[:5],
+            "total_audio_s": total_audio_s,
+            "throughput_audio_s_per_s": total_audio_s / wall if wall > 0 else float("nan"),
+            "ttfa_s": _stats([r["ttfa_s"] for r in ok]),
+            "e2e_s": _stats([r["e2e_s"] for r in ok]),
+            "rtf": _stats([r["rtf"] for r in ok]),
+            "step_latency_ms": _stats(gap_vals),
+            "step_latency_ms_by_idx": step_latency_by_idx,
+            "num_steps": len(all_gaps),
+            # per-utterance-length breakdown; n per text = num_prompts / len(texts)
+            "by_text": {
+                t: {
+                    "audio_s": _stats([r["audio_s"] for r in ok if r.get("text") == t]),
+                    "e2e_s": _stats([r["e2e_s"] for r in ok if r.get("text") == t]),
+                    "rtf": _stats([r["rtf"] for r in ok if r.get("text") == t]),
+                }
+                for t in sorted({r.get("text") for r in ok})
+            },
+        }
+        print(json.dumps(agg, indent=2, ensure_ascii=False), flush=True)
+        return agg
+    finally:
+        print("  shutting down server", flush=True)
+        try:
+            os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+        except Exception:
+            proc.terminate()
+        try:
+            proc.wait(timeout=60)
+        except subprocess.TimeoutExpired:
+            try:
+                os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
+            except Exception:
+                proc.kill()
+            proc.wait(timeout=20)
+        deploy.unlink(missing_ok=True)
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--model", default=DEFAULT_MODEL)
+    p.add_argument(
+        "--text",
+        dest="texts",
+        nargs="+",
+        default=DEFAULT_TEXTS,
+        help="One or more utterance texts. With several, requests cycle through "
+        "them so a single server launch covers all utterance lengths; results "
+        "are bucketed per text.",
+    )
+    p.add_argument("--ref-audio", default=DEFAULT_REF_AUDIO)
+    p.add_argument("--ref-text", default=DEFAULT_REF_TEXT)
+    p.add_argument("--concurrency", type=int, nargs="+", default=[1, 8])
+    p.add_argument(
+        "--step-size",
+        type=int,
+        nargs="+",
+        default=[15],
+        help="codec_chunk_frames values to sweep (the streaming step size).",
+    )
+    p.add_argument("--num-prompts", type=int, default=30)
+    p.add_argument("--warmup", type=int, default=2)
+    p.add_argument("--port", type=int, default=8091)
+    p.add_argument(
+        "--server-timeout",
+        type=float,
+        default=1800.0,
+        help="Startup budget; first run also downloads weights and builds the TRT engine.",
+    )
+    p.add_argument("--output-dir", default="./perf_steps")
+    p.add_argument("--cuda-visible-devices", default="0")
+    p.add_argument(
+        "--trt",
+        choices=["0", "1"],
+        default=None,
+        help="Pin COSYVOICE3_TRT. 0=force torch estimator, 1=allow TensorRT. Default: unset.",
+    )
+    p.add_argument(
+        "--min-audio-s",
+        type=float,
+        default=1.0,
+        help="Minimum expected audio seconds per request for the correctness check.",
+    )
+    p.add_argument(
+        "--seed",
+        type=int,
+        default=42,
+        help="Fixed seed for reproducible (byte-identical) audio. Pass --seed '' "
+        "equivalent (any non-int) to disable -- there is no unset sentinel here "
+        "by design, since the default sweep is meant to be exactly reproducible.",
+    )
+    return p.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    out = Path(args.output_dir).resolve()
+    out.mkdir(parents=True, exist_ok=True)
+
+    all_results = []
+    failed = False
+    for conc in args.concurrency:
+        for chunk_frames in args.step_size:
+            try:
+                result = run_combo(args, conc, chunk_frames, out)
+                all_results.append(result)
+                if result["errors"] > 0 or result["correctness_problems"] > 0:
+                    failed = True
+            except Exception as e:
+                print(f"  combo c={conc} chunk={chunk_frames} FAILED: {e}", flush=True)
+                failed = True
+
+    out_path = out / "steps_results.json"
+    with open(out_path, "w", encoding="utf-8") as f:
+        json.dump({"model": args.model, "texts": args.texts, "runs": all_results}, f, indent=2, ensure_ascii=False)
+    print(f"\nResults written to {out_path}", flush=True)
+
+    print("\n=== SUMMARY ===")
+    header = (
+        f"{'conc':>4} {'step':>4} {'ok':>4} {'corr':>4} "
+        f"{'TTFA med(ms)':>12} {'step med(ms)':>12} {'step p95(ms)':>12} {'RTF mean+-sd':>14}"
+    )
+    print(header)
+    print("-" * len(header))
+    for r in all_results:
+        print(
+            f"{r['concurrency']:>4} {r['step_size']:>4} {r['ok']:>4} {r['correctness_problems']:>4} "
+            f"{r['ttfa_s']['median'] * 1e3:>12.1f} "
+            f"{r['step_latency_ms']['median']:>12.1f} {r['step_latency_ms']['p95']:>12.1f} "
+            f"{r['rtf']['mean']:>6.3f}+-{r['rtf']['stdev']:>5.3f}"
+        )
+        for t, stats in r.get("by_text", {}).items():
+            print(
+                f"      text {t[:24]!r}: e2e {stats['e2e_s']['mean'] * 1e3:8.1f}"
+                f"+-{stats['e2e_s']['stdev'] * 1e3:6.1f} ms  (n={stats['e2e_s']['n']}, "
+                f"audio {stats['audio_s']['mean']:.1f}s)"
+            )
+
+    if failed:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

docker/Dockerfile.cuda

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai:v0.28.0
+ARG BASE_IMAGE=vllm/vllm-openai:v0.29.0
 FROM ${BASE_IMAGE}
 
 ARG COMMON_WORKDIR=/app

docker/Dockerfile.npu

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_VERSION=v0.28.0
+ARG VLLM_ASCEND_VERSION=v0.29.0
 ARG VLLM_ASCEND_DEVICE_SUFFIX=
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_VERSION}${VLLM_ASCEND_DEVICE_SUFFIX}
 

docker/Dockerfile.npu.ci

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_VERSION=v0.28.0
+ARG VLLM_ASCEND_VERSION=v0.29.0
 ARG VLLM_ASCEND_DEVICE_SUFFIX=
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_VERSION}${VLLM_ASCEND_DEVICE_SUFFIX}
 
@@ -44,6 +44,7 @@ RUN python3 -m pip install -U pip && \
       "onnx>=1.16.0" \
       "librosa>=0.10.0" \
       "datasets>=3.0.0" \
+      "torchmetrics==1.9.0" \
       --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
       --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
       --default-timeout=60

docker/Dockerfile.rocm

@@ -1,16 +1,22 @@
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.28.0
-FROM ${BASE_IMAGE} AS base
+# Keep the ROCm runtime aligned with the vLLM release targeted by
+# docker/Dockerfile.ci. A stale base image can build successfully while failing
+# at runtime when vLLM-Omni imports APIs added by the newer vLLM release.
+ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.29.0
+
+# Keep the dependency-heavy portion independent from the source tree. The
+# registry cache is keyed on these inputs, while the final source copy can be
+# linked onto the cached image without materializing its large ROCm layers.
+FROM ${BASE_IMAGE} AS deps
 
-# Declare a variable to know if we want to use the nightly build or the stable build.
 # NOTE: REMINDER to vLLM-Omni rebase maintainer
-# Remember to set `USE_NIGHTLY_BUILD` to 0, when switching back to stable vLLM docker image
-# 1. If vLLM-omni maintainer is forced to use custom commits
-#    during rebasing, they can change the two variables
-# 2. Whenever vLLM upstream has released a stable version,
-#    we should swap over to use stable release ASAP.
-#    We should avoid relying on custom commits.
+# BASE_IMAGE pins the default prebuilt runtime when USE_NIGHTLY_BUILD=0.
+# VLLM_VERSION_OR_COMMIT_HASH is consumed only by the opt-in source rebuild when
+# USE_NIGHTLY_BUILD=1. Keep both aligned with docker/Dockerfile.ci's
+# VLLM_BASE_TAG. This is enforced by tests/buildkite/test_rocm_dockerfile.py. An
+# intentional custom source pin must update that regression in the same change,
+# and should return to the stable release as soon as one is available.
 ARG USE_NIGHTLY_BUILD=0
-ARG VLLM_VERSION_OR_COMMIT_HASH=v0.28.0
+ARG VLLM_VERSION_OR_COMMIT_HASH=v0.29.0
 ARG ARG_PYTORCH_ROCM_ARCH
 ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}}
 
@@ -38,6 +44,10 @@ RUN if [ "${USE_NIGHTLY_BUILD}" = "1" ]; then \
         rm -rf vllm/.git; \
     fi
 
+# Fail during image construction instead of letting every AMD test fail at
+# runtime when the base image and vLLM-Omni target incompatible vLLM APIs.
+RUN python3 -c "import vllm; from vllm.v1.kv_cache_interface import compute_layout_strides; from vllm.v1.kv_cache_layout import KVCacheLayout; print(f'Validated vLLM {vllm.__version__}')"
+
 # Build TorchCodec after any optional nightly vLLM reinstall so it targets the
 # final torch/ROCm ABI. Keep this layer before COPY . so source changes do not
 # invalidate the expensive native build. FFmpeg development headers are
@@ -58,9 +68,24 @@ RUN --mount=type=cache,target=/root/.cache/pip \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-# Step 3: Copy vllm-omni code and install without uv
-RUN mkdir -p ${COMMON_WORKDIR}/vllm-omni
-COPY . ${COMMON_WORKDIR}/vllm-omni
+# Step 3: Install project dependencies against a placeholder package tree.
+# The editable install keeps resolving imports from this directory after the
+# final COPY --link replaces the placeholder with the real source tree.
+WORKDIR ${COMMON_WORKDIR}/vllm-omni
+RUN mkdir -p vllm_omni && touch vllm_omni/__init__.py
+COPY pyproject.toml setup.py ./
+COPY requirements/ requirements/
+ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0
+
+RUN uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" \
+        --no-cache-dir -e ".[dev]" --no-build-isolation && \
+    python3 -m pip uninstall -y onnxruntime && \
+    python3 -m pip install --no-cache-dir --force-reinstall --no-deps \
+        "onnxruntime-rocm>=1.22.2" && \
+    python3 -c "import importlib.metadata as m; import onnxruntime as ort; assert 'ROCMExecutionProvider' in ort.get_available_providers(), ort.get_available_providers(); assert m.version('onnxruntime-rocm'); assert not any(d.metadata['Name'] == 'onnxruntime' for d in m.distributions())" && \
+    python3 -c "from torchcodec.decoders import VideoDecoder"
+
+RUN ln -sf /usr/bin/python3 /usr/bin/python
 
 # This is a workaround to ensure pytest exits with the correct status code in CI tests.
 RUN printf '%s\n' \
@@ -79,19 +104,13 @@ RUN printf '%s\n' \
     '    os._exit(_exit_code)' \
     > ${COMMON_WORKDIR}/vllm-omni/conftest.py
 
-# Some optional dependencies pull in the CPU ONNX Runtime distribution. The
-# CPU and ROCm wheels own the same import path and cannot coexist, so remove
-# the CPU wheel and reinstall the ROCm wheel after dependency resolution.
-RUN cd ${COMMON_WORKDIR}/vllm-omni && \
-    uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" \
-        --no-cache-dir ".[dev]" --no-build-isolation && \
-    python3 -m pip uninstall -y onnxruntime && \
-    python3 -m pip install --no-cache-dir --force-reinstall --no-deps \
-        "onnxruntime-rocm>=1.22.2" && \
-    python3 -c "import importlib.metadata as m; import onnxruntime as ort; assert 'ROCMExecutionProvider' in ort.get_available_providers(), ort.get_available_providers(); assert m.version('onnxruntime-rocm'); assert not any(d.metadata['Name'] == 'onnxruntime' for d in m.distributions())" && \
-    python3 -c "from torchcodec.decoders import VideoDecoder"
-
-RUN ln -sf /usr/bin/python3 /usr/bin/python
+# Keep source changes in one lightweight linked layer. Do not add RUN, COPY, or
+# ADD instructions that depend on the resulting filesystem after this point;
+# doing so would force BuildKit to download and extract the cached ROCm layers.
+FROM deps AS base
+ARG COMMON_WORKDIR=/app
+WORKDIR ${COMMON_WORKDIR}/vllm-omni
+COPY --link . .
 
 FROM base AS test
 

docker/Dockerfile.rocm.dockerignore

@@ -0,0 +1,35 @@
+# Keep the linked source layer small and deterministic.
+.git
+.venv
+/build
+dist
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+.mypy_cache
+
+# Distribution / packaging
+.Python
+/build/
+cmake-build-*/
+CMakeUserPresets.json
+develop-eggs/
+/dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+rust/target/

docker/Dockerfile.xpu

@@ -1,4 +1,4 @@
-# Target vLLM version: v0.28.0. Keep VLLM_VERSION aligned with the target vLLM
+# Target vLLM version: v0.29.0. Keep VLLM_VERSION aligned with the target vLLM
 # release on every rebase.
 #
 # This file layers vLLM-Omni on top of the published upstream XPU image, which
@@ -18,7 +18,7 @@
 #
 # .buildkite/intel/scripts/run-xpu-test.sh automates that pull-then-build fallback.
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai-xpu
-ARG VLLM_VERSION=v0.28.0
+ARG VLLM_VERSION=v0.29.0
 ARG VLLM_BASE=${VLLM_BASE_IMAGE}:${VLLM_VERSION}
 
 FROM ${VLLM_BASE} AS vllm-omni
@@ -40,7 +40,6 @@ ENV VLLM_OMNI_TARGET_DEVICE=xpu
 # Keep the upstream oneCCL packages paired with the XPU torch runtime.
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install --no-build-isolation \
-    --extra-index-url https://mirrors.ustc.edu.cn/pypi/simple/ \
     ".[dev]" && \
     uv pip uninstall triton && \
     uv pip install triton-xpu==3.7.2 --reinstall

docs/.nav.yml

@@ -71,6 +71,7 @@ nav:
       - Overview and Compatibility:
         - Overview: user_guide/diffusion_features.md
         - Feature Compatibility: user_guide/feature_compatibility.md
+      - Device-side Video Postprocessing: user_guide/diffusion/device_side_video_postprocess.md
       - CPU Offloading:
         - Overview: user_guide/diffusion/cpu_offload.md
         - Model-Level: user_guide/diffusion/offloader/module_offload.md
@@ -135,6 +136,7 @@ nav:
           - Mooncake Store Connector: design/feature/omni_connectors/mooncake_store_connector.md
           - Mooncake Transfer Engine Connector: design/feature/omni_connectors/mooncake_transfer_engine_connector.md
           - Mori Transfer Engine Connector: design/feature/omni_connectors/mori_transfer_engine_connector.md
+          - NIXL Connector: design/feature/omni_connectors/nixl_connector.md
           - Shared Memory Connector: design/feature/omni_connectors/shared_memory_connector.md
           - Yuanrong Store Connector: design/feature/omni_connectors/yuanrong_connector.md
           - Yuanrong Transfer Engine Connector: design/feature/omni_connectors/yuanrong_transfer_engine_connector.md
@@ -159,6 +161,7 @@ nav:
         - Cache-DiT: design/feature/cache_dit.md
         - TeaCache: design/feature/teacache.md
         - Diffusion Continuous Batching: design/feature/diffusion_continuous_batching.md
+        - Mooncake KV Transfer: design/feature/mooncake-kv-transfer.md
     - Infrastructure and Performance:
       - Prometheus Metrics: design/metrics.md
       - Speech Generation Performance Optimizations: design/qwen3_omni_tts_performance_optimization.md

docs/README.md

@@ -56,7 +56,7 @@ vLLM-Omni is flexible and easy to use with:
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
 - Omni-modality models (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
-- TTS models (e.g. Qwen3-TTS, IndexTTS 2.5, dots.tts, CosyVoice3)
+- TTS models (e.g. Qwen3-TTS, IndexTTS 2.5, CosyVoice3)
 - Diffusion models — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
 - Robot-policy and action models (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
 

docs/api/README.md

@@ -5,6 +5,9 @@
 Main entry points for vLLM-Omni inference and serving.
 
 - [vllm_omni.entrypoints.async_omni.AsyncOmni][]
+- [vllm_omni.entrypoints.async_omni_base.AsyncOmniBase][]
+- [vllm_omni.entrypoints.duplex_omni.DuplexOmni][]
+- [vllm_omni.entrypoints.duplex_omni.DuplexSessionHandle][]
 - [vllm_omni.engine.cfg_companion_tracker.CfgCompanionTracker][]
 - [vllm_omni.entrypoints.cli.benchmark.base.OmniBenchmarkSubcommandBase][]
 - [vllm_omni.entrypoints.cli.benchmark.main.OmniBenchmarkSubcommand][]
@@ -17,12 +20,13 @@ Main entry points for vLLM-Omni inference and serving.
 
 ## Clients
 
-Client libraries that talk to a running vLLM-Omni server (the full-duplex Realtime client and its per-model session presets).
+Client libraries for full-duplex sessions: the Realtime WebSocket client, the in-process client over `DuplexOmni`, and the per-model session presets.
 
 - [vllm_omni.clients.duplex.AudioDelta][]
 - [vllm_omni.clients.duplex.AudioFormat][]
 - [vllm_omni.clients.duplex.ConnectionResumed][]
 - [vllm_omni.clients.duplex.DuplexClient][]
+- [vllm_omni.clients.duplex.DuplexClientBase][]
 - [vllm_omni.clients.duplex.DuplexClientError][]
 - [vllm_omni.clients.duplex.DuplexConnectionError][]
 - [vllm_omni.clients.duplex.DuplexEvent][]
@@ -44,6 +48,7 @@ Client libraries that talk to a running vLLM-Omni server (the full-duplex Realti
 - [vllm_omni.clients.duplex.TextDelta][]
 - [vllm_omni.clients.duplex.TranscriptDelta][]
 - [vllm_omni.clients.duplex.WebSocketTransport][]
+- [vllm_omni.clients.inline_duplex.InlineDuplexClient][]
 
 ## Inputs
 

docs/cli/bench/serve.md

@@ -23,7 +23,9 @@ You can use `vllm bench serve --omni --help=all` to get descriptions of all para
   The API endpoint exposed externally, to which clients send their requests.
 
 - `--dataset-name`
-  The name of the dataset used; random-mm indicates generating random multimodal inputs (images, videos, audio), while `omniinteract` replays official OmniInteract videos.
+  The name of the dataset used; `random-mm` generates random multimodal inputs (images, videos, audio),
+  `omniinteract` replays official OmniInteract videos, and `videomme` loads Video-MME MCQ videos
+  (`lmms-eval/Video-MME`, MiniCPM OmniEvalKit packing).
 
 - `--num-prompts`
   The total number of requests to send, an integer.
@@ -316,8 +318,12 @@ arrives, checkpointing the response's history position so a later committed user
 outputs are ineligible and omitted from the official manifest; `audio_clipped_bytes` records output beyond the rounded video
 horizon.
 
-TTFT, TTFP, and RTF start at client receipt of `response.created`. TPOT/ITL use engine stage-0 timing; ITL is emitted only when
-every token interval is present.
+Per-response TTFT and TTFP start when the server begins executing the native model-turn request that owns the response. RTF
+continues to use client receipt of `response.created` through the last audio packet, divided by emitted audio duration. Global
+TTFT, TTFP, and RTF cover the complete input-stream window. TPOT/ITL use engine stage-0 timing; ITL is emitted only when every
+token interval is present within a continuous generation segment. Model-unit pacing and gaps between generation segments are
+excluded from TPOT/ITL. Raw request metrics retain `response_created_to_first_text_ms` and
+`response_created_to_first_audio_ms` as client-envelope diagnostics.
 
 The checked-in local performance configuration measures four deterministic cases from each OmniInteract subset (12 videos
 total), with no benchmark warmups and a maximum concurrency of two. Each subset also sends one readiness request before its
@@ -339,6 +345,57 @@ transcript, event, and result artifacts without errors. A valid LISTEN-only case
 chunks. Official-manifest eligibility is reported separately because clipped or cancelled output is a benchmark-quality
 signal, not a transport failure. This local performance test does not score answer accuracy.
 
+### Video-MME Benchmark
+
+Video-MME (`--dataset-name videomme`) scores multiple-choice video QA. Default packing is
+OmniEvalKit MiniCPM `minicpm-frames` (up to 96 sampled frames as `image_url`). Pass a local
+mirror with `--dataset-path` / `--videomme-parquet` + `--videomme-video-dir`, or a Hugging
+Face dataset id (`lmms-eval/Video-MME` by default; any `org/name` is accepted when
+`--dataset-name videomme` is explicit). Relative `--videomme-video-dir` values are resolved
+to absolute `file://` URLs. For those URLs, start the server with
+`--allowed-local-media-path` covering the video root; otherwise use `--videomme-inline-local-video`.
+
+For MiniCPM-o 4.5, the default deployment allows 64 images per request. To run the
+96-frame recipe, start the server with a stage 0 override (replace `/path/to/Video-MME`
+with the local video root shared by the benchmark client and server):
+
+```bash
+vllm serve openbmb/MiniCPM-o-4_5 --omni \
+  --allowed-local-media-path /path/to/Video-MME \
+  --stage-overrides '{"0":{"limit_mm_per_prompt":{"image":96,"audio":64,"video":1}}}'
+```
+
+The override applies to this server invocation only. Alternatively, use
+`--videomme-max-frames 64` in the benchmark to stay within the default image limit;
+this evaluates a different frame-sampling setting from the 96-frame recipe.
+
+Run the benchmark against that server:
+
+```bash
+vllm bench serve --omni \
+  --backend openai-chat-omni \
+  --dataset-name videomme \
+  --dataset-path /path/to/Video-MME \
+  --videomme-pack-mode minicpm-frames \
+  --videomme-max-frames 96 \
+  --model openbmb/MiniCPM-o-4_5 \
+  --endpoint /v1/chat/completions \
+  --num-prompts 8 \
+  --max-concurrency 1 \
+  --save-result
+```
+
+Accuracy keys (`videomme_accuracy`, per-duration / domain / task breakdowns) are written into
+the saved JSON. `videomme_accuracy` excludes HTTP failures; `videomme_accuracy_incl_http_fail`
+counts them as wrong. `videomme_submitted` / `videomme_unique_question_ids` show when
+oversampling made the request count differ from unique-question coverage.
+`videomme_skipped_rows` counts rows skipped during sampling because of missing or unreadable
+media or invalid question fields. The opt-in accuracy runner rejects skipped rows, HTTP
+failures, duplicate question IDs, and missing gold answers. This checks the sampled subset;
+it does not establish that a supplied local mirror contains the entire official dataset.
+Use `--videomme-save-eval-items` (or `VIDEOMME_SAVE_EVAL_ITEMS=1`) for per-request
+`videomme_eval_items` rows.
+
 ### Multi-Modal Benchmark
 
 <details class="admonition abstract" markdown="1">

docs/cli/serve.md

@@ -1,5 +1,67 @@
 # vllm-omni serve
 
+## Multiple API frontends
+
+For local EngineCore pipelines with one or more stages, `--api-server-count` starts
+multiple API frontend processes that share one parent-owned set of stage
+engines:
+
+```bash
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+    --api-server-count 2 \
+    --port 8091
+```
+
+This mode is intended for workloads whose request parsing, multimodal input
+processing, or response encoding can bottleneck a single frontend. It
+currently requires local multi-process EngineCore stages with one local
+process group per replica. A stage may still use multiple `num_replicas`.
+It cannot be combined with diffusion, headless or
+remote stages, intra-stage data parallelism, Ray, fault tolerance, elastic
+expert parallelism, sleep mode, or runtime LoRA updating. Runtime voice upload
+and deletion are also disabled because those mutations are process-local;
+built-in voices, inline reference audio, and voices restored at startup remain
+available. The `/v1/omni/sleep` and `/v1/omni/wakeup` control routes return HTTP
+409 for the same reason: their bookkeeping is process-local while stage engines
+are shared.
+
+A single-stage model pipeline is supported through the normal launch command;
+this is distinct from the distributed `--stage-id` / `--headless` launch mode.
+Every stage in the pipeline must be an EngineCore stage. Mixed AR-to-DiT
+pipelines, including MiniMax-H3, remain unsupported even when their first
+stage is an EngineCore stage. Diffusion support requires per-frontend channels
+and shared-engine lifecycle handling in the diffusion backend.
+
+### Workloads and performance validation
+
+Each API process owns its frontend processor, Orchestrator, request state,
+and per-replica EngineCore channels. Requests return through the channel of
+the originating API process; model weights and GPU stage engines are shared.
+This distributes frontend and orchestration work across CPU processes.
+
+| Workload | Potential benefit | Limitation |
+| ---------- | ------------------- | ------------ |
+| Concurrent cold image/video inputs on supported EngineCore pipelines | Parallel frontend processing when one API process is CPU-limited | A single cold request and GPU computation may be unchanged |
+| Many replicas producing frequent text/audio outputs | Less API/Orchestrator work and GIL contention per process | Routing remains serial within each Orchestrator |
+| Warm inputs or low-concurrency, GPU-limited generation | Additional frontend capacity | More API processes may add overhead without improving throughput |
+
+Here, a cold input means media or preprocessing caches miss, not that model
+weights must be loaded or GPU kernels compiled. Each frontend has its own
+processor/cache state, so cache duplication, CPU thread contention, and memory
+usage also matter. Multi-API serving does not itself change the output queue,
+move response encoding to an executor, or change the model's numerical scheduler.
+
+Compare `--api-server-count 1`, `2`, and higher counts with the same hardware,
+model/stage configuration, media, sampling parameters, output lengths, and
+offered load. Keep cold-cache and warmed-cache runs separate and report repeated
+measurements. Record CPU and request counts for every API worker, input and
+output queue delay, GPU utilization, first-output latency, and end-to-end latency.
+Confirm requests reach every worker: connection reuse can concentrate traffic
+on one process. Queue-only or single-API experiments do not establish a
+multi-API throughput gain. The multi-replica workload in
+[#4680](https://github.com/vllm-project/vllm-omni/issues/4680) remains a relevant
+validation target.
+
 ## Stage-based CLI quickstart
 
 The stage-based CLI is designed for deployments that require launching each pipeline stage in an isolated process
@@ -61,3 +123,10 @@ CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
 ## Arguments
 
 --8<-- "docs/generated/argparse_omni/omni_serve.inc.md"
+
+Async video jobs use process-local metadata and task handles. With multiple API
+workers, creating, listing, retrieving, downloading, and deleting these jobs
+returns HTTP 409. This restriction is independent of the current diffusion
+stage launch restriction: supporting multi-worker diffusion also requires a
+shared job lifecycle. Missing frontend topology state returns HTTP 503 for
+operations that require process-local state protection.

docs/configuration/environment_variables.md

@@ -70,6 +70,7 @@ The lifecycle labels used below are:
 | `OMNI_DIFFUSION_PROMPT_EMBED_CACHE_SIZE` | Positive integer; default `32` entries | Each diffusion runner; resolved during model setup | A valid environment value overrides the explicit cache size. A non-integer logs a warning; a non-positive value is ignored. | Experimental |
 | `OMNI_DIFFUSION_SESSION_STATE_MANAGER` | Boolean with the same accepted spellings as the prompt cache; default disabled | Experimental diffusion session manager; model setup | A recognized environment value overrides the explicit enable setting. An unrecognized value is ignored. | Experimental |
 | `OMNI_DIFFUSION_SESSION_STATE_MANAGER_MAX_SESSIONS` | Positive integer; default `64` | Experimental diffusion session manager; model setup | A valid environment value overrides the explicit maximum. A non-integer logs a warning; a non-positive value is ignored. | Experimental |
+| `VLLM_OMNI_AR_DIFFUSION_KV_GATHER` | `1` enables; default `0` (off) | AR-diffusion KV manager at allocation and attention at dispatch; set before worker startup | Only the exact value `1` enables contiguous KV gathering. History staging additionally requires `ARDiffusionKVConfig.reuse_history_staging`. | Experimental |
 
 Backend names for `DIFFUSION_ATTENTION_BACKEND` are the members of
 `DiffusionAttentionBackendEnum`, such as `FLASH_ATTN`, `TORCH_SDPA`,
@@ -82,14 +83,23 @@ depends on the installed kernels and model path.
 | --- | --- | --- | --- | --- |
 | `SPEAKER_SAMPLES_DIR` | Filesystem path; default `~/.cache/vllm-omni/speakers` | Speech server; read when speaker storage initializes | Environment-only setting. The directory is created; filesystem errors propagate. | Stable |
 | `SPEAKER_MAX_UPLOADED` | Integer; default `1000` | Speech server; read when speaker storage initializes | Environment-only setting. A non-integer logs a warning and uses `1000`; range is not otherwise validated. | Stable |
+| `VLLM_OMNI_ABORT_TIMEOUT` | Float seconds; default `2` | Engine abort wait for Videos DELETE and `generate()` cancel/error cleanup; read when `async_omni` imports | Environment-only setting. A non-float raises `ValueError` during import. | Experimental |
 | `VLLM_OMNI_ASYNC_OUTPUT_TIMEOUT` | Float seconds; default `600` | Diffusion engine async-output wait in `step_streaming`; resolved per call on the request path, not at import | Environment-only setting. A non-float or `<=0` value warns once and uses the default. | Experimental |
-| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `1`, `true`, `yes` or `on` enables; default `0` (off) | Orchestration loop and the serving-side final-output drain; read once when the `Orchestrator` is constructed | Environment-only setting. Values are stripped and case-normalized; any unrecognized value leaves the legacy poll loop selected. | Experimental |
+| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `1`, `true`, `yes` or `on` enables; defaults to on for Qwen3-TTS and off for other pipelines | Pipeline default computed from `pipeline_config.model_type` at engine initialization; env override resolved at `Orchestrator` construction and separately when the serving-side final-output drain starts | An explicit env value wins; otherwise both consumers use the engine's pipeline default. Values are stripped and case-normalized; any unrecognized value selects the legacy poll loop. Set before server startup. | Experimental |
 | `VLLM_OMNI_INPUT_WAIT_TIMEOUT_S` | Float seconds; default `600`; `<=0` disables | Full-payload input coordinator, not async-chunk transfer; read when the scheduler module imports in each worker | Environment-only setting. A non-float logs a warning and uses `600`. | Stable operational control |
 | `VLLM_OMNI_ORCH_MONITOR_PATH` | Filesystem path; default `<current-working-directory>/vllm_omni_orch_monitor_<timestamp>.json` | Orchestrator monitor enabled by `--enable-orch-monitor`; read when the monitor is created | Environment-only path override. Parent directories are created; write errors are logged. | Diagnostic |
+| `VLLM_OMNI_SPEAKER_REGISTRATION_POLICY` | `overwrite` or `immutable`; default `overwrite` | Speech server; read when speaker storage initializes | Environment-only setting. `immutable` rejects re-registering an existing uploaded voice name until it is deleted; any other value raises `ValueError` at startup. | Experimental |
 | `VLLM_OMNI_VIDEO_SYNC_TIMEOUT` | Float seconds; default `600` | Synchronous Videos API; read when the API server module imports | Environment-only setting. A non-float raises `ValueError` during import. | Experimental |
 | `VLLM_VIDEO_ASYNC_CHUNK` | `on` or `off`; default `on` | Streaming video output; read on attribute access | Environment-only setting. Values are trimmed and case-normalized; an invalid value warns once and uses `on`. | Experimental |
 | `VLLM_VIDEO_AUDIO_DELTA_MODE` | `fast` or `slow`; default `fast` | Streaming video audio deltas; read on attribute access | Environment-only setting. Values are trimmed and case-normalized; an invalid value warns once and uses `fast`. | Experimental |
 
+### NIXL stage transfer
+
+| Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
+| --- | --- | --- | --- | --- |
+| `VLLM_OMNI_NIXL_LEASE_S` | Float seconds; default `3600` | Unclaimed producer payload expiry; connector construction | Nonempty environment value overrides `lease_seconds`; empty falls back to config/default. Invalid floats raise `ValueError`. Claimed READ allocations never expire by time. | Experimental |
+| `VLLM_OMNI_NIXL_XFER_TIMEOUT_S` | Float seconds; default `300` | Receiver READ wait; connector construction | Nonempty environment value overrides `transfer_timeout_s`; empty falls back to config/default. Invalid floats raise `ValueError`. Timeout does not cancel DMA or release active allocations. | Experimental |
+
 ### Server storage
 
 Storage names use Pydantic's nested-settings delimiter (`__`). They are read
@@ -112,6 +122,7 @@ settings.
 
 | Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
 | --- | --- | --- | --- | --- |
+| `VLLM_OMNI_FUSED_QK_NORM_ROPE_MIN_TOKENS` | Non-negative integer; unset or empty means each consumer's own default (Boogu-Image: `2048`) | Token gate of the shared `fused_qk_norm_rope` op: a rotary table is packed for the fused kernel only when it spans at least this many positions (`B*S`); read at forward time | Environment-only. `0` = always fuse; a very large value disables the fused path. Any other value raises `ValueError` on the first forward. The crossover is host-dependent; re-benchmark before overriding. | Experimental performance control |
 | `VLLM_OMNI_SKIP_NVFP4_NAN_CLAMP` | Boolean truthy spellings: `1`, `true`, `yes`, `on`; default false | ModelOpt NVFP4 compatibility patch; read when `vllm_omni.patch` imports | Environment-only escape hatch. Any other value means false. Set only to diagnose the upstream NaN-scale issue. | Diagnostic and temporary |
 | `VLLM_OMNI_USE_QUACK_FP8` | Boolean truthy spellings: `1`, `true`, `yes`, `on`; unset means hardware auto-detection | FP8 scaled matrix multiplication; evaluated when quack capability is selected | A set value overrides auto-detection. Any non-truthy value forces quack off. If quack cannot load, vLLM-Omni warns and falls back to FlashInfer. | Experimental performance control |
 

docs/configuration/stage_configs.md

@@ -17,6 +17,7 @@ Common `PipelineConfig` fields include:
 | ------- | ------------- |
 | `model_type` | Pipeline identifier used during model and config resolution. |
 | `default_deploy_config_name` | Bundled deploy YAML loaded when the user does not pass `deploy_config`. |
+| `duplex_plugin` | Dotted path of the model's `DuplexModelPlugin`. Set only for full-duplex models; it makes `vllm-omni serve` run the model through `DuplexOmni` (a server whose every surface runs on a duplex session) and the engine host a `DuplexOrchestrator` with the plugin loaded. |
 | `model_arch` | Default Hugging Face architecture for the pipeline. |
 | `hf_architectures` | Architecture names used to identify checkpoints whose `model_type` is shared. |
 | `hf_config_predicate` | Optional predicate used to select between pipelines with otherwise identical HF metadata. |
@@ -38,7 +39,7 @@ Common `StagePipelineConfig` fields include:
 | `custom_process_input_func` | Processor applied to this stage's incoming payload. |
 | `custom_process_next_stage_input_func` | Processor used for full-payload handoff to the next stage. |
 | `async_chunk_process_next_stage_input_func` | Processor used for async chunk handoff. |
-| `sampling_constraints` | Model-owned sampling constraints that deploy defaults cannot override. |
+| `sampling_constraints` | Model-owned sampling constraints that deploy defaults cannot override. Scalar values replace deploy defaults; required `stop_token_ids` extend and deduplicate them. |
 
 To add or change topology, define and register a new pipeline variant. Use
 deploy YAML only for runtime placement, resource sizing, connectors, and other
@@ -52,7 +53,7 @@ The new deploy schema lives under `vllm_omni/deploy/` and is paired with a froze
 | ------- | ------ | ---------- | --------- | ------------- |
 | `base_config` | str (path) | optional | — | Overlay parent (relative or absolute). `stages:` / `platforms:` deep-merged by stage_id; other scalars overlay-wins. Intended for user-authored overlays; prod yamls stay flat. |
 | `async_chunk` | bool | optional | `true` | Enable chunked streaming between stages. Pin to `false` if the pipeline runs end-to-end. |
-| `session_mode` | str | optional | `"turn"` | Session behavior. MiniCPM-o 4.5 deploy YAMLs set `"duplex"` so `/v1/realtime` and chat share the same profile. |
+| `session_mode` | str | optional | `"turn"` | Session behavior. For pipelines with a `duplex_plugin`, explicitly select `"duplex"` for the duplex engine or `"turn"` for the ordinary online serving stack. The shipped MiniCPM-o 4.5 default remains `"duplex"`. See [Full Duplex](../serving/full_duplex_api.md#enable-full-duplex). |
 | `active_stream_window` | int | optional | `0` | Number of active downstream stream slots; `0` preserves all-stream cycling. |
 | `duplex_session` | dict | optional | runtime defaults | Full-duplex session lifecycle, buffering, replay, and capacity limits. |
 | `connectors` | dict | optional | `null` | Named connector specs (`{name, extra}`). Referenced by each stage's `input_connectors` / `output_connectors`. See [Connector schema](#connector-schema). |
@@ -97,7 +98,7 @@ Each entry under `stages:` accepts any `StageDeployConfig` field directly (no ne
 | `devices` | str \| null | optional | `null` | Device list assigned to this stage. The number of device ids must equal this stage's local world size (`tensor_parallel_size` × local data-parallel size × `pipeline_parallel_size`, or `num_replicas` × that product for a replica pool); a mismatch fails early — see the note below. |
 | `output_connectors` | dict \| null | optional | `null` | Keyed by `to_stage_<n>`; values are names registered under top-level `connectors:`. |
 | `input_connectors` | dict \| null | optional | `null` | Keyed by `from_stage_<n>`; values are names registered under top-level `connectors:`. |
-| `default_sampling_params` | dict \| null | optional | `null` | Baseline sampling params. Deep-merged with pipeline `sampling_constraints` (pipeline wins). |
+| `default_sampling_params` | dict \| null | optional | `null` | Baseline sampling params. Merged with pipeline `sampling_constraints`; scalar constraints win, while required `stop_token_ids` are appended and deduplicated. |
 | `engine_extras` | dict | optional | `{}` | Catch-all for engine fields not listed above; deep-merged across overlays and forwarded to the stage engine. |
 
 **Note:** a stage's `devices` count must equal its local world size (`tensor_parallel_size` × `data_parallel_size_local` × `pipeline_parallel_size`, falling back to global `data_parallel_size` when the local size is unset), or `num_replicas` × that product for a replica pool. A mismatch fails early and names the offending stage. A top-level `--tensor-parallel-size` is broadcast to every stage, so it can make a single-GPU stage violate this contract ([issue #5003](gh-issue:5003)); fix that case with `--stage-overrides` (set `tensor_parallel_size` and `devices` together per stage) or set TP only on the multi-GPU stage.
@@ -289,3 +290,96 @@ vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/depl
 
 !!! important
     We are actively iterating on the definition of deployment configurations, and we welcome feedback from users and developers.
+
+## Qwen3-TTS with Model Runner V2
+
+Qwen3-TTS runs the native CUDA Model Runner V2 pipeline by default on vLLM
+0.29.0. MRV2 is an experimental feature for this model: the bundled default
+profile selects it on CUDA only, and its scheduler and delivery paths are
+still being qualified. Set `model_runner: v1` in a copy of the deploy config
+to opt out; the `platforms:` sections of `qwen3_tts.yaml` keep V1 on NPU, XPU,
+ROCm and MUSA. Select one of these deployment profiles:
+
+| Profile | Runner | Code2Wav graph batches | Intended use |
+| --- | --- | --- | --- |
+| `qwen3_tts.yaml` | V2 (default) | Existing defaults | Shipped default; experimental |
+| `qwen3_tts_mrv2.yaml` | V2 | B1 | Explicit MRV2 profile (same runner selection as the default) |
+| `qwen3_tts_high_concurrency_mrv2.yaml` | V2 | B1, B2 | Opt-in throughput tuning |
+| `qwen3_tts_high_concurrency_mrv2_b4.yaml` | V2 | B1, B2, B3, B4 | Experimental throughput / buffered playback |
+| `qwen3_tts_high_concurrency.yaml` | V1 | Existing defaults | V1 high-concurrency control |
+
+```bash
+# MRV2 is the default; pass a copy with `model_runner: v1` to force V1.
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base --omni \
+  --deploy-config /path/to/qwen3_tts_v1.yaml
+```
+
+The V2 profiles bound Talker prefill to 512 tokens per step and select the
+Talker AR runner and the Code2Wav generation
+runner together. Native inter-stage delivery carries codec payloads directly;
+request-owned snapshots preserve buffers through asynchronous completion and
+CUDA graph reuse. Terminal completion waits for upstream stage metrics before
+releasing request state. Platform sections retain V1 on NPU, XPU, ROCm and MUSA;
+this change does not qualify MRV2 on those backends or enable other model families.
+Selecting V2 for a stage without `supports_native_mrv2_data_plane` emits a warning
+when pipeline and deployment settings are merged. That stage retains the legacy
+transport path; the warning does not establish support for that combination.
+
+MRV2 model hooks use capability declarations rather than architecture or stage
+names. Omni lifecycle flags select the model state, and `_returns_tuple` declares
+the capture output contract. The optional batched predictor hook is `mtp`, with
+an explicit `mtp_output_key` (a string or two-part payload key), optional
+`mtp_validity_key`, and `mtp_graph_safe`/`mtp_disable_graph` capture controls.
+Its inputs are token IDs, embeddings, previous hidden states and per-row
+conditioning; it returns updated embeddings and prediction codes. Models may
+supply `mtp_sampling_params` and `get_mtp_seed(sampling_params)` for model-local explicit seeds, and declare
+`mtp_accepts_per_row_generators`, `mtp_accepts_req_infos`, or `mtp_sample_uniforms`
+(with `mtp_sample_steps` and `mtp_sample_vocab_size`) as needed. Qwen3-TTS retains
+its existing `talker_mtp` entry point for V1.
+
+### Included performance work
+
+- Request snapshots have a fast path for immutable scalar leaves, including
+  waveform lists. Mutable containers, aliases and cycles keep deep-copy
+  semantics. This is not the historical shallow `dict(prompt)` experiment.
+- The API reference cache holds owned float32 arrays with a default capacity
+  of 1024 entries and a 512 MiB waveform-payload budget. Cache hits return
+  independent lists. Qwen3-TTS model artifact caches also default to 1024
+  entries. These caches have different owners and eviction policies; equal
+  entry limits do not make them a single coherent cache.
+- Talker state stays on GPU where its lifecycle allows it. BOS/EOS projections
+  are cached in their projection dtype and invalidated on weight loading.
+- Code2Wav packs CPU codec inputs before device transfer. Optional B2/B4 graph
+  buckets batch compatible requests without sharing their per-request state.
+- Native output materialization runs in a bounded worker and drains before
+  closing the data plane. The existing control-only shortcut avoids launching
+  model work for a step that has only lifecycle events.
+
+MTP prefix re-prefill remains disabled in the high-concurrency V2 profile.
+Adding more graph shapes increases compilation cost and has not established a
+stable end-to-end gain for this extracted version.
+
+### Decoder batches and playback
+
+Changing `decode_cudagraph_batch_sizes` selects the captured batch buckets.
+Keep `decode_batch_max_size` consistent with the intended maximum too; the
+stateful graph path currently groups according to the captured buckets, while
+the stateless path also uses the explicit maximum.
+
+B4 remains experimental. First-packet latency alone does not establish
+uninterrupted playback. Validate inter-chunk arrival times, buffering, WER and
+speaker similarity before adopting either batching preset for a production
+workload. Floating-point decoder outputs can differ across batch sizes; this PR
+does not claim bitwise or quality equivalence.
+
+### Optional MPS deployment
+
+NVIDIA MPS is an optional operator setting for colocated CUDA processes, not a
+YAML option or a library default. This PR does not establish a throughput or
+first-packet latency benefit from MPS. Measure the exact deployment with and
+without MPS before enabling it.
+
+Use only assigned GPUs and an independent MPS pipe directory. A private MPS
+server does not provide exclusive GPU ownership or MIG isolation. For a
+single-GPU deployment, explicitly place both stages on that GPU; the supplied
+high-concurrency profile places its two stages on different GPUs by default.

docs/contributing/ci/ci_settings.md

@@ -15,7 +15,8 @@ Canonical layout (prefer these paths for new changes):
 │   │   ├── skip_ci.py               # skip-ci decision (docs / skip-mark / CI YAML paths)
 │   │   ├── upload_pipeline.py       # Bootstrap + test-pipeline uploader (CUDA/NPU)
 │   │   └── resolve_skip_ci.sh       # Shell helpers for AMD/Intel bootstrap
-│   └── ci_mirror_hardwares.yml      # CUDA uploader presets (referenced by name only)
+│   ├── ci_mirror_hardwares.yml      # CUDA/NPU hardware presets (referenced by name only)
+│   └── ci_source_file_dependencies.yml  # L2–L5 path-prefix presets (referenced by name only)
 ├── cuda/                            # Primary NVIDIA CUDA CI
 │   ├── pipeline.yml                 # Bootstrap entry (hook upload)
 │   ├── bootstrap-upload-steps.yml   # Bootstrap child steps (upload_pipeline --upload)
@@ -34,6 +35,7 @@ Canonical layout (prefer these paths for new changes):
 ├── amd/
 │   ├── test-amd-ready.yml           # L2 job definitions (template input)
 │   ├── test-amd-merge.yml           # L3 job definitions
+│   ├── test-amd-nightly.yml         # Nightly full-model jobs (experimental)
 │   ├── test-template-amd-omni.j2    # Renders final pipeline.yaml
 │   └── scripts/
 │       ├── bootstrap-amd-omni.sh    # Entry: skip-ci → Jinja → upload
@@ -63,7 +65,7 @@ Canonical layout (prefer these paths for new changes):
 | -------- | ---------------- | -------------- | ---------------- | -------------------- |
 | **CUDA** | `cuda/pipeline.yml` | `test-ready.yml`, `test-merge.yml`, `test-nightly.yml`, `test-weekly.yml` | `upload_pipeline.py --upload` (expands uploader-only keys) | omit `mirror_hardwares` (from `-m`) / preset string + `MIRROR_HW` |
 | **NPU** | `npu/pipeline-npu.yml` | `test-npu-ready.yml`, `test-npu-nightly.yml` | `upload_pipeline.py --upload` | `mirror_hardwares: a2b3_npu_1` / `a2b3_npu_4` / `a3_npu_2` |
-| **AMD** | `amd/scripts/bootstrap-amd-omni.sh` | `test-amd-ready.yml`, `test-amd-merge.yml` | Jinja (`test-template-amd-omni.j2`) → `pipeline upload` | `agent_pool` + `mirror_hardwares: [amdproduction]` (array, template filter) |
+| **AMD** | `amd/scripts/bootstrap-amd-omni.sh` | `test-amd-ready.yml`, `test-amd-merge.yml`, `test-amd-nightly.yml` | Jinja (`test-template-amd-omni.j2`) → `pipeline upload` | `agent_pool` + `mirror_hardwares: [amdproduction]` (array, template filter) |
 | **Intel** | `intel/scripts/bootstrap-intel-omni.sh` | `intel/pipeline-intel.yml` (steps inline) | Direct `pipeline upload` | Inline `agents.queue` on each step |
 
 ## Platform configuration style
@@ -108,19 +110,21 @@ Canonical layout (prefer these paths for new changes):
 
     Inferred form: unset/`empty` `MIRROR_HW` matches `H100` or `L4` in `-m` (both present → H100); no match skips the step. `MIRROR_HW=b200` must appear in `-m` or the step is skipped. The uploader does **not** rewrite pytest `-m`; B200 collection comes from the YAML expression and from tests that declare `B200` in `hardware_test` / `hardware_marks` (for example `res={"cuda": ["H100", "B200"]}`). `MIRROR_HW` must be empty or `b200` (case-insensitive); unknown values (for example `b20o`) **fail the upload**. A CUDA preset string such as `mirror_hardwares: h100_4` is skipped when `MIRROR_HW=b200`. Unset `MIRROR_HW` keeps string presets. NPU presets ignore `MIRROR_HW`. A `mirror_hardwares` name that is not a key in `ci_mirror_hardwares.yml` **fails the upload**.
 
+    **Path filter in YAML:** `source_file_dependencies: <preset>` (string, or a list of preset names)—preset names in [`common/ci_source_file_dependencies.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/ci_source_file_dependencies.yml). Required on **E2E Test** leaf jobs in L2/L3, and on L4/L5 leaf jobs that should be diff-gated on PR labels; see [Step filtering](#step-filtering).
+
     **Conventions**
 
     - **`depends_on`:** leaf jobs depend on `upload-ready-pipeline`, `upload-merge-pipeline`, etc.
     - **`group` / `label`:** `:card_index_dividers:` groups; labels like `Diffusion · Qwen Image Test`.
     - **`commands`:** `timeout … pytest …` with markers and `--run-level` for the pipeline level.
-    - **`source_file_dependencies`:** required on **E2E Test** leaf jobs in L2/L3; see [Step filtering](#step-filtering).
+    - **`source_file_dependencies`:** required on **E2E Test** leaf jobs in L2/L3, and on L4/L5 leaf jobs that should be diff-gated on PR labels; name a job key from `ci_source_file_dependencies.yml` (see [Step filtering](#step-filtering)). The key must list every script the step runs; `commands` is not scanned.
 
     **Adding a job**
 
     1. Pick the level file (`test-ready.yml` / `test-merge.yml` / `test-nightly.yml` / `test-weekly.yml`).
     2. Add a step under the right **group** (usually **E2E Test** for model pytest).
-    3. Set `label`, `commands`, `mirror_hardwares`, `depends_on: upload-ready-pipeline` (or `upload-merge-pipeline` / `upload-nightly-pipeline` / `upload-weekly-pipeline`).
-    4. For L2/L3 E2E, add `source_file_dependencies` (pytest + model + deploy YAML prefixes).
+    3. Set `label`, `commands`, `mirror_hardwares`, `depends_on: upload-<level>-pipeline`.
+    4. For L2–L5 jobs that should be diff-gated, add a job key in `common/ci_source_file_dependencies.yml` and set `source_file_dependencies: <key>` on the step. YAML anchors hold model business-code paths only. The job key (`{model}_function` / `_perf` / `_accuracy` / `_reliability` / `_doc` / `_cov`) aliases that anchor and lists the pytest / example / benchmark scripts the step actually runs. The uploader does **not** parse `commands` for path filters—omitting a script from the key means editing that script will not select the job.
     5. Dry-run:
 
     ```bash
@@ -144,6 +148,8 @@ Canonical layout (prefer these paths for new changes):
     - A5: `a5_npu_2`, `a5_npu_4`, `a5_npu_8`, `a5_npu_16`
     - 310P: `310p_npu_1`, `310p_npu_2`, `310p_npu_4`
 
+    **Path filter in YAML:** same `source_file_dependencies` presets as CUDA, on **L4** leaf jobs in `test-npu-nightly.yml`. Filtering applies on the PR `nightly-test` label; `main` + `NIGHTLY=1` keeps every job. If no job-key prefix matches, a change to the pipeline YAML being uploaded, or to any path under `source_filter_fallback`, keeps every job in that upload. See [Step filtering](#step-filtering).
+
     **Conventions**
 
     - **`depends_on: upload-ready-pipeline`** (or `upload-nightly-pipeline`) ties jobs to bootstrap upload keys.
@@ -153,7 +159,7 @@ Canonical layout (prefer these paths for new changes):
 
     1. Edit `test-npu-ready.yml` (L2) or `test-npu-nightly.yml` (L4).
     2. Add a step with `mirror_hardwares` (add a new preset in `ci_mirror_hardwares.yml` first if needed).
-    3. Set `commands` to your pytest file and markers.
+    3. Set `commands` to your pytest file and markers. For L4, set `source_file_dependencies` to a job key from `ci_source_file_dependencies.yml` that aliases the model anchor and lists that pytest file.
     4. Dry-run:
 
     ```bash
@@ -166,9 +172,9 @@ Canonical layout (prefer these paths for new changes):
 
     **Bootstrap:** [`amd/scripts/bootstrap-amd-omni.sh`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/amd/scripts/bootstrap-amd-omni.sh)—skip-ci, diff filtering, Jinja render, then `buildkite-agent pipeline upload`.
 
-    **Test YAML (data):** `amd/test-amd-ready.yml` (L2 / `ready`), `test-amd-merge.yml` (L3 / `merge-test` and `main`). On a PR carrying both labels, AMD combines both suites behind one image build. PR builds without either tier label retain the legacy L2 fallback for compatibility with the `amd-test` trigger. `DEBUG_TEST_YAML` remains an explicit override. AMD has no L4 file yet, so `nightly-test` does not select an additional suite.
+    **Test YAML (data):** `amd/test-amd-ready.yml` (L2 / `ready`), `test-amd-merge.yml` (L3 / `merge-test` and ordinary `main`), and experimental `test-amd-nightly.yml` (`full_model` / `nightly-test` or `main` with `NIGHTLY=1`). Multiple PR tier labels combine their suites behind one image build. PR builds that reach the bootstrap without a tier label retain the legacy ready-suite fallback. `DEBUG_TEST_YAML` accepts `ready`, `merge`, and `nightly` as an explicit override. During nightly burn-in every AMD nightly leaf is non-blocking.
 
-    **Trigger boundary:** AMD label handling has two layers. First, the external `vllm-omni-amd-ci` Buildkite pipeline condition decides whether a GitHub label event creates a build. Only after that build starts does this repository's bootstrap inspect all current PR labels and select L2, L3, or both. Repository-side selection therefore cannot make a `merge-test` event start AMD CI by itself. The external condition must admit both `ready` and `merge-test`, while preserving the legacy `amd-test` trigger and non-PR `main` / scheduled builds. For PR builds without a `DEBUG_TEST_YAML` override, an unreadable label set fails the bootstrap instead of silently selecting the wrong tier. Debug overrides skip label lookup and are validated directly; invalid values, including whitespace-only values, fail the bootstrap. Keep `nightly-test` out of the AMD condition until an AMD L4 suite exists; otherwise it would start a build without providing nightly coverage.
+    **Trigger boundary:** AMD label handling has two layers. First, the external `vllm-omni-amd-ci` Buildkite pipeline condition decides whether a GitHub label event creates a build. Only after that build starts does this repository's bootstrap inspect all current PR labels and select the ready, merge, nightly, or combined suites. Repository-side selection therefore cannot start AMD CI by itself. AMD-only nightly validation uses both `amd-test` and `nightly-test`: the external condition should accept an event for either label only when the complete PR label set contains both. `amd-test` admits the AMD pipeline, while `nightly-test` makes the bootstrap select `NIGHTLY_TESTS:test-amd-nightly.yml`. Existing `ready` / `merge-test` behavior and non-PR `main` / scheduled builds remain separate. For PR builds without a `DEBUG_TEST_YAML` override, an unreadable label set fails the bootstrap instead of silently selecting the wrong tier. Debug overrides skip label lookup and are validated directly; invalid values, including whitespace-only values, fail the bootstrap. Until the paired-label condition is configured, use an authorized manual Buildkite build with `DEBUG_TEST_YAML=nightly` and verify that the bootstrap reports only the nightly suite.
 
     **Rendering:** [`test-template-amd-omni.j2`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/amd/test-template-amd-omni.j2) wraps data steps with `amd-build` image build and `amd_<agent_pool>` queues. Do **not** hand-edit generated `pipeline.yaml`.
 
@@ -184,7 +190,7 @@ Canonical layout (prefer these paths for new changes):
 
     **Adding a job**
 
-    1. Edit `test-amd-ready.yml` or `test-amd-merge.yml`.
+    1. Edit `test-amd-ready.yml`, `test-amd-merge.yml`, or `test-amd-nightly.yml`.
     2. Copy a neighboring block: `label`, `agent_pool`, `mirror_hardwares`, `commands`, optional `grade`.
     3. Regenerate via bootstrap / Jinja; update `skip_ci.py` if you add a new YAML path.
 
@@ -213,7 +219,7 @@ PR diffs drive **two independent skip layers**. Both read changed files from git
 | Layer | Script | When | What is skipped | Where you configure |
 | ----- | ------ | ---- | --------------- | ------------------- |
 | **Bootstrap** | [`skip_ci.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/skip_ci.py) + [`upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/upload_pipeline.py) | Before child test pipelines upload (`cuda/pipeline.yml`, `npu/pipeline-npu.yml`, AMD/Intel bootstraps) | Entire default CI, or whole L2/L3 upload for a platform | Whitelists in `skip_ci.py`; bootstrap `if` injected by step `key` in `upload_pipeline.py` |
-| **Step filter** | [`upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/upload_pipeline.py) | While uploading CUDA L2/L3 YAML | Individual Buildkite steps inside `test-ready.yml` / `test-merge.yml` | `source_file_dependencies` on each step or group |
+| **Step filter** | [`upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/upload_pipeline.py) | While uploading CUDA L2–L5 YAML | Individual Buildkite steps inside `test-ready.yml` / `test-merge.yml` / `test-nightly.yml` / `test-weekly.yml` | `source_file_dependencies` key on each step (presets in `ci_source_file_dependencies.yml`) |
 
 **Changed files** (both layers):
 
@@ -240,7 +246,7 @@ Label triggers (`ready`, `merge-test`) are unchanged—diff-aware logic only red
 | Non-qualifying skip-mark and no CI YAML | normal CI | all on |
 | Diff unavailable | normal CI | all on |
 
-**`skip_all` exceptions (CUDA/NPU bootstrap only):** PR labels (`nightly-test`, `merge-test`, `weekly-test`, `ready`, …) do **not** revive jobs under `skip_all`. On `main`, scheduled `NIGHTLY=1` still builds the image and uploads L4; `WEEKLY=1` or `NON_CRITICAL=1` still builds the image and uploads L5 (CUDA). `WEEKLY=1` on `main` also uploads L2/L3 with `--e2e`.
+**`skip_all` exceptions:** For CUDA/NPU, PR labels (`nightly-test`, `merge-test`, `weekly-test`, `ready`, …) do **not** revive jobs under `skip_all`; on `main`, scheduled `NIGHTLY=1` still builds the image and uploads L4, while `WEEKLY=1` or `NON_CRITICAL=1` still builds the image and uploads L5 (CUDA). `WEEKLY=1` on `main` also uploads L2/L3 with `--e2e`. AMD applies `skip_ci.py gate` only to selected ready/merge specs, so an explicitly selected AMD nightly spec (`nightly-test`, `NIGHTLY=1`, or a nightly debug override) remains runnable even for a docs-only diff.
 
 **Yaml-gated nightly/weekly:** L4/L5 upload steps keep their normal label / `NIGHTLY` / `WEEKLY` / `NON_CRITICAL` conditions (for example PR `nightly-test` still uploads nightly). Only L2/L3 upload steps are matrix-gated.
 
@@ -257,6 +263,7 @@ Register new files in `L2_YAML_FILES`, `L3_YAML_FILES`, or `L45_YAML_FILES` in `
 | L3 | `.buildkite/cuda/test-merge.yml` | cuda |
 | L3 | `.buildkite/amd/test-amd-merge.yml` | amd |
 | L4/L5 | `.buildkite/cuda/test-nightly.yml`, `test-weekly.yml` | cuda |
+| L4/L5 | `.buildkite/amd/test-amd-nightly.yml` | amd |
 | L4/L5 | `.buildkite/npu/test-npu-nightly.yml` | npu |
 
 ##### Yaml-gated category branches
@@ -289,7 +296,7 @@ Docs/skip-mark mixed with any row below follows the same matrix.
 | CUDA ready + AMD merge | G4 | `cuda/l2` + `amd/l3` |
 | CUDA ready + AMD ready | G2 | `cuda/l2` + `amd/l2` |
 | CUDA merge + AMD merge | G3 | `cuda/l3` + `amd/l3` |
-| Only nightly / weekly / npu-nightly YAML | G1 | (none) |
+| Only nightly / weekly / amd-nightly / npu-nightly YAML | G1 | (none) |
 | CUDA ready + nightly | G5 | `cuda/l2` |
 | CUDA merge + nightly | G6 | `cuda/l3` |
 | CUDA ready + merge + nightly | G7 | `cuda/l2` + `cuda/l3` |
@@ -306,40 +313,38 @@ Unit coverage: `tests/buildkite/test_skip_ci.py`.
 
 #### Step filtering {#step-filtering}
 
-CUDA **L2** (`.buildkite/cuda/test-ready.yml`) and **L3** (`.buildkite/cuda/test-merge.yml`) only. Bootstrap upload entry: `upload_pipeline.py --upload .buildkite/cuda/bootstrap-upload-steps.yml`.
+CUDA **L2** (`.buildkite/cuda/test-ready.yml`), **L3** (`.buildkite/cuda/test-merge.yml`), **L4** (`.buildkite/cuda/test-nightly.yml`), **L5** (`.buildkite/cuda/test-weekly.yml`), and NPU **L4** (`.buildkite/npu/test-npu-nightly.yml`). Bootstrap upload entry: `upload_pipeline.py --upload .buildkite/cuda/bootstrap-upload-steps.yml` (or `npu/bootstrap-upload-steps.yml`).
+
+**When filtering applies:** **PR label** uploads (`ready`, `merge-test`, `nightly-test`, `weekly-test`) and **post-merge `main` L3** (`test-merge.yml`). Scheduled **`main` + env** uploads that pass `--all` (`NIGHTLY=1`, `WEEKLY=1` / `NON_CRITICAL=1`) or `--e2e` (`WEEKLY=1` on L2/L3) keep every selected job and still strip the key. If **no** job-key prefix matches the diff, a change to the pipeline YAML being uploaded, or any path under the `source_filter_fallback` key in `ci_source_file_dependencies.yml`, keeps every job in that upload. `source_filter_fallback` is not a job key — do not attach it to a step. When any job-key prefix already matches, normal filtering wins (the fallback does not expand the selection).
 
 **Uploader-only keys** — removed before Buildkite sees the YAML; never used at runtime on agents:
 
 | Key | Purpose |
 | --- | ------- |
-| `source_file_dependencies` | List of path **prefixes**. If any changed file equals a prefix or starts with `prefix/`, keep the step (or group); otherwise omit it. |
+| `source_file_dependencies` | Named preset (or list of presets) from [`ci_source_file_dependencies.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/ci_source_file_dependencies.yml). The uploader expands only that key; it does **not** scan `commands`. If any changed file equals a listed prefix or starts with `prefix/`, keep the step (or group); otherwise omit it. Inline path-prefix lists are still accepted. |
 | `mirror_hardwares` | Expand to `agents` + `plugins` (+ optional `image`) from `ci_mirror_hardwares.yml`. |
 
 ### Policy
 
-- **Always uploaded** (no key): groups outside **E2E Test**—Simple Test, Diffusion unit tests, Engine/Model Executor, Distributed, Custom Pipeline, Entrypoints (L2), LoRA / Entrypoints (L3).
-- **Diff-gated**: every **E2E Test** leaf job. List the smallest prefix set per step—pytest file(s), model code under `vllm_omni/model_executor/models/` or `vllm_omni/diffusion/models/`, plus `stage_input_processors/` and `vllm_omni/deploy/*.yaml` when applicable. A **group** may define the key instead; the whole group drops if no prefix matches.
+- **Always uploaded** (no key): L2/L3 groups outside **E2E Test**—Simple Test, Diffusion unit tests, Engine/Model Executor, Distributed, Custom Pipeline, Entrypoints (L2), LoRA / Entrypoints (L3).
+- **Diff-gated**: every **E2E Test** leaf job in L2/L3, L4 nightly leaf jobs (CUDA and NPU), and L5 weekly jobs whose `if` includes the `weekly-test` label (Reliability / Perf). Jobs gated only by `WEEKLY=1` or `NON_CRITICAL=1` have no `source_file_dependencies`. Registry layout: YAML anchors are model business code only (`vllm_omni/model_executor/models/`, `vllm_omni/diffusion/models/`, plus `stage_input_processors/` and `vllm_omni/deploy/*.yaml` when applicable). Job keys alias the anchor and **also** list the scripts that job runs (pytest files, `run_cov_split.sh --offline` / `--online` targets, example README runners). Feature keys stay cross-cutting. A **group** may define the key instead; the whole group drops if no prefix matches.
 
 ### YAML examples
 
 ```yaml
       - label: "Diffusion · Qwen Image Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen_image.py
-          - vllm_omni/diffusion/models/qwen_image/
+        source_file_dependencies: diffusion_qwen_image_function
+        # diffusion_qwen_image_function: *qwen_image + tests/e2e/online_serving/test_qwen_image.py
         commands:
           - pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' ...
         mirror_hardwares: h100_1
 
       - label: "TTS · Qwen3-TTS CustomVoice Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
-          - vllm_omni/model_executor/models/qwen3_tts/
-          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
-          - vllm_omni/deploy/qwen3_tts.yaml
+        source_file_dependencies: tts_qwen3_tts_function
+        # tts_qwen3_tts_function: *qwen3_tts + tests/e2e/online_serving/test_qwen3_tts_customvoice.py
         commands:
           - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py ...
-        mirror_hardwares: l4_4
+        mirror_hardwares: l4_1
 ```
 
 ### Local dry-run
@@ -443,15 +448,16 @@ plugin, which provides the agent. `l4_*` jobs run on the EKS `l4-k8s` queue
 still needs `mount-buildkite-agent: true`.
 
 List both `run_cov_split.sh` and `pyproject.toml` in every opted-in job's
-`source_file_dependencies` — both change what the job measures, so without them a
+`source_file_dependencies` preset — both change what the job measures, so without them a
 change there is filtered out of normal PR builds and only surfaces in a later
 nightly. `tests/buildkite/test_upload_pipeline.py` covers the filter behavior with
 a synthetic job (it does not pin real merge labels).
-Editing only the surrounding CI YAML still does not schedule them, so a PR that
-touches just the wiring needs a full E2E run (or the commands run on a GPU host)
-to produce artifacts. When checking a new model's
-artifacts, compare `lines-covered` between the online and offline XML rather than
-just confirming both files exist.
+If no job-key prefix matches, editing the pipeline YAML being uploaded, or any
+path under the `source_filter_fallback` registry key, keeps every job in that
+upload (including coverage jobs) so the wiring can be validated before merge.
+When a job-key prefix already matches, bypass does not expand the selection. A
+change to a different pipeline YAML still does not schedule them. When checking a new model's artifacts, compare `lines-covered`
+between the online and offline XML rather than just confirming both files exist.
 
 ### Validation checklist
 

docs/contributing/ci/test_execution_guide.md

@@ -177,10 +177,10 @@ Failed jobs: 1/2
     cd tests
 
     # Stability: Qwen3-Omni
-    pytest -s -v dfx/stability/scripts/test_stability_qwen3_omni.py -m slow
+    pytest -s -v dfx/stability/scripts/run_stability_qwen3_omni.py -m slow
 
     # Stability: Wan2.2 (v1/videos diffusion benchmark loop)
-    pytest -s -v dfx/stability/scripts/test_stability_wan22.py -m slow
+    pytest -s -v dfx/stability/scripts/run_stability_wan22.py -m slow
 
     # Reliability: Qwen3-Omni (H100 × 2)
     pytest -s -v dfx/reliability/test_reliability_qwen3_omni.py -m slow

docs/contributing/ci/test_system_overview.md

@@ -182,6 +182,8 @@ CUDA **L4** (`full_model`) jobs are defined once in [`.buildkite/cuda/test-night
 - **Nightly:** scheduled `main` builds with `NIGHTLY=1` (and PR labels such as `nightly-test` / `omni-test`). Default fleet is typically H100.
 - **Pre-release:** the same L4 suite runs again on extra GPU SKUs before a release (for example B200 via `MIRROR_HW=b200`), so `full_model` coverage is not limited to the nightly machine type.
 
+AMD has an experimental, non-blocking `full_model` nightly vertical slice in [`.buildkite/amd/test-amd-nightly.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/amd/test-amd-nightly.yml). It selects ROCm Qwen3-Omni function-expansion, accuracy, documentation-example, and AITER-on smoke coverage on the MI300 pool. AMD-only PR validation uses `amd-test` to admit the external AMD pipeline and `nightly-test` to select this suite; scheduled `main` builds select it with `NIGHTLY=1`. Keep it non-blocking while the lane accumulates stability and completion-time evidence.
+
 How count-form `mirror_hardwares` and `MIRROR_HW` pick a SKU: [CI Settings](./ci_settings.md).
 
 ### L2 / L3 diff-aware CI (CUDA)

docs/contributing/ci/test_writing_guide.md

@@ -555,7 +555,7 @@ L5 level testing focuses on the performance of model services under ***long-runn
 
 - ***Trigger Timing***: **`Weekly`** (weekly) or **`Days before Release`** (several days before a major release). Due to long execution times, the frequency is lower.
 - ***Run Command***:
-    - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_<model>.py` alongside a matching JSON config)
+    - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/run_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/run_stability_wan22.py` (or add `run_stability_<model>.py` alongside a matching JSON config)
     - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_<model>.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`).
 - ***Script Example***:
 
@@ -564,7 +564,7 @@ L5 level testing focuses on the performance of model services under ***long-runn
 
 ##### Stability
 
-When you want to add L5-level stability test cases, add or extend the appropriate JSON file under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` for Omni bench traffic, or `test_wan22.json` for diffusion `/v1/videos` workloads). Pair the JSON with `tests/dfx/stability/scripts/test_stability_<model>.py`. The following illustrates the Qwen3-Omni shape:
+When you want to add L5-level stability test cases, add or extend the appropriate JSON file under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` for Omni bench traffic, or `test_wan22.json` for diffusion `/v1/videos` workloads). Pair the JSON with `tests/dfx/stability/scripts/run_stability_<model>.py`. The following illustrates the Qwen3-Omni shape:
 
 ```json
 {

docs/contributing/profiling.md

@@ -22,7 +22,7 @@ profiler_config:
 Supported fields:
 
 | Field | Description |
-|---|---|
+| --- | --- |
 | `profiler` | Profiler backend. Supported values: `torch`, `cuda`. Use `torch` for `trace.json`, Excel operator tables, and optional memory snapshots. Use `cuda` for Nsight Systems only. |
 | `torch_profiler_dir` | Output directory for torch-profiler artifacts. Required when `profiler: torch`. |
 | `torch_profiler_use_gzip` | Compress `trace_rank*.json` into `trace_rank*.json.gz`. |
@@ -287,6 +287,52 @@ Recommended viewers:
 
 For upstream background on the underlying vLLM profiling infrastructure, see the [vLLM profiling guide](https://docs.vllm.ai/en/stable/contributing/profiling/).
 
+## 6. Profiling Omni AR/TTS stages
+
+For Omni AR and TTS stages, put `profiler_config` under each stage that should
+be captured. The serving API then controls a narrow window with
+`POST /start_profile` and `POST /stop_profile`; model initialization and
+warmup stay outside the trace.
+
+Example stage override for a host-stack torch trace:
+
+```bash
+PROFILE='{"profiler":"torch","torch_profiler_dir":"/tmp/qwen3_tts_profile",'\
+'"torch_profiler_with_stack":true,"torch_profiler_record_shapes":false,'\
+'"torch_profiler_use_gzip":true,"torch_profiler_dump_cuda_time_total":true}'
+
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni \
+  --deploy-config vllm_omni/deploy/qwen3_tts_high_concurrency.yaml \
+  --stage-overrides "{\"0\":{\"profiler_config\":$PROFILE},\"1\":{\"profiler_config\":$PROFILE}}"
+```
+
+After one or more warmup requests, capture one short measured request:
+
+```bash
+curl -sS -X POST http://127.0.0.1:8000/start_profile \
+  -H 'Content-Type: application/json' -d '{"stages":[0,1]}'
+# send one fixed /v1/audio/speech or /v1/chat/completions request here
+curl -sS -X POST http://127.0.0.1:8000/stop_profile \
+  -H 'Content-Type: application/json' -d '{"stages":[0,1]}'
+```
+
+For an Nsight Systems CUDA timeline, use the same stage override with
+`{"profiler":"cuda"}` and wrap the server:
+
+```bash
+nsys profile --trace=cuda,nvtx,osrt --sample=process-tree \
+  --cpuctxsw=process-tree --trace-fork-before-exec=true \
+  --capture-range=cudaProfilerApi --capture-range-end=stop-shutdown \
+  --cuda-graph-trace=node -o /tmp/qwen3_tts_nsys \
+  vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni ...
+```
+
+For Omni AR workers, `profiler="cuda"` requires the native CUDA profiler
+wrapper and uses `cudaProfilerStart/Stop`; the report is only valid if the
+`/start_profile` and `/stop_profile` calls both succeed. Use `nsys stats` for
+CUDA API, GPU kernel, OS runtime, and NVTX summaries. Treat profiler latency as
+diagnostic only; use a separate no-profiler run for performance numbers.
+
 ## 7. Orchestrator Monitor
 
 > **Warning:** Diagnostic only. Adds lightweight counters on the orchestrator poll loop and per-replica queue depth. Disable in production unless you are actively debugging orchestrator saturation.
@@ -314,9 +360,11 @@ If unset, the monitor writes `./vllm_omni_orch_monitor_<MMDDHHMM>.json` at shutd
 Each 1-second window records:
 
 | Series | Description |
-|---|---|
+| --- | --- |
 | `windows.duration_s` | Wall time covered by the window |
 | `windows.loop_idle` / `windows.loop_active` | Orchestrator poll-loop iterations with no work vs. active forwarding |
+| `windows.dispatch_queue_size` | Ready-queue size sampled at each monitor window boundary |
+| `windows.dispatch_queue_high_water` | Maximum Event Driven ready-queue size observed in each window |
 | `replicas.<stage,replica>.outputs_queue_size` | MP client `outputs_queue` backlog for that replica |
 | `replicas.<stage,replica>.inflight` | Requests currently bound/routed to the replica |
 
@@ -331,12 +379,16 @@ orchestrator wakes only once per 0.5 s reconcile timeout, while a busy one still
 records one iteration per routed output. Window counts and the `loop_active_pct`
 summary are therefore not comparable across the two modes.
 
+For Event Driven runs, use `dispatch_queue_high_water` rather than only the
+window-boundary `dispatch_queue_size`: a serial dispatcher can drain a burst
+before the window closes, hiding a transient ready-queue backlog in the latter.
+
 ### Relationship to other diagnostics
 
 This monitor is intentionally separate from the existing profiling tools:
 
 | Tool | Scope | Output |
-|---|---|---|
+| --- | --- | --- |
 | `--enable-diffusion-pipeline-profiler` | Diffusion stage functions (`vae.decode`, `diffuse`, …) | Per-request `stage_durations` / logs |
 | `--enable-ar-profiler` | AR stage generation time | Per-request `stage_durations` |
 | `profiler_config` (`torch` / `cuda`) | GPU/CPU kernels inside a stage worker | `trace.json`, Nsight, operator tables |

docs/design/architecture_overview.md

@@ -131,13 +131,13 @@ for the validated batching, attention, and quantization combinations.
 
 ### MiniMax-H3: conditioning, denoising, and decode
 
-MiniMax-H3 illustrates logical component and phase boundaries between text
-encoding, task-specific denoising, and media decoding:
-`Text Encoder → DiT → VAE Decoder`. The current registered topology co-locates
-these components in one diffusion stage. These boundaries are useful for
-profiling and offload; independent placement would require a future
-multi-stage `PipelineConfig`. The stage mapping follows the implementation and
-serving recipe; the [release overview](https://minimaxi.com/blog/minimax-h3)
+MiniMax-H3 combines text and reference-media encoding, task-specific denoising,
+and media decoding. Standard serving co-locates these components in one diffusion
+stage. The opt-in
+[disaggregated topology](https://github.com/vllm-project/vllm-omni/blob/main/recipes/MiniMaxAI/MiniMax-H3-Disaggregated.md)
+runs Qwen and the video/audio VAE encoders in Stage 0, then sends their conditioning
+to a Stage 1 containing the DiTs and VAE decoders. The stage mapping follows the
+implementation and serving recipe; the [release overview](https://minimaxi.com/blog/minimax-h3)
 notes that a detailed technical report is forthcoming.
 
 #### Model architecture by component
@@ -149,16 +149,15 @@ notes that a detailed technical report is forthcoming.
 | **VAE Decoder** | Video VAE and audio VAE decode the generated latents and assemble the synchronized output | Decoded video frames and stereo waveform, with FPS and audio sample rate metadata |
 
 Both DiT partitions remain behind one task-selection point; `task` chooses
-which partition runs. In the current implementation, the three logical
-components execute inside one diffusion stage. This is different from
-Qwen3-Omni's three independently scheduled stages, even though both models
-produce synchronized audio and video-like outputs.
+which partition runs.
 
 #### Representative component execution
 
+The following table describes the standard single-stage deployment.
+
 | Logical component | Batching | Attention and execution | Parallelism | Quantization |
 | --- | --- | --- | --- | --- |
-| **Text Encoder** | The current co-located pipeline follows the diffusion request scheduler; independent prompt/reference batching would require a separate multi-stage topology | Qwen3-VL attention and multimodal preprocessing | `--text-encoder-tp-size N` shards the encoder across the first `N` DiT ranks; independent placement is not part of the current topology | BF16/FP32 baseline; the H3 DiT FP8 path does not quantize the text encoder |
+| **Text Encoder** | Follows the diffusion request scheduler | Qwen3-VL attention and multimodal preprocessing | `--text-encoder-tp-size N` shards the encoder across the first `N` DiT ranks | BF16/FP32 baseline; the H3 DiT FP8 path does not quantize the text encoder |
 | **DiT** | The current H3 implementation executes one generation request per diffusion batch; use concurrency for service-level throughput | cuDNN attention for the validated two-GPU consumer profile; TRTLLM attention or FlashAttention-4 on supported Blackwell profiles | TP2 plus text-encoder TP and Ulysses/VAE parallel groups on multi-GPU profiles; DLO/CPU offload trade memory for transfer time | Online FP8 applies to eligible DiT linears and is incompatible with layerwise offload |
 | **VAE Decoder** | Decode follows each generation request; tile/patch work can be distributed even when denoising is not batched | VAE decode kernels rather than DiT attention | VAE patch parallelism and native tiled decode within the diffusion stage | BF16/FP32 baseline; the documented H3 FP8 path leaves both VAEs unchanged |
 
@@ -324,7 +323,7 @@ inputs, fields, and ownership rules are described below.
 flowchart TB
     layer1["Layer 1 · Authoring inputs<br/>PipelineConfig + DeployConfig"]
     layer2["Layer 2 · Production resolution boundary<br/>resolve_omni_config()<br/>(StageConfigFactory + VllmOmniConfig internally)"]
-    layer3["Layer 3 · Startup hand-off<br/>OmniConfigResolution<br/>PipelineConfig + temporary OmegaConf stage bridge"]
+    layer3["Layer 3 · Startup hand-off<br/>OmniConfigResolution<br/>PipelineConfig + typed stage configs"]
     layer4["Layer 4 · Runtime launch planning<br/>StageRuntime"]
     layer5["Layer 5 · Engine materialization<br/>VllmConfig / OmniDiffusionConfig"]
 
@@ -336,10 +335,10 @@ The production resolution boundary is `resolve_omni_config()` in
 It delegates typed construction to `StageConfigFactory.create_from_model()` and
 `VllmOmniConfig.from_pipeline_config()`, then returns an
 `OmniConfigResolution` consumed by both `AsyncOmniEngine` and headless startup.
-Until `StageRuntime` consumes typed stage configs directly, this envelope carries
-the effective `PipelineConfig` alongside OmegaConf-compatible `stage_configs` as
-a temporary runtime bridge; both views describe the same resolved topology,
-including injected stages.
+This envelope carries the effective `PipelineConfig` alongside typed
+`stage_configs`, which `StageRuntime` consumes directly. Both views describe the
+same resolved topology, including injected stages. Backend arguments are
+projected from these typed stages when each engine is initialized.
 
 The legacy `stage_args` YAML path has been removed. Model topology now resolves
 through `PipelineConfig`, with runtime overrides supplied by `DeployConfig`.
@@ -361,7 +360,7 @@ The important ownership rules are:
 3. CLI and Python overrides are applied at the resolution boundary, with
    per-stage overrides taking precedence over global values where supported.
 4. `OmniConfigResolution` is the sole startup hand-off. Its `pipeline_config`
-   and temporary `stage_configs` compatibility view must describe the same
+   and typed `stage_configs` must describe the same
    topology.
 5. `StageRuntime` owns launch planning and replica lifecycle. `ReplicaInitPlan`
    is runtime-private state, not a user configuration object.

docs/design/feature/attention_backend_selection.md

@@ -76,6 +76,34 @@ preserve common `self` or `cross` policies. A model also decides whether its
 path is compatible with automatic TRTLLM selection, because only the model
 knows whether masking and packing satisfy the kernel contract.
 
+### Model-owned implementation specializations
+
+Use `Attention(..., impl_overrides={backend_name: implementation_class})`
+when a model needs different computation within an existing backend contract.
+The model defines the implementation in its own directory and passes it when
+constructing the layer:
+
+```python
+# MyModelVSAImpl is a model-owned subclass of FastVideoVSAImpl.
+self.attention = Attention(
+    num_heads=num_heads,
+    head_size=head_size,
+    softmax_scale=head_size**-0.5,
+    causal=False,
+    role="self",
+    impl_overrides={"FASTVIDEO_VSA": MyModelVSAImpl},
+)
+```
+
+The override is applied after role/platform selection, keyed by the selected
+backend's `get_name()`. It must subclass the **selected implementation**;
+incompatible platform implementations raise `TypeError`. Unselected overrides
+have no effect. Backend capabilities, constructor options, and shared
+parallel dispatch remain unchanged and must be honored by the specialization.
+
+Use a new backend for a new provider/capability contract. `custom_attention`
+instead owns communication and requires `skip_sequence_parallel=True`.
+
 ## Adding or changing a backend
 
 An implementation change is complete only when it:
@@ -91,8 +119,9 @@ An implementation change is complete only when it:
 
 ## FastVideo VSA model metadata
 
-FastVideo VSA support is scoped to `FastVideo/FastWan2.2-TI2V-5B-Diffusers`, whose text-to-video and image-to-video modes both use `Wan22Pipeline`. It operates on a flattened DiT sequence but partitions tokens in
-the original latent video grid. Wan integrations therefore attach the
+FastWan's text-to-video and image-to-video modes both use `Wan22Pipeline`.
+It operates on a flattened DiT sequence but partitions tokens in the original
+latent video grid. Wan integrations therefore attach the
 post-patch `(T, H, W)` shape as `vsa_dit_seq_shape` attention metadata. The
 backend validates that `T * H * W` equals the sequence length, derives the
 runtime block count, and selects the configured top-k key/value blocks for
@@ -104,3 +133,8 @@ the checkpoint contains those weights. Checkpoints without the projection do
 not allocate or execute it. When top-k selects every block, native checkpoints
 route to SDPA, while FastVideo DMD checkpoints preserve the VSA all-block path
 to retain checkpoint semantics.
+
+MiniMax-H3 supplies `MiniMaxH3VSAImpl` through `impl_overrides`.
+`models/minimax_h3/attention/fastvideo_h3.py` owns its prefix layout, video tiling, and
+learned compression gate. Pooling, block-map construction, and tile64 provider
+calls are shared through `attention/ops/block_sparse.py`.

docs/design/feature/cache_dit.md

@@ -180,17 +180,52 @@ cache_dit.enable_cache(
 
 > **Note:** For single transformer with multiple block lists, `refresh_context` works the same as standard models.
 
+### Example 3: Sequential-CFG Diffusion Pipeline (MammothModa2)
+
+MammothModa2's DiT stage runs on the diffusion runner, which enables the
+configured backend at startup and transfers ownership to the pipeline through
+the request-scoped protocol (`adopt_cache_dit_backend` / `is_cache_dit_enabled`);
+`forward()` then reconciles per-request state through
+`RequestScopedCacheDiTRuntime` (MiniMax H3 precedent).
+
+**Key difference:** The enabler caches only the repeated main-layer stack and
+marks the pipeline as requiring paired CFG forwards, because cache-dit's
+separate-CFG accounting distinguishes cond/uncond passes purely by forward
+parity:
+
+```python
+pipeline._cache_dit_requires_paired_cfg = True
+block_adapter = BlockAdapter(
+    transformer=pipeline.gen_transformer,
+    blocks=[pipeline.gen_transformer.layers],  # Refiners stay outside
+    forward_pattern=[ForwardPattern.Pattern_3],
+    has_separate_cfg=True,
+    check_forward_pattern=True,
+)
+```
+
+Per request, the pipeline calls `runtime.prepare(spec)` for CFG requests
+(refreshing the step context) and `runtime.prepare(None)` for `guidance=1.0`
+requests (disabling hooks — their single forward per step cannot be expressed
+by the parity accounting).
+
 ### Registering Custom Implementations
 
-After writing your custom enabler, register it in `CUSTOM_DIT_ENABLERS` in
+After writing your custom enabler, register it in
+`CUSTOM_DIT_ENABLERS` via `register_custom_dit_enablers()` in
 `vllm_omni/diffusion/cache/cachedit/model_specific.py`:
 
 ```python
-CUSTOM_DIT_ENABLERS = {
-    "Wan22Pipeline": enable_cache_for_wan22,
-    "LongCatImagePipeline": enable_cache_for_longcat_image,
-    "YourCustomPipeline": enable_cache_for_your_model,  # Add here
-}
+CUSTOM_DIT_ENABLERS.update(
+    {
+        "Wan22Pipeline": enable_cache_for_wan22,
+        "Cosmos3OmniDiffusersPipeline": enable_cache_for_cosmos3,
+        "Krea2Pipeline": enable_cache_for_krea2,
+        "Magi2Pipeline": enable_cache_for_magi2,
+        "MammothModa2DiTPipeline": enable_cache_for_mammothmoda2,
+        "YourCustomPipeline": enable_cache_for_your_model,  # Add here
+    }
+)
 ```
 
 ---
@@ -274,6 +309,7 @@ Complete examples in the codebase:
 | **Wan2.2** | `cachedit.model_specific::enable_cache_for_wan22` | Single or dual-transformer | Auto-detects mode based on transformer_2 presence |
 | **LongCat** | `cachedit.config::CacheDiTAdapterConfig` | Declarative block adapter | Two block lists in one transformer |
 | **BAGEL** | `cachedit.model_specific::BagelCachedAdapter` | Custom cached adapter | Complex architecture |
+| **MammothModa2** | `cachedit.model_specific::enable_cache_for_mammothmoda2` | Sub-stack + paired CFG | Generation-runner pipeline; lifecycle owned by `pipeline_mammothmoda2_dit.py` |
 
 ---
 

docs/design/feature/disaggregated_inference.md

@@ -9,6 +9,7 @@ Backend-specific setup lives in separate docs:
 - [MooncakeStoreConnector](omni_connectors/mooncake_store_connector.md)
 - [MooncakeTransferEngineConnector](omni_connectors/mooncake_transfer_engine_connector.md)
 - [MoriTransferEngineConnector](omni_connectors/mori_transfer_engine_connector.md)
+- [NixlConnector](omni_connectors/nixl_connector.md)
 - [YuanrongConnector](omni_connectors/yuanrong_connector.md)
 - [YuanrongTransferEngineConnector](omni_connectors/yuanrong_transfer_engine_connector.md)
 
@@ -25,6 +26,7 @@ Current connectors operate in D2H2D (device to host to device) mode.
 | Multi node (Mooncake Store) | MooncakeStoreConnector | TCP-based, requires Mooncake Master + metadata server. |
 | Multi node (Mooncake RDMA) | MooncakeTransferEngineConnector | RDMA/TCP direct transfer with managed memory pool. Fastest. |
 | Multi node (Mori RDMA) | MoriTransferEngineConnector | RDMA direct transfer via Mori IOEngine. |
+| Multi node (NIXL) | NixlConnector | NIXL `READ` over UCX; the only backend covering Intel XPU. |
 | Multi node (Yuanrong) | YuanrongConnector | Requires Yuanrong Datasystem + etcd. |
 | Ascend NPU P2P (Yuanrong TE) | YuanrongTransferEngineConnector | Uses Yuanrong TransferEngine directly. Configure NPU device IPv4 and `memory_pool_device: "npu"`. |
 

docs/design/feature/host_weight_runtime.md

@@ -42,6 +42,11 @@ V1 does not include:
 
 ## Motivation and use cases
 
+For investigating CPU memory retained by dependency tensor materialization,
+see the [standalone safetensors diagnostic](https://github.com/vllm-project/vllm-omni/blob/main/benchmarks/host_weight_runtime/README.md).
+It distinguishes repeated dependency calls from reuse of cached views and does
+not establish per-request HWR leakage.
+
 Model loading can create the same final host representation repeatedly. This
 is especially expensive when loading performs checkpoint decoding, tensor
 renaming, TP slicing, quantization, packing, or scale construction before GPU
@@ -313,8 +318,15 @@ One process per exact identity owns a build; other workers wait and then acquire
 leases for the published artifact. Publication is invisible until all payloads
 and metadata are validated, hashed, fsynced, and atomically renamed.
 
-`coordination_timeout_seconds` bounds filesystem lock acquisition. It does not
-cancel synchronous validation, a producer that has already started, or atomic
+`coordination_timeout_seconds` bounds domain-initialization and lookup/build
+lock acquisition. Store construction and each later resolution or publication
+operation have separate budgets from the same wait policy, rather than one
+end-to-end startup deadline. A domain-init timeout follows the retryable domain
+failure policy below. After contention ends, a fresh runtime construction can
+retry initialization; the timed-out runtime retains its original failure.
+
+The coordination budget does not cancel filesystem I/O, synchronous validation,
+a producer that has already started, or atomic
 publication. A hung in-process producer therefore blocks its owning process and
 must be handled by external process supervision. Enforceable producer
 cancellation requires a future process-isolated producer contract.

docs/design/feature/model_local_kv_caches.md

@@ -0,0 +1,132 @@
+# Model-local KV caches
+
+Some models keep their attention KV as HuggingFace `transformers` cache objects
+instead of using the engine's paged KV manager. Each such model declares what it
+holds by implementing `model_local_kv_specs()`, and
+`OmniGPUModelRunner.load_model` sums and logs the result after load.
+
+## What is and is not already accounted for
+
+This distinction decides what the protocol is for, and an earlier revision of
+this page got it wrong by claiming none of this memory was accounted for.
+
+CUDA-graph capture happens *during* `load_model` -- MiMo in its constructor,
+Qwen3-TTS in `load_weights`. `determine_available_memory()` then samples
+per-process GPU memory through NVML and subtracts it from the requested budget
+(`vllm_omni/worker/base.py`). So the **graph-resident** portion is already
+charged. Subtracting the declared total from the KV budget would double-count
+it.
+
+What is *not* charged is the **per-request** portion, which is allocated after
+profiling. That half is not small: Qwen3-TTS's codec decoder is 4.44 MiB per
+request, so a server at `max_num_seqs=256` reaches roughly 1.1 GiB that no
+per-stage number predicted.
+
+The consumer therefore reports and does not subtract.
+
+## What the four models declare
+
+| Model | Cache | Bounded by | Per row | Rows |
+|---|---|---|---|---|
+| Qwen3-TTS codec decoder (async-chunk only) | sliding `DynamicCache` | `sliding_window - 1 = 71` | 4.44 MiB | `max_num_seqs`, x2 for the working copy |
+| Qwen3-TTS graph pool | retained captures | same | 4.44 MiB | fixed, from captured shapes |
+| MiMo-Audio local transformer | `DynamicCache` | `group_size + max(delay_pattern) = 11` | 704 KiB | `max_num_seqs` |
+| MiMo-Audio graph pool | captured | same | 704 KiB | fixed 261 (bucket sum) |
+| MiniCPM-o Whisper encoder (streaming only) | `EncoderDecoderCache` | `embed_positions` rows = 1500 | 140.62 MiB | fixed 1, per session |
+| ming_flash_omni talker | `StaticCache` | hardcoded `max_cache_len = 2048` | 24.00 MiB | fixed 1 |
+
+**No cache is bounded by `max_model_len`.** Each has its own mechanism: a
+sliding window, a decode-loop trip count, an encoder frame limit, a constant.
+
+**ming's geometry does not exist in this repo.** `self._llm_config` is a
+`Qwen2Config` from the checkpoint, so layers, kv-heads, head-dim and dtype are
+only knowable after load. That is what makes this a runtime query rather than a
+table.
+
+## Lifetime is not multiplicity
+
+`scope` records when memory appears and goes. It does **not** decide how wide
+the allocation is, and an earlier revision that derived scaling from it was
+wrong in two different directions at once:
+
+- ming's talker is per-call but serialized -- `forward` takes
+  `runtime_additional_information[0]` and runs the whole AR loop inline -- so
+  scaling it by `max_num_seqs` over-reported it by that factor.
+- MiniCPM-o's encoder is per *session*, not per sequence, so `max_num_seqs` is
+  simply the wrong number for it.
+
+So width is declared explicitly, by naming the driver. `RowDriver` has two
+values, because two are all the known caches need:
+
+- `FIXED` -- a count the model controls, with `rows_fixed` and a required
+  `rows_reason`. `rows_fixed=1` on a cache that looks per-request is a claim a
+  reviewer must be able to check.
+- `MAX_NUM_SEQS` -- one row per in-flight sequence; only the engine knows the
+  value.
+
+A third value belongs here when a model actually has a cache the engine widens
+by some other number. An earlier revision added one speculatively, for a single
+declarer, and paid for it with a class hierarchy, an engine-capacity object and
+a conditional-annotation field.
+
+`rows` counts row-equivalents at peak and deliberately does not distinguish
+"one allocation N rows wide" from "N allocations of one row". Those cost the
+same, and a revision that tried to carry both got the object topology wrong in
+three of four declarers: MiniCPM-o keeps one cache object per session while
+asserting batch size 1, and both graph pools keep one distinct object per
+captured bucket. The layout belongs in `allocation_note`, where it cannot be
+mistaken for arithmetic.
+
+## Declaring nothing is a valid declaration
+
+A declaration must be inert when its path is. Qwen3-TTS holds KV only in
+async-chunk mode -- stateless decoding runs `_forward_exact`, which calls the
+transformer without `use_cache` -- so the stateless path declares nothing.
+Getting this wrong is easy in both directions, and both have happened here: one
+revision hung the declaration off the CUDA-graph wrapper and reported zero under
+`enforce_eager`, where the eager async-chunk path does allocate; the next
+declared unconditionally and invented roughly a gigabyte for stateless
+deployments.
+
+Where the model cannot see the deciding setting, it declares the honest unit
+and says so in `allocation_note`. MiniCPM-o's encoder cache exists only on the
+streaming path and is one object per session; the session cap is not visible
+from the model, so it declares one session rather than scaling by a number it
+would have to guess at.
+
+## Finding the declarers
+
+`collect_model_local_kv_specs()` walks `named_modules()`. Two wrinkles:
+
+- The runner's `self.model` may be a `CUDAGraphWrapper`, which is not an
+  `nn.Module` but forwards attribute access to its runnable. The walk therefore
+  works either way, but entering through both the wrapper and its runnable would
+  count a root-level declarer twice. The collector calls the supported
+  `unwrap()` accessor first.
+- An owner that is not a submodule at all (Qwen3-TTS's graph wrapper, ming's
+  `MingAudioGenerator`) is reached by one explicit hop from the module that
+  holds it.
+
+## Adding a model
+
+Return one entry per distinct allocation. Prefer `spec_from_hf_config()` and
+pass the same config object the cache is built from; it raises rather than
+guessing when an attribute is missing. `capacity_source` is diagnostic text --
+never branch on it.
+
+## Known limits
+
+Row counts taken from live state after warmup miss caches `transformers` has
+not yet materialized, which makes Qwen3-TTS's graph-pool figure a floor for the
+xvec path.
+
+MiMo's graph-pool cache has no reachable Python object -- it is a local inside
+the captured frame -- so its width comes from the captured bucket list rather
+than from tensors. Its two entries are also mutually exclusive per call: a call
+either replays a bucket or runs eagerly. The sum is the right ceiling for the
+process and the wrong number for a single request.
+
+MiMo captures every bucket gated only on `torch.cuda.is_available()`
+(`mimo_audio_llm.py:670`), so those 179.44 MiB stay resident even when the
+deploy config sets `enforce_eager: true`. Gating it is a one-line change that
+needs a model boot to verify, so it is noted rather than bundled here.

docs/design/feature/mooncake-kv-transfer.md

@@ -0,0 +1,89 @@
+# Mooncake KV Transfer
+
+## PR summary
+
+This change adds an opt-in AR-to-DiT KV handoff using vLLM's native
+`KVTransferConfig`, Scheduler-owned diffusion KV pages, and
+`MooncakeConnector`. The existing `OmniKVTransferManager` path remains the
+fallback when `kv_transfer_config` is absent.
+
+The orchestrator creates one transfer ticket per request and resolves the
+bound AR replica's Mooncake endpoint after generation. DiT allocates the
+destination pages, waits for every worker rank to finish loading, and releases
+the source and destination pages only after the connector lifecycle completes.
+Request registration failures are rolled back before Worker dispatch. Receive
+timeouts and cancellation terminate the request without recycling in-flight
+pages: the scheduler retains them until every CFG row has completed on every
+rank. Late completion is polled only while such requests remain. If a peer
+never completes, its pages remain reserved until shutdown; other requests can
+continue using the remaining capacity. Invalid pages, missing Worker ranks,
+and failures without a proven rollback still stop the replica before teardown.
+The example sets `transfer_timeout: 60.0` seconds explicitly. Cancellation
+queued during a synchronous receive is processed when that receive completes
+or times out; it does not interrupt the remote write.
+
+## Reference configuration
+
+| Stage | Replicas | Devices | Parallelism | KV cache |
+| --- | ---: | --- | --- | ---: |
+| AR | 2 | 0-3 | TP=2 per replica | 2 GiB per worker |
+| DiT | 2 | 4-7 | TP=2 per replica, EP enabled | 1 GiB per worker |
+
+The acceptance workload uses 512x512 images, two denoising steps, guidance
+scale 2.0, and `max_num_seqs=1`.
+
+For a four-GPU debug run, set both stages to one replica and assign AR TP=2 to
+devices `0,1` and DiT TP=2 to devices `2,3`.
+
+## Experiment
+
+The final comparison used one node with eight NVIDIA RTX PRO 6000 Blackwell
+Server Edition GPUs, TCP Mooncake transport, the same 50-prompt/seed T2I
+dataset, concurrency 8, and two independent 100-request runs per
+implementation. Each side completed 200/200 requests without an OOM or request
+failure.
+
+| Implementation | Run 1 (req/s) | Run 2 (req/s) | Mean (req/s) |
+| --- | ---: | ---: | ---: |
+| `OmniKVTransferManager` | 4.4103 | 4.4497 | 4.4300 |
+| Native `MooncakeConnector` | 4.4457 | 4.4699 | 4.4578 |
+
+Native throughput is **0.63% higher** than the legacy mean. The absolute
+performance difference is below the 1% acceptance threshold.
+
+For accuracy, a non-zero transferred prefix and the resulting attention output
+were compared bit-for-bit with the legacy manager path:
+
+```text
+prefix_bits_equal    True
+attention_bits_equal True
+max_abs_diff         0.0
+```
+
+The one-AR/one-DiT TP=2 debug topology also completed 20/20 sequential
+requests. A separate real-TCP transport check transferred both CFG rows from a
+TP=1 producer to a strict-Ulysses SP=2 consumer with exact values on both
+target ranks.
+
+## Scope and limitations
+
+- The fully validated end-to-end topology is 2x AR TP=2 plus 2x DiT TP=2.
+- Heterogeneous TP=1 to SP=2 reshaping is transport-tested, but arbitrary mixed
+  TP/CFG/SP end-to-end combinations have not been exhaustively validated.
+- CFG-parallel end-to-end deployment is not part of this acceptance run.
+- Native AR-to-DiT transfer currently requires `async_chunk: false`.
+- TCP is covered by the acceptance run. The supplementary single-node RDMA
+  measurements use a separate short workload; they do not establish RDMA
+  support for multi-node deployment, which remains out of scope.
+- The native path currently validates the two-stage `AR(0) -> DiT(1)` graph. Other stage graphs are rejected during startup until
+  their producer/consumer edge mapping is implemented.
+- Sleep mode is disabled for the native paged path because registered KV pages
+  must remain mapped for the connector lifetime.
+
+- Ascend NPU adaptation and its performance validation are maintained in a
+  separate follow-up PR; the GPU acceptance results above do not validate NPU.
+- The homogeneous piecewise CUDA batching optimization is retained. A native
+  paged-attention CUDA graph capture/replay regression covers two CFG rows,
+  strided QKV, mixed causal/image spans, changed inputs, and output stability.
+  It must be run on CUDA hardware; CPU unit tests do not establish graph or
+  end-to-end performance acceptance.

docs/design/feature/offloader/README.md

@@ -41,7 +41,8 @@ Every backend implements `OffloadBackend`:
 - `is_enabled()` reports lifecycle state.
 
 `disable()` does not promise to restore the pipeline's original device
-placement. The caller owns any subsequent rematerialization.
+placement. It must restore usable parameter and buffer storage before releasing
+backend-owned host weights. The caller owns subsequent device placement.
 
 An enable failure must remove every partially installed hook. Rank-local
 backends restore ordinary tensors so the pipeline can be retried. A failed
@@ -54,9 +55,19 @@ must use distinct hook names and remove only hooks they own.
 ## Discovery and topology
 
 `resolve_offload_plan(pipeline, config)` is the single topology entry point.
-Every backend calls it once when `enable()` starts and then consumes the
-resolved components, block stacks, resident heads, and staged-residency flags
-it returns. Backends do not re-read model declarations.
+The generic model-level and ordinary layerwise paths call it once when
+`enable()` starts and consume its component selection and block stacks.
+Ordinary layerwise offload also respects a resolved `skip_reason`, preserving
+the legacy missing-DiT no-op without placing auxiliary components. Non-block
+DiT and encoder state is placed by resolved block tensor identity, so the
+backend does not rediscover block containers by attribute name.
+
+Two compatibility paths remain outside this completed generic cutover:
+model-level pipelines implementing `SupportsModelCpuOffload` still own their
+custom phase lifecycle (RFC #6648 J4), and distributed layerwise offload still
+reads declarations for nested-submodule staging and loader-owned mmap paths
+(J3). The custom model-level protocol is delegated to before generic plan
+resolution; it must not be subjected to the generic encoder/DiT swap checks.
 
 Pipeline component discovery prefers `SupportsComponentDiscovery` declarations:
 
@@ -73,10 +84,22 @@ Block topology comes from the pipeline `OffloadPlan` first: `block_attrs` maps
 each DiT path to its ordered block containers, `encoder_block_attrs` declares
 streamable encoder stacks, `on_demand_component_paths` marks pipeline-managed
 residency, `resident_dit_paths` marks the DiTs that may hold resident layers,
-and `encoder_dlo_weight_replication` marks the encoders whose loader-produced
-weights are safe for AllGather. DiTs absent from the plan fall back to
-`_layerwise_offload_blocks_attrs` (including the deprecated singular-name
-compatibility path).
+`encoder_dlo_weight_replication` marks the encoders whose loader-produced
+weights are safe for AllGather, and `offload_submodules` maps a DiT submodule
+that owns its residency to its own block container. DiTs absent from the plan
+fall back to `_layerwise_offload_blocks_attrs` (including the deprecated
+singular-name compatibility path).
+
+An undeclared DiT submodule joins the plan only when it is large enough that
+keeping it resident would defeat streaming; its block container is then found
+by scanning well-known attribute names, which warns once per submodule class.
+Size is computed from parameter shapes and dtypes, including meta parameters,
+so resolving before mmap loading preserves the same residency decision.
+Block ownership validation includes nested components: a block shared with a
+parent, sibling, or encoder is rejected before any hooks or storage are changed.
+A submodule with no block container must implement the `load_to_device` /
+`offload_to_cpu` lifecycle, and the resolver rejects it before any component of
+the pipeline is placed or hooked.
 
 The resolver is pure: it moves no tensor, installs no hook, writes no module
 attribute, and reads no process group — multi-rank facts come from
@@ -102,3 +125,17 @@ remains responsible for transfer, synchronization, and storage ownership.
 
 The diffusion [Offloader module design](../../module/diffusion/offloader.md)
 describes how these feature contracts fit into the larger diffusion runtime.
+
+## Backend acceptance tests
+
+`tests/diffusion/offloader/test_backend_plan_contract.py` covers the two
+behaviors the generic cutover changed. A DiT attribute aliasing a streamed
+block must keep the ring's host residency, so placement cannot follow attribute
+names; the CUDA case fails against the previous name-based placement. Both
+backends must then run with their selector helpers and the model declaration
+made unreadable, proving execution consumes the resolved plan alone.
+
+Selection errors, rollback, residency, and enable/disable cycles stay in
+`test_plan_resolver.py`, `test_layerwise_backend.py` and
+`test_sequential_backend.py`. Model-specific phases and executed tied aliases
+remain the J4 contract suite.

docs/design/feature/omni_async_output_materialization.md

@@ -318,18 +318,18 @@ The same behavior applies to the Base and VoiceDesign Qwen3-TTS checkpoints
 because they use the same Talker implementation.
 
 !!! warning
-    Do not pass `--no-async-chunk` or enable prefix caching when you want this
-    optimization. Either change causes the runner to fall back to synchronous
-    output construction. Enabling async chunk alone cannot activate the feature
-    for a model that has not opted into async Omni output.
+    Do not pass `--no-async-chunk` when you want this optimization. Disabling
+    async chunk causes the runner to fall back to synchronous output
+    construction. Enabling async chunk alone cannot activate the feature for a
+    model that has not opted into async Omni output.
 
 !!! note "Prefix cache compatibility"
-    Async Omni output materialization and Omni prefix caching cannot currently
-    run together. `_should_use_async_omni_output()` returns `False` whenever
-    `self.omni_prefix_cache` is present, so prefix caching selects synchronous
-    output materialization. Supporting both features requires snapshotting or
-    otherwise synchronizing prefix-cache merge and update state before the
-    background output path can consume it safely.
+    Prefix cache and async Omni output can run together. `save_outputs` returns
+    a `step_id`; the output builder later calls `materialize(sid, req_ids)` with
+    the save-time request list, which may be a step late relative to the live
+    `input_batch`. `_should_use_async_omni_output()` does not disable itself
+    when the cache is present. See
+    [Automatic Prefix Caching](prefix_caching.md#implementation).
 
 ## Compatibility and Fallbacks
 
@@ -341,10 +341,9 @@ runtime conditions hold:
 | AR async scheduling is enabled | The optimization relies on the scheduler advancing while the prior output is materialized |
 | `async_chunk` is enabled | The feature targets incremental downstream Omni payloads |
 | The model stage opts in with `use_async_omni_output` | Models must declare that their output lifecycle is safe to defer |
-| Omni prefix cache is disabled | Prefix-cache merge and update ordering currently requires synchronous materialization |
 | Speculative decoding is disabled | Speculative output state is not included in this deferred path |
 | Routed-expert output is disabled | Routed-expert extraction currently requires the synchronous path |
-| Postprocess is absent or explicitly runs eagerly | State needed by the next decode step must be updated before the runner returns |
+| Postprocess is absent or explicitly runs eagerly | State needed by the next decode step must be updated before the runner returns. Eager postprocess sees the scheduled slice (`None, None` combined tensors), not the prefix-cache-merged full prompt; it must only consume the tail |
 
 These checks are evaluated per stage on runners based on `GPUARModelRunner`.
 An unsupported combination does not prevent serving; it only falls back to
@@ -380,6 +379,8 @@ so Omni payload materialization remains synchronous.
 - `tests/worker/test_gpu_ar_model_runner.py`: Snapshot, guard, connector
   ordering, and background error-propagation tests.
 - [Async Chunk](async_chunk.md): Inter-stage chunking and scheduling design.
+- [Automatic Prefix Caching](prefix_caching.md#implementation): Prefix-cache
+  `step_id` / `materialize` contract used by the deferred output builder.
 - [Qwen3-Omni optimization blog](https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization):
   Optimization context and controlled performance results.
 - [PR #4476](https://github.com/vllm-project/vllm-omni/pull/4476): Feature

docs/design/feature/omni_connectors/nixl_connector.md

@@ -0,0 +1,135 @@
+# NixlConnector
+
+## When to Use
+
+Multi-node or intra-node stage transfer over NIXL, which brokers RDMA/shared-memory
+transports through a single API. Useful when the deployment already standardises on
+NIXL for KV transfer, or when the target accelerator has no Mooncake/Mori backend --
+NIXL reaches Intel XPU through UCX's Level Zero support.
+
+## Mechanism
+
+Uses vLLM's `NixlWrapper` (`vllm.distributed.nixl_utils`) to register the producer's
+tensors and let the consumer pull them with a NIXL `READ`.
+
+- Data Plane: NIXL agent-to-agent `READ`, GPU-to-GPU where the backend allows it.
+- Control Plane: a ZMQ ROUTER socket atomically claims the source allocation before
+  returning descriptors. Forwarded `put()` metadata includes the generation and
+  endpoint; the consumer claims that generation before submitting any READ.
+
+Transfer metadata uses schema version 1. Tensor descriptors are grouped by NIXL
+memory type (`DRAM`, `VRAM`, and so on), while each descriptor retains its global
+tensor index so mixed CPU/accelerator payloads can be reconstructed in order.
+
+Payloads are not restricted to tensors. A single tensor or a list of tensors is
+transferred as-is; a nested structure has its tensor leaves extracted and shipped
+alongside a msgpack-encoded skeleton; anything else is msgpack-encoded into one
+uint8 tensor. Mapping and sequence structure is preserved. `msgspec.Struct` values,
+including `OmniPayloadStruct`, are normalised to mappings so their tensor leaves use
+the direct NIXL path rather than CPU object serialisation.
+
+Each NIXL agent uses strict thread synchronization. A consumer performs one short
+metadata query per scheduler poll (10ms by default), which prevents an unavailable
+producer from blocking unrelated requests. The regular transfer-completion handshake
+keeps the configured transfer timeout.
+
+## Installation
+
+CUDA hosts can install the published wheel:
+
+```bash
+pip install nixl
+```
+
+For Intel XPU, use `docker/Dockerfile.xpu`. Its upstream vLLM XPU base image
+provides a matching torch-xpu, UCX with Level Zero support, and NIXL runtime.
+
+## Configuration
+
+```yaml
+connectors:
+  nixl_connector:
+    name: NixlConnector
+    extra:
+      host: "auto"
+      zmq_port: 50061
+      backends: ["UCX"]
+
+stages:
+  - stage_id: 0
+    output_connectors:
+      to_stage_1: nixl_connector
+
+  - stage_id: 1
+    input_connectors:
+      from_stage_0: nixl_connector
+```
+
+Parameters:
+
+- `host`: address the producer binds its handshake socket to (`"auto"` to detect).
+- `zmq_port`: handshake port. If a producer omits it, an ephemeral port is opened
+  for ownership claims and returned in `put()` metadata. The value in the deploy YAML is a
+  base port. Purpose, replica, rank, and producer-stage offsets are added centrally,
+  so colocated connectors and tensor-parallel workers receive distinct endpoints
+  without one config entry per rank.
+- `sender_host` / `sender_zmq_port`: consumer-side override naming the producer's
+  handshake endpoint. Only needed when the consumer cannot learn it from metadata.
+  In replicated deployments, the producer endpoint is carried with each request and
+  selected at `get()` time; it is not mutable connector-wide state.
+  Explicit sender ports are preserved, not recomputed using the receiver replica.
+  Intermediate NIXL stages retain their incoming receiver configuration and a
+  separate outgoing bind configuration, so one worker can receive and publish.
+- `backends`: NIXL backends to register memory with. Defaults to `["UCX"]`.
+- `receive_device`: forces where received tensors land. By default the consumer
+  keeps the producer's device *type* but uses its own current device of that type.
+- `memory_type`: overrides the NIXL memory type, otherwise `DRAM` for CPU tensors
+  and `VRAM` for accelerator tensors.
+- `lease_seconds`: how long a `put()` payload stays registered while waiting to be
+  read (default 3600). The consumer reports completion, so this only bounds payloads
+  nobody has claimed. An internal reaper releases only unclaimed registrations.
+  `VLLM_OMNI_NIXL_LEASE_S` overrides it.
+- `transfer_timeout_s`: how long a `get()` waits for its `READ` to complete
+  (default 300). NIXL 1.3 has no transfer cancellation API, so a timed-out transfer's
+  buffers and registrations remain owned by the connector until NIXL reports a
+  terminal state; `close()` waits for that state before releasing them.
+  `VLLM_OMNI_NIXL_XFER_TIMEOUT_S` overrides it.
+
+### Source ownership and failure limits
+
+Claims and expiry use the same lock. Completion identifies both a payload
+generation and a unique READ claim; duplicate or stale ACKs cannot release a
+different reader or replacement payload. Replacing a claimed key is rejected.
+Terminal errors also release claims; timeout, unknown status, and active sibling
+transfers retain the complete destination bundle and source claim until terminal.
+Zero-element tensors retain their shape/dtype/skeleton positions but never create
+DMA descriptors or registrations.
+
+A lost metadata response, lost completion ACK, or abandoned consumer can retain a
+claim indefinitely. NIXL 1.3 cannot prove remote cancellation, so neither TTL nor
+`cleanup()` frees those allocations. Producer `close()` rejects new work and
+retains its agent, listener and claimed allocations; a later `close()` finishes
+teardown after claims drain. Permanently abandoned claims remain until process
+exit. This favors memory safety over bounded shutdown/memory usage. Only trusted
+peers may access this unauthenticated control plane. Both endpoints must use the
+claim-aware protocol; rolling compatibility with older GET_META clients is not
+provided. Legacy externally-owned metadata without a generation is accepted only
+under the caller's lifetime guarantee, not managed as a leased allocation here.
+
+## Validation
+
+The native CUDA smoke test requires NIXL and at least two visible CUDA devices:
+
+```bash
+UCX_RCACHE_MAX_UNRELEASED=1024 pytest -o addopts='' -q -s \
+  tests/distributed/omni_connectors/test_nixl_connector_native.py
+```
+
+It starts independent producer and consumer processes, loads the real UCX backend,
+performs the ZMQ `GET_META` handshake and a native NIXL `READ` from GPU 0 to GPU 1,
+checks mixed CPU/GPU tensor leaves from an `OmniPayloadStruct`, sends `XFER_DONE`,
+and verifies that producer registrations are released. This validates same-host,
+cross-GPU operation. Inter-node UCX transport is supported by the design but was not
+validated by this test.
+
+For more details, refer to the [NIXL repository](https://github.com/ai-dynamo/nixl).

docs/design/feature/prefix_caching.md

@@ -1,6 +1,5 @@
 # Automatic Prefix Caching in Omni Models
 
-
 ---
 
 ## Table of Contents
@@ -9,6 +8,8 @@
 - [High-Level Approach](#high-level-approach)
 - [Example](#example)
 - [What About Multimodal Inputs?](#what-about-multimodal-inputs)
+- [Implementation](#implementation)
+- [Related Files](#related-files)
 
 ---
 
@@ -27,6 +28,7 @@ vLLM implements automatic prefix caching for managing its kv-cache, which is bes
     This document describes vLLM-Omni's mechanism for caching tensor outputs that are meant to be passed between stages, when requests have common prefixes, similar to the way in which vLLM has prefix caching for the kv-cache. This works in conjunction with vLLM's multimodal encoder caching, but is distinct. See the final section for a concrete example for how they tie together in practice.
 
 ### High-Level Approach
+
 !!! note "Note 2"
     Prior to reading this section, it's recommended to take a look at the design documents in vLLM for [Automatic Prefix Caching](https://docs.vllm.ai/en/latest/features/automatic_prefix_caching/), which will make some of the concepts more clear.
 
@@ -38,8 +40,10 @@ The main focus of vLLM-Omni's approach to prefix caching stage outputs is to bui
 
 With this in mind, consider the set of blocks in a 2D layout, where the row represents the index of blocks being considered, and the columns represent the slots corresponding to tokens within each block. Since we know the `num_blocks` and `block_size` from our kv cache config, if we want to cache a tensor with feature size `D`, we can preallocate a CPU tensor of size `(num_blocks, block_size, D)`, and use the same block index and slot mapping to retrieve the corresponding feature vector.
 
+Host footprint: each cached key costs `num_blocks × block_size × D × dtype_bytes` of **pinned** CPU memory (pinned so device→host can overlap compute), allocated on the first `save_outputs` that sees the key — the first real request pays the `cudaHostAlloc`. Measured on a Qwen3-Omni deployment: thinker `__hidden_states__` `[15092, 16, 2048]` bf16 ≈ 0.92 GiB (plus the same again for each `hidden_states.layer_*` key a model exposes), Qwen3-TTS talker `codes.audio` `[16180, 16, 16]` ≈ 33 MiB. Budget host RAM for the stage accordingly.
 
 ### Example
+
 !!! note "Note 3"
     Prefix caching in vLLM-Omni currently is only supported on AutoRegressive stages with one kv-cache group. Configure it with the pipeline-wide `enable_prefix_caching` field in the deploy config.
 
@@ -56,7 +60,7 @@ The prefix cache flow is then outlined below.
 
 2. Say we process the request `The quick brown fox was tired and slept beneath the shady tree`, which is 12 tokens and evenly divides into 3 blocks as shown below.
 
-```
+```text
          [  The quick brown fox  ] [  was tired and slept ] [beneath the shady tree ]
 Block 1: |<--- block tokens ---->|
 Block 2: |<------- prefix ------>| |<--- block tokens --->|
@@ -65,10 +69,9 @@ Block 3: |<------------------ prefix -------------------->| |<--- block tokens -
 
 When the request processes, we inspect the multimodal outputs and identify the `mm_feature` tensor, which will be of shape `(seq_len, feature_dim)`, i.e., `(12, 16)` in this example. We note that the first axis is dependent on the `seq_len` and add a new cache_tensor of shape `(num_blocks, block_size, feature_dim)` to our multimodal cache for tensors.
 
+1. If we lay out the cache as a 2D tensor of shape (`num_blocks`, `block_size`), we'll have something like the following:
 
-3. If we lay out the cache as a 2D tensor of shape (`num_blocks`, `block_size`), we'll have something like the following:
-
-```
+```text
 0: [  The quick brown fox  ]
 1: [  was tired and slept  ]
 2: [beneath the shady tree ]
@@ -78,7 +81,8 @@ When the request processes, we inspect the multimodal outputs and identify the `
 ```
 
 Or, if we flatten it down to 1D,
-```
+
+```text
 0: The
 1: quick
 2: brown
@@ -90,7 +94,8 @@ Or, if we flatten it down to 1D,
 ```
 
 which we can think of as row indices into the hidden states tensor if we view it as the 2D shape `(num_blocks x block_size, feature_dim)`. That is, the analogous flattened (from 3D -> 2D) mapping of the cache for hidden states becomes the following.
-```
+
+```text
 0: <hidden states vector of len 2 corresponding to 'The'>
 1: <hidden states vector of len 2 corresponding to 'quick'>
 2: <hidden states vector of len 2 corresponding to 'brown'>
@@ -103,10 +108,9 @@ which we can think of as row indices into the hidden states tensor if we view it
 
 Similarly, for the multimodal outputs cache, the flattened coordinates are the same, but the `mm_feature` maps to vectors of length `16` instead of the hidden size of `2`. Note that in practice, we may have multiple  multimodal output tensors per forward pass, which may have different names and different feature dimensions.
 
+1. Now, say that we receive a new request `The quick brown fox jumped over the dog`.
 
-4. Now, say that we receive a new request `The quick brown fox jumped over the dog`.
-
-```
+```text
          [  The quick brown fox  ] [  jumped over the dog ]
 Block 1: |<--- block tokens ---->|
 Block 2: |<------- prefix ------>| |<--- block tokens --->|
@@ -116,7 +120,7 @@ Here, we will have a cache hit for `Block 1` which will be detected by vLLM base
 
 Since we have the block indices / slot mappings from the kv cache manager, we can simply mirror the mappings and leverage the same indices for the cached hidden states and multimodal outputs. This allows us to look up the correct tensors from our externally maintained 3D caches.
 
-```
+```text
 0: [  The quick brown fox  ] < already in the cache
 1: [  was tired and slept  ]
 2: [beneath the shady tree ]
@@ -129,11 +133,11 @@ Since we have the block indices / slot mappings from the kv cache manager, we ca
 
 Finally, to pass the full hidden states and multimodal outputs to the next stage, we simply concatenate the cached contents with the corresponding new tensors computed from the current forward call.
 
-
 ### What About Multimodal Inputs?
+
 It's also useful to consider the case about how Omni prefix caching is handled when we have multimodal inputs that don't cleanly end on block boundaries, as well as how this works with multimodal encoder caching in vLLM. For example:
 
-```
+```text
          [   Im0  Im1  Im2  Im3  ] [ Im4  Im5 foo <empty> ]
 Block 1: |<--- block tokens ---->|
 Block 2: |<------- prefix ------>| |<--- block tokens --->|
@@ -147,9 +151,9 @@ In reality, this isn't a big problem for correctness, because vLLM also maintain
 - The hash describing the image data starting at position 0 and with length 6
 - In vLLM's encoder cache, a mapping from the image hash above to the encoder output
 
-
 To understand what happens, say we get the following input as a second request:
-```
+
+```text
          [   Im0  Im1  Im2  Im3  ] [  Im4  Im5 bar  baz  ]
 Block 1: |<--- block tokens ---->|
 Block 2: |<------- prefix ------>| |<--- block tokens --->|
@@ -162,3 +166,162 @@ Because we have multimodal data in a scheduled span that isn't fully precomputed
 When we pass our multimodal tensors to the language model component in the same stage, we'll then expect the same outputs, because the prefix caching behaviors in vLLM-Omni / vLLM match, so the LLM will use vLLM's KV cache manager's prefix caching to correctly handle the attention information for `Block 1` while calculating the outputs for `Block 2`, giving us the correct results for processing `Block 2` with the context of `Block 1`.
 
 Finally, we look up the output hidden states/multimodal tensors corresponding to the prefix cache hit `Block 1` and concatenate it with the forward pass result to get the final result, which is expected to be identical to the full hidden states when prefix caching is disabled.
+
+### Implementation
+
+The block/slot model is `vllm_omni/core/prefix_cache/`.
+`OmniPrefixCacheManager` owns slot occupancy, the request-task table,
+hit spans, and merge.
+`OmniPrefixCacheController` moves data: a reusable `StagingBufferPool` for
+this step's device→host copy, and writes into the durable `PrefixBlockPool`.
+The state lock covers those tables only.
+
+Miss is not an error (this step's forward slice only). A hit span whose
+hidden rows are absent is fatal. For mm keys the rule is looser: a model may
+emit a key only for some requests, so a hit span with no rows behind an mm key
+reads zeros from the pool for those positions rather than raising. Abort still
+writes: once a hash entered this step's batch it must land in the cache.
+
+`enable_prefix_caching` is refused on KV-consumer stages (`kv_role` of
+`kv_consumer` or `kv_both`). KV received from a producer is reported as
+`num_computed_tokens` too, and the manager cannot tell it from a local hit.
+Producer-only stages are unaffected. Pooling stages never save, so they get
+no cache. This gate is one function, `stage_prefix_cache_config`, called by
+both the GPU and the NPU model runner at kv-cache init.
+
+Which stages may set `enable_prefix_caching: true`:
+
+| Stage | `enable_prefix_caching` | Why |
+| --- | --- | --- |
+| AR stage with one full-attention kv group whose hidden states / per-token mm feed the next stage (Qwen3-Omni thinker and talker) | supported | The case the cache is built for: full-prompt hidden states are merged from the pool on a hit. |
+| AR stage that sets `requires_full_prefix_cached_hidden_states = False`, optionally with `deferred_prefix_cache_mm_keys` (Qwen3-TTS talker, Higgs v3 talker) | supported | Hidden is not cached; deferred codec rows are written once on finish. |
+| Pooling stage | ignored | Never saves; the gate returns no config. |
+| `kv_role: kv_consumer` / `kv_both` | refused at kv-cache init (`OmniPrefixCacheUnmatchError`) | Producer KV shows up as `num_computed_tokens` and is indistinguishable from a local hit. |
+| Speculative decoding on the stage | refused at kv-cache init | Under async scheduling vLLM keeps `num_computed_tokens_cpu` optimistic (all drafts accepted) during the forward and corrects it afterwards; `step_slots_cpu` would mirror rows at the wrong slots. Not verified; refused as a whole. |
+| `prefix_match_unit` smaller than `block_size` | refused at kv-cache init | Sub-block hits make `num_computed_tokens` unaligned; the hit registry only mirrors whole blocks. |
+| Hybrid / sliding-window / multi-group kv cache (e.g. a talker with `attention_type: sliding_recompute`) | refused at kv-cache init | The cache mirrors exactly one full-attention block table. |
+| Attention backend whose kernel block size differs from `--block-size` (FlashInfer / FlashMLA / CutlassMLA with a block size they do not list natively), or decode context parallel | refused at first step (`FullAttentionGroupView`) | `step_slots_cpu` computes `table[req, pos // block_size] * block_size + pos % block_size` over allocator block ids; hybrid kernel blocks and DCP token striping change that row layout. FlashAttention / Triton accept any multiple of 16 and never split blocks. |
+| Codec decoder / Code2Wav stages (Qwen3-Omni stage 2, Qwen3-TTS stage 1) | keep `false` | Nothing downstream consumes their hidden states; the cache would only add device→host copies. Not validated. |
+| Diffusion stages | n/a | No vLLM KV cache to mirror. |
+
+Hit spans come from `scheduled_new_reqs` only, as in the pre-refactor cache:
+
+- A new request with a (partial) prefix hit is the normal path: the hit
+  blocks are read from the pool, the rest is this step's rows, and the
+  divergent tail is written under its own blocks. The gather starts on the
+  prefetch thread at `new_step_starts`; a same-step hit (vLLM hashes blocks
+  at schedule time, so `b` can hit blocks `a` computes in the same forward)
+  cannot plan until `a`'s write is registered and starts at `save_outputs`
+  instead. Either way it runs before the next step can hand those blocks
+  to a new tenant. vLLM frees blocks one step before `finished_req_ids`
+  arrives, so a finished request whose last step materializes late may
+  still find a hit slot reassigned. Each `(slot, key)` carries a write
+  version, bumped whenever a new write claims it. A planned read captures
+  that version and is registered in `_pending_reads`; a later write that
+  reclaims those slots copy-on-writes the still-`COMMITTED` rows into the
+  ref before it overwrites the pool, so a delayed fetch serves the original
+  tenant. A version mismatch with no preserved copy raises for live and
+  finished alike (the pool rows are a newer tenant's). Production defaults
+  stay opt-in until preempt/resume hit spans are reconstructed.
+- `async_chunk` continuation: when the next upstream chunk arrives, the same
+  request id re-enters `scheduled_new_reqs` with `num_computed_tokens` equal
+  to what it already computed itself. Ids already in `live_reqs` are skipped
+  for hit marking: those rows were delivered in earlier steps and re-emitting
+  them would duplicate output. A `delivered_upto` span for this case is
+  Phase 2.
+- Preemption + reschedule: vLLM resets `num_computed_tokens` to 0 on
+  preemption and re-runs prefix matching on resume, so the resumed request
+  can come back with a fresh hit. With the V1 model runner it arrives
+  through `scheduled_cached_reqs` (id in `resumed_req_ids`, `new_block_ids`
+  replaces the table); with the V2 runner it re-enters `scheduled_new_reqs`
+  while still in `live_reqs`. Neither path marks an omni hit span: the
+  resumed request gets only the rows it recomputes, and its still-open
+  deferred write keeps appending (a slot written twice keeps the later
+  chunk). Cache integrity holds either way — the hit blocks already have
+  rows, from this request or the one it hit. Stages that need full prompt
+  hidden states should be sized so preemption does not occur while prefix
+  caching is on. Same as before this refactor; tracked for Phase 2.
+
+Two write paths, split by `ModelCachePolicy.deferred_keys`:
+
+- Immediate (`JOIN_NEXT_STEP`): save launches a whole-step device→host into
+  a staging slot; the committer waits that event and writes the CPU pool.
+  The next save waits `done` (the pool write): a reused slot must never
+  leave a pending pool write behind, or a delayed hit read of the old rows
+  would find nothing recoverable.
+- Deferred (`JOIN_ON_FINISH`): mm whose first dim is this step's token count
+  stays on a per-request GPU clone; `_WriteChunk`s append across steps;
+  finish/abort (or GPU-byte-budget pressure) forces the copy. One open
+  WriteTask per request; a budget flush may close it mid-request, in which
+  case the next save opens a new one (`write_n` + 1) and a hit reads both,
+  so one long request cannot pin the whole budget.
+
+A `WriteTask` moves through `TaskState` only via `transition()`, one step at
+a time along a strict chain:
+`PENDING` (registered, not queued) → `QUEUED` (on a copy queue) →
+`COPYING` (one thread owns the copy stage) → `HOST_READY` (host rows ready) →
+`WRITTEN` (in the CPU pool); `FAILED` is reachable from any non-terminal
+state. Skipping a step raises. The worker claims `COPYING` in the same `_wake`
+critical section as the queue pop, so `escalate` never re-queues a task it can
+see is already claimed. `host_ready` / `done` are wait primitives set by the
+transitions into `HOST_READY` / `WRITTEN` / `FAILED`. GPU-clone bytes are
+charged once per clone on a `_BudgetTicket` pinned by every task that views
+it, and uncharged when the last holder releases (idempotent per tid).
+
+Mm whose first dim equals the unpadded scheduled length *or* the CUDA-graph
+padded length is registered on first sighting. Talker `codes.audio` is a
+cat of scheduled rows and stays unpadded while hidden is padded; both must
+open a pool key, whether immediate or deferred.
+Leftover mm (lists, `codes.ref`, any tensor whose first dim is not this
+step's token count) is copied to CPU at save without truncating that first
+dim. That leftover copy is the async-builder read replica for this step; it
+does not write the pool or carry abort/preempt occupancy.
+
+```python
+cache.register_policy(ModelCachePolicy.from_model(model))   # load_model
+cache.new_step_starts(scheduler_output)   # before _update_states
+sid = cache.save_outputs(hidden, mm_outputs, num_tokens_unpadded=n,
+                         num_tokens_padded=n_pad)
+outs = cache.materialize(sid, req_ids)    # or discard_step(sid)
+```
+
+Each step id is consumed exactly once. `req_ids` must be a subset of the save
+snapshot. At most `staging_depth` unused step ids may exist at once: every
+`save_outputs` claims one staging slot, including saves with only leftover
+mm that copy no device→host page. A slot is also held by each immediate
+write that views it, until the committer's pool write. A later save waits
+for `materialize`/`discard_step` or that pool write to free a slot;
+`staging_claim_timeout_s` then errors with the unused ids and the task count.
+`join`/`join_host_ready` use the same bound and raise with the stuck task's
+id and state instead of hanging the caller.
+`materialize` may run on the async output builder after the engine has
+entered the next step; leftover mm (not written to the pool) is copied
+to CPU at `save_outputs` so the builder never reads live CUDA-graph
+buffers. See
+[Async Omni Output Materialization](omni_async_output_materialization.md).
+
+The cache is constructed only on the last pipeline-parallel rank
+(`_ensure_omni_prefix_cache`). Other ranks skip it: they never call
+`save_outputs`, so a hit table there would resolve to absent slots.
+
+Threads, locks, and what each may block on:
+
+| Thread | Role | May block on | Must not hold while blocked |
+| --- | --- | --- | --- |
+| Engine | `new_step_starts`, `save_outputs` | previous-step `join` (`done`) / finished-write `join_host_ready`; `reserve()` GPU-byte flush; staging-slot claim; `dispatch()` / finish-abort `escalate()` (eager mode: the copy + pool write run inline) | `_state_lock` |
+| Async output builder | `materialize` (may overlap the next engine step) | this step's `step_d2h_event`; `join` (`done`); deferred `fetch_host` | `_state_lock` |
+| Committer | `_worker_loop`: wait device→host / deferred copy / pool write | `_wake.wait`; `step_d2h_event` or copy-stream sync | never takes `_state_lock` |
+| Prefetch pool | hit-span gather during forward | `join` (`done`); deferred `fetch_host` | `_state_lock` |
+
+| Lock | Covers | Does not cover |
+| --- | --- | --- |
+| manager `_state_lock` | occupancy tables, step contexts, hit spans, task registration, pool-key publish (`install_key`) | join, GPU-byte flush, copy, `step_d2h_event` wait, pool-key allocation, eager `dispatch()` / `escalate()` |
+| controller `_lock` / `_wake` | task registry, queues, GPU-clone byte budget | device→host / pool-write body (released before `synchronize`) |
+| `WriteTask.lock` | `state`, `reassigned`, `append_chunk` | waiting on `host_ready` / `done` (those are events) |
+
+### Related Files
+
+- `vllm_omni/core/prefix_cache/`
+- `vllm_omni/worker/gpu_model_runner.py` (`_ensure_omni_prefix_cache`)
+- `tests/core/test_prefix_cache.py`
+- [Async Omni Output Materialization](omni_async_output_materialization.md)

docs/design/feature/realtime_ar_diffusion.md

@@ -143,8 +143,20 @@ fraction to select additional resident sessions up to the model-declared cap.
 All model-owned reservations are deducted before the paged self-attention pool
 is allocated.
 
-LingBot reports its persistent full-horizon image-condition tensor. DreamZero
-reports the measured 603 MiB per-session Wan VAE causal-convolution state.
+LingBot advances a session-owned causal Wan VAE encoder for each condition
+block: the opening pixel frame is followed by four zero pixel frames per later
+latent frame. It retains one current condition block, with separate committed
+and in-flight encoder histories so a failed block cannot overwrite committed
+context. The state reservation includes both encoder histories, derived from
+the encoder's convolution input grids, and the streaming decoder cache. These
+caches are released through the same close/reset hooks. Condition encoding
+currently requires the unpatched, non-tiled Wan encoder path; the existing
+decoder's tiling fallback remains independent. Real-weight GPU parity with a
+full-horizon encode still requires validation because the pointwise posterior
+projection now runs per block.
+
+DreamZero reports the measured 603 MiB per-session Wan VAE causal-convolution
+state.
 
 ## Concurrency and routing limits
 
@@ -178,6 +190,24 @@ For LingBot World v2:
 - `lingbot_world/pipeline.py` constructs conditioning, owns small non-KV session state, and
   produces one block plus the standard metadata envelope.
 
+## Ulysses sequence parallelism
+
+LingBot supports pure Ulysses sequence parallelism for both direct and
+AR-Diffusion execution. With Ulysses degree greater than one, hidden tokens,
+camera features, token-expanded timestep modulation, and RoPE tables are sharded
+together. Without SP, timestep modulation retains the frame-broadcast path.
+Self-attention performs the sequence-to-head all-to-all before reading or writing
+paged KV, while static text K/V uses the same local head shard. Text K/V shards
+own compact storage; cross-attention exchanges query/output layouts to use these
+shards. This retains the shared cache geometry and reduces text K/V storage at
+the cost of two all-to-all calls per layer.
+
+The output head projects local tokens before gathering the flow values. For
+the 14B model this reduces the gathered width from 5120 to 64; frame modulation
+uses each shard's global token offset, including shards that split a frame. Only
+`ulysses_mode="strict"` is supported; `advanced_uaa`, Ring, and AllGather-KV modes
+remain unsupported for this model.
+
 ## Non-goals
 
 This contract does not currently provide:

docs/design/feature/sequence_parallel.md

@@ -77,7 +77,9 @@ Use it when plain Ulysses-SP would otherwise fail because:
 
 - `auto_pad=True` pads sequence tokens in `_sp_plan` and requires attention backends that support `attention_mask`.
 - `advanced_uaa` does not depend on mask-based token padding inside Ulysses attention. It is therefore a better fit for non-divisible head counts and uneven Ulysses shard sizes.
-- `auto_pad=True` remains incompatible with Ring attention because the ring backend does not consume `attention_mask`.
+- `auto_pad=True` remains incompatible with Ring attention because the ring backend does not consume arbitrary
+  `attention_mask` values. A model with contiguous suffix padding may instead publish `valid_kv_length`; Ring trims
+  each circulated K/V block against that global valid prefix.
 - `advanced_uaa` is still experimental and hybrid mode remains limited by Ring's equal-shape requirement.
 
 ---
@@ -416,7 +418,7 @@ _sp_plan = {
 | Constraint | Description |
 |------------|-------------|
 | **Attention Backend Compatibility** | The attention backends must support `attention_mask`. Currently only `TORCH_SDPA` and `FLASH_ATTN` (default for diffusion models) are supported. |
-| **Ring Attention Limitation** | Ring attention does not support `attention_mask`. Therefore, when using `auto_pad=True`, the combination of Ulysses + Ring attention is not feasible. |
+| **Ring Attention Limitation** | Ring attention does not support arbitrary `attention_mask` values, so `auto_pad=True` remains incompatible. Contiguous suffix padding is supported when the model publishes the global `valid_kv_length` prefix. |
 
 1. Enable `auto_pad=True` for all sequence-dimension inputs in `_sp_plan`:
 ```python

docs/design/fullduplex-personaplex.md

@@ -20,7 +20,7 @@ The target is the engine-native path:
   -> PersonaPlex Talker
   -> streaming PersonaPlex Code2Wav
   -> PersonaPlex data-plane projector
-  -> response.audio.delta + response.audio_transcript.delta
+  -> response.output_audio.delta + response.output_audio_transcript.delta
 ```
 
 The standalone `/api/chat` and `/v1/audio/duplex` server that accompanied the
@@ -197,7 +197,7 @@ model-neutral native results:
 
 It owns per-request audio and text cursors so a cumulative output cannot replay
 old audio. The generic projector turns each native result into the Realtime
-`response.audio.delta` / `response.audio_transcript.delta` pair (and
+`response.output_audio.delta` / `response.output_audio_transcript.delta` pair (and
 `response.output_text.delta` for text) under one `response_id`; the full
 mapping is the name map in the
 [Realtime Duplex API](../serving/realtime_duplex_api.md) serving guide.
@@ -271,3 +271,111 @@ The required evidence is:
 
 Audio that is empty, all zero, non-finite, or only a protocol `listen` event is
 not a successful end-to-end result.
+
+### Paced multi-session load driver
+
+The existing driver keeps its two-session admission and slot-reuse validation
+when `--sessions` is omitted. Passing `--sessions N` selects a separate,
+bounded load mode; it does not replace the lifecycle test or change server
+admission capacity. Start an existing PersonaPlex server configured to admit
+the requested number of sessions, then run from a source checkout with the
+vLLM-Omni client dependencies installed:
+
+```bash
+python -m tests.e2e.online_serving.personaplex_realtime_duplex \
+  --model nvidia/personaplex-7b-v1 \
+  --input-wav /path/to/input.wav \
+  --sessions 2 --load-frames 1000 --drain-s 2 \
+  --server-revision SERVER_COMMIT --server-hardware A100-80GB \
+  --output-dir /tmp/personaplex-load-n2
+```
+
+Use a new output directory for every run. The driver refuses to overwrite an
+existing `load-result.json`. The server labels are supplied by the operator,
+not detected or verified by the client. Record the server command, package
+versions, checkpoint revision and hardware alongside a published measurement.
+
+All attempted sessions finish admission before admitted sessions share a
+common input start. A rejected session remains a failed row in the report;
+it is not removed from the denominator. Each session receives at most
+`--load-frames` input frames, without looping the WAV, and uses 80 ms pacing.
+A stalled send cannot cause a catch-up burst. The timeline records send
+schedule lateness so an overloaded client is visible rather than mistaken
+for a faster server. This mode measures steady concurrency, not join/leave
+interference; the original lifecycle mode still checks slot reuse.
+Load mode does not probe an additional N+1 session for `resource_exhausted`;
+the lifecycle mode owns that admission-limit check. A per-session
+`client_pacing_warning` and the summary's `client_pacing_warning_sessions`
+identify send-lateness p99 above one nominal frame (80 ms). This is a warning,
+not an extra acceptance gate: read it alongside server-facing measurements
+so a slow client is not mistaken for a slow server.
+
+The fixed drain window is part of the measurement configuration. Audio
+arriving during cleanup is excluded, so closing cannot rescue a session that
+missed its observation deadline. Cleanup is attempted for admitted sessions
+even when audio validation fails. Protocol errors, missing/invalid audio,
+shared response IDs and cleanup failures make the report fail and the CLI
+exit nonzero. Cancellation closes transports and propagates to the caller.
+`--cleanup-timeout-s` (default 5 s) bounds session-close send and acknowledgement
+together, independently of `--timeout-s`. The subsequent WebSocket close
+handshake has the same separate bound; an unresponsive close acknowledgement
+cannot hold cancellation for the full inference timeout.
+
+`load-result.json` retains per-session send and audio-packet timelines using
+the client's monotonic clock. It does not export handshake credentials or
+raw audio payloads. Metric definitions are explicit:
+
+Both modes validate each nonempty audio packet's strict base64 encoding, PCM16
+sample alignment, 24 kHz sample rate and response identity. Valid aggregate
+audio cannot hide a malformed packet. In load mode, rejected packets make the session fail
+and appear in `invalid_audio_packets` with their event index, receive time and
+fixed diagnostic code; they do not contribute samples or timing intervals.
+Empty audio deltas remain legal and contribute no samples. This check does
+not change the public client's decoding. The lifecycle scenario and dispatch
+remain the same, but malformed packets that aggregate checks once tolerated
+now fail that mode as well.
+
+The probe accepts both the current OpenAI Realtime audio event name
+`response.output_audio.delta` and the legacy `response.audio.delta`. Validation
+uses the wire payload directly rather than relying on a particular client
+library's alias table, so a server-side event-name migration cannot silently
+turn real audio into a zero-output measurement.
+
+| Field | Meaning |
+| --- | --- |
+| `client_audio_packet_interval_ms` | Consecutive nonempty packet arrivals: median, linear p99, max, and zero-based index of the first maximal interval. Interval index 0 ends at packet 1. One packet can contain multiple codec frames; this is not server tick latency. |
+| `client_first_audio_after_stream_start_ms` | First audio receipt minus first input send start; admission is excluded. |
+| `client_stream_rtf` | First input send start to last audio receipt, divided by received audio duration. This includes real-time input pacing and is not an inference-only RTF. |
+| `client_send_lateness_ms` | Send start minus its original shared schedule, clipped at zero. |
+| `frame_deficit` | Integer input-frame minus output-frame count from the shared session validation; missing when valid whole-frame accounting is unavailable. |
+| `output_frames`, `voiced_frames`, `silent_frames`, `frame_coverage_ratio`, `audio_rms`, `audio_chunks` | Session-validation statistics retained on success and on acceptance failure once computed. |
+| `acceptance_check` | Failed local check, with numeric diagnostics in `error`; other exceptions retain phase/type only, without remote handshake or metadata text. |
+| `audio_underrun_s`, `audio_underrun_event_count`, `audio_continuity_ok` | Shared `compute_continuity_stats` on valid PCM16 packet arrivals, with its default 100 ms underrun threshold; absent audio gives missing values rather than a continuity pass. |
+
+No output gives missing latency/RTF values, not zero. By default RTF is
+diagnostic; `--max-client-rtf` adds an explicit ceiling. The existing frame
+coverage, audible-frame and scheduler-data-plane checks still apply. A longer
+`--drain-s` can demonstrate eventual completion under backlog, but it must not
+be reported as real-time capacity when the fixed-window run failed or client
+RTF exceeded the stated target. Likewise, the first live session can pay
+one-time model/codec/JIT costs; either use the server's configured duplex
+warmup path or label that run as cold-start evidence before comparing steady
+capacity. Passing client-only metrics does not certify model quality or a
+real-time capacity target on another GPU. Begin with N=1 and N=2; higher N is a
+capacity experiment, not a promised property of an A100 or of this patch.
+Continuity is a client-arrival diagnostic, not proof of speaker playback and
+not an additional pass/fail gate. Packet-interval and send-lateness summaries
+retain their documented linear p99; the older client interval helper reports
+rounded nearest-rank p95 instead and is not substituted for that schema.
+
+Deterministic driver and localhost WebSocket checks need no model or GPU:
+
+```bash
+python -m pytest tests/e2e/online_serving/test_personaplex_load_driver.py \
+  -m 'core_model and cpu' --run-level core_model -q
+```
+
+These checks test the driver against controlled protocol responses. Run the
+load-driver command against real weights separately before reporting model-serving
+performance. Server tick timestamps and generic TTFT/TPOT aggregation remain
+the responsibility of the existing duplex metrics paths, not this driver.