Merge main and align text encoder TP regression documentation

Signed-off-by: Sy03 <1370724210@qq.com>

.buildkite/amd/scripts/bootstrap-amd-omni.sh

@@ -14,6 +14,8 @@ set -euo pipefail
 # shellcheck disable=SC1091
 source .buildkite/common/scripts/resolve_skip_ci.sh
 
+AMD_SUITE_SELECTOR=".buildkite/amd/scripts/select_test_suites.py"
+
 if [[ -z "${RUN_ALL:-}" ]]; then
     RUN_ALL=0
 fi
@@ -30,31 +32,20 @@ if [[ -z "${AMD_MIRROR_HW:-}" ]]; then
     AMD_MIRROR_HW="amdproduction"
 fi
 
-fail_fast() {
-    DISABLE_LABEL="ci-no-fail-fast"
-    # If BUILDKITE_PULL_REQUEST != "false", then we check the PR labels using curl and jq
-    if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then
-        PR_LABELS=$(curl -s "https://api.github.com/repos/vllm-project/vllm-omni/pulls/$BUILDKITE_PULL_REQUEST" | jq -r '.labels[].name')
-        if [[ $PR_LABELS == *"$DISABLE_LABEL"* ]]; then
-            echo false
-        else
-            echo true
-        fi
-    else
-        echo false  # not a PR or BUILDKITE_PULL_REQUEST not set
+PR_LABELS=""
+# Explicit debug suites do not depend on PR labels.
+if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" && -z "${DEBUG_TEST_YAML:-}" ]]; then
+    if ! PR_LABELS=$(curl -fsSL \
+        "https://api.github.com/repos/vllm-project/vllm-omni/pulls/$BUILDKITE_PULL_REQUEST" \
+        | jq -r '.labels[].name'); then
+        echo "ERROR: Could not read PR labels; refusing to select an incorrect AMD test tier." >&2
+        exit 1
     fi
-}
+fi
 
-check_run_all_label() {
-    RUN_ALL_LABEL="ready-run-all-tests"
-    # If BUILDKITE_PULL_REQUEST != "false", then we check the PR labels using curl and jq
-    if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then
-        PR_LABELS=$(curl -s "https://api.github.com/repos/vllm-project/vllm-omni/pulls/$BUILDKITE_PULL_REQUEST" | jq -r '.labels[].name')
-        if [[ $PR_LABELS == *"$RUN_ALL_LABEL"* ]]; then
-            echo true
-        else
-            echo false
-        fi
+fail_fast() {
+    if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
+        echo true
     else
         echo false  # not a PR or BUILDKITE_PULL_REQUEST not set
     fi
@@ -64,6 +55,64 @@ if [[ -z "${COV_ENABLED:-}" ]]; then
     COV_ENABLED=0
 fi
 
+resolve_test_specs() {
+    local selected_specs
+    if ! selected_specs=$(python3 "$AMD_SUITE_SELECTOR" \
+        --branch "$BUILDKITE_BRANCH" \
+        --labels "$PR_LABELS" \
+        --debug-test-yaml "${DEBUG_TEST_YAML:-}" \
+        --nightly "$NIGHTLY"); then
+        exit 1
+    fi
+
+    TEST_SPECS=()
+    while IFS= read -r suite_spec; do
+        [[ -z "$suite_spec" ]] && continue
+        TEST_SPECS+=("$suite_spec")
+    done <<< "$selected_specs"
+}
+
+filter_test_specs_by_skip_ci() {
+    local suite_spec level decision
+    local skip_all=0
+    local -a runnable_specs=()
+
+    for suite_spec in "${TEST_SPECS[@]}"; do
+        case "$suite_spec" in
+            READY_TESTS:*) level="l2" ;;
+            MERGE_TESTS:*) level="l3" ;;
+            NIGHTLY_TESTS:*)
+                # Explicit PR and scheduled nightly selections must survive
+                # L2/L3 diff gating, including docs-only main commits.
+                runnable_specs+=("$suite_spec")
+                continue
+                ;;
+            *)
+                echo "ERROR: unknown AMD test suite spec '$suite_spec'" >&2
+                exit 1
+                ;;
+        esac
+
+        if decision=$(python3 "$SKIP_CI_PY" gate amd "$level"); then
+            echo "Skipping AMD $level suite due to $decision decision."
+            [[ "$decision" == "skip-all" ]] && skip_all=1
+        else
+            runnable_specs+=("$suite_spec")
+        fi
+    done
+
+    TEST_SPECS=("${runnable_specs[@]}")
+    if [[ ${#TEST_SPECS[@]} -eq 0 ]]; then
+        if [[ $skip_all -eq 1 ]]; then
+            buildkite-agent annotate \
+                ":memo: CI skipped — docs or pytest skip-mark changes only" \
+                --style "info" 2>/dev/null || true
+        fi
+        echo "No AMD suites remain after skip-ci filtering."
+        exit 0
+    fi
+}
+
 upload_pipeline() {
     echo "Uploading pipeline..."
     # Install minijinja
@@ -84,7 +133,7 @@ upload_pipeline() {
     # (WIP) Use pipeline generator instead of jinja template
     if [ -e ".buildkite/amd/pipeline_generator/pipeline_generator.py" ]; then
         python -m pip install click pydantic
-        python .buildkite/amd/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW"
+        python .buildkite/amd/pipeline_generator/pipeline_generator.py --run_all="$RUN_ALL" --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW"
         buildkite-agent pipeline upload .buildkite/amd/pipeline.yaml
         exit 0
     fi
@@ -97,49 +146,12 @@ upload_pipeline() {
 
     cd .buildkite/amd
 
-    # Select test definition file: merge suite for main, ready suite for PRs.
-    # For debugging, DEBUG_TEST_YAML accepts a comma-separated list containing
-    # "merge" and/or "ready" (case-insensitive). Multiple suites are combined
-    # before rendering so they share one amd-build step.
-    if [[ -n "${DEBUG_TEST_YAML:-}" ]]; then
-        declare -a DEBUG_TEST_SPECS=()
-        declare -A SEEN_DEBUG_TESTS=()
-        IFS=',' read -ra REQUESTED_DEBUG_TESTS <<< "${DEBUG_TEST_YAML,,}"
-
-        for requested_test in "${REQUESTED_DEBUG_TESTS[@]}"; do
-            # Permit readable values such as "ready, merge," and ignore the
-            # empty item produced by a trailing comma.
-            requested_test="${requested_test//[[:space:]]/}"
-            [[ -z "$requested_test" ]] && continue
-
-            if [[ -n "${SEEN_DEBUG_TESTS[$requested_test]:-}" ]]; then
-                echo "ERROR: duplicate DEBUG_TEST_YAML suite '$requested_test'" >&2
-                exit 1
-            fi
-            SEEN_DEBUG_TESTS[$requested_test]=1
-
-            case "$requested_test" in
-                ready)
-                    DEBUG_TEST_SPECS+=("READY_TESTS:test-amd-ready.yml")
-                    ;;
-                merge)
-                    DEBUG_TEST_SPECS+=("MERGE_TESTS:test-amd-merge.yml")
-                    ;;
-                *)
-                    echo "ERROR: DEBUG_TEST_YAML entries must be 'merge' or 'ready', got '$requested_test'" >&2
-                    exit 1
-                    ;;
-            esac
-        done
-
-        if [[ ${#DEBUG_TEST_SPECS[@]} -eq 0 ]]; then
-            echo "ERROR: DEBUG_TEST_YAML did not contain a test suite" >&2
-            exit 1
-        elif [[ ${#DEBUG_TEST_SPECS[@]} -eq 1 ]]; then
-            TEST_YAML="${DEBUG_TEST_SPECS[0]#*:}"
-        else
-            TEST_YAML=$(mktemp "${TMPDIR:-/tmp}/amd-debug-tests.XXXXXX.yml")
-            python - "$TEST_YAML" "${DEBUG_TEST_SPECS[@]}" <<'PY'
+    # Multiple label-selected or debug-selected suites share one image build.
+    if [[ ${#TEST_SPECS[@]} -eq 1 ]]; then
+        TEST_YAML="${TEST_SPECS[0]#*:}"
+    else
+        TEST_YAML=$(mktemp "${TMPDIR:-/tmp}/amd-selected-tests.XXXXXX.yml")
+        python - "$TEST_YAML" "${TEST_SPECS[@]}" <<'PY'
 import sys
 
 import yaml
@@ -172,13 +184,8 @@ for suite_spec in suite_specs:
 with open(output_path, "w", encoding="utf-8") as output_file:
     yaml.safe_dump(combined, output_file, sort_keys=False)
 PY
-        fi
-        echo "DEBUG_TEST_YAML override: using ${DEBUG_TEST_SPECS[*]}"
-    elif [[ $BUILDKITE_BRANCH == "main" ]]; then
-        TEST_YAML="test-amd-merge.yml"
-    else
-        TEST_YAML="test-amd-ready.yml"
     fi
+    echo "AMD test suites: ${TEST_SPECS[*]}"
 
     (
         set -x
@@ -198,7 +205,7 @@ PY
             > pipeline.yaml
     )
     cat pipeline.yaml
-    if [[ "$TEST_YAML" == "${TMPDIR:-/tmp}/amd-debug-tests."*.yml ]]; then
+    if [[ "$TEST_YAML" == "${TMPDIR:-/tmp}/amd-selected-tests."*.yml ]]; then
         rm -f -- "$TEST_YAML"
     fi
     buildkite-agent artifact upload pipeline.yaml
@@ -219,12 +226,10 @@ if [[ $BUILDKITE_BRANCH == "main" ]]; then
     file_diff=$(get_diff_main)
 fi
 
-# Early exit: unified skip-ci (docs / skip marks) and CI-yaml-only level targeting.
-if [[ $BUILDKITE_BRANCH == "main" ]]; then
-    gate_bootstrap_ci amd l3
-else
-    gate_bootstrap_ci amd l2
-fi
+# Resolve PR tier labels before skip-ci so ready/merge suites can be filtered
+# independently while an explicitly selected nightly suite passes through.
+resolve_test_specs
+filter_test_specs_by_skip_ci
 
 patterns=(
     "docker/Dockerfile"
@@ -274,14 +279,6 @@ for file in $file_diff; do
     fi
 done
 
-# Check for ready-run-all-tests label
-LABEL_RUN_ALL=$(check_run_all_label)
-if [[ $LABEL_RUN_ALL == true ]]; then
-    RUN_ALL=1
-    NIGHTLY=1
-    echo "Found 'ready-run-all-tests' label. Running all tests including optional tests."
-fi
-
 # Decide whether to use precompiled wheels
 # Relies on existing patterns array as a basis.
 if [[ -n "${VLLM_USE_PRECOMPILED:-}" ]]; then

.buildkite/amd/scripts/build-ci-image.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+set -euo pipefail
+
+# Build and push the ROCm CI image with a registry-backed BuildKit cache. The
+# AMD builders are ephemeral, so a local Docker cache cannot survive between
+# jobs. Keeping the cache beside the runtime image also reuses the registry
+# credentials already configured on the amd-cpu agents.
+
+REGISTRY="${ROCM_CI_IMAGE_REGISTRY:-rocm/vllm-omni}"
+DOCKERFILE="docker/Dockerfile.rocm"
+BUILDER_NAME="vllm-omni-rocm-builder"
+ROCM_ARCH="${PYTORCH_ROCM_ARCH:-gfx942}"
+
+if [[ -z "${BUILDKITE_COMMIT:-}" ]]; then
+    echo "ERROR: BUILDKITE_COMMIT is not set" >&2
+    exit 1
+fi
+
+dependency_files=(
+    "$DOCKERFILE"
+    "${DOCKERFILE}.dockerignore"
+    pyproject.toml
+    setup.py
+    tools/install_torchcodec_rocm.sh
+)
+while IFS= read -r dependency_file; do
+    dependency_files+=("$dependency_file")
+done < <(find requirements -maxdepth 1 -type f -name '*.txt' -print | LC_ALL=C sort)
+
+for dependency_file in "${dependency_files[@]}"; do
+    if [[ ! -f "$dependency_file" ]]; then
+        echo "ERROR: cache-key input does not exist: $dependency_file" >&2
+        exit 1
+    fi
+done
+
+CACHE_KEY=$(
+    {
+        printf 'rocm-arch=%s\n' "$ROCM_ARCH"
+        for dependency_file in "${dependency_files[@]}"; do
+            printf 'path=%s\n' "$dependency_file"
+            sha256sum "$dependency_file"
+        done
+    } | sha256sum | cut -c1-16
+)
+CACHE_REF="${REGISTRY}:rocm-deps-cache-${CACHE_KEY}"
+IMAGE_REF="${REGISTRY}:${BUILDKITE_COMMIT}"
+
+echo "ROCm dependency cache: ${CACHE_REF}"
+echo "ROCm CI image: ${IMAGE_REF}"
+
+# The docker-container driver supports exporting the full cache to a registry;
+# the default docker driver on CI does not.
+if ! docker buildx inspect "$BUILDER_NAME" >/dev/null 2>&1; then
+    docker buildx create --name "$BUILDER_NAME" --driver docker-container
+fi
+docker buildx use "$BUILDER_NAME"
+docker buildx inspect --bootstrap
+
+docker buildx build --push --pull --progress=plain \
+    --target test \
+    --build-arg "ARG_PYTORCH_ROCM_ARCH=${ROCM_ARCH}" \
+    --cache-from "type=registry,ref=${CACHE_REF}" \
+    --cache-to "type=registry,ref=${CACHE_REF},mode=max,compression=zstd,ignore-error=true" \
+    --file "$DOCKERFILE" \
+    --tag "$IMAGE_REF" .

.buildkite/amd/scripts/select_test_suites.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Select AMD ready/merge/nightly suites after Buildkite creates a build.
+
+This repository-side selector does not decide which GitHub label events start
+the external ``vllm-omni-amd-ci`` pipeline. Its Buildkite PR build condition
+must admit the relevant label event before this code can inspect the labels.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Iterable
+
+SUITE_SPECS = {
+    "ready": "READY_TESTS:test-amd-ready.yml",
+    "merge": "MERGE_TESTS:test-amd-merge.yml",
+    "nightly": "NIGHTLY_TESTS:test-amd-nightly.yml",
+}
+
+
+def _parse_debug_suites(value: str) -> tuple[str, ...]:
+    suites = tuple(item.strip().lower() for item in value.split(",") if item.strip())
+    if not suites:
+        raise ValueError("DEBUG_TEST_YAML did not contain a test suite")
+
+    seen: set[str] = set()
+    for suite in suites:
+        if suite not in SUITE_SPECS:
+            raise ValueError(f"DEBUG_TEST_YAML entries must be 'merge', 'nightly', or 'ready', got {suite!r}")
+        if suite in seen:
+            raise ValueError(f"duplicate DEBUG_TEST_YAML suite {suite!r}")
+        seen.add(suite)
+    return suites
+
+
+def select_amd_test_suites(
+    *,
+    branch: str,
+    labels: Iterable[str],
+    debug_test_yaml: str = "",
+    nightly: bool = False,
+) -> tuple[str, ...]:
+    """Return ordered suite names while preserving legacy unlabeled PRs."""
+
+    if debug_test_yaml:
+        return _parse_debug_suites(debug_test_yaml)
+    if branch == "main":
+        return ("nightly",) if nightly else ("merge",)
+
+    label_set = {label.strip() for label in labels if label.strip()}
+    selected = tuple(
+        suite
+        for label, suite in (
+            ("ready", "ready"),
+            ("merge-test", "merge"),
+            ("nightly-test", "nightly"),
+        )
+        if label in label_set
+    )
+    # AMD historically uploaded the ready suite for every admitted PR build.
+    # Preserve that fallback when no explicit tier label is present.
+    return selected or ("ready",)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--branch", required=True)
+    parser.add_argument("--labels", default="")
+    parser.add_argument("--debug-test-yaml", default="")
+    parser.add_argument("--nightly", type=int, choices=(0, 1), default=0)
+    args = parser.parse_args()
+
+    labels = tuple(args.labels.splitlines())
+    try:
+        suites = select_amd_test_suites(
+            branch=args.branch,
+            labels=labels,
+            debug_test_yaml=args.debug_test_yaml,
+            nightly=bool(args.nightly),
+        )
+    except ValueError as exc:
+        parser.error(str(exc))
+
+    label_set = {label.strip() for label in labels if label.strip()}
+    if args.branch != "main" and not args.debug_test_yaml and not ({"ready", "merge-test", "nightly-test"} & label_set):
+        print(
+            "No AMD ready/merge/nightly label found; preserving the legacy ready-suite fallback.",
+            file=sys.stderr,
+        )
+
+    for suite in suites:
+        print(SUITE_SPECS[suite])
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.buildkite/amd/test-amd-merge.yml

@@ -9,15 +9,26 @@ steps:
   steps:
     - label: "Simple · Model Executor Test · Shard %N/%t"
       agent_pool: mi300_1
-      parallelism: 2
+      parallelism: 3
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      timeout_in_minutes: 90
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+        - "pytest -sv tests/model_executor -m 'core_model and cpu and not omni' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+
+    - label: "Simple · Omni Processor Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 60
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/model_executor/models/test_omni_processing.py -m 'core_model and cpu and omni'"
 
     - label: "Simple · Diffusion Test · Shard %N/%t"
       agent_pool: mi300_1
@@ -25,6 +36,7 @@ steps:
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      timeout_in_minutes: 45
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM

.buildkite/amd/test-amd-nightly.yml

@@ -0,0 +1,66 @@
+# AMD MI300 `full_model` nightly coverage during burn-in.
+# All jobs are non-blocking until the lane has stable runtime evidence.
+# Historical build #11646 was red in Function Expansion and Accuracy (one
+# async_chunk similarity mismatch; Seed-TTS WER 0.376480 > 0.35).
+
+env:
+  VLLM_WORKER_MULTIPROC_METHOD: spawn
+  HF_HUB_DOWNLOAD_TIMEOUT: 300
+  HF_HUB_ETAG_TIMEOUT: 60
+
+steps:
+  # MI325 is the repository's legacy ROCm pytest marker. The AMD template maps
+  # agent_pool to the current MI300 Buildkite queues independently of that mark.
+  - group: ":full_moon: Qwen3-Omni MI300 Nightly"
+    steps:
+      - label: "Qwen3-Omni Function Expansion"
+        agent_pool: mi300_2
+        depends_on: amd-build
+        mirror_hardwares: [amdproduction]
+        grade: NonBlocking
+        timeout_in_minutes: 180
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_expansion.py -m "full_model and rocm and MI325 and cards_2" --run-level "full_model"
+
+      - label: "Qwen3-Omni Accuracy"
+        agent_pool: mi300_2
+        depends_on: amd-build
+        mirror_hardwares: [amdproduction]
+        grade: NonBlocking
+        timeout_in_minutes: 180
+        artifact_paths:
+          - tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json
+        commands:
+          - export SEED_TTS_WER_EVAL=1
+          # PyTorch ROCm uses the torch.cuda device namespace.
+          - export SEED_TTS_EVAL_DEVICE=cuda:1
+          - pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model and rocm and MI325 and cards_2" --run-level "full_model"
+          # Native tests run from the image copy under /app/vllm-omni, while
+          # Buildkite's artifact hook searches the checkout workspace.
+          - |
+            if [[ "$$PWD" != "$${BUILDKITE_BUILD_CHECKOUT_PATH:?}" ]]; then
+              artifact_dir="$${BUILDKITE_BUILD_CHECKOUT_PATH}/tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc"
+              mkdir -p "$$artifact_dir"
+              cp -v tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json "$$artifact_dir/"
+            fi
+
+      - label: "Qwen3-Omni Documentation Examples"
+        agent_pool: mi300_2
+        depends_on: amd-build
+        mirror_hardwares: [amdproduction]
+        grade: NonBlocking
+        timeout_in_minutes: 120
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - pytest -s -v tests/examples/online_serving/test_qwen3_omni.py -m "full_model and rocm and MI325 and cards_2" --run-level "full_model"
+
+      - label: "Qwen3-Omni AITER-on Smoke"
+        agent_pool: mi300_2
+        depends_on: amd-build
+        mirror_hardwares: [amdproduction]
+        grade: NonBlocking
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ROCM_USE_AITER=1
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_expansion.py::test_text_to_text_audio_001 -m "full_model and rocm and MI325 and cards_2" --run-level "full_model"

.buildkite/amd/test-amd-ready.yml

@@ -9,26 +9,39 @@ steps:
   steps:
     - label: "Simple · Model Executor Test · Shard %N/%t"
       agent_pool: mi300_1
-      parallelism: 2
+      parallelism: 3
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      timeout_in_minutes: 90
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+        - "pytest -sv tests/model_executor -m 'core_model and cpu and not omni' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
 
-    - label: "Simple · Diffusion Test"
+    - label: "Simple · Omni Processor Test"
       agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      timeout_in_minutes: 60
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/model_executor/models/test_omni_processing.py -m 'core_model and cpu and omni'"
+
+    - label: "Simple · Diffusion Test · Shard %N/%t"
+      agent_pool: mi300_1
+      parallelism: 4
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 45
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "pytest -sv tests/diffusion -m 'core_model and cpu'  --ignore=tests/diffusion/cache/test_teacache_extractors.py"
+        - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
 
     - label: "Simple · Engine&Entrypoints Test"
       agent_pool: mi300_1
@@ -69,6 +82,27 @@ steps:
 
 - group: ":card_index_dividers: Diffusion Test"
   steps:
+    # Keep real FlashAttention execution out of the CPU-marked Simple lane.
+    # AITER may need to build several kernels on a cold worker, so run one
+    # pytest worker with job-local build directories and a bounded timeout.
+    - label: "ROCm · FlashAttention/AITER GPU Coverage"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: NonBlocking
+      timeout_in_minutes: 60
+      commands:
+        - export VLLM_ROCM_USE_AITER=1
+        - export AITER_JIT_DIR="/tmp/vllm-omni-aiter-$$BUILDKITE_JOB_ID"
+        - export TORCH_EXTENSIONS_DIR="/tmp/vllm-omni-torch-extensions-$$BUILDKITE_JOB_ID"
+        - mkdir -p "$$AITER_JIT_DIR" "$$TORCH_EXTENSIONS_DIR"
+        - >-
+          timeout --signal=TERM --kill-after=2m 55m
+          pytest -s -v -n 1
+          tests/diffusion/attention/
+          -m 'core_model and rocm and MI325 and cards_1'
+          --run-level "core_model"
+
     - label: "Diffusion · Batch Test"
       agent_pool: mi300_1
       depends_on: amd-build
@@ -107,6 +141,11 @@ steps:
         - export VLLM_LOGGING_LEVEL=DEBUG
         - export VLLM_WORKER_MULTIPROC_METHOD=spawn
         - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
+        - >-
+          timeout 20m pytest -s -v
+          tests/diffusion/models/ltx2/test_ltx2_transformer_ulysses.py
+          -m 'core_model and rocm and cards_2'
+          --run-level "core_model"
 
     - label: "Diffusion GPU Worker Test"
       agent_pool: mi300_2
@@ -122,7 +161,13 @@ steps:
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
-        - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
+        # Preserve legacy one-GPU tests without cards_1 while excluding every
+        # registered multi-GPU marker from this one-GPU job.
+        - >-
+          timeout 15m pytest -sv tests/diffusion/models/
+          -m 'core_model and cuda and not
+          (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
+          --run-level "core_model"
 
     - label: "Diffusion · Wan22 Test"
       agent_pool: mi300_1
@@ -149,6 +194,39 @@ steps:
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
 
+- label: "ROCm · Engine & Model Executor GPU Coverage (R2-01)"
+  agent_pool: mi300_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: NonBlocking
+  timeout_in_minutes: 20
+  artifact_paths:
+    - "artifacts/rocm-r2-01/**/*"
+  commands:
+    # Native AMD tests run from /app/vllm-omni, while Buildkite uploads
+    # artifacts relative to its checkout directory.
+    - export R2_01_ARTIFACT_DIR="$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/rocm-r2-01"
+    - mkdir -p "$$R2_01_ARTIFACT_DIR"
+    # The CUDA ready job also covers tests/worker/test_batched_omni_output.py.
+    # Keep the initial ROCm burn-in scoped to the requested directories.
+    # Verbose pytest output and JUnit retain node IDs without a second
+    # collection-only pass over the same GPU-backed test tree.
+    # The CUDA/L4-only CosyVoice3 ragged-flow parity test cold-JITs multiple
+    # AITER kernels on ROCm and is covered by the CUDA ready lane.
+    - >-
+      pytest -s -v -ra
+      tests/engine/ tests/model_executor/
+      -m 'core_model and cuda and not
+      (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
+      -k 'not test_code2wav_streaming_batch_matches_ragged_flow_numerics'
+      --run-level "core_model"
+      --durations=0
+      --junitxml="$$R2_01_ARTIFACT_DIR/pytest.xml"
+      2>&1 | tee "$$R2_01_ARTIFACT_DIR/pytest.log"
+    - >-
+      tail -n 100 "$$R2_01_ARTIFACT_DIR/pytest.log" >
+      "$$R2_01_ARTIFACT_DIR/pytest-summary.txt"
+
 - label: "Engine Test"
   agent_pool: mi300_1
   depends_on: amd-build
@@ -161,6 +239,71 @@ steps:
               pytest -s -v tests/engine/test_async_omni_engine_abort.py
       '
 
+- label: "ROCm · Entrypoints GPU Coverage (R2-02)"
+  agent_pool: mi300_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: NonBlocking
+  timeout_in_minutes: 30
+  env:
+    VLLM_WORKER_MULTIPROC_METHOD: spawn
+  artifact_paths:
+    - "artifacts/rocm-r2-02/**/*"
+  commands:
+    # Native AMD tests run from /app/vllm-omni, while Buildkite uploads
+    # artifacts relative to its checkout directory.
+    - export R2_02_ARTIFACT_DIR="$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/rocm-r2-02"
+    - mkdir -p "$$R2_02_ARTIFACT_DIR"
+    - >-
+      ps -eo pid=,ppid=,lstart=,stat=,comm= >
+      "$$R2_02_ARTIFACT_DIR/processes-before.txt"
+    # Verbose pytest output and JUnit retain selected node IDs without a
+    # second collection-only pass over the same GPU-backed test tree.
+    # The inner timeout leaves five minutes for teardown verification and
+    # artifact generation before the Buildkite step timeout.
+    - >-
+      timeout --signal=TERM --kill-after=1m 25m
+      pytest -s -v -ra
+      tests/entrypoints/
+      -m 'core_model and cuda and not
+      (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
+      --run-level "core_model"
+      --durations=0
+      --junitxml="$$R2_02_ARTIFACT_DIR/pytest.xml"
+      2>&1 | tee "$$R2_02_ARTIFACT_DIR/pytest.log"
+    - >-
+      tail -n 100 "$$R2_02_ARTIFACT_DIR/pytest.log" >
+      "$$R2_02_ARTIFACT_DIR/pytest-summary.txt"
+    - |
+      sleep 5
+      ps -eo pid=,ppid=,lstart=,stat=,comm= > "$$R2_02_ARTIFACT_DIR/processes-after.txt"
+      # Compare PID plus process start time so PID reuse cannot hide a leak.
+      # Double dollars preserve awk field references through Buildkite interpolation.
+      awk '
+        NR == FNR {
+          if ($$9 != "ps") {
+            baseline[$$1 "|" $$3 "|" $$4 "|" $$5 "|" $$6 "|" $$7] = 1
+          }
+          next
+        }
+        $$9 != "ps" {
+          identity = $$1 "|" $$3 "|" $$4 "|" $$5 "|" $$6 "|" $$7
+          if (!(identity in baseline)) {
+            print
+            leaked = 1
+          }
+        }
+        END {
+          if (leaked) {
+            print "status=FAIL leaked_processes_detected=1"
+            exit 1
+          }
+          print "status=PASS leaked_processes=0"
+        }
+      ' "$$R2_02_ARTIFACT_DIR/processes-before.txt" \
+        "$$R2_02_ARTIFACT_DIR/processes-after.txt" \
+        2>&1 | tee "$$R2_02_ARTIFACT_DIR/process-cleanup.txt"
+
 # - label: "Omni Model Test Qwen3-Omni"
 #   agent_pool: mi300_2
 #   depends_on: amd-build
@@ -190,6 +333,12 @@ steps:
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      # A ROCm hardware hang aborts the interpreter with 134. Let Buildkite
+      # create one fresh pod instead of retrying on the poisoned GPU process.
+      retry:
+        automatic:
+          - exit_status: 134
+            limit: 1
       commands:
         - pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"
 

.buildkite/amd/test-template-amd-omni.j2

@@ -3,7 +3,8 @@
    Last synced: 2026-08-04
    Modifications: Use the shared MI300 queues and Omni command-passing contract
 #}
-{% set docker_image_amd = "rocm/vllm-omni:$BUILDKITE_COMMIT" %}
+{% set docker_registry_amd = "rocm/vllm-omni" %}
+{% set docker_image_amd = docker_registry_amd ~ ":$BUILDKITE_COMMIT" %}
 {% set default_working_dir = "/app/vllm-omni" %}
 
 {# MI300 runs tests natively in Kubernetes; Docker-in-Docker is unavailable. #}
@@ -74,11 +75,12 @@ steps:
         depends_on: ~
         soft_fail: false
         commands:
-          - "docker build -f docker/Dockerfile.rocm -t {{ docker_image_amd }} --target test --build-arg ARG_PYTORCH_ROCM_ARCH=gfx942 --progress plain ."
-          - "docker push {{ docker_image_amd }}"
+          - "bash .buildkite/amd/scripts/build-ci-image.sh"
         key: "amd-build"
         env:
           DOCKER_BUILDKIT: "1"
+          PYTORCH_ROCM_ARCH: "gfx942"
+          ROCM_CI_IMAGE_REGISTRY: "{{ docker_registry_amd }}"
         retry:
           automatic:
             - exit_status: -1  # Agent was lost
@@ -156,6 +158,14 @@ steps:
 {% for path in step.artifact_paths %}
           - "{{ path }}"
 {% endfor %}
+{% endif %}
+{% if step.retry %}
+        retry:
+          automatic:
+{% for retry_rule in step.retry.automatic %}
+            - exit_status: {{ retry_rule.exit_status }}
+              limit: {{ retry_rule.limit }}
+{% endfor %}
 {% endif %}
         {% if step.timeout_in_minutes %}
         timeout_in_minutes: {{ step.timeout_in_minutes }}
@@ -224,6 +234,14 @@ steps:
 {% for path in step.artifact_paths %}
       - "{{ path }}"
 {% endfor %}
+{% endif %}
+{% if step.retry %}
+    retry:
+      automatic:
+{% for retry_rule in step.retry.automatic %}
+        - exit_status: {{ retry_rule.exit_status }}
+          limit: {{ retry_rule.limit }}
+{% endfor %}
 {% endif %}
     {% if step.timeout_in_minutes %}
     timeout_in_minutes: {{ step.timeout_in_minutes }}

.buildkite/common/ci_mirror_hardwares.yml

@@ -17,6 +17,8 @@
 #   a2b3_npu_1  — ascend-a2b3, resource_class npu-1 (A2/B3 CI image)
 #   a2b3_npu_4  — ascend-a2b3, resource_class npu-4
 #   a3_npu_2    — ascend-a3, resource_class npu-2 (A3 CI image tag …:$BUILDKITE_COMMIT-a3)
+#   a5_npu_2    — ascend-a5, resource_class npu-2
+#   310p_npu_1  — ascend-310p, resource_class npu-1
 #
 # B200 (b200-k8s): matches vLLM device ``b200-k8s`` from ci-infra
 # (AgentQueue.B200_K8S + b200_plugin_template). Queue routes the K8s pool;
@@ -26,6 +28,9 @@
 # packed onto g6.12xlarge L4 nodes (``vllm.ci/gpu-pool=l4x4``). GPU / CPU /
 # memory / shm requests scale with card count (1 GPU = 10 CPU / 40Gi / 24Gi shm).
 # Retry covers lost agents / killed pods, not pytest or OOM (exit 1 / 137).
+# The Kubernetes HF secret deliberately uses VLLM_CI_HF_TOKEN. Scheduled
+# build-level HF_TOKEN values can override pod env entries with the same name;
+# upload_pipeline.py restores the secret-backed value inside the job shell.
 
 x-npu-k8s-plugins: &npu_k8s_plugins
   - kubernetes:
@@ -84,7 +89,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /fsx/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -135,7 +140,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /fsx/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -186,7 +191,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /fsx/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -237,7 +242,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /fsx/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -271,7 +276,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /root/.cache/huggingface
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -306,7 +311,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /root/.cache/huggingface
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -341,7 +346,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /root/.cache/huggingface
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -376,7 +381,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /root/.cache/huggingface
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -426,7 +431,7 @@ mirror_hardwares:
                     value: "0"
                   - name: HF_HOME
                     value: /raid/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -477,7 +482,7 @@ mirror_hardwares:
                     value: "0"
                   - name: HF_HOME
                     value: /raid/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -528,7 +533,7 @@ mirror_hardwares:
                     value: "0"
                   - name: HF_HOME
                     value: /raid/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -579,7 +584,7 @@ mirror_hardwares:
                     value: "0"
                   - name: HF_HOME
                     value: /raid/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -646,3 +651,52 @@ mirror_hardwares:
       resource_class: "npu-16"
     image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
     plugins: *npu_k8s_plugins
+
+  a5_npu_2:
+    agents:
+      queue: "ascend-a5"
+      resource_class: "npu-2"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a5"
+    plugins: *npu_k8s_plugins
+
+  a5_npu_4:
+    agents:
+      queue: "ascend-a5"
+      resource_class: "npu-4"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a5"
+    plugins: *npu_k8s_plugins
+
+  a5_npu_8:
+    agents:
+      queue: "ascend-a5"
+      resource_class: "npu-8"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a5"
+    plugins: *npu_k8s_plugins
+
+  a5_npu_16:
+    agents:
+      queue: "ascend-a5"
+      resource_class: "npu-16"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a5"
+    plugins: *npu_k8s_plugins
+
+  310p_npu_1:
+    agents:
+      queue: "ascend-310p"
+      resource_class: "npu-1"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-310p"
+    plugins: *npu_k8s_plugins
+
+  310p_npu_2:
+    agents:
+      queue: "ascend-310p"
+      resource_class: "npu-2"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-310p"
+    plugins: *npu_k8s_plugins
+
+  310p_npu_4:
+    agents:
+      queue: "ascend-310p"
+      resource_class: "npu-4"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-310p"
+    plugins: *npu_k8s_plugins

.buildkite/common/ci_source_file_dependencies.yml

@@ -0,0 +1,471 @@
+# Uploader-only path-prefix presets for CUDA L2–L5 and NPU L4 test YAML.
+# Referenced as ``source_file_dependencies: omni_qwen3_omni_function`` (or a list of
+# keys) on a step; ``upload_pipeline.py`` expands the key(s) only.
+#
+# YAML anchors above the registry hold model business-code paths. Job keys
+# ({model}_function / _perf / _accuracy / _reliability / _doc / _cov) alias
+# those anchors and add the scripts the job actually runs. Feature keys below
+# are unchanged. ``source_filter_fallback`` is not a job key: when no job-key
+# prefix matches, a matching diff keeps every job for that upload.
+#
+# YAML anchors are nested lists; the loader flattens them.
+
+x-qwen3-tts: &qwen3_tts
+  - vllm_omni/engine/async_omni_engine.py
+  - vllm_omni/entrypoints/omni_base.py
+  - vllm_omni/worker/gpu_model_runner.py
+  - vllm_omni/model_executor/models/qwen3_tts/
+  - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+  - vllm_omni/model_executor/models/common/snake_activation.py
+  - vllm_omni/model_executor/models/common/alias_free_activation.py
+  - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+  - vllm_omni/deploy/qwen3_tts.yaml
+  - vllm_omni/entrypoints/openai/serving_speech.py
+  - vllm_omni/entrypoints/openai/tts_adapters/
+  - vllm_omni/engine/stage_pool.py
+  - vllm_omni/worker_v2/
+  - vllm_omni/core/sched/
+  - vllm_omni/distributed/omni_connectors/
+  - vllm_omni/config/stage_config.py
+  - vllm_omni/config/omni_config.py
+  - vllm_omni/engine/orchestrator.py
+  - vllm_omni/engine/stage_engine_core_proc.py
+  - vllm_omni/deploy/qwen3_tts_mrv2.yaml
+  - vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2.yaml
+  - vllm_omni/deploy/qwen3_tts_high_concurrency_mrv2_b4.yaml
+
+x-minicpmo-4-5: &minicpmo_4_5
+  - vllm_omni/engine/async_omni_engine.py
+  - vllm_omni/entrypoints/omni_base.py
+  - vllm_omni/model_executor/models/minicpmo_4_5/
+  - vllm_omni/model_executor/models/step_audio2/
+  - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
+  - vllm_omni/model_executor/models/cosyvoice3/utils.py
+  - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+  - vllm_omni/deploy/minicpmo_4_5.yaml
+  - vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja
+
+x-minicpmo-4-5-duplex: &minicpmo_4_5_duplex
+  - vllm_omni/deploy/minicpmo_4_5_duplex.yaml
+  - vllm_omni/engine/duplex/
+  - vllm_omni/engine/duplex_omni_engine.py
+  - vllm_omni/engine/duplex_orchestrator.py
+  - vllm_omni/engine/omni_engine_base.py
+  - vllm_omni/entrypoints/duplex/
+  - vllm_omni/entrypoints/duplex_omni.py
+  - vllm_omni/entrypoints/async_omni_base.py
+  - vllm_omni/protocol/
+  - vllm_omni/clients/
+  - vllm_omni/outputs/duplex.py
+  - vllm_omni/model_executor/duplex_sampling.py
+  - vllm_omni/experimental/fullduplex/
+  - vllm_omni/core/sched/
+  - vllm_omni/outputs/multimodal_accumulation.py
+  - vllm_omni/engine/stage_init_utils.py
+  - vllm_omni/engine/orchestrator.py
+  - vllm_omni/entrypoints/cli/serve.py
+  - vllm_omni/entrypoints/openai/api_server.py
+  - vllm_omni/entrypoints/openai/duplex_capability.py
+  - vllm_omni/config/stage_config.py
+
+x-qwen2-5-omni: &qwen2_5_omni
+  - vllm_omni/engine/async_omni_engine.py
+  - vllm_omni/entrypoints/omni_base.py
+  - vllm_omni/model_executor/models/qwen2_5_omni/
+  - vllm_omni/model_executor/stage_input_processors/qwen2_5_omni.py
+  - vllm_omni/deploy/qwen2_5_omni.yaml
+
+x-qwen3-omni: &qwen3_omni
+  - vllm_omni/engine/async_omni_engine.py
+  - vllm_omni/entrypoints/omni_base.py
+  - vllm_omni/entrypoints/openai/serving_chat.py
+  - vllm_omni/model_executor/models/qwen3_omni/
+  - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+  - vllm_omni/model_executor/models/common/snake_activation.py
+  - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
+  - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
+  - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
+  - vllm_omni/deploy/qwen3_omni_moe.yaml
+  - vllm_omni/deploy/qwen3_omni_moe_thinking.yaml
+
+x-wan22: &wan22
+  - vllm_omni/diffusion/models/wan2_2/
+  - vllm_omni/diffusion/models/dmd2/
+
+x-cosmos3: &cosmos3
+  - vllm_omni/diffusion/models/cosmos3/
+  - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_wan.py
+  - vllm_omni/experimental/world_models/
+  - vllm_omni/model_extras/cosmos3.py
+
+x-qwen-image: &qwen_image
+  - vllm_omni/diffusion/models/qwen_image/
+  - vllm_omni/diffusion/models/dmd2/
+  - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py
+
+x-hunyuan-image3: &hunyuan_image3
+  - vllm_omni/diffusion/models/hunyuan_image3/
+  - vllm_omni/model_executor/models/hunyuan_image3/
+  - vllm_omni/deploy/hunyuan_image3_dit.yaml
+
+x-minimax-h3: &minimax_h3
+  - vllm_omni/diffusion/models/minimax_h3/
+  - vllm_omni/diffusion/layers/indexed_modulation.py
+
+x-hunyuan-video: &hunyuan_video
+  - vllm_omni/diffusion/models/hunyuan_video/
+  - vllm_omni/model_executor/models/hunyuan_video/
+
+x-ltx2: &ltx2
+  - vllm_omni/diffusion/models/ltx2/
+  - vllm_omni/model_extras/ltx2.py
+
+x-sana-video: &sana_video
+  - vllm_omni/diffusion/models/sana_video/
+  - vllm_omni/model_extras/sana_video.py
+
+x-z-image: &z_image
+  - vllm_omni/diffusion/models/z_image/
+
+x-stable-audio: &stable_audio
+  - vllm_omni/diffusion/models/stable_audio/
+
+x-lingbot-video: &lingbot_video
+  - vllm_omni/diffusion/models/lingbot_video/
+  - vllm_omni/model_extras/lingbot_video.py
+
+x-lingbot-world: &lingbot_world
+  - vllm_omni/diffusion/models/lingbot_world/
+
+
+source_file_dependencies:
+  # Hardware presets and shared test helpers. Not a job key — do not attach this
+  # to a step. Used only when no job-key prefix matches: then a hit keeps every job.
+  source_filter_fallback:
+    - .buildkite/common/ci_mirror_hardwares.yml
+    - tests/helpers/
+
+  # --- features ---
+  diffusion_distributed_attention:
+    - tests/diffusion/distributed/
+    - tests/diffusion/attention/test_attention_sp.py
+    - vllm_omni/diffusion/distributed/
+    - vllm_omni/diffusion/attention/
+    - vllm_omni/diffusion/data.py
+    - vllm_omni/diffusion/models/
+
+  custom_pipeline:
+    - tests/e2e/features/custom_pipeline/
+    - tests/e2e/features/helpers/custom_pipeline.py
+    - vllm_omni/diffusion/
+    - vllm_omni/engine/
+    - vllm_omni/entrypoints/
+    - vllm_omni/inputs/
+    - vllm_omni/outputs/
+
+  rlhf_verl_omni:
+    - tests/e2e/features/rlhf_test/test_verl_omni_e2e.py
+    - tests/e2e/features/helpers/verl_omni_server.py
+    - tests/e2e/features/helpers/custom_pipeline.py
+    - vllm_omni/diffusion/
+    - vllm_omni/engine/
+    - vllm_omni/entrypoints/
+    - vllm_omni/inputs/
+    - vllm_omni/outputs/
+
+  # Tiny-model diffusion jobs (shared across many model trees under models/).
+  diffusion_tiny_model:
+    - tests/model_tests/diffusion/
+    - vllm_omni/diffusion/models/
+
+  diffusion_offloader:
+    - tests/diffusion/offloader/
+    - vllm_omni/diffusion/offloader/
+
+  diffusion_quantization:
+    - tests/diffusion/quantization/
+    - vllm_omni/diffusion/quantization/
+
+  # --- models ---
+  # --- omni / tts jobs ---
+  omni_qwen3_omni_function:
+    - *qwen3_omni
+    - tests/e2e/offline_inference/test_qwen3_omni.py
+    - tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py
+    - tests/e2e/offline_inference/test_qwen3_omni_autoround_w4a16_expansion.py
+    - tests/e2e/offline_inference/test_qwen3_omni_modelopt_nvfp4_w4a4.py
+    - tests/e2e/online_serving/test_qwen3_omni.py
+    - tests/e2e/online_serving/test_qwen3_omni_expansion.py
+    - tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
+
+  omni_qwen3_omni_accuracy:
+    - *qwen3_omni
+    - tests/e2e/accuracy/qwen3_omni/
+
+  omni_qwen3_omni_perf:
+    - *qwen3_omni
+    - tests/dfx/perf/scripts/run_benchmark.py
+    - tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json
+    - tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
+    - tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
+    - tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json
+
+  omni_qwen3_omni_reliability:
+    - *qwen3_omni
+    - tests/dfx/reliability/test_reliability_qwen3_omni.py
+
+  omni_qwen3_omni_doc:
+    - *qwen3_omni
+    - examples/online_serving/qwen3_omni/
+    - examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py
+    - tests/examples/online_serving/test_qwen3_omni.py
+
+  omni_qwen2_5_omni_doc:
+    - *qwen2_5_omni
+    - examples/online_serving/qwen2_5_omni/
+    - examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py
+    - tests/examples/online_serving/test_qwen2_5_omni.py
+
+  omni_minicpmo_4_5_function:
+    - *minicpmo_4_5
+    - tests/e2e/offline_inference/test_minicpmo_4_5.py
+    - tests/e2e/online_serving/test_minicpmo_4_5.py
+    - tests/e2e/online_serving/test_minicpmo_4_5_expansion.py
+
+  omni_minicpmo_4_5_accuracy:
+    - *minicpmo_4_5
+    - tests/e2e/accuracy/minicpmo_4_5/
+
+  omni_minicpmo_4_5_perf:
+    - *minicpmo_4_5
+    - tests/dfx/perf/scripts/run_benchmark.py
+    - tests/dfx/perf/tests/test_minicpmo_4_5_ready.json
+    - tests/dfx/perf/tests/test_minicpmo_4_5.json
+
+  omni_minicpmo_4_5_doc:
+    - *minicpmo_4_5
+    - examples/online_serving/minicpmo/realtime_web/
+    - examples/online_serving/minicpmo/realtime_duplex_demo.py
+    - tests/examples/test_minicpmo_realtime_web_server.py
+    - tests/examples/test_minicpmo_realtime_web_static.py
+    - tests/examples/test_minicpmo_realtime_duplex_simple_demo.py
+
+  omni_minicpmo_4_5_duplex_function:
+    - *minicpmo_4_5
+    - *minicpmo_4_5_duplex
+    - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
+    - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+    - tests/e2e/online_serving/test_duplex_client_live.py
+    - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
+    - tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
+    - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
+    - tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py
+    - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
+    - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
+    - tests/assets/minicpmo_4_5/
+    - tests/clients/
+    - tests/engine/duplex/
+    - tests/engine/test_duplex_omni_engine.py
+    - tests/engine/test_duplex_orchestrator.py
+    - tests/entrypoints/duplex/
+    - tests/entrypoints/test_duplex_omni.py
+    - tests/entrypoints/openai_api/test_duplex_api_server.py
+    - tests/model_executor/models/minicpmo_4_5/duplex/
+    - tests/protocol/
+
+  omni_minicpmo_4_5_duplex_perf:
+    - *minicpmo_4_5
+    - *minicpmo_4_5_duplex
+    - tests/dfx/perf/scripts/run_benchmark.py
+    - tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json
+
+  tts_qwen3_tts_function:
+    - *qwen3_tts
+    - tests/e2e/features/prefill_phase/
+    - tests/e2e/offline_inference/test_qwen3_tts_base.py
+    - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
+    - tests/e2e/online_serving/test_qwen3_tts_base.py
+    - tests/e2e/online_serving/test_qwen3_tts_base_expansion.py
+    - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
+    - tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py
+
+  tts_qwen3_tts_perf:
+    - *qwen3_tts
+    - tests/dfx/perf/scripts/run_benchmark.py
+    - tests/dfx/perf/tests/test_tts.json
+
+  tts_qwen3_tts_cov:
+    - *qwen3_tts
+    - .buildkite/common/scripts/run_cov_split.sh
+    - pyproject.toml
+    - tests/e2e/offline_inference/test_qwen3_tts_base.py
+    - tests/e2e/online_serving/test_qwen3_tts_base.py
+
+  # --- diffusion jobs ---
+  diffusion_wan22_function:
+    - *wan22
+    - tests/e2e/offline_inference/test_wan22_t2v.py
+    - tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py
+    - tests/e2e/offline_inference/test_device_postprocess_equivalence.py
+    - tests/e2e/online_serving/test_wan22_t2v.py
+    - tests/e2e/online_serving/test_wan22_expansion.py
+
+  diffusion_wan22_accuracy:
+    - *wan22
+    - tests/e2e/accuracy/wan22_i2v/
+    - tests/e2e/accuracy/test_diffusers_backend_similarity.py
+
+  diffusion_wan22_perf:
+    - *wan22
+    - tests/dfx/perf/scripts/run_diffusion_benchmark.py
+    - tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json
+
+  diffusion_wan22_reliability:
+    - *wan22
+    - tests/dfx/reliability/test_reliability_wan22.py
+
+  diffusion_image_to_video_doc:
+    - *wan22
+    - *hunyuan_video
+    - *ltx2
+    - *sana_video
+    - *lingbot_video
+    - examples/offline_inference/image_to_video/
+    - tests/examples/offline_inference/test_image_to_video.py
+
+  diffusion_cosmos3_function:
+    - *cosmos3
+    - tests/e2e/online_serving/test_cosmos3.py
+
+  diffusion_cosmos3_accuracy:
+    - *cosmos3
+    - tests/e2e/accuracy/test_cosmos3_similarity.py
+
+  diffusion_cosmos3_perf:
+    - *cosmos3
+    - tests/dfx/perf/scripts/run_diffusion_benchmark.py
+    - tests/dfx/perf/tests/test_cosmos3_vllm_omni.json
+
+  diffusion_qwen_image_function:
+    - *qwen_image
+    - tests/e2e/online_serving/test_qwen_image.py
+    - tests/e2e/online_serving/test_qwen_image_expansion.py
+    - tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py
+
+  diffusion_qwen_image_accuracy:
+    - *qwen_image
+    - tests/e2e/accuracy/test_qwen_image.py
+    - tests/e2e/accuracy/test_diffusers_backend_similarity.py
+
+  diffusion_qwen_image_perf:
+    - *qwen_image
+    - tests/dfx/perf/scripts/run_diffusion_benchmark.py
+    - tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
+
+  diffusion_text_to_image_doc:
+    - *qwen_image
+    - *z_image
+    - examples/offline_inference/text_to_image/
+    - examples/online_serving/text_to_image/
+    - tests/examples/offline_inference/test_text_to_image.py
+    - tests/examples/online_serving/test_text_to_image.py
+    - vllm_omni/diffusion/models/nextstep_1_1/
+    - vllm_omni/diffusion/models/longcat_image/
+    - vllm_omni/diffusion/models/ovis_image/
+    - vllm_omni/diffusion/models/omnigen2/
+    - vllm_omni/diffusion/models/sd3/
+    - vllm_omni/diffusion/models/flux/
+    - vllm_omni/diffusion/models/flux2/
+    - vllm_omni/diffusion/models/flux2_klein/
+    - vllm_omni/diffusion/models/hidream_image/
+    - vllm_omni/diffusion/models/krea2/
+    - vllm_omni/diffusion/models/lingbot_video/
+
+  diffusion_hunyuan_image3_function:
+    - *hunyuan_image3
+    - tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py
+
+  diffusion_hunyuan_image3_accuracy:
+    - *hunyuan_image3
+    - tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py
+
+  diffusion_hunyuan_image3_perf:
+    - *hunyuan_image3
+    - tests/dfx/perf/scripts/run_diffusion_benchmark.py
+    - tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json
+    - tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json
+    - tests/dfx/perf/tests/test_hunyuan_image_tp4.json
+
+  diffusion_hunyuan_image3_reliability:
+    - *hunyuan_image3
+    - tests/dfx/reliability/test_reliability_hunyuan_image.py
+
+  diffusion_minimax_h3_function:
+    - *minimax_h3
+    - tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py
+    - tests/e2e/online_serving/minimax_h3/
+    - tests/diffusion/models/minimax_h3/
+    - vllm_omni/diffusion/lora/
+    - vllm_omni/diffusion/offloader/
+    - vllm_omni/diffusion/distributed/
+    - vllm_omni/entrypoints/openai/
+
+  diffusion_minimax_h3_accuracy:
+    - *minimax_h3
+    - tests/e2e/accuracy/minimax_h3/
+
+  diffusion_minimax_h3_perf:
+    - *minimax_h3
+    - tests/dfx/perf/scripts/run_diffusion_benchmark.py
+    - tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json
+
+  diffusion_hunyuan_video_function:
+    - *hunyuan_video
+    - tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
+
+  diffusion_hunyuan_video_accuracy:
+    - *hunyuan_video
+    - tests/e2e/accuracy/hunyuanvideo15_t2v/
+    - tests/e2e/accuracy/hunyuanvideo15_i2v/
+
+  diffusion_hunyuan_video_perf:
+    - *hunyuan_video
+    - tests/dfx/perf/scripts/run_diffusion_benchmark.py
+    - tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
+    - tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json
+
+  diffusion_z_image_function:
+    - *z_image
+    - tests/e2e/online_serving/test_z_image.py
+    - tests/e2e/online_serving/test_zimage_expansion.py
+
+  diffusion_sana_video_function:
+    - *sana_video
+    - tests/e2e/online_serving/test_sana_video_expansion.py
+    - tests/e2e/accuracy/sana_video/
+
+  diffusion_stable_audio_function:
+    - *stable_audio
+    - tests/e2e/offline_inference/test_stable_audio_expansion.py
+    - tests/e2e/online_serving/test_stable_audio_online_expansion.py
+
+  diffusion_lingbot_function:
+    - *lingbot_video
+    - *lingbot_world
+    - tests/e2e/online_serving/test_lingbot_video.py
+    - tests/e2e/online_serving/test_lingbot_video_moe.py
+    - tests/e2e/offline_inference/test_lingbot_world_v2.py
+
+
+  # Invalid-param jobs: model business code only, plus the reliability scripts.
+  reliability_invalid_param_h100:
+    - *qwen3_omni
+    - *minicpmo_4_5
+    - *qwen_image
+    - *wan22
+    - tests/dfx/reliability/invalid_param_test/
+
+  reliability_invalid_param_l4:
+    - *qwen3_tts
+    - *stable_audio
+    - tests/dfx/reliability/invalid_param_test/

.buildkite/common/scripts/skip_ci.py

@@ -44,6 +44,7 @@ L3_YAML_FILES: dict[str, str] = {
 L45_YAML_FILES: dict[str, str] = {
     ".buildkite/cuda/test-nightly.yml": "cuda",
     ".buildkite/cuda/test-weekly.yml": "cuda",
+    ".buildkite/amd/test-amd-nightly.yml": "amd",
     ".buildkite/npu/test-npu-nightly.yml": "npu",
 }
 

.buildkite/common/scripts/upload_pipeline.py

@@ -10,8 +10,17 @@ Bootstrap mode (``bootstrap-upload-steps.yml``):
   - Detect docs-only, pytest skip-mark-only, or combined skip-ci from git diff.
   - When only CI level YAML changes, enable **L2/L3** upload steps for affected levels only.
 
-Test pipeline mode (e.g. test-merge.yml):
-  - Drop steps whose ``source_file_dependencies`` do not match changed files.
+Test pipeline mode (e.g. test-merge.yml, test-nightly.yml, test-weekly.yml):
+  - Drop steps whose ``source_file_dependencies`` do not match changed files
+    (string/list keys from ci_source_file_dependencies.yml, or inline path
+    prefixes). Filtering applies on **PR label** uploads and on post-merge
+    ``main`` L3 uploads. Scheduled ``main`` uploads that pass ``--all``
+    (NIGHTLY/WEEKLY) or ``--e2e`` keep every selected job and still strip
+    the field. If no job-key prefix matches, a change to the pipeline YAML
+    being uploaded or to a path under the ``source_filter_fallback`` registry
+    key keeps every job so command, env, and hardware edits can be validated
+    before merge. When any job-key prefix already matches, normal filtering
+    wins.
   - Expand uploader-only ``mirror_hardwares`` into ``agents`` (+ optional ``image``
     for NPU) + ``plugins`` (see ci_mirror_hardwares.yml).
   - Omit ``mirror_hardwares`` to compose ``{chip}_{n}`` from pytest ``-m`` SKU
@@ -70,7 +79,15 @@ from tests.helpers.mark import (  # noqa: E402
 
 LOG = "upload_pipeline"
 BOOTSTRAP_STEPS_FILENAME = "bootstrap-upload-steps.yml"
-BOOTSTRAP_IMAGE_BUILD_KEYS = frozenset({"image-build", "image-build-a2", "image-build-a3"})
+BOOTSTRAP_IMAGE_BUILD_KEYS = frozenset(
+    {
+        "image-build",
+        "image-build-a2",
+        "image-build-a3",
+        "image-build-a5",
+        "image-build-310p",
+    }
+)
 BOOTSTRAP_UPLOAD_IF_KEYS = {
     "upload-ready-pipeline": "ready",
     "upload-merge-pipeline": "merge",
@@ -79,20 +96,19 @@ BOOTSTRAP_UPLOAD_IF_KEYS = {
 }
 E2E_GROUP_MARKER = "E2E Test"
 CI_MIRROR_HARDWARES_PATH = ROOT / ".buildkite/common/ci_mirror_hardwares.yml"
+CI_SOURCE_FILE_DEPENDENCIES_PATH = ROOT / ".buildkite/common/ci_source_file_dependencies.yml"
+# Registry key whose prefixes bypass source filtering. Paths live in
+# ci_source_file_dependencies.yml; do not attach this key to a job.
+SOURCE_FILTER_FALLBACK_KEY = "source_filter_fallback"
+CUDA_HF_TOKEN_ENV = "VLLM_CI_HF_TOKEN"
+CUDA_HF_TOKEN_EXPORT = f'if [ -n "$${{{CUDA_HF_TOKEN_ENV}:-}}" ]; then export HF_TOKEN="$${{{CUDA_HF_TOKEN_ENV}}}"; fi'
 
 # Bootstrap Buildkite ``if`` expressions.
 # ``*_MAIN_IF``: main + env schedule. ``*_LABEL_IF``: PR label (and/or composed with MAIN).
 # ``*_UPLOAD_IF``: full gate for uploading that child pipeline.
 NIGHTLY_MAIN_IF = 'build.branch == "main" && build.env("NIGHTLY") == "1"'
 NIGHTLY_LABEL_IF = (
-    f"({NIGHTLY_MAIN_IF}) || "
-    '(build.branch != "main" && ('
-    'build.pull_request.labels includes "nightly-test" || '
-    'build.pull_request.labels includes "omni-test" || '
-    'build.pull_request.labels includes "tts-test" || '
-    'build.pull_request.labels includes "diffusion-x2iat-test" || '
-    'build.pull_request.labels includes "diffusion-x2v-test"'
-    "))"
+    f'({NIGHTLY_MAIN_IF}) || (build.branch != "main" && build.pull_request.labels includes "nightly-test")'
 )
 WEEKLY_E2E_IF = 'build.branch == "main" && build.env("WEEKLY") == "1"'
 WEEKLY_MAIN_IF = 'build.branch == "main" && (build.env("WEEKLY") == "1" || build.env("NON_CRITICAL") == "1")'
@@ -238,7 +254,7 @@ def _render_bootstrap_pipeline(
     return yaml.safe_dump(doc, sort_keys=False)
 
 
-# --- Test pipeline (test-ready.yml, test-merge.yml) ---
+# --- Test pipeline (test-ready.yml, test-merge.yml, test-nightly.yml) ---
 
 
 @lru_cache(maxsize=1)
@@ -254,6 +270,117 @@ def _load_mirror_hardwares() -> dict[str, dict[str, Any]]:
     return presets
 
 
+@lru_cache(maxsize=1)
+def _load_source_file_dependencies() -> dict[str, list[str]]:
+    def flatten(value: Any, *, key: str) -> list[str]:
+        """Flatten YAML-anchor nested lists into a de-duplicated prefix list."""
+        if isinstance(value, str):
+            if not value.strip():
+                raise ValueError(f"empty path in source_file_dependencies[{key!r}]")
+            return [value]
+        if isinstance(value, list):
+            flattened: list[str] = []
+            seen: set[str] = set()
+            for item in value:
+                for path in flatten(item, key=key):
+                    if path not in seen:
+                        seen.add(path)
+                        flattened.append(path)
+            return flattened
+        raise ValueError(
+            f"source_file_dependencies[{key!r}] must be a list of path prefixes, got {type(value).__name__}",
+        )
+
+    if not CI_SOURCE_FILE_DEPENDENCIES_PATH.is_file():
+        raise FileNotFoundError(
+            f"missing CI source_file_dependencies registry: {CI_SOURCE_FILE_DEPENDENCIES_PATH}",
+        )
+    doc = yaml.safe_load(CI_SOURCE_FILE_DEPENDENCIES_PATH.read_text(encoding="utf-8"))
+    if not isinstance(doc, dict):
+        raise ValueError(f"invalid CI source_file_dependencies registry: {CI_SOURCE_FILE_DEPENDENCIES_PATH}")
+    presets = doc.get("source_file_dependencies")
+    if not isinstance(presets, dict):
+        raise ValueError(f"source_file_dependencies must be a mapping in {CI_SOURCE_FILE_DEPENDENCIES_PATH}")
+    loaded: dict[str, list[str]] = {}
+    for key, value in presets.items():
+        if not isinstance(key, str) or not key.strip():
+            raise ValueError(
+                f"source_file_dependencies keys must be non-empty strings in {CI_SOURCE_FILE_DEPENDENCIES_PATH}",
+            )
+        loaded[key] = flatten(value, key=key)
+    return loaded
+
+
+_REGISTRY_KEY_RE = re.compile(r"^[a-z][a-z0-9_]*$")
+
+
+def _parse_command_text(step: dict[str, Any]) -> str:
+    """Parse step ``commands`` / ``command`` into a single string."""
+    raw = step.get("commands", step.get("command"))
+    if raw is None:
+        return ""
+    if isinstance(raw, str):
+        return raw
+    if isinstance(raw, list):
+        return "\n".join(str(item) for item in raw if item is not None)
+    return str(raw)
+
+
+def _resolve_source_file_dependencies(step: dict[str, Any]) -> list[str] | None:
+    """Expand a registry key (or list of keys / inline path prefixes)."""
+    deps = step.get("source_file_dependencies")
+    if deps is None:
+        return None
+
+    def dedupe(*groups: list[str]) -> list[str]:
+        merged: list[str] = []
+        seen: set[str] = set()
+        for group in groups:
+            for path in group:
+                if path not in seen:
+                    seen.add(path)
+                    merged.append(path)
+        return merged
+
+    def lookup(key: str) -> list[str]:
+        if key == SOURCE_FILTER_FALLBACK_KEY:
+            raise ValueError(
+                f"source_file_dependencies {key!r} bypasses filtering and cannot be attached to a step "
+                f"({_get_step_label(step)!r})",
+            )
+        registry = _load_source_file_dependencies()
+        paths = registry.get(key)
+        if paths is None:
+            known = ", ".join(sorted(registry))
+            raise ValueError(
+                f"unknown source_file_dependencies {key!r} in step {_get_step_label(step)!r}; known: {known}",
+            )
+        return list(paths)
+
+    prefixes: list[str]
+    if isinstance(deps, str):
+        prefixes = lookup(deps) if _REGISTRY_KEY_RE.fullmatch(deps) else [deps]
+    elif isinstance(deps, list):
+        if not all(isinstance(item, str) for item in deps):
+            raise ValueError(
+                f"source_file_dependencies must be a string key or list of strings in step {_get_step_label(step)!r}",
+            )
+        keyish = [bool(_REGISTRY_KEY_RE.fullmatch(item)) for item in deps]
+        if deps and all(keyish):
+            prefixes = dedupe(*(lookup(key) for key in deps))
+        elif any(keyish):
+            raise ValueError(
+                f"source_file_dependencies in step {_get_step_label(step)!r} mixes registry keys and path prefixes",
+            )
+        else:
+            prefixes = list(deps)
+    else:
+        raise ValueError(
+            f"source_file_dependencies must be a string key or list in step {_get_step_label(step)!r}",
+        )
+    return prefixes
+
+
 @lru_cache(maxsize=1)
 def _cuda_mirror_chips() -> tuple[str, ...]:
     """Lowercase CUDA SKUs that have at least one ``{chip}_{n}`` preset.
@@ -268,7 +395,7 @@ def _cuda_mirror_chips() -> tuple[str, ...]:
     return tuple(sorted(chips, key=lambda chip: (-len(chip), chip)))
 
 
-def _read_cards_marks(expr: str) -> tuple[set[int], bool]:
+def _parse_cards_marks(expr: str) -> tuple[set[int], bool]:
     """Parse registered ``cards_n`` / ``not cards_n`` from a pytest ``-m`` expr.
 
     Returns positive card counts and whether any ``not cards_*`` is present.
@@ -285,8 +412,8 @@ def _read_cards_marks(expr: str) -> tuple[set[int], bool]:
     return positives, has_not_cards
 
 
-def _read_hardware_marks(expr: str) -> set[str]:
-    """Return lowercase positive CUDA SKUs that have a mirror preset.
+def _parse_hardware_marks(expr: str) -> set[str]:
+    """Parse lowercase positive CUDA SKUs that have a mirror preset.
 
     ``not H100`` is ignored. SKUs with no ``{chip}_*`` preset (e.g. H200) are dropped.
     """
@@ -328,36 +455,22 @@ def _get_mirror_hw_selector() -> str:
     return selector
 
 
-def _read_pytest_marks(commands: Any) -> tuple[set[str], int | Literal["max"] | None]:
-    """Read pytest ``-m`` from step commands: CUDA SKU chips and cards count.
+def _parse_pytest_marks(step: dict[str, Any]) -> tuple[set[str], int | Literal["max"] | None]:
+    """Parse pytest ``-m`` from step commands: CUDA SKU chips and cards count.
 
     *chips* are lowercase SKUs from positive markers (``not H100`` is ignored).
     *cards* is the max positive ``cards_n``, ``"max"`` when only ``not cards_*``
     is present, or ``None``.
     """
-    chunks: list[str] = []
-
-    def _collect(value: Any) -> None:
-        if value is None:
-            return
-        if isinstance(value, str):
-            chunks.append(value)
-        elif isinstance(value, list):
-            for part in value:
-                _collect(part)
-        else:
-            chunks.append(str(value))
-
-    _collect(commands)
-    text = "\n".join(chunks)
+    text = _parse_command_text(step)
 
     chips: set[str] = set()
     positives: set[int] = set()
     has_not_cards = False
     for match in _PYTEST_MARKER_ARG.finditer(text):
         expr = match.group(1) or match.group(2) or match.group(3) or ""
-        chips |= _read_hardware_marks(expr)
-        card_counts, not_cards = _read_cards_marks(expr)
+        chips |= _parse_hardware_marks(expr)
+        card_counts, not_cards = _parse_cards_marks(expr)
         positives.update(card_counts)
         has_not_cards = has_not_cards or not_cards
 
@@ -474,7 +587,7 @@ def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any] | None:
     if "mirror_hardwares" not in step:
         if has_pool:
             return step
-        chips, cards = _read_pytest_marks(step.get("commands"))
+        chips, cards = _parse_pytest_marks(step)
         if not chips and cards is None:
             return step
         preset_name = _compose_mirror_hardware_name(chips, cards, step_label=step_label)
@@ -501,6 +614,17 @@ def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any] | None:
 
     expanded = copy.deepcopy(preset)
     merged = {key: value for key, value in step.items() if key != "mirror_hardwares"} | expanded
+    # A scheduled build's HF_TOKEN may override a pod env entry with the same
+    # name. The preset injects the Kubernetes secret under a collision-proof
+    # alias; restore the conventional name after Buildkite has merged env.
+    if any(preset_name == chip or preset_name.startswith(f"{chip}_") for chip in _cuda_mirror_chips()):
+        commands = merged.get("commands")
+        if isinstance(commands, list):
+            merged["commands"] = [CUDA_HF_TOKEN_EXPORT, *commands]
+        elif commands is None:
+            merged["commands"] = [CUDA_HF_TOKEN_EXPORT]
+        else:
+            merged["commands"] = [CUDA_HF_TOKEN_EXPORT, commands]
     # Preset retry (K8S_RETRY on l4_*) must not clobber a step that opted out.
     if "retry" in step:
         merged["retry"] = step["retry"]
@@ -531,11 +655,7 @@ def _process_test_steps(
             processed.append(step)
             continue
 
-        deps = step.get("source_file_dependencies")
-        if deps is not None and not isinstance(deps, list):
-            raise ValueError(
-                f"source_file_dependencies must be a list in step {_get_step_label(step)!r}",
-            )
+        deps = _resolve_source_file_dependencies(step)
         if changed_files is not None and deps is not None and not _match_source_file(changed_files, deps):
             _log(f"skip {_get_step_label(step)!r} (no changes under {deps})")
             continue
@@ -574,10 +694,25 @@ def _select_e2e_group_steps(steps: list[Any]) -> list[Any]:
     return selected
 
 
+def _any_source_dependency_match(steps: list[Any], changed_files: list[str]) -> bool:
+    """True when any step with ``source_file_dependencies`` matches *changed_files*."""
+    for step in steps:
+        if not isinstance(step, dict):
+            continue
+        deps = _resolve_source_file_dependencies(step)
+        if deps is not None and _match_source_file(changed_files, deps):
+            return True
+        nested = step.get("steps")
+        if isinstance(nested, list) and _any_source_dependency_match(nested, changed_files):
+            return True
+    return False
+
+
 def _render_test_pipeline(
     doc: dict[str, Any],
     changed_files: list[str] | None,
     *,
+    pipeline_path: Path | None = None,
     e2e_only: bool = False,
 ) -> dict[str, Any]:
     """Filter steps by PR diff and strip uploader-only ``source_file_dependencies`` metadata."""
@@ -590,6 +725,13 @@ def _render_test_pipeline(
         return doc
     if e2e_only:
         steps = _select_e2e_group_steps(steps)
+    # Bypass is a fallback: only when no job-key prefix matched.
+    if changed_files is not None and pipeline_path is not None:
+        if not _any_source_dependency_match(steps, changed_files):
+            bypass = _source_filter_fallback_reason(changed_files, pipeline_path)
+            if bypass is not None:
+                _log(f"keep all jobs (no source_file_dependencies match; bypassed by {bypass})")
+                changed_files = None
     steps = _process_test_steps(steps, changed_files)
     return {**doc, "steps": steps}
 
@@ -597,6 +739,60 @@ def _render_test_pipeline(
 # --- Entry (read file → bootstrap or test render → YAML string) ---
 
 
+def _repo_relative_posix(path: Path) -> str:
+    resolved = path.resolve()
+    try:
+        return resolved.relative_to(ROOT).as_posix()
+    except ValueError:
+        return path.as_posix().replace("\\", "/")
+
+
+def _source_filter_fallback_prefixes() -> list[str]:
+    """Shared uploader/registry prefixes from ``source_filter_fallback``. Not a job key."""
+    registry = _load_source_file_dependencies()
+    prefixes = registry.get(SOURCE_FILTER_FALLBACK_KEY)
+    if not prefixes:
+        raise ValueError(
+            f"source_file_dependencies[{SOURCE_FILTER_FALLBACK_KEY!r}] must list shared uploader paths "
+            f"in {CI_SOURCE_FILE_DEPENDENCIES_PATH}",
+        )
+    return prefixes
+
+
+def _source_filter_fallback_reason(changed_files: list[str] | None, pipeline_path: Path) -> str | None:
+    """Return the changed path that should disable source filtering, if any."""
+    if not changed_files:
+        return None
+    changed = set(changed_files)
+    pipeline_rel = _repo_relative_posix(pipeline_path)
+    if pipeline_rel in changed:
+        return pipeline_rel
+    prefixes = _source_filter_fallback_prefixes()
+    for path in changed_files:
+        if _match_source_file([path], prefixes):
+            return path
+    return None
+
+
+def _changed_files_for_source_filter(
+    ctx,
+    *,
+    force_all: bool,
+    e2e_only: bool,
+) -> list[str] | None:
+    """Return the diff to filter against, or None to keep every step.
+
+    ``--all`` / ``--e2e`` disable filtering (scheduled NIGHTLY/WEEKLY full
+    uploads and weekly E2E sweeps). Post-merge ``main`` L3 and PR-label
+    uploads filter against the commit/PR diff. Pipeline YAML /
+    ``source_filter_fallback`` matches are applied later in
+    ``_render_test_pipeline`` only when no job-key prefix already matched.
+    """
+    if force_all or e2e_only:
+        return None
+    return ctx.changed_files
+
+
 def _render_pipeline(
     path: Path,
     *,
@@ -614,13 +810,18 @@ def _render_pipeline(
 
     text = path.read_text(encoding="utf-8")
     ctx = resolve_ci_context_from_git()
-    changed_files = None if force_all or e2e_only else ctx.changed_files
+    changed_files = _changed_files_for_source_filter(ctx, force_all=force_all, e2e_only=e2e_only)
 
     doc = yaml.safe_load(text)
     if not isinstance(doc, dict):
         raise ValueError(f"invalid pipeline YAML: {path}")
 
-    doc = _render_test_pipeline(doc, changed_files, e2e_only=e2e_only)
+    doc = _render_test_pipeline(
+        doc,
+        changed_files,
+        pipeline_path=path,
+        e2e_only=e2e_only,
+    )
     return yaml.safe_dump(doc, sort_keys=False)
 
 

.buildkite/cuda/bootstrap-upload-steps.yml

@@ -22,7 +22,8 @@ steps:
     agents:
       queue: "cpu_queue_premerge"
 
-  # L3 Test — PR merge-test label (diff-aware) or main+WEEKLY=1 (--e2e, full E2E group)
+  # L3 Test — PR merge-test label or post-merge on main (both diff-aware);
+  # main+WEEKLY=1 uploads the E2E group only (--e2e).
   - label: "Upload Merge Pipeline"
     depends_on: image-build
     key: upload-merge-pipeline
@@ -36,20 +37,30 @@ steps:
     agents:
       queue: "cpu_queue_premerge"
 
-  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
+  # L4 Test — main+NIGHTLY=1 (full pipeline), or PR nightly-test label (diff-aware)
   - label: "Upload Nightly Pipeline"
     depends_on: image-build
     key: upload-nightly-pipeline
     commands:
-      - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-nightly.yml
+      - |
+        EXTRA=""
+        if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
+          EXTRA="--all"
+        fi
+        python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-nightly.yml
     agents:
       queue: "cpu_queue_premerge"
 
-  # L5 Test — main+WEEKLY=1 / NON_CRITICAL=1 (scheduled), or PR with weekly-test label
+  # L5 Test — main+WEEKLY=1 / NON_CRITICAL=1 (full pipeline), or PR weekly-test label (diff-aware)
   - label: "Upload Weekly Pipeline"
     depends_on: image-build
     key: upload-weekly-pipeline
     commands:
-      - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-weekly.yml
+      - |
+        EXTRA=""
+        if [ "${BUILDKITE_BRANCH}" = "main" ]; then
+          EXTRA="--all"
+        fi
+        python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-weekly.yml
     agents:
       queue: "cpu_queue_premerge"

.buildkite/cuda/test-merge.yml

@@ -1,5 +1,6 @@
 # CUDA CI L3 (`advanced_model` e2e) plus L1 CPU Simple Test (`core_model and cpu`).
-# Uploaded by Upload Merge Pipeline: PR label `merge-test`, or post-merge on main.
+# Uploaded by Upload Merge Pipeline: PR label `merge-test`, or post-merge on main
+# (both diff-aware via source_file_dependencies).
 
 env:
   VLLM_WORKER_MULTIPROC_METHOD: spawn
@@ -65,7 +66,9 @@ steps:
         # 0.26 startup regression is understood and fixed.
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"
+          # Turn-based server VAD over /v1/realtime is deferred to RFC #7181 PR 5
+          # (see the P0 thread on #7413); the test stays in the tree and runs again there.
+          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model" --deselect tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py::TestQwen3OmniRealtimeWebSocket::test_server_vad_multi_turn_without_client_commit
         mirror_hardwares: h100_2
 
       - label: "Entrypoint Test with L4"
@@ -78,44 +81,18 @@ steps:
     depends_on: upload-merge-pipeline
     steps:
       - label: "TTS · Qwen3-TTS CustomVoice Test"
-        source_file_dependencies:
-          - tests/helpers/media.py
-          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
-          - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
-          - vllm_omni/model_executor/models/qwen3_tts/
-          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-          - vllm_omni/model_executor/models/common/snake_activation.py
-          - vllm_omni/model_executor/models/common/alias_free_activation.py
-          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
-          - vllm_omni/deploy/qwen3_tts.yaml
-          - vllm_omni/entrypoints/openai/serving_speech.py
-          - vllm_omni/entrypoints/openai/tts_adapters/
-          - vllm_omni/engine/stage_pool.py
+        source_file_dependencies: tts_qwen3_tts_function
         commands:
           - |
             timeout 40m bash -c "
               export VLLM_LOGGING_LEVEL=DEBUG
               export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
-              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py tests/e2e/features/prefill_phase/test_chunked_prefill_tail.py -m 'advanced_model and cuda' --run-level 'advanced_model'
             "
         mirror_hardwares: l4_1
 
       - label: "TTS · Qwen3-TTS Base Test"
-        source_file_dependencies:
-          - tests/helpers/media.py
-          - tests/e2e/online_serving/test_qwen3_tts_base.py
-          - tests/e2e/offline_inference/test_qwen3_tts_base.py
-          - vllm_omni/model_executor/models/qwen3_tts/
-          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-          - vllm_omni/model_executor/models/common/snake_activation.py
-          - vllm_omni/model_executor/models/common/alias_free_activation.py
-          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
-          - vllm_omni/deploy/qwen3_tts.yaml
-          - vllm_omni/entrypoints/openai/serving_speech.py
-          - vllm_omni/entrypoints/openai/tts_adapters/
-          - vllm_omni/engine/stage_pool.py
-          - .buildkite/common/scripts/run_cov_split.sh
-          - pyproject.toml
+        source_file_dependencies: tts_qwen3_tts_cov
         # 40 -> 60: the coverage split runs the model twice.
         timeout_in_minutes: 60
         commands:
@@ -130,35 +107,14 @@ steps:
         mirror_hardwares: l4_1
 
       - label: "Omni · Qwen3-Omni Test"
-        source_file_dependencies:
-          - tests/e2e/offline_inference/test_qwen3_omni.py
-          - tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py
-          - tests/e2e/online_serving/test_qwen3_omni.py
-          - tests/helpers/runtime.py
-          - vllm_omni/entrypoints/openai/serving_chat.py
-          - vllm_omni/model_executor/models/qwen3_omni/
-          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-          - vllm_omni/model_executor/models/common/snake_activation.py
-          - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
-          - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
-          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
-          - vllm_omni/deploy/qwen3_omni_moe.yaml
-          - vllm_omni/deploy/qwen3_omni_moe_thinking.yaml
+        source_file_dependencies: omni_qwen3_omni_function
         timeout_in_minutes: 50
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py -m 'advanced_model and cuda' --run-level 'advanced_model'
         mirror_hardwares: h100_2
 
       - label: "Omni · MiniCPM-o 4.5 Test"
-        source_file_dependencies:
-          - tests/e2e/offline_inference/test_minicpmo_4_5.py
-          - tests/e2e/online_serving/test_minicpmo_4_5.py
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/models/step_audio2/
-          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
-          - vllm_omni/model_executor/models/cosyvoice3/utils.py
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
-          - vllm_omni/deploy/minicpmo_4_5.yaml
+        source_file_dependencies: omni_minicpmo_4_5_function
         timeout_in_minutes: 50
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'
@@ -171,59 +127,14 @@ steps:
         mirror_hardwares: h100_2
 
       - label: "Omni · MiniCPM-o 4.5 Duplex Test"
-        source_file_dependencies:
-          - tests/assets/minicpmo_4_5/response_required_16k.wav
-          - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
-          - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
-          - tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
-          - vllm_omni/deploy/minicpmo_4_5.yaml
-          - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
-          - vllm_omni/engine/duplex/
-          - vllm_omni/entrypoints/duplex/
-          # Public duplex client library + its unit suites: client-side
-          # changes must re-run the duplex e2e group. The live client probe
-          # (test_duplex_client_live.py, single GPU, core tier) runs in the
-          # ready pipeline's duplex group.
-          - vllm_omni/clients/
-          - tests/clients/
-          - vllm_omni/entrypoints/duplex_request_client.py
-          - vllm_omni/outputs/duplex.py
-          - vllm_omni/model_executor/duplex_sampling.py
-          - tests/engine/duplex/
-          - tests/entrypoints/duplex/
-          - tests/model_executor/models/minicpmo_4_5/duplex/
-          # Legacy Realtime probe client + video stacking used by the e2e drivers.
-          - vllm_omni/experimental/fullduplex/
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/models/step_audio2/
-          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
-          - vllm_omni/model_executor/models/cosyvoice3/utils.py
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
-          - vllm_omni/core/sched/
-          - vllm_omni/outputs/multimodal_accumulation.py
-          - vllm_omni/engine/stage_init_utils.py
-          - vllm_omni/engine/async_omni_engine.py
-          - vllm_omni/engine/orchestrator.py
-          - vllm_omni/entrypoints/cli/serve.py
-          - vllm_omni/entrypoints/openai/api_server.py
-          - vllm_omni/entrypoints/openai/duplex_capability.py
-          - vllm_omni/config/stage_config.py
+        source_file_dependencies: omni_minicpmo_4_5_duplex_function
         timeout_in_minutes: 50
         commands:
           - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model'
         mirror_hardwares: h100_1
 
       - label: "Diffusion · Qwen Image Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen_image.py
-          - vllm_omni/diffusion/models/qwen_image/
-          - vllm_omni/diffusion/models/dmd2/
-          - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py
+        source_file_dependencies: diffusion_qwen_image_function
         commands:
           - |
             timeout 40m bash -c '
@@ -233,11 +144,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Diffusion · Wan22 Test"
-        source_file_dependencies:
-          - tests/e2e/offline_inference/test_wan22_t2v.py
-          - tests/e2e/online_serving/test_wan22_t2v.py
-          - vllm_omni/diffusion/models/wan2_2/
-          - vllm_omni/diffusion/models/dmd2/
+        source_file_dependencies: diffusion_wan22_function
         commands:
           - |
             timeout 40m bash -c "
@@ -247,9 +154,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Diffusion · HunyuanVideo-1.5 Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
-          - vllm_omni/diffusion/models/hunyuan_video/
+        source_file_dependencies: diffusion_hunyuan_video_function
         commands:
           - |
             timeout 40m bash -c "
@@ -259,17 +164,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Diffusion · MiniMax-H3 L3 DLO DP2 and FastH3"
-        source_file_dependencies:
-          - tests/e2e/online_serving/minimax_h3/
-          - tests/helpers/assertions.py
-          - tests/helpers/mark.py
-          - tests/helpers/media.py
-          - tests/helpers/runtime.py
-          - vllm_omni/diffusion/models/minimax_h3/
-          - vllm_omni/diffusion/lora/
-          - vllm_omni/diffusion/offloader/
-          - vllm_omni/diffusion/distributed/
-          - vllm_omni/entrypoints/openai/
+        source_file_dependencies: diffusion_minimax_h3_function
         # Keep four server lifecycles in one Buildkite job, but never in one
         # pytest process. DLO uses multiprocessing shared-memory queues; the
         # parent's resource_tracker can retain a stale /psm_* entry during

.buildkite/cuda/test-nightly.yml

@@ -12,138 +12,145 @@ steps:
   - group: ":card_index_dividers: Omni Model Test"
     key: nightly-omni-test-group
     depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     steps:
 
       - label: ":full_moon: Omni · Function Test with H100 · Single-GPU"
+        source_file_dependencies:
+          - omni_qwen3_omni_function
+          - omni_minicpmo_4_5_function
         timeout_in_minutes: 120
         commands:
           - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
 
       - label: ":full_moon: Omni · Function Test with H100 · 2-GPU"
+        source_file_dependencies:
+          - omni_qwen3_omni_function
+          - omni_minicpmo_4_5_function
         timeout_in_minutes: 90
         commands:
           - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_2" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
 
       - label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
+        source_file_dependencies: omni_minicpmo_4_5_duplex_function
         timeout_in_minutes: 50
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni and cards_1" --run-level "full_model"
-        mirror_hardwares: h100_1
+          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and B200 and omni and cards_1" --run-level "full_model"
 
       - label: ":full_moon: Omni · Doc Test with L4 · 4-GPU"
+        source_file_dependencies: omni_qwen2_5_omni_doc
         timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          - pytest -s -v tests/examples/ -m "full_model and omni and L4 and cards_4" --run-level "full_model"
-        mirror_hardwares: l4_4
+          - pytest -s -v tests/examples/ -m "full_model and omni and L4 and B200 and cards_4" --run-level "full_model"
 
       - label: ":full_moon: Omni · Doc Test with H100 · 2-GPU"
+        source_file_dependencies: omni_qwen3_omni_doc
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/examples/ -m "full_model and omni and H100 and cards_2" --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/examples/ -m "full_model and omni and H100 and B200 and cards_2" --run-level "full_model"
 
       - label: ":full_moon: Omni · Accuracy Test"
+        source_file_dependencies: omni_qwen3_omni_accuracy
         timeout_in_minutes: 180
         commands:
           - export SEED_TTS_WER_EVAL=1
           - export SEED_TTS_EVAL_DEVICE=cuda:1
           - |
             set +e
-            pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model" --run-level full_model
+            pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model and H100 and B200 and cards_2" --run-level full_model
             EXIT=$$?
             buildkite-agent artifact upload "tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_2
 
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
+        source_file_dependencies: omni_minicpmo_4_5_accuracy
         timeout_in_minutes: 180
         commands:
           - export SEED_TTS_WER_EVAL=1
           - export SEED_TTS_EVAL_DEVICE=cuda:1
           - |
             set +e
-            pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and cuda" --run-level full_model
+            pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level full_model
             EXIT=$$?
             buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_2
 
       - label: ":full_moon: Omni · Perf Test · No Async Chunk"
+        source_file_dependencies: omni_qwen3_omni_perf
         key: nightly-omni-performance-no-async-chunk
         timeout_in_minutes: 300
         commands:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json -m "H100 and B200 and cards_2"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_2
 
       - label: ":full_moon: Omni · Perf Test · Async Chunk"
+        source_file_dependencies: omni_qwen3_omni_perf
         key: nightly-omni-performance-async-chunk
         timeout_in_minutes: 180
         commands:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and full_model"
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and B200 and full_model and cards_2"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_2
 
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"
+        source_file_dependencies: omni_minicpmo_4_5_perf
         key: nightly-omni-performance-minicpmo-4-5
         timeout_in_minutes: 180
         commands:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json -m "H100 and B200 and cards_1"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_1
 
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test"
+        source_file_dependencies: omni_minicpmo_4_5_duplex_perf
         key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts
         timeout_in_minutes: 180
         commands:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json -m "H100 and B200 and cards_1"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_1
 
       - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100"
+        source_file_dependencies: omni_qwen3_omni_function
         timeout_in_minutes: 45
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model" --run-level "core_model"
-        mirror_hardwares: h100_4
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model and H100 and B200 and cards_4" --run-level "core_model"
 
   - group: ":card_index_dividers: TTS Model Test"
     key: nightly-tts-test-group
     depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     steps:
 
       - label: ":full_moon: TTS · Function Test with L4"
+        source_file_dependencies: tts_qwen3_tts_function
         timeout_in_minutes: 120
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
           - export VLLM_USE_DEEP_GEMM="0"
           - export VLLM_MOE_USE_DEEP_GEMM="0"
-          - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
-        mirror_hardwares: l4_1
+          - pytest -s -v tests/e2e/ -m "full_model and L4 and B200 and tts and cards_1" --run-level "full_model" --ignore=tests/e2e/accuracy
 
       - label: ":full_moon: TTS · Perf Test · Single-GPU"
+        source_file_dependencies: tts_qwen3_tts_perf
         key: nightly-tts-performance-single
         timeout_in_minutes: 180
         commands:
@@ -151,13 +158,13 @@ steps:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and cards_1"
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and B200 and cards_1"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_1
 
       - label: ":full_moon: TTS · Perf Test · 2-GPU"
+        source_file_dependencies: tts_qwen3_tts_perf
         key: nightly-tts-performance-2gpu
         timeout_in_minutes: 180
         commands:
@@ -165,11 +172,10 @@ steps:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and cards_2"
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and B200 and cards_2"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-        mirror_hardwares: h100_2
 
   - group: ":card_index_dividers: Tiny Model Tests [Multi-GPU]"
     key: nightly-tiny-model-test-group
@@ -180,6 +186,7 @@ steps:
     steps:
 
       - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 2-GPU"
+        source_file_dependencies: diffusion_tiny_model
         timeout_in_minutes: 30
         # NOTE: Single-GPU (core_model) tiny model tests run on the ready label,
         # so nightly only adds the multi-GPU parallelism configurations.
@@ -189,91 +196,85 @@ steps:
         # for the CI, since these tests should pass irrespective of whether or not we use
         # the full weights or not.
         commands:
-          - pytest -sv tests/model_tests/diffusion -m 'full_model and cuda and cards_2' --run-level "core_model"
-        mirror_hardwares: l4_2
+          - pytest -sv tests/model_tests/diffusion -m 'full_model and L4 and B200 and cuda and cards_2' --run-level "core_model"
 
       - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 4-GPU"
+        source_file_dependencies: diffusion_tiny_model
         timeout_in_minutes: 30
         # CFG+TP extra_test_groups request 2×2=4 devices (see get_required_device_count).
         commands:
-          - pytest -sv tests/model_tests/diffusion -m 'full_model and cuda and cards_4' --run-level "core_model"
-        mirror_hardwares: l4_4
+          - pytest -sv tests/model_tests/diffusion -m 'full_model and L4 and B200 and cuda and cards_4' --run-level "core_model"
 
   - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
     key: nightly-diffusion-x2iat-group
     depends_on: upload-nightly-pipeline
     if: >-
       build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2iat-test"
+      build.pull_request.labels includes "nightly-test"
     steps:
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU"
+        source_file_dependencies: diffusion_qwen_image_function
         timeout_in_minutes: 120
         commands:
           - >-
             pytest -sv
             tests/e2e/online_serving/test_qwen_image_expansion.py
+            -m "full_model and diffusion and H100 and B200 and cards_1"
+            --run-level "full_model"
+
+      - label: ":robot_face: Robot Policy OpenPI · π0.5 · H100 · Single-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - pip install --no-deps "openpi-client @ git+https://github.com/Physical-Intelligence/openpi.git@215abfb217dbac7d5f1273282331b9b1866c0479#subdirectory=packages/openpi-client"
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_pi05_expansion.py
             -m "full_model and diffusion and H100 and cards_1"
             --run-level "full_model"
         mirror_hardwares: h100_1
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU"
+        source_file_dependencies: diffusion_qwen_image_function
         timeout_in_minutes: 120
         commands:
           - >-
             pytest -sv
             tests/e2e/online_serving/test_qwen_image_expansion.py
-            -m "full_model and diffusion and H100 and cards_2"
+            -m "full_model and diffusion and H100 and B200 and cards_2"
             --run-level "full_model"
-        mirror_hardwares: h100_2
 
-      - label: ":full_moon: Diffusion X2I(&A&T) · Boogu Edit CFG · 3-GPU"
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · Single-GPU"
+        source_file_dependencies: diffusion_qwen_image_function
         timeout_in_minutes: 120
         commands:
-          - >-
-            pytest -sv
-            tests/e2e/online_serving/test_boogu_image_edit_expansion.py
-            -k "test_double_guidance_cfg_parallel and cfg_parallel_3_full"
-            -m "slow and diffusion and cuda and H100 and cards_3"
-            --run-level "full_model"
-        mirror_hardwares: h100_3
+          - pytest -sv tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py -m "full_model and diffusion and L4 and B200 and cards_1" --run-level "full_model"
 
-      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · 4-GPU"
-        timeout_in_minutes: 120
-        commands:
-          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4 and cards_4" --run-level "full_model" --ignore=tests/e2e/accuracy
-        mirror_hardwares: l4_4
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
+        source_file_dependencies: diffusion_text_to_image_doc
         timeout_in_minutes: 60
         commands:
-          - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100 and B200 and (cards_1 or cards_2)" --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test"
+        source_file_dependencies: diffusion_qwen_image_accuracy
         timeout_in_minutes: 180
         commands:
           - export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=120
-          - pytest -s -v tests/e2e/accuracy/test_qwen_image.py --run-level full_model
-          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model
-        mirror_hardwares: h100_1
+          - pytest -s -v tests/e2e/accuracy/test_qwen_image.py -m "H100 and B200 and cards_1" --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m "full_model and H100 and B200 and cards_1" --run-level full_model
 
       - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
+        source_file_dependencies: diffusion_hunyuan_image3_accuracy
         timeout_in_minutes: 180
         commands:
           - export HUNYUAN_IMAGE3_MODEL="tencent/HunyuanImage-3.0-Instruct"
           - export HUNYUAN_IMAGE3_DEVICES="0,1,2,3"
-          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m full_model --run-level full_model
-        mirror_hardwares: h100_4
-
-      - label: ":full_moon: HiDream-O1-Image · Accuracy Test"
-        timeout_in_minutes: 180
-        commands:
-          - pytest -s -v tests/e2e/accuracy/test_hidream_o1_image.py -m full_model --run-level full_model
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m "full_model and H100 and B200 and cards_4" --run-level full_model
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · Single-GPU"
+        source_file_dependencies: diffusion_qwen_image_perf
         key: nightly-diffusion-x2iat-performance-qwen-image-single
         timeout_in_minutes: 180
         commands:
@@ -281,168 +282,182 @@ steps:
           - export CACHE_DIT_VERSION=1.5.0
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and cards_1"
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and B200 and cards_1"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
-        mirror_hardwares: h100_1
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · 4-GPU"
+        source_file_dependencies: diffusion_qwen_image_perf
         key: nightly-diffusion-x2iat-performance-qwen-image-4gpu
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
           - export CACHE_DIT_VERSION=1.5.0
+          # Do not pin DIFFUSION_ATTENTION_BACKEND: H100 auto-selects FLASH_ATTN;
+          # B200 rejects explicit FLASH_ATTN without FA4 and uses CUDNN/TRTLLM.
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and cards_4"
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and B200 and cards_4"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
-        mirror_hardwares: h100_4
 
   - group: ":card_index_dividers: Diffusion X2V Model Test"
     key: nightly-diffusion-x2v-group
     depends_on: upload-nightly-pipeline
     if: >-
       build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2v-test"
+      build.pull_request.labels includes "nightly-test"
     steps:
 
       - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · Single-GPU"
+        source_file_dependencies: diffusion_wan22_function
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and cards_1" --run-level "full_model"
-        mirror_hardwares: h100_1
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and B200 and cards_1" --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · 2-GPU"
+        source_file_dependencies: diffusion_wan22_function
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and cards_2" --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and B200 and cards_2" --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · Single-GPU"
+        source_file_dependencies: diffusion_wan22_function
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and cards_1" --run-level "full_model"
-        mirror_hardwares: h100_1
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and B200 and cards_1" --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · 2-GPU"
+        source_file_dependencies: diffusion_wan22_function
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and cards_2" --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and B200 and cards_2" --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test"
+        source_file_dependencies: diffusion_hunyuan_video_function
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and cuda" --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level "full_model"
 
-      - label: ":full_moon: Diffusion X2V · MAGI-2 Preview Serving Smoke"
-        timeout_in_minutes: 180
+      - label: ":full_moon: Diffusion X2V · LingBot Function Test"
+        source_file_dependencies: diffusion_lingbot_function
+        timeout_in_minutes: 120
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_magi2.py -m "advanced_model and diffusion and cuda and H100 and cards_4" --run-level "advanced_model"
-        mirror_hardwares: h100_4
+          - >-
+            pytest -s -v
+            tests/e2e/online_serving/test_lingbot_video.py
+            tests/e2e/online_serving/test_lingbot_video_moe.py
+            tests/e2e/offline_inference/test_lingbot_world_v2.py
+            -m "full_model and diffusion and H100 and B200 and cards_1"
+            --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2V · Doc Test"
+        source_file_dependencies: diffusion_image_to_video_doc
         timeout_in_minutes: 60
         commands:
-          - pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m "full_model and example and H100" --run-level "full_model"
-        mirror_hardwares: h100_1
+          - pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m "full_model and example and H100 and B200 and cards_1" --run-level "full_model"
 
-      - label: ":full_moon: Diffusion X2V · Other Function Test"
+      - label: ":full_moon: Diffusion X2V · Wan2.2 Autoround Function Test"
+        source_file_dependencies: diffusion_wan22_function
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level "full_model"
+
+      - label: ":full_moon: Diffusion X2V · Wan2.2 Device-Postprocess Equivalence Test"
+        source_file_dependencies: diffusion_wan22_function
+        timeout_in_minutes: 90
+        commands:
+          - |
+            VLLM_OMNI_DEVICE_POSTPROCESS_MODEL=Wan-AI/Wan2.2-TI2V-5B-Diffusers \
+              pytest -s -v tests/e2e/offline_inference/test_device_postprocess_equivalence.py -m "full_model and H100 and B200 and cuda and cards_1" --run-level "full_model"
 
       - label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test"
+        source_file_dependencies: diffusion_wan22_accuracy
         timeout_in_minutes: 180
         commands:
-          - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model
-          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m "full_model and H100 and B200 and (cards_1 or cards_2)" --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m "full_model and H100 and B200 and cards_1" --run-level full_model
 
       - label: ":full_moon: Diffusion X2V · MiniMax H3 I2VA/Ref2VA Accuracy Test"
+        source_file_dependencies: diffusion_minimax_h3_accuracy
         timeout_in_minutes: 180
         commands:
-          - pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py -m full_model --run-level full_model
-        mirror_hardwares: h100_4
+          - pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py -m "full_model and H100 and B200 and cards_4" --run-level full_model
 
-      - label: ":full_moon: Diffusion X2V · Other Accuracy Test"
+      - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Accuracy Test"
+        source_file_dependencies: diffusion_hunyuan_video_accuracy
         timeout_in_minutes: 180
         commands:
-          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model
-          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model
-        mirror_hardwares: h100_2
+          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m "full_model and H100 and B200 and (cards_1 or cards_2)" --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m "full_model and H100 and B200 and (cards_1 or cards_2)" --run-level full_model
 
       - label: ":full_moon: Diffusion X2V · Perf Test · Single-GPU"
+        source_file_dependencies: diffusion_wan22_perf
         key: nightly-diffusion-x2v-performance-single
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_1"
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and B200 and cards_1"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
-        mirror_hardwares: h100_1
 
       - label: ":full_moon: Diffusion X2V · Perf Test · 2-GPU"
+        source_file_dependencies:
+          - diffusion_wan22_perf
+          - diffusion_cosmos3_perf
         key: nightly-diffusion-x2v-performance-2gpu
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_2"
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and B200 and cards_2"
             EXIT1=$$?
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json -m "H100 and cards_2"
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json -m "H100 and B200 and cards_2"
             EXIT2=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi
             exit $$EXIT2
-        mirror_hardwares: h100_2
 
       - label: ":full_moon: Diffusion X2V · Perf Test · HunyuanVideo-1.5"
+        source_file_dependencies: diffusion_hunyuan_video_perf
         key: nightly-diffusion-x2v-performance-hunyuanvideo15
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json -m "H100 and B200 and (cards_1 or cards_2)"
             EXIT1=$$?
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json -m "H100 and B200 and (cards_1 or cards_2)"
             EXIT2=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi
             exit $$EXIT2
-        mirror_hardwares: h100_2
 
       - label: ":full_moon: Diffusion X2V · Perf Test · MiniMax-H3"
+        source_file_dependencies: diffusion_minimax_h3_perf
         key: nightly-diffusion-x2v-performance-minimax-h3
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json -m "H100 and B200 and cards_4"
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
-        mirror_hardwares: h100_4
 
   - group: ":card_index_dividers: Diffusion Test"
     key: nightly-diffusion-test-group
@@ -450,59 +465,59 @@ steps:
     if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     steps:
       - label: ":full_moon: Diffusion · Distributed Test with L4 · 2-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_2' --run-level "full_model"
-        mirror_hardwares: l4_2
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_2' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · Distributed Test with L4 · 3-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_3' --run-level "full_model"
-        mirror_hardwares: l4_3
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_3' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · Distributed Test with L4 · 4-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_4' --run-level "full_model"
-        mirror_hardwares: l4_4
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_4' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · Distributed Test with H100 · 2-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and cards_2' --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and B200 and cards_2' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · Distributed Test with H100 · 4-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and cards_4' --run-level "full_model"
-        mirror_hardwares: h100_4
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and B200 and cards_4' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · Offloader Test"
+        source_file_dependencies: diffusion_offloader
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/offloader -m "full_model and cuda"
-        mirror_hardwares: l4_1
+          - pytest -sv tests/diffusion/offloader -m "full_model and L4 and B200 and cuda and cards_1"
 
       - label: ":full_moon: Diffusion · Quantization Test with H100 · Single-GPU"
+        source_file_dependencies: diffusion_quantization
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model"
-        mirror_hardwares: h100_1
+          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100 and B200 and cards_1' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · MiniMax H3 FP8 Quality Test with H100 · 2-GPU"
+        source_file_dependencies: diffusion_minimax_h3_function
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py -m 'full_model and cuda and H100' --run-level "full_model"
-        mirror_hardwares: h100_2
+          - pytest -sv tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py -m 'full_model and cuda and H100 and B200 and cards_2' --run-level "full_model"
 
       - label: ":full_moon: Diffusion · Quantization Test with L4"
+        source_file_dependencies: diffusion_quantization
         timeout_in_minutes: 60
         commands:
           - pip install "vllm-gguf-plugin==0.0.4"
-          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model"
-        mirror_hardwares: l4_1
+          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4 and B200 and cards_1' --run-level "full_model"
 
   # NOTE: ":email: Nightly Collection & Email" is deprecated and will be removed soon.
   # Do not add new depends_on entries or extend this step.

.buildkite/cuda/test-ready.yml

@@ -38,6 +38,12 @@ steps:
     depends_on: upload-ready-pipeline
     steps:
 
+      - label: "Diffusion · FlashAttention Test"
+        timeout_in_minutes: 20
+        commands:
+          - timeout 15m pytest -s -v tests/diffusion/attention/ -m 'core_model and cuda and L4 and cards_1' --run-level "core_model"
+        mirror_hardwares: l4_1
+
       - label: "Diffusion · Offloader Test"
         commands:
           - timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
@@ -54,16 +60,19 @@ steps:
         mirror_hardwares: l4_1
 
       - label: "Diffusion · Distributed Test · Single-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         commands:
           - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_1' --run-level "core_model"
         mirror_hardwares: l4_1
 
       - label: "Diffusion · Distributed Test · 2-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         commands:
           - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_2' --run-level "core_model"
         mirror_hardwares: l4_2
 
       - label: "Diffusion · Distributed & Attention Test · 4-GPU"
+        source_file_dependencies: diffusion_distributed_attention
         commands:
           - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_4' --run-level "core_model"
           - timeout 20m pytest -sv tests/diffusion/attention/test_attention_sp.py -m 'core_model and cuda and L4 and cards_4' --run-level "core_model"
@@ -76,7 +85,7 @@ steps:
 
       - label: "Diffusion · Other Test"
         commands:
-          - timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"
+          - timeout 15m pytest -sv tests/diffusion/*.py tests/diffusion/ar_diffusion -m 'core_model and cuda' --run-level "core_model"
         mirror_hardwares: l4_1
 
   - group: ":card_index_dividers: Tiny Model Tests [Single GPU]"
@@ -93,7 +102,7 @@ steps:
     commands:
       - |
         timeout 15m bash -c "
-                  pytest -sv tests/engine/ tests/model_executor/ -m 'core_model and cuda' --run-level 'core_model'
+                  pytest -sv tests/engine/ tests/model_executor/ tests/worker/test_batched_omni_output.py tests/worker_v2/ -m 'core_model and cuda' --run-level 'core_model'
         "
     mirror_hardwares: l4_1
 
@@ -120,29 +129,14 @@ steps:
     depends_on: upload-ready-pipeline
     steps:
       - label: "Custom Pipeline Test"
-        source_file_dependencies:
-          - tests/e2e/features/custom_pipeline/
-          - tests/e2e/features/helpers/custom_pipeline.py
-          - vllm_omni/diffusion/
-          - vllm_omni/engine/
-          - vllm_omni/entrypoints/
-          - vllm_omni/inputs/
-          - vllm_omni/outputs/
+        source_file_dependencies: custom_pipeline
         commands:
           - timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"
         mirror_hardwares: l4_1
 
 
       - label: "RLHF · VeRL-Omni E2E Test"
-        source_file_dependencies:
-          - tests/e2e/features/rlhf_test/test_verl_omni_e2e.py
-          - tests/e2e/features/helpers/verl_omni_server.py
-          - tests/e2e/features/helpers/custom_pipeline.py
-          - vllm_omni/diffusion/
-          - vllm_omni/engine/
-          - vllm_omni/entrypoints/
-          - vllm_omni/inputs/
-          - vllm_omni/outputs/
+        source_file_dependencies: rlhf_verl_omni
         commands:
           - |
             timeout 20m bash -c "
@@ -151,17 +145,7 @@ steps:
         mirror_hardwares: l4_1
 
       - label: "Omni · Qwen3-Omni Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen3_omni.py
-          - tests/helpers/runtime.py
-          - vllm_omni/entrypoints/openai/serving_chat.py
-          - vllm_omni/model_executor/models/qwen3_omni/
-          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-          - vllm_omni/model_executor/models/common/snake_activation.py
-          - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
-          - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
-          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
-          - vllm_omni/deploy/qwen3_omni_moe.yaml
+        source_file_dependencies: omni_qwen3_omni_function
         commands:
           - |
             timeout 20m bash -c "
@@ -170,14 +154,7 @@ steps:
         mirror_hardwares: h100_2
 
       - label: "Omni · MiniCPM-o 4.5 Offline Test"
-        source_file_dependencies:
-          - tests/e2e/offline_inference/test_minicpmo_4_5.py
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/models/step_audio2/
-          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
-          - vllm_omni/model_executor/models/cosyvoice3/utils.py
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
-          - vllm_omni/deploy/minicpmo_4_5.yaml
+        source_file_dependencies: omni_minicpmo_4_5_function
         commands:
           - |
             timeout 20m bash -c "
@@ -186,14 +163,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Omni · MiniCPM-o 4.5 Online Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_minicpmo_4_5.py
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/models/step_audio2/
-          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
-          - vllm_omni/model_executor/models/cosyvoice3/utils.py
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
-          - vllm_omni/deploy/minicpmo_4_5.yaml
+        source_file_dependencies: omni_minicpmo_4_5_function
         commands:
           - |
             timeout 20m bash -c "
@@ -202,48 +172,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Omni · MiniCPM-o 4.5 Duplex Test"
-        source_file_dependencies:
-          - tests/assets/minicpmo_4_5/response_required_16k.wav
-          - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
-          - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
-          - tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
-          - vllm_omni/deploy/minicpmo_4_5.yaml
-          - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
-          - vllm_omni/engine/duplex/
-          - vllm_omni/entrypoints/duplex/
-          # Public duplex client library + its unit suites: client-side
-          # changes must re-run the duplex e2e group, which includes the
-          # live client probe (test_duplex_client_live.py, single GPU).
-          - vllm_omni/clients/
-          - tests/clients/
-          - tests/e2e/online_serving/test_duplex_client_live.py
-          - vllm_omni/entrypoints/duplex_request_client.py
-          - vllm_omni/outputs/duplex.py
-          - vllm_omni/model_executor/duplex_sampling.py
-          - tests/engine/duplex/
-          - tests/entrypoints/duplex/
-          - tests/model_executor/models/minicpmo_4_5/duplex/
-          # Legacy Realtime probe client + video stacking used by the e2e drivers.
-          - vllm_omni/experimental/fullduplex/
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/models/step_audio2/
-          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
-          - vllm_omni/model_executor/models/cosyvoice3/utils.py
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
-          - vllm_omni/core/sched/
-          - vllm_omni/outputs/multimodal_accumulation.py
-          - vllm_omni/engine/stage_init_utils.py
-          - vllm_omni/engine/async_omni_engine.py
-          - vllm_omni/engine/orchestrator.py
-          - vllm_omni/entrypoints/cli/serve.py
-          - vllm_omni/entrypoints/openai/api_server.py
-          - vllm_omni/entrypoints/openai/duplex_capability.py
-          - vllm_omni/config/stage_config.py
+        source_file_dependencies: omni_minicpmo_4_5_duplex_function
         commands:
           - |
             timeout 35m bash -c "
@@ -253,29 +182,14 @@ steps:
 
       - label: "Omni · MiniCPM-o 4.5 Perf Test"
         timeout_in_minutes: 60
-        source_file_dependencies:
-          - tests/dfx/perf/tests/test_minicpmo_4_5_ready.json
-          - vllm_omni/deploy/minicpmo_4_5.yaml
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+        source_file_dependencies: omni_minicpmo_4_5_perf
         commands:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda'
         mirror_hardwares: h100_1
 
       - label: "TTS · Qwen3-TTS CustomVoice Test"
-        source_file_dependencies:
-          - tests/helpers/media.py
-          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
-          - vllm_omni/model_executor/models/qwen3_tts/
-          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-          - vllm_omni/model_executor/models/common/snake_activation.py
-          - vllm_omni/model_executor/models/common/alias_free_activation.py
-          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
-          - vllm_omni/deploy/qwen3_tts.yaml
-          - vllm_omni/entrypoints/openai/serving_speech.py
-          - vllm_omni/entrypoints/openai/tts_adapters/
-          - vllm_omni/engine/stage_pool.py
+        source_file_dependencies: tts_qwen3_tts_function
         commands:
           - |
             timeout 20m bash -c "
@@ -285,10 +199,7 @@ steps:
         mirror_hardwares: l4_1
 
       - label: "Diffusion · Wan22 Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_wan22_t2v.py
-          - vllm_omni/diffusion/models/wan2_2/
-          - vllm_omni/diffusion/models/dmd2/
+        source_file_dependencies: diffusion_wan22_function
         commands:
           - |
             timeout 40m bash -c "
@@ -298,12 +209,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Diffusion · Cosmos3 Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_cosmos3.py
-          - vllm_omni/diffusion/models/cosmos3/
-          - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_wan.py
-          - vllm_omni/experimental/world_models/
-          - vllm_omni/model_extras/cosmos3.py
+        source_file_dependencies: diffusion_cosmos3_function
         commands:
           - |
             timeout 40m bash -c "
@@ -314,14 +220,7 @@ steps:
 
       - label: "Diffusion · MiniMax-H3 DLO DP2 T2VA Test"
         timeout_in_minutes: 90
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py
-          - tests/helpers/assertions.py
-          - tests/helpers/runtime.py
-          - vllm_omni/diffusion/models/minimax_h3/
-          - vllm_omni/diffusion/offloader/
-          - vllm_omni/diffusion/distributed/
-          - vllm_omni/entrypoints/openai/
+        source_file_dependencies: diffusion_minimax_h3_function
         commands:
           - |
             timeout 80m bash -c "
@@ -331,11 +230,7 @@ steps:
         mirror_hardwares: h100_2
 
       - label: "Diffusion · Qwen Image Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen_image.py
-          - vllm_omni/diffusion/models/qwen_image/
-          - vllm_omni/diffusion/models/dmd2/
-          - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py
+        source_file_dependencies: diffusion_qwen_image_function
         commands:
           - |
             timeout 40m bash -c "
@@ -345,9 +240,7 @@ steps:
         mirror_hardwares: h100_1
 
       - label: "Diffusion · HunyuanVideo-1.5 Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
-          - vllm_omni/diffusion/models/hunyuan_video/
+        source_file_dependencies: diffusion_hunyuan_video_function
         commands:
           - |
             timeout 40m bash -c "

.buildkite/cuda/test-weekly.yml

@@ -29,36 +29,42 @@ steps:
     steps:
 
       - label: "Reliability Test · qwen3-omni"
+        source_file_dependencies: omni_qwen3_omni_reliability
         timeout_in_minutes: 180
         commands:
           - pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow"
         mirror_hardwares: h100_2
 
       - label: "Reliability Test · wan22"
+        source_file_dependencies: diffusion_wan22_reliability
         timeout_in_minutes: 180
         commands:
           - pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow"
         mirror_hardwares: h100_2
 
       - label: "Reliability Test · hunyuan-image"
+        source_file_dependencies: diffusion_hunyuan_image3_reliability
         timeout_in_minutes: 180
         commands:
           - pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow"
         mirror_hardwares: h100_4
 
       - label: "Reliability Test · Invalid parameters · H100 · Single-GPU"
+        source_file_dependencies: reliability_invalid_param_h100
         timeout_in_minutes: 90
         commands:
           - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100 and cards_1"
         mirror_hardwares: h100_1
 
       - label: "Reliability Test · Invalid parameters · H100 · 2-GPU"
+        source_file_dependencies: reliability_invalid_param_h100
         timeout_in_minutes: 90
         commands:
           - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100 and cards_2"
         mirror_hardwares: h100_2
 
       - label: "Reliability Test · Invalid parameters · L4"
+        source_file_dependencies: reliability_invalid_param_l4
         timeout_in_minutes: 120
         commands:
           - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"
@@ -70,6 +76,7 @@ steps:
     steps:
 
       - label: ":full_moon: Omni · Perf Test · vLLM Text"
+        source_file_dependencies: omni_qwen3_omni_perf
         key: weekly-omni-performance-vllm-text
         timeout_in_minutes: 180
         commands:
@@ -96,6 +103,7 @@ steps:
         mirror_hardwares: h100_2
 
       - label: ":full_moon: Omni · Perf Test · Multi-Replica"
+        source_file_dependencies: omni_qwen3_omni_perf
         key: weekly-omni-performance-multi-replicas
         timeout_in_minutes: 180
         commands:
@@ -183,13 +191,14 @@ steps:
       # from downgrading transformers; see
       # examples/offline_inference/text_to_speech/README.md.
       - label: "TTS · Gepard-1.0 · L4"
-        timeout_in_minutes: 60
+        timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
           - printf '%s\n' "transformers>=5.5.3" "huggingface-hub>=1.0" > /tmp/gepard-overrides.txt
           - uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" "nemo_toolkit[tts]==2.7.3" --overrides /tmp/gepard-overrides.txt
           - >-
             pytest -s -v tests/e2e/offline_inference/test_gepard_tts.py
+            tests/e2e/online_serving/test_gepard_tts.py
             -m "tts and cuda"
             --run-level "full_model"
         mirror_hardwares: l4_1
@@ -221,24 +230,277 @@ steps:
             --run-level "full_model"
         mirror_hardwares: l4_4
 
-      - label: "Diffusion · H100 · Single-GPU"
-        timeout_in_minutes: 150
+      # Split the old shared H100 Single-GPU sweep: one pytest process collected
+      # every slow+cards_1 diffusion file and exceeded 150 minutes.
+      - label: "Diffusion · Qwen-Image · H100 · Single-GPU"
+        timeout_in_minutes: 120
         commands:
           - >-
             pytest -sv
-            tests/e2e/ -m "slow and diffusion and H100 and cards_1"
+            tests/e2e/online_serving/test_qwen_image_edit.py
+            tests/e2e/online_serving/test_qwen_image_edit_expansion.py
+            tests/e2e/online_serving/test_qwen_image_layered.py
+            tests/e2e/online_serving/test_qwen_image_layered_expansion.py
+            tests/e2e/accuracy/test_qwen_image_edit.py
+            tests/e2e/accuracy/test_qwen_image_layered.py
+            -m "slow and diffusion and H100 and cards_1"
             --run-level "full_model"
         mirror_hardwares: h100_1
 
-      - label: "Diffusion · H100 · 2-GPU"
+      - label: "Diffusion · Wan2.1 VACE · H100 · Single-GPU"
         timeout_in_minutes: 90
         commands:
           - >-
             pytest -sv
-            tests/e2e/ -m "slow and diffusion and H100 and cards_2"
+            tests/e2e/online_serving/test_wan_2_1_vace_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · LongCat · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_longcat_image_expansion.py
+            tests/e2e/online_serving/test_longcat_image_edit_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Bagel · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_bagel.py
+            tests/e2e/online_serving/test_bagel_expansion.py
+            tests/e2e/offline_inference/test_bagel.py
+            tests/e2e/offline_inference/test_bagel_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Boogu · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_boogu_image.py
+            tests/e2e/online_serving/test_boogu_image_edit.py
+            tests/e2e/online_serving/test_boogu_image_expansion.py
+            tests/e2e/online_serving/test_boogu_image_edit_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Flux · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_flux_2_dev_expansion.py
+            tests/e2e/offline_inference/test_flux1_schnell_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · LTX · H100 · Single-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_ltx.py
+            tests/e2e/online_serving/test_ltx25.py
+            tests/e2e/accuracy/ltx/test_ltx_official_similarity.py
+            tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Krea2 · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_krea2_expansion.py
+            tests/e2e/offline_inference/test_krea2_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · HiDream-I1 · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_hidream_i1_full_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · SANA-Video · H100 · Single-GPU"
+        source_file_dependencies: diffusion_sana_video_function
+        timeout_in_minutes: 180
+        commands:
+          - export SANA_VIDEO_GOLDEN_BASE_URL=https://vllm-public-assets.s3.us-west-2.amazonaws.com/omni-assets/sana-video/v2
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_sana_video_expansion.py
+            tests/e2e/accuracy/sana_video/
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · SenseNova-U1 · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/offline_inference/test_sensenova_u1_expansion.py
+            tests/e2e/offline_inference/test_sensenova_u1_text2img_expansion.py
+            tests/e2e/offline_inference/test_sensenova_u1_img2img_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · MammothModa2 · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/offline_inference/test_mammoth_moda2_expansion.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Helios · H100 · Single-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/accuracy/test_video_streaming_output_similarity.py
+            -m "slow and diffusion and H100 and cards_1"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      # Split the old shared H100 2-GPU sweep: one pytest process collected every
+      # slow+cards_2 diffusion file and exceeded 90 minutes.
+      - label: "Diffusion · Qwen-Image · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_qwen_image_edit_expansion.py
+            tests/e2e/online_serving/test_qwen_image_layered_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
             --run-level "full_model"
         mirror_hardwares: h100_2
 
+      - label: "Diffusion · Wan2.1 VACE · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_wan_2_1_vace_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · LongCat · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_longcat_image_expansion.py
+            tests/e2e/online_serving/test_longcat_image_edit_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · Bagel · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_bagel_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · Boogu · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_boogu_image_expansion.py
+            tests/e2e/online_serving/test_boogu_image_edit_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · Flux · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_flux_2_dev_expansion.py
+            tests/e2e/offline_inference/test_flux1_kontext_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · GLM-Image · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_glm_image_expansion.py
+            tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · LTX · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_ltx.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · Krea2 · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/offline_inference/test_krea2_expansion.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · DreamZero · H100 · 2-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_dreamzero_expansion.py
+            tests/e2e/accuracy/test_dreamzero.py
+            -m "slow and diffusion and H100 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · H100 · 3-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/ -m "slow and diffusion and H100 and cards_3"
+            --run-level "full_model"
+        mirror_hardwares: h100_3
+
       - label: "Diffusion · H100 · 4-GPU"
         timeout_in_minutes: 90
         commands:
@@ -246,4 +508,33 @@ steps:
             pytest -sv
             tests/e2e/ -m "slow and diffusion and H100 and cards_4"
             --run-level "full_model"
+            --ignore=tests/e2e/online_serving/test_magi2.py
+        mirror_hardwares: h100_4
+
+      # Dedicated: HiDream-O1 accuracy needs a long TP2 run; the shared H100
+      # 2-GPU sweep is only 90 minutes.
+      - label: "Diffusion · HiDream-O1-Image Accuracy · H100 · 2-GPU"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/e2e/accuracy/test_hidream_o1_image.py -m "slow and diffusion and cuda and H100 and cards_2" --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      # Dedicated: Boogu CFG-parallel size 3 needs 3 GPUs; weekly has no H100 3-GPU sweep.
+      - label: "Diffusion · Boogu Edit CFG · H100 · 3-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_boogu_image_edit_expansion.py
+            -k "test_double_guidance_cfg_parallel and cfg_parallel_3_full"
+            -m "slow and diffusion and cuda and H100 and cards_3"
+            --run-level "full_model"
+        mirror_hardwares: h100_3
+
+      # Dedicated: MAGI-2 Preview serving smoke needs ~30 min server init; the
+      # shared H100 4-GPU sweep is only 90 minutes.
+      - label: "Diffusion · MAGI-2 Preview Serving Smoke · H100 · 4-GPU"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_magi2.py -m "slow and diffusion and cuda and H100 and cards_4" --run-level "full_model"
         mirror_hardwares: h100_4

.buildkite/intel/pipeline-intel.yml

@@ -10,7 +10,7 @@ steps:
           DOCKER_BUILDKIT: "1"
           # Buildkite will automatically replace this with the actual commit hash
           VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.28.0"
+          VLLM_VERSION: "v0.29.0"
         priority: 100
         timeout_in_minutes: 60
         soft_fail: false

.buildkite/npu/bootstrap-upload-steps.yml

@@ -3,10 +3,10 @@
 steps:
   - label: ":buildkit: Build and Push NPU Test Image - A2"
     key: image-build-a2
-    agents:
+    agents: &npu-image-build-agents
       queue: "ascend-a2b3"
       resource_class: "npu-1"
-    plugins:
+    plugins: &npu-image-build-plugins
       - kubernetes:
           metadata:
             annotations:
@@ -46,136 +46,67 @@ steps:
       IMAGE_NAME: "vllm-omni-ci-npu"
       IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
       BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
-    command: |
-      set -ex
-
-      echo "--- Building and pushing NPU Test Image"
-      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-      echo "buildkitd address: $${BUILDKITD_ADDR}"
-
-      if ! command -v buildctl &> /dev/null; then
-        echo "Installing buildctl..."
-        mkdir -p /tmp/buildkit
-        BUILDKIT_VERSION="v0.29.0"
-        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
-        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
-        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
-      fi
-
-      export DOCKER_CONFIG=/home/user/.docker
-      buildctl \
-        --addr="$${BUILDKITD_ADDR}" \
-        --tlscacert=/home/user/.docker/ca.pem \
-        --tlscert=/home/user/.docker/cert.pem \
-        --tlskey=/home/user/.docker/key.pem \
-        build \
-        --frontend dockerfile.v0 \
-        --local context=. \
-        --local dockerfile=./docker \
-        --opt filename=Dockerfile.npu.ci \
-        --secret id=dockerconfig,src=/home/user/.docker/config.json \
-        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
-        --progress=plain
-
-      echo "--- Image pushed successfully"
-      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+    command: bash .buildkite/npu/scripts/build-npu-image.sh Dockerfile.npu.ci "$${VLLM_IMAGE_TAG}"
 
   - label: ":buildkit: Build and Push NPU Test Image - A3"
     key: image-build-a3
-    agents:
-      queue: "ascend-a2b3"
-      resource_class: "npu-1"
-    plugins:
-      - kubernetes:
-          metadata:
-            annotations:
-              vault.hashicorp.com/agent-init-first: "true"
-              vault.hashicorp.com/agent-inject: "true"
-              vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
-              vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
-              vault.hashicorp.com/agent-inject-perms-config.json: "0400"
-              vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
-              vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
-              vault.hashicorp.com/agent-pre-populate-only: "true"
-              vault.hashicorp.com/agent-run-as-group: "1000"
-              vault.hashicorp.com/agent-run-as-user: "1000"
-              vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
-              vault.hashicorp.com/role: ascend-gha-runners
-              vault.hashicorp.com/secret-volume-path: /home/user/.docker/
-              vault.hashicorp.com/tls-skip-verify: "true"
-          podSpecPatch:
-            volumes:
-            - name: token-vol
-              projected:
-                defaultMode: 420
-                sources:
-                - serviceAccountToken:
-                    audience: api
-                    expirationSeconds: 600
-                    path: token
+    agents: *npu-image-build-agents
+    plugins: *npu-image-build-plugins
     env:
       VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-a3"
       IMAGE_NAME: "vllm-omni-ci-npu"
       IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
       BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
-    command: |
-      set -ex
+    command: bash .buildkite/npu/scripts/build-npu-image.sh Dockerfile.npu.ci "$${VLLM_IMAGE_TAG}" -a3
 
-      echo "--- Building and pushing NPU Test Image"
-      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-      echo "buildkitd address: $${BUILDKITD_ADDR}"
+  - label: ":buildkit: Build and Push NPU Test Image - A5"
+    key: image-build-a5
+    agents: *npu-image-build-agents
+    plugins: *npu-image-build-plugins
+    env:
+      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-a5"
+      IMAGE_NAME: "vllm-omni-ci-npu"
+      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
+      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
+    command: bash .buildkite/npu/scripts/build-npu-image.sh Dockerfile.npu.ci "$${VLLM_IMAGE_TAG}" -a5
 
-      if ! command -v buildctl &> /dev/null; then
-        echo "Installing buildctl..."
-        mkdir -p /tmp/buildkit
-        BUILDKIT_VERSION="v0.29.0"
-        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
-        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
-        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
-      fi
-
-      export DOCKER_CONFIG=/home/user/.docker
-      buildctl \
-        --addr="$${BUILDKITD_ADDR}" \
-        --tlscacert=/home/user/.docker/ca.pem \
-        --tlscert=/home/user/.docker/cert.pem \
-        --tlskey=/home/user/.docker/key.pem \
-        build \
-        --frontend dockerfile.v0 \
-        --local context=. \
-        --local dockerfile=./docker \
-        --opt filename=Dockerfile.npu.ci.a3 \
-        --secret id=dockerconfig,src=/home/user/.docker/config.json \
-        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
-        --progress=plain
-
-      echo "--- Image pushed successfully"
-      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+  - label: ":buildkit: Build and Push NPU Test Image - 310P"
+    key: image-build-310p
+    agents: *npu-image-build-agents
+    plugins: *npu-image-build-plugins
+    env:
+      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-310p"
+      IMAGE_NAME: "vllm-omni-ci-npu"
+      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
+      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
+    command: bash .buildkite/npu/scripts/build-npu-image.sh Dockerfile.npu.ci "$${VLLM_IMAGE_TAG}" -310p
 
   - label: "Upload Ready Pipeline"
     depends_on:
       - image-build-a2
       - image-build-a3
+      - image-build-a5
+      - image-build-310p
     key: upload-ready-pipeline
     agents:
       queue: "cpu_queue_premerge"
     command: |
       python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/test-npu-ready.yml
 
-  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
+  # L4 Test — main+NIGHTLY=1 (scheduled), or PR nightly-test label
   - label: "Upload Nightly Pipeline"
     depends_on:
       - image-build-a2
       - image-build-a3
+      - image-build-a5
+      - image-build-310p
     key: upload-nightly-pipeline
     commands:
-      - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/test-npu-nightly.yml
+      - |
+        EXTRA=""
+        if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
+          EXTRA="--all"
+        fi
+        python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/npu/test-npu-nightly.yml
     agents:
       queue: "cpu_queue_premerge"

.buildkite/npu/pipeline-npu-a3.yml

@@ -96,13 +96,7 @@
   #   key: upload-nightly-pipeline
   #   if: >-
   #     (build.branch == "main" && build.env("NIGHTLY") == "1") ||
-  #     (build.branch != "main" && (
-  #       build.pull_request.labels includes "nightly-test" ||
-  #       build.pull_request.labels includes "omni-test" ||
-  #       build.pull_request.labels includes "tts-test" ||
-  #       build.pull_request.labels includes "diffusion-x2iat-test" ||
-  #       build.pull_request.labels includes "diffusion-x2v-test"
-  #     ))
+  #     (build.branch != "main" && build.pull_request.labels includes "nightly-test")
   #   commands:
   #     - buildkite-agent pipeline upload .buildkite/npu/test-npu-nightly.yml
   #   agents:

.buildkite/npu/scripts/build-npu-image.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -ex
+
+dockerfile=${1:?Dockerfile is required}
+image_tag=${2:?Image tag is required}
+device_suffix=${3:-}
+
+image_name=${IMAGE_NAME:-vllm-omni-ci-npu}
+image_registry=${IMAGE_REGISTRY:-swr.cn-southwest-2.myhuaweicloud.com/modelfoundry}
+buildkitd_addr=${BUILDKITD_ADDR:-tcp://buildkitd-service.buildkitd:1234}
+
+echo "--- Building and pushing NPU Test Image"
+echo "Image: ${image_registry}/${image_name}:${image_tag}"
+echo "Dockerfile: ${dockerfile}"
+
+if ! command -v buildctl &> /dev/null; then
+  mkdir -p /tmp/buildkit
+  buildkit_version=v0.29.0
+  wget -q \
+    "https://github.com/moby/buildkit/releases/download/${buildkit_version}/buildkit-${buildkit_version}.linux-arm64.tar.gz" \
+    -O /tmp/buildkit.tar.gz
+  tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
+  cp /tmp/buildkit/bin/buildctl /usr/local/bin/
+fi
+
+build_args=()
+if [[ -n "${device_suffix}" ]]; then
+  build_args+=(--opt "build-arg:VLLM_ASCEND_DEVICE_SUFFIX=${device_suffix}")
+fi
+
+export DOCKER_CONFIG=/home/user/.docker
+buildctl \
+  --addr="${buildkitd_addr}" \
+  --tlscacert=/home/user/.docker/ca.pem \
+  --tlscert=/home/user/.docker/cert.pem \
+  --tlskey=/home/user/.docker/key.pem \
+  build \
+  --frontend dockerfile.v0 \
+  --local context=. \
+  --local dockerfile=./docker \
+  --opt "filename=${dockerfile}" \
+  "${build_args[@]}" \
+  --secret id=dockerconfig,src=/home/user/.docker/config.json \
+  --output "type=image,name=${image_registry}/${image_name}:${image_tag},push=true" \
+  --progress=plain
+
+echo "--- Image pushed successfully"
+echo "${image_registry}/${image_name}:${image_tag}"

.buildkite/npu/test-npu-nightly.yml

@@ -4,10 +4,10 @@ steps:
     depends_on: upload-nightly-pipeline
     if: >-
       build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "omni-test"
+      build.pull_request.labels includes "nightly-test"
     steps:
       - label: ":full_moon: Omni · Perf Test · No Async Chunk"
+        source_file_dependencies: omni_qwen3_omni_perf
         key: nightly-omni-performance-no-async-chunk
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4
@@ -24,6 +24,7 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
       - label: ":full_moon: Omni · Perf Test · Async Chunk"
+        source_file_dependencies: omni_qwen3_omni_perf
         key: nightly-omni-performance-async-chunk
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4
@@ -40,6 +41,7 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
       - label: ":full_moon: Omni · MiniCPM-o 4.5 Test"
+        source_file_dependencies: omni_minicpmo_4_5_function
         mirror_hardwares: a3_npu_2
         env:
           VLLM_WORKER_MULTIPROC_METHOD: spawn
@@ -48,6 +50,7 @@ steps:
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model'
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Test"
+        source_file_dependencies: omni_minicpmo_4_5_duplex_function
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_2
         env:
@@ -62,6 +65,7 @@ steps:
             -m "npu and A3 and omni"
             --run-level full_model
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
+        source_file_dependencies: omni_minicpmo_4_5_accuracy
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_2
         env:
@@ -82,6 +86,7 @@ steps:
             buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
             exit $$EXIT
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"
+        source_file_dependencies: omni_minicpmo_4_5_perf
         key: nightly-omni-performance-minicpmo-4-5-npu
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_2
@@ -98,6 +103,7 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test (A2)"
+        source_file_dependencies: omni_minicpmo_4_5_perf
         key: nightly-omni-performance-minicpmo-4-5-npu-a2
         timeout_in_minutes: 180
         mirror_hardwares: a2b3_npu_1
@@ -115,6 +121,7 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test"
+        source_file_dependencies: omni_minicpmo_4_5_duplex_perf
         key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts-npu
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_2
@@ -133,9 +140,10 @@ steps:
   - group: ":card_index_dividers: TTS Model Test"
     key: nightly-tts-test-group
     depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     steps:
       - label: ":full_moon: TTS · Perf Test"
+        source_file_dependencies: tts_qwen3_tts_perf
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_2
         env:
@@ -151,15 +159,15 @@ steps:
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
-  - group: ":card_index_dividers: Diffusion I2I Model Test"
-    key: nightly-HunyuanImage3-x2i-test-group
+  - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
+    key: nightly-diffusion-x2iat-group
     depends_on: upload-nightly-pipeline
     if: >-
       build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2iat-test"
+      build.pull_request.labels includes "nightly-test"
     steps:
       - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
+        source_file_dependencies: diffusion_hunyuan_image3_accuracy
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4
         env:
@@ -169,15 +177,8 @@ steps:
           DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
         commands:
           - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m "full_model and A3" --run-level full_model
-  - group: ":card_index_dividers: Diffusion Benchmark Perf Test"
-    key: nightly-diffusion-benchmark-group
-    depends_on: upload-nightly-pipeline
-    if: >-
-      build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2iat-test"
-    steps:
       - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 CFGP=2"
+        source_file_dependencies: diffusion_hunyuan_image3_perf
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4
         env:
@@ -194,6 +195,7 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
       - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 SP=2"
+        source_file_dependencies: diffusion_hunyuan_image3_perf
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4
         env:
@@ -210,6 +212,7 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
       - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=4"
+        source_file_dependencies: diffusion_hunyuan_image3_perf
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4
         env:
@@ -230,10 +233,41 @@ steps:
     depends_on: upload-nightly-pipeline
     if: >-
       build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2v-test"
+      build.pull_request.labels includes "nightly-test"
     steps:
-      - label: ":full_moon: Diffusion X2V · Function Test"
+      - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test"
+        source_file_dependencies: diffusion_hunyuan_video_function
+        timeout_in_minutes: 180
+        mirror_hardwares: a2b3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+          HF_HUB_OFFLINE: "1"
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "npu and A2" --run-level "full_model"
+      - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Perf Test"
+        source_file_dependencies: diffusion_hunyuan_video_perf
+        key: nightly-diffusion-x2v-performance-hunyuanvideo15
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_2
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+          HF_HUB_OFFLINE: "1"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+      - label: ":full_moon: Diffusion X2V · Wan 2.2 Function Test"
+        source_file_dependencies: diffusion_wan22_function
         mirror_hardwares: a2b3_npu_4
         env:
           VLLM_WORKER_MULTIPROC_METHOD: spawn
@@ -241,7 +275,8 @@ steps:
           HF_TOKEN: "${HF_TOKEN}"
         commands:
           - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"
-      - label: ":full_moon: Diffusion X2V · Perf Test"
+      - label: ":full_moon: Diffusion X2V · Wan 2.2 Perf Test"
+        source_file_dependencies: diffusion_wan22_perf
         key: nightly-diffusion-x2v-performance
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_4

.buildkite/npu/test-npu-ready.yml

@@ -1,26 +1,59 @@
 steps:
-  - label: "Platforms Test"
-    key: ready-platforms-test
+  - group: ":card_index_dividers: Simple Test"
     depends_on: upload-ready-pipeline
-    mirror_hardwares: a3_npu_2
-    commands:
-      - >-
-        pytest -s -v
-        tests/platforms/npu/test_minicpmo_code2wav_npugraph.py
-        -m 'core_model and npu and A3 and omni'
-        --run-level 'core_model'
+    steps:
+      - label: "Platforms Test"
+        key: ready-platforms-test
+        mirror_hardwares: a3_npu_2
+        commands:
+          - >-
+            pytest -s -v
+            tests/platforms/npu/test_minicpmo_code2wav_npugraph.py
+            -m 'core_model and npu and A3 and omni'
+            --run-level 'core_model'
 
-  - label: "Diffusion NPU layer tests"
-    key: ready-diffusion-npu-layer-tests
-    depends_on: upload-ready-pipeline
-    mirror_hardwares: a3_npu_2
-    commands:
-      - >-
-        pytest -s -v
-        tests/diffusion/layers/test_activation.py
-        tests/diffusion/layers/test_fused_qk_norm_rope_npu.py
-        -m 'core_model and npu and A3'
-        --run-level 'core_model'
+      - label: "Diffusion NPU layer tests - A2"
+        key: ready-diffusion-npu-layer-tests-a2
+        mirror_hardwares: a2b3_npu_1
+        commands:
+          - >-
+            pytest -s -v
+            tests/diffusion/layers/test_activation.py
+            -m 'core_model and npu and A2'
+            --run-level 'core_model'
+
+      - label: "Diffusion NPU layer tests - A3"
+        key: ready-diffusion-npu-layer-tests
+        mirror_hardwares: a3_npu_2
+        commands:
+          - >-
+            pytest -s -v
+            tests/diffusion/layers/test_activation.py
+            tests/diffusion/layers/test_fused_qk_norm_rope_npu.py
+            tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_swiglu.py
+            tests/diffusion/models/minimax_h3/test_minimax_h3_modulation.py
+            -m 'core_model and npu and A3'
+            --run-level 'core_model'
+
+      - label: "Diffusion NPU layer tests - A5"
+        key: ready-diffusion-npu-layer-tests-a5
+        mirror_hardwares: a5_npu_2
+        commands:
+          - >-
+            pytest -s -v
+            tests/diffusion/layers/test_activation.py
+            -m 'core_model and npu and A5'
+            --run-level 'core_model'
+
+      - label: "Diffusion NPU layer tests - 310P"
+        key: ready-diffusion-npu-layer-tests-310p
+        mirror_hardwares: 310p_npu_1
+        commands:
+          - >-
+            pytest -s -v
+            tests/diffusion/layers/test_activation.py
+            -m 'core_model and npu and 310P'
+            --run-level 'core_model'
 
   - group: ":card_index_dividers: Omni Model Test"
     key: ready-omni-group

.claude/skills/production-add-diffusion-model/references/production-validation.md

@@ -340,7 +340,7 @@ On supported GPU hosts, wrap a test or soak with the repository monitor:
 
 ```bash
 bash tests/dfx/stability/scripts/resource_monitor.sh run --backend gpu -- \
-  pytest -s -v tests/dfx/stability/scripts/test_stability_<model>.py -m slow
+  pytest -s -v tests/dfx/stability/scripts/run_stability_<model>.py -m slow
 ```
 
 Check the target revision's monitor backends. Do not claim CPU/NPU monitoring

.claude/skills/review-pr/SKILL.md

@@ -123,10 +123,11 @@ it changes again, report the churn and wait for a stable target.
 
 For a trusted PR head, materialize the pinned head in an isolated detached
 worktree. A worktree freezes identity but is not a security sandbox. Treat fork
-heads as untrusted unless the user and environment policy explicitly establish
-otherwise: execute them only in a disposable, secret-free sandbox with restricted
-filesystem, network, and resources; without one, use static SHA-addressed reads
-and CI evidence only. For a local review, freeze the committed, index, worktree,
+heads as untrusted until the user and environment policy explicitly establish
+trust: execute their code only after that trust is recorded for this host,
+with the trusted SHA noted in the review state and secrets kept out of the
+execution scope; without recorded trust, use static SHA-addressed reads and CI
+evidence only. For a local review, freeze the committed, index, worktree,
 and NUL-safe in-scope untracked contents. Follow
 [review-execution.md](references/process/review-execution.md) for trust gates,
 state fingerprints, and byte-for-byte staleness checks.
@@ -196,7 +197,7 @@ style preferences as simplification findings.
 
 Before each validation group, verify the frozen SHA plus the tracked, index,
 untracked, and ignored-file fingerprint, or recreate a pristine snapshot. On a
-trusted head or inside the required sandbox, run an import/version preflight,
+head the user explicitly trusted for this host, run an import/version preflight,
 then the narrowest relevant tests and low-cost static checks. Bind every result
 to the head SHA, snapshot fingerprint, and environment fingerprint. Never run
 imports, tests, builds, hooks, or repo-configurable tooling from an untrusted

.claude/skills/review-pr/references/checks/perf-verification.md

@@ -22,6 +22,19 @@ Keep exact commands and report variability rather than a cherry-picked run.
 Use the repository's benchmark when it covers the claim; otherwise use the
 smallest reproducible workload that reaches the changed production path.
 
+## Isolation per claim
+
+Isolate one claim per comparison. When a PR stacks several changes (a fused or
+restructured load path, a distilled step schedule, a new cache), request one
+column per claim at equal steps/workload instead of one blended base-vs-head
+delta, and name the unchanged stage or metric that pins equivalence.
+
+When the changed path is runtime-switchable (feature flag or environment
+variable), run every column on the same head and toggle the switch rather than
+diffing against a base commit that cannot separate the stacked changes:
+base, head with the switch off, head with the switch on, then head with the
+switch on plus each further claim.
+
 ## Evidence table
 
 | Dimension | Typical evidence |
@@ -34,6 +47,100 @@ smallest reproducible workload that reaches the changed production path.
 Every optimization needs correctness or quality evidence. A faster result on a
 different workload, precision, seed, or topology is not a valid comparison.
 
+## E2e stage-attribution table
+
+For an end-to-end latency claim, request the stage-attribution table the
+repository's perf PRs use:
+
+- Rows carry the benchmark's own field names: total e2e, per-stage execution,
+  postprocess, response encoding, and a request-to-output roll-up. Add a stage
+  handoff/idle row when the pipeline spans stages — connector transfer and
+  scheduling gaps are invisible in per-stage latency deltas.
+- Sanity rows pin comparability: inference steps, output frames/pixels/samples,
+  and generated tokens held identical across columns.
+- Report P50 and P100 from the stated warmup and measured counts. One
+  population per number: never mix cold-start and warm samples into one value.
+- Name the baseline and candidate commit SHAs, and close with an isolation
+  statement naming the metric that stayed unchanged ("X remained effectively
+  unchanged, isolating the gain to Y").
+
+When the serving path caches, captures, or regenerates at runtime (CUDA-graph
+bucket growth, cache rebuilds, recompilation), medians hide the resulting tail.
+Also request the event count and the maximum observed stall.
+
+Component-level tables pair with, not replace, the e2e table: per-op reference
+vs candidate timings (with the numerical-equality result per op) plus one
+complete-pipeline check under the production shapes.
+
+## Proportionality
+
+When a PR adds a model and explicitly labels its numbers as smoke observations
+rather than claims, do not demand the attribution machinery. Ask only for
+evidence protecting the model's stated operating contract — for a
+realtime/duplex model that is sustained cadence over a realistic session
+(enough ticks to leave warmup), with any mid-session capture/regeneration
+event and its stall reported, since a cadence that collapses mid-session is a
+functional failure, not a perf claim.
+
+## Realtime/duplex world models
+
+Request the operating-contract evidence as serving metrics, not claims:
+
+- Sustained per-tick latency over one realistic session: median, max, and the
+  argmax tick.
+- Stall events with their causes.
+- Peak memory at first versus last tick — session-owned state must not grow
+  per tick.
+- Control latency from a submitted action to the tick that applies it.
+- Realtime-versus-offline parity under the same controls, using the
+  repository's video-similarity helpers. Parity plotted over tick index is the
+  drift-over-horizon curve and the honest measure of session lifetime;
+  distribution-level suites (FVD, VBench) belong to model evaluation, not the
+  serving integration.
+
+Watch the contract boundaries the model documents: if a control input is
+tick-API-only, the parity pair must use a horizon both paths can run.
+
+## Speech-generation (TTS) models
+
+The streaming contract outranks the average — a synthesis that pauses
+mid-stream is broken for realtime use regardless of its averages. Request:
+
+- The benchmark's own speech metrics over one long utterance (enough audio to
+  leave warmup, tens of seconds): `audio_ttfp` (time to first packet),
+  `audio_rtf` (real-time factor; above 1.0 cannot keep up), `e2el`, and
+  `ttft`/`tpot` where tokens precede audio.
+- Mid-utterance stalls as the benchmark's continuity fields, not prose:
+  `audio_underrun` percentiles and the `audio_continuity_ok` rate
+  (`vllm_omni/benchmarks/metrics/metrics.py`), computed by
+  `vllm_omni/benchmarks/audio_continuity.py` — the companion to `audio_ttfp`
+  and `audio_rtf`. A synthesis that bursts chunks fast enough on average but
+  underruns mid-stream fails the streaming contract even when the averages
+  pass.
+- Quality as a paired comparison against the reference implementation under
+  the same text, speaker prompt, and seed; report sample rate and frame
+  count, and assert output completeness — truncated or dropped trailing audio
+  is a known failure class, not a quality nuance.
+- For realtime or duplex TTS: interruption (barge-in) latency and the
+  per-session memory trend across turns.
+
+## Omni models
+
+Apply the e2e stage-attribution table directly to the voice-to-voice path of
+omni (speech-to-speech and multimodal AR) models:
+
+- TTFT for text output, `audio_ttfp` for audio output, and per-stage
+  ITL/TPOT cadence across the perception-understanding-generation chain.
+- Cross-modal parity — the omni analogue of realtime-versus-offline parity:
+  the same semantic request through text and audio inputs, compared on output
+  content or timing.
+- For duplex omni sessions: turn-taking latency, interruption handling, the
+  concurrent session count actually validated, and a per-turn memory trend —
+  session-owned KV state must not grow per turn. When either turn produces
+  audio, continuity is judged with the same `audio_underrun` and
+  `audio_continuity_ok` fields as TTS, since turn-taking latency alone does
+  not expose mid-utterance underruns.
+
 ## Reviewer verification ladder
 
 1. Run base/head A/B on suitable hardware when affordable.

.claude/skills/review-pr/references/process/general-checks.md

@@ -121,6 +121,19 @@ Search bounded callers and sibling implementations before reporting dead code,
 duplication, or a missed consumer. Keep new abstractions only when they have a
 distinct live caller, invariant, or compatibility need.
 
+Verify absence and equality claims with whole-file evidence. A line window, a
+grep for one symbol, or a commit message proves presence, never absence: to
+assert that a file matches another version or lacks a change, diff or hash the
+full files (for example `git diff <sha1>:<path> <sha2>:<path>` or the contents
+API's size and SHA fields) before anchoring the finding. A verification
+shortcut that would itself be wrong to run is not evidence.
+
+For a value written or finalized in more than one place, establish which site
+executes first on each entry path — and whether the earlier site makes the
+later ones unreachable — before naming where a fix belongs. Enumerating all
+call sites correctly still misses the bug when a path the enumeration ranked
+last actually finalizes first and clears the state the proposed fix checks.
+
 ## Finding bar
 
 Anchor each finding to a changed `path:line`; name the trigger or call path,

.claude/skills/review-pr/references/process/review-execution.md

@@ -39,13 +39,13 @@ isolated detached worktree. Run every source read, `rg` search, import, and test
 from that snapshot, not the caller's checkout. A detached worktree provides
 snapshot isolation, not a security boundary.
 
-Treat a fork head as untrusted unless the user and environment policy explicitly
-establish trust. Never run its imports, tests, builds, hooks, package setup, or
-repo-configurable linters/plugins on the reviewer host. Execute them only in a
-disposable sandbox or VM with no credentials or inherited secrets, no host agent
-or service sockets, minimal read-only host mounts, disabled or explicitly
-allowlisted network, resource/time limits, and destruction after the run. If
-that boundary is unavailable, limit the review to the remote diff, SHA-addressed
+Treat a fork head as untrusted until the user and environment policy
+explicitly establish trust. Without recorded trust, never run its imports,
+tests, builds, hooks, package setup, or repo-configurable linters/plugins on
+the reviewer host. Once the user explicitly trusts this pinned commit for
+this host, record the trusted SHA with the review state and execute with
+credentials, agent sockets, and other secrets out of scope. Without recorded
+trust, limit the review to the remote diff, SHA-addressed
 reads such as `git show <head_sha>:<path>`, and existing CI evidence; report all
 executable validation as a gap.
 

.claude/skills/vllm-omni-test/SKILL.md

@@ -468,10 +468,10 @@ When the user asks for **invalid parameter validation**, **invalid request bodie
 | **Invalid parameters Test · H100** | `pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"` | Diffusion / omni / video invalid-param tests with `H100` hardware mark |
 | **Invalid parameters Test · L4** | `pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"` | TTS / lighter models marked `L4` |
 
-- **Trigger:** `build.env("WEEKLY") == "1"` or PR label `weekly-test`.
+- **Trigger:** `build.env("WEEKLY") == "1"` (all weekly jobs) or PR label `weekly-test` (Reliability / Perf, then filtered by `source_file_dependencies`).
 - **Default:** extending an existing `invalid_param_test` script needs **no YAML edit** — the weekly steps already sweep the whole directory.
 - **Edit YAML only** when adding a **new hardware queue**, a **new top-level script** that must run in isolation, or a **model-specific weekly shard** (mirror neighboring reliability steps).
-- Weekly steps **do not** use `source_file_dependencies`.
+- Weekly Reliability / Perf jobs declare `source_file_dependencies`. Filtering applies only on the `weekly-test` label; `WEEKLY=1` on `main` keeps every job in those groups. CPU coverage (`WEEKLY=1` only) and the E2E Tests group (`NON_CRITICAL=1` only) do **not** use this field.
 
 **Example — append to `test_invalid_image_generation.py`:**
 
@@ -644,7 +644,7 @@ If the test is not already collected by an existing pipeline command (for exampl
 | **L2** | `test-ready.yml` with `core_model` + `source_file_dependencies` | merge / nightly pipelines |
 | **Invalid param** | `test-weekly.yml` — **Invalid parameters Test** group (`-m "slow and H100"` / `-m "slow and L4"`). Usually **no YAML edit** when appending cases to existing `invalid_param_test` scripts | `test-ready.yml`, `test-merge.yml`, `test-nightly.yml` |
 
-`test-nightly.yml` shards use **explicit pytest file paths** in `commands` (and PR labels like `diffusion-x2iat-test`); they generally **do not** use `source_file_dependencies`. Only suggest merge/ready YAML when the requested level is L3/L2 (or the user explicitly asks for multi-level CI).
+`test-nightly.yml` shards use **explicit pytest file paths** in `commands` plus `source_file_dependencies`; PR trigger is `nightly-test` (path filter), or `NIGHTLY=1` on `main` (all jobs). Only suggest merge/ready YAML when the requested level is L3/L2 (or the user explicitly asks for multi-level CI).
 
 **`test-nightly.yml` top-level groups** (each uses `full_model` + type marker in pytest `-m`):
 
@@ -653,7 +653,7 @@ If the test is not already collected by an existing pipeline command (for exampl
 | **Omni Model Test** | `omni` | Function / doc / accuracy / **perf** for Qwen-Omni family |
 | **TTS Model Test** | `tts` | `-m "full_model and L4 and tts"` (and H100 variants when enabled); **Perf Test** uses `run_benchmark.py` + `test_tts.json` |
 | **Diffusion X2I(&A&T) Model Test** | `diffusion` | Image/audio/text diffusion; explicit file shards for **Function**; **Perf Test · <Model>** uses `run_diffusion_benchmark.py` + JSON config |
-| **Diffusion X2V Model Test** | `diffusion` | Video-only (Wan, HunyuanVideo, …); PR label `diffusion-x2v-test`; separate **Perf** steps when in-tree |
+| **Diffusion X2V Model Test** | `diffusion` | Video-only (Wan, HunyuanVideo, …); separate **Perf** steps when in-tree |
 
 Guidelines when editing YAML:
 

.claude/skills/vllm-omni-test/references/test-routing.md

@@ -37,7 +37,7 @@ Every model-centric e2e test must declare **exactly one** type marker:
 | **Diffusion X2I(&A&T) Model Test** | x2**i** / x2**a** / x2**t** — image, audio, text (non-video) | Qwen-Image*, BAGEL, FLUX, SD3, Z-Image, LongCat, DreamZero |
 | **Diffusion X2V Model Test** | x2**v** — video only | Wan2.2, HunyuanVideo 1.5, Wan VACE |
 
-PR labels for selective nightly runs: `diffusion-x2iat-test`, `diffusion-x2v-test` (plus `nightly-test` / `NIGHTLY=1`).
+PR label for L4 on a branch: `nightly-test` (jobs then filtered by `source_file_dependencies`). Scheduled `main` uses `NIGHTLY=1` and runs every nightly job. L5 Reliability / Perf use `weekly-test` with the same path filter; CPU coverage and the weekly E2E group are env-only (`WEEKLY=1` / `NON_CRITICAL=1`) and are not source-filtered.
 
 ## Level and Marker Mapping
 
@@ -238,7 +238,7 @@ pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and L4"
 pytest -s -v dfx/reliability/invalid_param_test/test_invalid_image_generation.py::test_images_generations_invalid_requests -m "slow and H100"
 ```
 
-**Trigger:** `WEEKLY=1` or PR label `weekly-test`. Appending cases to existing scripts usually needs **no YAML edit** (directory sweep). No `source_file_dependencies` on weekly steps.
+**Trigger:** `WEEKLY=1` (all jobs) or PR label `weekly-test` (Reliability / Perf, filtered by `source_file_dependencies`). Appending cases to existing scripts usually needs **no YAML edit** (directory sweep).
 
 ### Platform-targeted examples
 

.github/CODEOWNERS

@@ -67,6 +67,11 @@
 # File-level overrides within entrypoints/
 /vllm_omni/entrypoints/openai/errors.py @alex-jw-brooks @NickCao
 /vllm_omni/entrypoints/cli/benchmark/ @alex-jw-brooks @Bounty-hunter
+# Shared wire codec: protocol/realtime (OpenAI Realtime GA) and protocol/duplex
+# (vLLM-Omni's extension of it) — RFC #6592 P0a. Layer-neutral by design — the
+# engine depends on it — so it is owned by the serving-boundary document rather
+# than by any one consumer. — docs/design/module/entrypoints.md
+/vllm_omni/protocol/ @alex-jw-brooks @linyueqian @NickCao
 
 # Error contracts — docs/design/module/error_contracts.md
 /vllm_omni/errors.py @alex-jw-brooks @NickCao

CONTRIBUTING.md

@@ -2,6 +2,8 @@
 
 You may find information about contributing to vLLM-Omni on [Contributing](https://vllm-omni.readthedocs.io/en/latest/contributing/).
 
+AI-assisted contributions must follow the [AI Usage Policy](docs/contributing/README.md#ai-usage-policy), including disclosure, human review, and validation requirements.
+
 Local `pre-commit` gates (SPDX, forbidden imports, `torch.cuda`, shellcheck,
 mypy, test marks, TTS adapter ratchet, markdownlint, Buildkite schema, and
 how to extend allowlists) are documented in the

README.md

@@ -58,7 +58,7 @@ vLLM-Omni is flexible and easy to use with:
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
 - **Omni-modality models** (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
-- **TTS models** (e.g. Qwen3-TTS, IndexTTS 2.5, dots.tts, CosyVoice3)
+- **TTS models** (e.g. Qwen3-TTS, IndexTTS 2.5, CosyVoice3)
 - **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
 - **Robot-policy and action models** (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
 

apps/ComfyUI-vLLM-Omni/README.md

@@ -13,7 +13,6 @@ It can send model inference requests to either a locally running vLLM-Omni servi
 > [!TIP]
 > If you run both ComfyUI and vLLM-Omni on the same device, you can create separate virtual environments and use different Python versions for them.
 
-
 ## Installation
 
 Copy this folder to the `custom_nodes` subfolder of your ComfyUI installation. Your directory should look like `ComfyUI/custom_nodes/ComfyUI-vLLM-Omni`.
@@ -21,9 +20,10 @@ Copy this folder to the `custom_nodes` subfolder of your ComfyUI installation. Y
 If you are running ComfyUI during copying, you should restart ComfyUI to load this extension.
 
 > [!TIP]
-> You can use utility websites such as https://download-directory.github.io/ to download a subdirectory of a repo. Also checkout community discussions (e.g., https://stackoverflow.com/questions/7106012/download-a-single-folder-or-directory-from-a-github-repository) for more info.
+> You can use utility websites such as <https://download-directory.github.io/> to download a subdirectory of a repo. Also checkout community discussions (e.g., <https://stackoverflow.com/questions/7106012/download-a-single-folder-or-directory-from-a-github-repository>) for more info.
 
 On the device and virtual environment you run ComfyUI, launch ComfyUI with
+
 ```bash
 cd ComfyUI
 
@@ -35,6 +35,7 @@ python main.py --cpu
 ```
 
 On the device and virtual environment you run vLLM-Omni, start a model service with
+
 ```bash
 vllm serve The_Model_ID_to_Serve --omni --port 8000
 ```
@@ -48,14 +49,31 @@ This extension offers the following nodes based on the output modalities (at **C
 
 - **Generate Image** for text-to-image and image-to-image tasks
 - **Generate Video** for text-to-video, first-frame/image-to-video, and reference-conditioned video
+- **Latent Mask Editing** for MiniMax-H3 latent-mask editing (source media plus video/audio noise masks)
+- **FastH3 Deployment** for routing text-to-video requests to a MiniMax-H3 server with FastH3 fused at startup
 - **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks
 - **TTS** and **TTS Voice Clone** for TTS tasks
+- **Generate Music** for lyrics-and-description-to-music generation
 
 This extension also offers example workflows (at **ComfyUI sidebar -> Templates -> vLLM-Omni**)
 
 > [!NOTE]
 > The node UI and feature designs are intended to match vLLM-Omni online serving interfaces. It cannot offer more than what the interfaces support.
 
+Every node carries the vLLM-Omni mark in its title bar and is tinted by what it outputs, so a graph is readable at a glance:
+
+| Colour | Nodes | What they produce |
+| --- | --- | --- |
+| Blue | Generate Image, Generate Video, Multimodality Understanding, TTS, TTS Voice Clone | A generated image, video, audio, or text. These are the only nodes that reach a server. |
+| Amber | AR / Diffusion / Multi-Stage Sampling Params | Sampling parameters that apply to any model |
+| Purple | Qwen TTS Params, Wan Video Params, MiniMax-H3 Video Params | Parameters that only one model family accepts |
+| Red | LoRA, FastH3 Deployment | Which weights the server is expected to have loaded |
+| Teal | Video References, Latent Mask Editing | Reference media |
+
+Recolouring a node by hand (right click -> Colors) overrides its tint, and the choice is kept.
+
+**Generate Video** takes a clip length in seconds (`duration`), not a frame count. Frames stay the wire unit and are derived with the node's `fps`, so the length is always measured against the rate that is actually served; models that accept only certain frame counts still round to their own lattice server-side. Graphs saved before this widget existed stored `num_frames` in its place and are converted on load, using the fps recorded alongside it -- the browser console names every node it rewrites.
+
 To build a simple workflow yourself,
 
 - Drag a generation node onto the canvas.
@@ -117,11 +135,146 @@ You can configure per-stage sampling parameters for multi-stage models.
 </p>
 
 > [!TIP]
-> Connect a **frame** image for first-frame / image-to-video (e.g., Wan I2V, MiniMax-H3 FL2VA).
+> Connect **frame** for backward-compatible first-frame / image-to-video behavior. For MiniMax-H3 FL2VA,
+> connect **first_frame**, **last_frame**, or both to condition the start frame, end frame, or both.
 >
 > For reference-conditioned generation (MiniMax-H3 Ref2VA), connect a **Video References** node instead.
 >
-> Do not use `frame` and `references` together. Task routing is automatic from which inputs you connect.
+> Do not combine `frame` with `first_frame` or `last_frame`, and do not combine any frame input with
+> `references`. Task routing is automatic from which inputs you connect.
+
+For MiniMax-H3 Ref2VA, **Video References** accepts up to 9 images (`image_1`–`image_9`),
+3 videos (`video_1`–`video_3`), and 3 audio clips (`audio_1`–`audio_3`), with at most
+12 connected inputs in total. Any mixture containing at least one image or video
+is supported; empty and audio-only references are rejected. For example, you can
+combine 6 images, 3 videos, and 3 audio clips in one request.
+
+Within each media type, references follow slot-number order, skipping unconnected
+slots. For example, connecting `image_2` and `image_9` sends `image_2` as the first
+image and `image_9` as the second. Each image slot uses the first image in its batch.
+Existing connections to `image_1`, `image_2`, `audio_1`, `audio_2`, `video_1`, and
+`video_2` remain valid in saved workflows.
+
+#### MiniMax-H3 Reference to Video
+
+Load [MiniMax-H3 Reference to Video](example_workflows/vLLM-Omni%20MiniMax-H3%20Reference%20to%20Video.json)
+from **Templates → ComfyUI-vLLM-Omni**, or drag the JSON onto the canvas.
+It adapts the [official ComfyUI R2V workflow](https://github.com/Comfy-Org/workflow_templates/blob/main/templates/video_minimax_h3_r2v.json)
+to remote vLLM-Omni execution. Model loading, sampling, and VAE decoding run on
+the server; ComfyUI loads references and saves the returned video.
+
+Use an extension version containing the audio-preserving output fix from
+[#7456](https://github.com/vllm-project/vllm-omni/pull/7456). Older versions can
+drop the server's audio while decoding the response.
+
+1. Start a Ref2VA-capable service using the [MiniMax-H3 recipe](../../recipes/MiniMaxAI/MiniMax-H3.md).
+   Set **Generate Video** to its `/v1` URL and served model name.
+2. Select your own image in **Load Image**. The template's filenames are placeholders
+   relative to ComfyUI's input directory; no sample assets or model weights are bundled.
+3. Connect **Load Video** and **Load Audio** when needed. Duplicate the loaders to
+   fill more reference slots, or disconnect the image when using video-only input.
+   Keep the references connected to **Generate Video** and leave `frame` disconnected.
+4. Match prompt tags such as `<Picture 1>`, `<Video 1>`, and `<Audio 1>` to the
+   connected references, counting each media type separately and skipping empty slots.
+5. Run the workflow. **Save Video** writes an MP4 under `output/video/` and preserves
+   the generated audio when the audio-output prerequisite is installed.
+
+The default is **1344×768, 24 FPS, 124 frames** (about 5.17 seconds), 50 sampling
+points, video flow shift 12, audio flow shift 3, and seed 42. The duration widget
+is set to 5.167 seconds, which converts to 124 frames at 24 FPS. Other H3 canvas presets
+are 1024×768, 768×768, 768×1024, and 768×1344. Keep frame counts at `17k+5` within
+the 4–15 second output range; examples are 107, 124, 209, and 345 frames.
+Reference video/audio clips must each be 2–15 seconds, with at most 15 seconds of
+reference video and at most 15 seconds of standalone reference audio per request.
+
+For **Ref2VA Turbo**, use the same template. Configure a Ref2VA-only server with
+`--lora-backend peft --lora-path "$TURBO_LORA" --task-type ref2va`, following the
+recipe's hardware setup. For `minimax_h3_ref2v_turbo_4step_v0.1_bf16.safetensors`,
+set **LoRA** `local_path` to that file's path on the server and connect it to
+**Generate Video**. Use `num_inference_steps=5`, `flow_shift=12`,
+`audio_flow_shift=3`, and LoRA `scale=1`. The API counts sigma points, so five
+points produce four denoiser evaluations. Use the Diffusers artifact, not its
+`_comfyui_` export. To return to base mode, disconnect LoRA and restore 50 steps.
+The 8-step v1.0 Ref2VA adapter instead requires nine sampling points and video
+flow shift 6. FastH3 is a separate startup-fused T2VA deployment.
+
+Validate the template's wiring and defaults locally with:
+
+```bash
+python -m pytest tests/e2e/features/comfyui/test_h3_reference_workflow.py -q
+```
+
+For a real-model check, run the workflow against H3 and inspect the saved file:
+
+```bash
+ffprobe -v error -show_entries stream=codec_type,width,height,r_frame_rate,sample_rate,channels \
+  -show_entries format=duration -of json "$OUTPUT_MP4"
+```
+
+Check for 24 FPS video, stereo audio, the requested canvas, and aligned duration;
+also play the result to assess reference conditioning and audio/video synchronization.
+Record the server commit, model/adapter, input assets, prompt, settings, and output
+alongside the result. Local schema or mocked-server checks do not establish H3
+generation quality or replace this real-model validation.
+
+### MiniMax H3 Text to Video
+
+Import [the H3 text-to-video template](example_workflows/MiniMax_H3_Text_to_Video.json)
+for remote video generation with audio. It includes connected base presets and
+optional Turbo sampling, H3 parameters, and Remote LoRA nodes. See the
+[workflow guide](docs/minimax-h3-t2v.md) for server setup, artifact-specific Turbo
+settings, and saved-video/audio validation. Set Remote LoRA’s `local_path` to
+the downloaded Turbo artifact on your server before enabling Turbo.
+
+#### H3 video upscale (WF-07)
+
+The **vLLM-Omni MiniMax H3 Video Upscale** template generates video remotely, upscales it with SeedVR2, and saves the original and upscaled videos with the generated audio and FPS. See [workflow setup](docs/wf07-h3-upscale.md).
+
+#### FastH3 text-to-video
+
+FastH3 is fused into MiniMax-H3 when the vLLM-Omni server starts; it is not a request-switchable LoRA. Download one adapter variant and start a non-offloaded FL2VA server. For the Dense / Data-Free profile:
+
+```bash
+export FASTH3_DIR=/path/to/fasth3
+hf download FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA \
+  dense-datafree/adapter_model.safetensors --local-dir "${FASTH3_DIR}"
+
+vllm serve /path/to/MiniMax-H3 \
+  --omni \
+  --task-type fl2va \
+  --lora-path "${FASTH3_DIR}/dense-datafree/adapter_model.safetensors" \
+  --port 8000
+```
+
+For the VSA / Data-Free profile, download `vsa-datafree/adapter_model.safetensors`, use that file as `--lora-path`, and add:
+
+```bash
+--diffusion-attention-backend FASTVIDEO_VSA \
+--fastvideo-vsa-topk 64
+```
+
+The VSA profile also requires a compatible `fastvideo-kernel` installation. See the [MiniMax-H3 FastH3 recipe](../../recipes/MiniMaxAI/MiniMax-H3.md#fasth3-adapter) for the full serving contract, supported parallel layouts, and kernel requirements.
+
+Open the **vLLM-Omni FastH3 Text to Video** template, then:
+
+- Set the server URL and served model name on **FastH3 Deployment**.
+- Connect its output to **Generate Video → fast_h3**. When connected, the deployment node's URL and model take precedence over the corresponding Generate Video widgets.
+- Keep `frame`, `references`, **LoRA**, and **MiniMax-H3 Video Params** disconnected. FastH3 Preview v1 supports T2VA only, is already fused, and owns both flow shifts.
+- A connected **Diffusion Sampling Params** node may set seed and other ordinary sampling options. The integration always enforces four denoising steps and 24 FPS for FastH3.
+
+The node records which server the workflow targets; it does not start one, nor switch adapters or attention backends on a running server.
+
+#### Latent-mask editing (MiniMax-H3)
+
+The [WF-05 template](example_workflows/vLLM-Omni%20MiniMax-H3%20Latent%20Mask%20Editing.json) contains inpainting, object removal, continuation, and extension examples in one graph. See the [workflow guide](docs/wf05-h3-latent-editing.md) for inputs, mask settings, dependencies, and preview limitations.
+
+Connect a **Latent Mask Editing** node to **Generate Video → latent_edit** to edit a source clip instead of generating from scratch. It uploads the source media and serializes the video/audio noise masks the MiniMax H3 API accepts:
+
+- `source_video` / `source_audio` — the media to edit.
+- `video_mask` — a ComfyUI mask image; `0` preserves a region, `1` regenerates it, fractional values blend. A 2D mask `[H, W]` is applied to every frame; a 3D mask `[T, H, W]` is treated as a temporal mask (one slice per frame) for continuation or extension. It is resized to the video latent grid.
+- `audio_mask` — a scalar in `[0, 1]`; `0` keeps the source audio, `1` regenerates it, fractional values blend.
+
+A non-trivial mask requires its matching source, and a source without a mask is rejected. This node only forwards inputs to the server; the served model must declare latent-mask editing support (MiniMax-H3).
 
 ### TTS (e.g., Qwen TTS series)
 
@@ -137,6 +290,17 @@ You can configure per-stage sampling parameters for multi-stage models.
 > [!TIP]
 > There is a dedicated node for VoiceClone tasks with reference audio input. Other simple text-to-speech tasks should use the regular TTS node.
 
+### Music generation (e.g., MiniMax Music 3)
+
+(Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni Music Generation**)
+
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-music-generation.jpg">
+    <img alt="vLLM-Omni Music Generation" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-music-generation.jpg" width=55%>
+  </picture>
+</p>
+
 ### Chaining multiple model services
 
 (Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni Chaining Services**)
@@ -152,6 +316,8 @@ You can configure per-stage sampling parameters for multi-stage models.
 
 Follow the [development convention and rules of vLLM-Omni](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/).
 
+Node tints and the title-bar mark are applied in `web/main.js`, keyed off each node's declared output types rather than a list of node names. A new node that returns an existing type is themed with no front-end change; a new output type needs one entry in `FAMILY_BY_OUTPUT` there.
+
 ## Limitation and Non-Goals
 
 - Single server mode only. No automatic load balancing or failover.
@@ -178,11 +344,11 @@ Whenever you find an issue or problem, please
 
 Features
 
-- https://github.com/dougbtv/comfyui-vllm-omni/ The official reference implementation for ComfyUI integration with vLLM-Omni's DALL-E compatible image generation API.
-- https://github.com/Comfy-Org/ComfyUI/tree/master/comfy_extras ComfyUI's built-in node implementations.
+- <https://github.com/dougbtv/comfyui-vllm-omni/> The official reference implementation for ComfyUI integration with vLLM-Omni's DALL-E compatible image generation API.
+- <https://github.com/Comfy-Org/ComfyUI/tree/master/comfy_extras> ComfyUI's built-in node implementations.
 
 UI/UX design references
 
-- https://github.com/sgl-project/sglang/pull/15271 SGLang Diffusion's official ComfyUI integration for image and video generation.
-- https://github.com/SXQBW/ComfyUI-Qwen-Omni A third party ComfyUI integration for Qwen Omni series.
-- https://github.com/flybirdxx/ComfyUI-Qwen-TTS https://github.com/DarioFT/ComfyUI-Qwen3-TTS Third  party ComfyUI integrations for Qwen TTS series.
+- <https://github.com/sgl-project/sglang/pull/15271> SGLang Diffusion's official ComfyUI integration for image and video generation.
+- <https://github.com/SXQBW/ComfyUI-Qwen-Omni> A third party ComfyUI integration for Qwen Omni series.
+- <https://github.com/flybirdxx/ComfyUI-Qwen-TTS> <https://github.com/DarioFT/ComfyUI-Qwen3-TTS> Third  party ComfyUI integrations for Qwen TTS series.

apps/ComfyUI-vLLM-Omni/__init__.py

@@ -1,4 +1,8 @@
-"""Top-level package for comfyui_vllm_omni."""  # noqa: N999  # This is not a python library intended to be imported
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+# ruff: noqa: N999
+
+"""Top-level package for comfyui_vllm_omni."""  # This is not a Python library intended to be imported.
 
 __all__ = [
     "NODE_CLASS_MAPPINGS",
@@ -13,9 +17,13 @@ __version__ = "0.0.1"
 from .comfyui_vllm_omni.nodes import (
     VLLMOmniARSampling,
     VLLMOmniDiffusionSampling,
+    VLLMOmniFastH3Deployment,
     VLLMOmniGenerateImage,
+    VLLMOmniGenerateMusic,
     VLLMOmniGenerateVideo,
+    VLLMOmniLatentMaskEditing,
     VLLMOmniMiniMaxH3Params,
+    VLLMOmniMiniMaxH3TemporalMask,
     VLLMOmniQwenTTSParams,
     VLLMOmniRemoteLoRA,
     VLLMOmniSamplingParamsList,
@@ -28,18 +36,22 @@ from .comfyui_vllm_omni.nodes import (
 
 # A dictionary that contains all nodes you want to export with their names
 NODE_CLASS_MAPPINGS = {
+    "VLLMOmniMiniMaxH3TemporalMask": VLLMOmniMiniMaxH3TemporalMask,
     # === Generation ===
     "VLLMOmniGenerateImage": VLLMOmniGenerateImage,
     "VLLMOmniGenerateVideo": VLLMOmniGenerateVideo,
     "VLLMOmniUnderstanding": VLLMOmniUnderstanding,
     "VLLMOmniTTS": VLLMOmniTTS,
+    "VLLMOmniGenerateMusic": VLLMOmniGenerateMusic,
     "VLLMOmniVoiceClone": VLLMOmniVoiceClone,
     "VLLMOmniVideoReferences": VLLMOmniVideoReferences,
+    "VLLMOmniLatentMaskEditing": VLLMOmniLatentMaskEditing,
     # === Params ===
     "VLLMOmniARSampling": VLLMOmniARSampling,
     "VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling,
     "VLLMOmniSamplingParamsList": VLLMOmniSamplingParamsList,
     "VLLMOmniRemoteLoRA": VLLMOmniRemoteLoRA,
+    "VLLMOmniFastH3Deployment": VLLMOmniFastH3Deployment,
     "VLLMOmniQwenTTSParams": VLLMOmniQwenTTSParams,
     "VLLMOmniWanParams": VLLMOmniWanParams,
     "VLLMOmniMiniMaxH3Params": VLLMOmniMiniMaxH3Params,
@@ -47,18 +59,22 @@ NODE_CLASS_MAPPINGS = {
 
 # A dictionary that contains the friendly/humanly readable titles for the nodes
 NODE_DISPLAY_NAME_MAPPINGS = {
+    "VLLMOmniMiniMaxH3TemporalMask": "MiniMax-H3 Temporal Mask",
     # === Generation ===
     "VLLMOmniGenerateImage": "Generate Image",
     "VLLMOmniGenerateVideo": "Generate Video",
     "VLLMOmniUnderstanding": "Multimodality Understanding",
     "VLLMOmniTTS": "TTS (Text to Speech)",
+    "VLLMOmniGenerateMusic": "Generate Music",
     "VLLMOmniVoiceClone": "TTS Voice Cloning",
     "VLLMOmniVideoReferences": "Video References",
+    "VLLMOmniLatentMaskEditing": "Latent Mask Editing",
     # === Params ===
     "VLLMOmniARSampling": "AR Sampling Params",
     "VLLMOmniDiffusionSampling": "Diffusion Sampling Params",
     "VLLMOmniSamplingParamsList": "Multi-Stage Sampling Params List",
     "VLLMOmniRemoteLoRA": "LoRA",
+    "VLLMOmniFastH3Deployment": "FastH3 Deployment",
     "VLLMOmniQwenTTSParams": "Qwen TTS Params",
     "VLLMOmniWanParams": "Wan Video Params",
     "VLLMOmniMiniMaxH3Params": "MiniMax-H3 Video Params",

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py

@@ -1,15 +1,25 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+import math
 from typing import Literal
 
 import torch
 from comfy_api.input import AudioInput, VideoInput
 
 from .utils.api_client import VLLMOmniClient
+from .utils.latent_mask import _align_frame_count, _video_latent_t
 from .utils.logger import get_logger
 from .utils.models import lookup_model_spec
 from .utils.types import (
+    MAX_REFERENCE_AUDIOS,
+    MAX_REFERENCE_IMAGES,
+    MAX_REFERENCE_VIDEOS,
     AudioFormat,
     AutoregressionSamplingParams,
     DiffusionSamplingParams,
+    FastH3Deployment,
+    LatentMaskEditing,
     MiniMaxH3ModelSpecificParams,
     QwenTTSModelSpecificParams,
     VideoReferences,
@@ -22,6 +32,23 @@ from .utils.validators import (
 
 logger = get_logger(__name__)
 
+FASTH3_INFERENCE_STEPS = 4
+FASTH3_FPS = 24
+# A FastH3 deployment may expose H3 under any --served-model-name, so the payload
+# spec has to be named outright rather than recovered from that alias.
+FASTH3_SPEC_MODEL = "MiniMax-H3"
+
+
+def _resolve_fast_h3_deployment(deployment: dict) -> tuple[str, str]:
+    if not isinstance(deployment, dict):
+        raise ValueError("FastH3 deployment must be provided by a FastH3 Deployment node.")
+
+    url = str(deployment.get("url") or "").strip().rstrip("/")
+    model = str(deployment.get("model") or "").strip()
+    if not url or not model:
+        raise ValueError("FastH3 deployment requires both URL and model.")
+    return url, model
+
 
 class _VLLMOmniGenerateBase:
     """Base class for vLLM-Omni generation nodes with shared functionality."""
@@ -162,14 +189,31 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
                 "width": ("INT", {"default": 832, "min": 1}),
                 "height": ("INT", {"default": 480, "min": 1}),
                 "fps": ("INT", {"default": 16, "min": 1}),
-                "num_frames": ("INT", {"default": 41, "min": 1}),
+                "duration": (
+                    "FLOAT",
+                    {
+                        "default": 4.0,
+                        "min": 0.1,
+                        "step": 0.1,
+                        "round": 0.001,
+                        "tooltip": (
+                            "Clip length in seconds, converted to frames with the fps above. "
+                            "Models that only accept certain frame counts (e.g. MiniMax-H3) round to "
+                            "their own lattice, so the served clip can be slightly longer than requested."
+                        ),
+                    },
+                ),
             },
             "optional": {
                 "frame": ("IMAGE",),
+                "first_frame": ("IMAGE",),
+                "last_frame": ("IMAGE",),
                 "references": ("VIDEO_REFERENCES",),
                 "sampling_params": ("SAMPLING_PARAMS",),
                 "lora": ("REMOTE_LORA",),
                 "model_params": ("VIDEO_PARAMS",),
+                "fast_h3": ("FASTH3_DEPLOYMENT",),
+                "latent_edit": ("LATENT_MASK_EDITING",),
             },
         }
 
@@ -178,12 +222,30 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
     FUNCTION = "generate"
 
     @classmethod
-    def VALIDATE_INPUTS(cls, url, model, frame=None, references=None, **_kwargs) -> str | Literal[True]:
+    def VALIDATE_INPUTS(
+        cls,
+        url,
+        model,
+        frame=None,
+        first_frame=None,
+        last_frame=None,
+        references=None,
+        fast_h3=None,
+        **_kwargs,
+    ) -> str | Literal[True]:
         base = super().VALIDATE_INPUTS(url, model)
         if base is not True:
             return base
         if frame is not None and references is not None:
             return "Provide only one of frame or references, not both."
+        if frame is not None and (first_frame is not None or last_frame is not None):
+            return "Provide either frame or first_frame/last_frame, not both."
+        if references is not None and (first_frame is not None or last_frame is not None):
+            return "Provide either first_frame/last_frame or references, not both."
+        if fast_h3 is not None and any(value is not None for value in (frame, first_frame, last_frame, references)):
+            return (
+                "FastH3 Preview supports T2VA only; disconnect frame, first_frame, last_frame, and references inputs."
+            )
         return True
 
     async def generate(
@@ -194,20 +256,72 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
         width: int,
         height: int,
         fps: int,
-        num_frames: int,
+        duration: float,
         negative_prompt: str | None = None,
         frame: torch.Tensor | None = None,
+        first_frame: torch.Tensor | None = None,
+        last_frame: torch.Tensor | None = None,
         references: dict | None = None,
         sampling_params: dict | list[dict] | None = None,
         model_params: dict | None = None,
         lora: dict | None = None,
+        fast_h3: dict | None = None,
+        latent_edit: dict | None = None,
         **kwargs,
     ):
         if kwargs:
             logger.info("Uncaught kwargs: %s", kwargs)
         logger.debug("Got sampling params: %s", sampling_params)
         logger.debug("Got model params: %s", model_params)
-        validate_model_and_sampling_params_types(model, sampling_params)
+
+        # Which spec builds the payload. Only a FastH3 deployment separates this
+        # from the served name; every other path keeps them equal.
+        spec_model = model
+
+        if fast_h3 is not None:
+            if any(value is not None for value in (frame, first_frame, last_frame, references)):
+                raise ValueError(
+                    "FastH3 Preview supports T2VA only; disconnect frame, first_frame, "
+                    "last_frame, and references inputs."
+                )
+            if lora is not None:
+                raise ValueError(
+                    "FastH3 is already fused into the selected server; disconnect the request-level LoRA input."
+                )
+
+            url, model = _resolve_fast_h3_deployment(fast_h3)
+            logger.info("Using FastH3 deployment at %s", url)
+            fps = FASTH3_FPS
+            # The served name is whatever the operator passed to --served-model-name.
+            # Left alone, lookup_model_spec would miss H3 for an alias such as
+            # "fasth3", drop the params builder, and send a t2va request carrying no
+            # aspect_ratio -- which the server refuses.
+            spec_model = FASTH3_SPEC_MODEL
+
+            if sampling_params is None:
+                sampling_params = DiffusionSamplingParams()
+            elif isinstance(sampling_params, list):
+                if len(sampling_params) != 1:
+                    raise ValueError("FastH3 expects a single diffusion sampling params group.")
+                sampling_params = sampling_params[0].__class__(sampling_params[0])
+            else:
+                sampling_params = sampling_params.__class__(sampling_params)
+            sampling_params["num_inference_steps"] = FASTH3_INFERENCE_STEPS
+
+            # FastH3 owns both modality shifts. Sending the ordinary H3 values
+            # from a connected H3 Params node would turn a deployment choice
+            # into a request-level override, which the server intentionally
+            # rejects when it differs from the fused adapter contract.
+            if model_params is not None:
+                model_params = model_params.__class__(model_params)
+                model_params.pop("flow_shift", None)
+                model_params.pop("audio_flow_shift", None)
+
+        # Frames stay the wire unit. Convert after the FastH3 branch above, so the
+        # duration is measured against the fps the server will actually apply.
+        num_frames = max(1, round(duration * fps))
+
+        validate_model_and_sampling_params_types(spec_model, sampling_params)
 
         # Currently, all video generation models are single-stage diffusion models
         if isinstance(sampling_params, list):
@@ -227,8 +341,11 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
         client = VLLMOmniClient(url)
         output = await client.generate_video(
             model=model,
+            spec_model=spec_model,
             prompt=prompt,
             frame=frame,  # frame present => fl2va / Wan I2V
+            first_frame=first_frame,
+            last_frame=last_frame,
             references=references,
             width=width,
             height=height,
@@ -238,6 +355,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
             sampling_params=sampling_params,
             lora=lora,
             model_params=model_params,
+            latent_edit=latent_edit,
         )
         return (output,)
 
@@ -418,6 +536,66 @@ class VLLMOmniTTS(_VLLMOmniGenerateBase):
         return (audio,)
 
 
+class VLLMOmniGenerateMusic(_VLLMOmniGenerateBase):
+    """Generate a song from lyrics and a musical description with MiniMax Music 3."""
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "url": ("STRING", {"default": "http://localhost:8000/v1"}),
+                "model": ("STRING", {"default": "MiniMaxAI/MiniMax-Music3"}),
+                "instructions": (
+                    "STRING",
+                    {
+                        "multiline": True,
+                        "display_name": "caption",
+                        "tooltip": "Music caption: describe genre, instruments, tempo and mood.",
+                    },
+                ),
+                "lyrics": ("STRING", {"multiline": True}),
+                "max_duration_seconds": (
+                    "FLOAT",
+                    {
+                        "default": 300.0,
+                        "min": 1,
+                        "max": 360,
+                        "step": 0.01,
+                        "tooltip": "Upper limit; rounded down to whole 25 Hz audio frames. May end earlier.",
+                    },
+                ),
+                "seed": ("INT", {"default": 0, "min": 0, "max": 2**53 - 1, "control_after_generate": True}),
+                "response_format": (["wav", "mp3", "flac", "opus"],),
+            },
+        }
+
+    RETURN_TYPES = ("AUDIO",)
+    RETURN_NAMES = ("audio",)
+    FUNCTION = "generate"
+
+    async def generate(
+        self,
+        url: str,
+        model: str,
+        lyrics: str,
+        instructions: str,
+        response_format: AudioFormat,
+        max_duration_seconds: float,
+        seed: int = 0,
+    ) -> tuple[AudioInput]:
+        audio = await VLLMOmniClient(url.rstrip("/")).generate_speech(
+            model=model,
+            input=lyrics,
+            instructions=instructions,
+            voice="default",
+            speed=1.0,
+            response_format=response_format,
+            max_new_tokens=int(max_duration_seconds * 25),
+            seed=seed,
+        )
+        return (audio,)
+
+
 class VLLMOmniVoiceClone(_VLLMOmniGenerateBase):
     @classmethod
     def INPUT_TYPES(cls):
@@ -695,6 +873,52 @@ class VLLMOmniRemoteLoRA:
         return (lora,)
 
 
+class VLLMOmniFastH3Deployment:
+    """Select a vLLM-Omni service that fused FastH3 at startup."""
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "url": (
+                    "STRING",
+                    {
+                        "default": "http://localhost:8000/v1",
+                        "tooltip": "URL of a vLLM-Omni server started with a FastH3 --lora-path.",
+                    },
+                ),
+                "model": (
+                    "STRING",
+                    {
+                        "default": "MiniMaxAI/MiniMax-H3",
+                        "tooltip": "Model name exposed by the FastH3 deployment.",
+                    },
+                ),
+            }
+        }
+
+    RETURN_TYPES = ("FASTH3_DEPLOYMENT",)
+    RETURN_NAMES = ("deployment",)
+    FUNCTION = "get_deployment"
+    CATEGORY = "vLLM-Omni"
+    DESCRIPTION = (
+        "Selects a server with FastH3 fused at startup. The connected Generate Video node uses T2VA, "
+        "four inference steps, and 24 FPS without sending a request-level LoRA."
+    )
+
+    @classmethod
+    def VALIDATE_INPUTS(cls, url, model) -> str | Literal[True]:
+        try:
+            _resolve_fast_h3_deployment({"url": url, "model": model})
+        except ValueError as exc:
+            return str(exc)
+        return True
+
+    def get_deployment(self, url: str, model: str):
+        url, model = _resolve_fast_h3_deployment({"url": url, "model": model})
+        return (FastH3Deployment({"url": url, "model": model}),)
+
+
 class VLLMOmniQwenTTSParams:
     @classmethod
     def INPUT_TYPES(cls):
@@ -790,6 +1014,10 @@ class VLLMOmniVideoReferences:
                 "audio_2": ("AUDIO",),
                 "video_1": ("VIDEO",),
                 "video_2": ("VIDEO",),
+                # Append ports to preserve connections in saved workflows.
+                **{f"image_{i}": ("IMAGE",) for i in range(3, MAX_REFERENCE_IMAGES + 1)},
+                **{f"audio_{i}": ("AUDIO",) for i in range(3, MAX_REFERENCE_AUDIOS + 1)},
+                **{f"video_{i}": ("VIDEO",) for i in range(3, MAX_REFERENCE_VIDEOS + 1)},
             },
         }
 
@@ -806,21 +1034,112 @@ class VLLMOmniVideoReferences:
         audio_2: AudioInput | None = None,
         video_1: VideoInput | None = None,
         video_2: VideoInput | None = None,
+        image_3: torch.Tensor | None = None,
+        image_4: torch.Tensor | None = None,
+        image_5: torch.Tensor | None = None,
+        image_6: torch.Tensor | None = None,
+        image_7: torch.Tensor | None = None,
+        image_8: torch.Tensor | None = None,
+        image_9: torch.Tensor | None = None,
+        audio_3: AudioInput | None = None,
+        video_3: VideoInput | None = None,
         **kwargs,
     ):
         if kwargs:
             logger.info("Uncaught kwargs: %s", kwargs)
         refs = VideoReferences()
-        if image_1 is not None:
-            refs["image_1"] = image_1
-        if image_2 is not None:
-            refs["image_2"] = image_2
-        if audio_1 is not None:
-            refs["audio_1"] = audio_1
-        if audio_2 is not None:
-            refs["audio_2"] = audio_2
-        if video_1 is not None:
-            refs["video_1"] = video_1
-        if video_2 is not None:
-            refs["video_2"] = video_2
+        for kind, values in (
+            ("image", (image_1, image_2, image_3, image_4, image_5, image_6, image_7, image_8, image_9)),
+            ("video", (video_1, video_2, video_3)),
+            ("audio", (audio_1, audio_2, audio_3)),
+        ):
+            for index, value in enumerate(values, start=1):
+                if value is not None:
+                    refs[f"{kind}_{index}"] = value
         return (refs,)
+
+
+class VLLMOmniLatentMaskEditing:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {},
+            "optional": {
+                "source_video": ("VIDEO",),
+                "source_audio": ("AUDIO",),
+                "video_mask": ("MASK",),
+                "audio_mask": ("FLOAT", {"default": -1.0, "min": -1.0, "max": 1.0, "step": 0.01}),
+            },
+        }
+
+    RETURN_TYPES = ("LATENT_MASK_EDITING",)
+    RETURN_NAMES = ("latent_edit",)
+    FUNCTION = "get_latent_edit"
+    CATEGORY = "vLLM-Omni"
+
+    def get_latent_edit(
+        self,
+        source_video: VideoInput | None = None,
+        source_audio: AudioInput | None = None,
+        video_mask: torch.Tensor | None = None,
+        audio_mask: float = -1.0,
+        **kwargs,
+    ):
+        if kwargs:
+            logger.info("Uncaught kwargs: %s", kwargs)
+        edit = LatentMaskEditing()
+        if source_video is not None:
+            edit["source_video"] = source_video
+        if source_audio is not None:
+            edit["source_audio"] = source_audio
+        if video_mask is not None:
+            edit["video_mask"] = video_mask
+        if audio_mask >= 0.0:
+            edit["audio_mask"] = audio_mask
+        return (edit,)
+
+
+class VLLMOmniMiniMaxH3TemporalMask:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "images": ("IMAGE",),
+                "source_fps": ("FLOAT", {"default": 24.0, "min": 0.01}),
+                "duration": ("FLOAT", {"default": 5.0, "min": 0.01, "max": 15.0}),
+                "mode": (["continuation", "extension"],),
+                "preserve_fraction": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0}),
+            }
+        }
+
+    RETURN_TYPES = ("MASK", "FLOAT", "IMAGE", "MASK")
+    RETURN_NAMES = ("mask", "preview_fps", "preview_images", "preview_mask")
+    FUNCTION = "build"
+    CATEGORY = "vLLM-Omni"
+
+    def build(self, images, source_fps, duration, mode, preserve_fraction=0.5):
+        if images.shape[0] <= 0 or not math.isfinite(source_fps) or source_fps <= 0:
+            raise ValueError("Source must contain frames and have a positive finite FPS.")
+        if not math.isfinite(duration) or duration <= 0 or not 0 <= preserve_fraction <= 1:
+            raise ValueError("Invalid duration or preserve_fraction.")
+        frames = _align_frame_count(max(1, round(duration * 24)))
+        source_seconds = images.shape[0] / source_fps
+        if mode == "extension":
+            if frames / 24 <= source_seconds:
+                raise ValueError("Extension output must be longer than the source; increase duration.")
+            boundary = source_seconds
+        elif mode == "continuation":
+            boundary = min(source_seconds, frames / 24) * preserve_fraction
+        else:
+            raise ValueError(f"Unknown temporal mask mode: {mode}")
+        available = min(frames, math.floor(boundary * 24 + 1e-8))
+        prefix = 0 if available < 5 else 5 + 17 * ((available - 5) // 17)
+        preserved = _video_latent_t(prefix) if prefix else 0
+        total = _video_latent_t(frames)
+        mask = torch.ones(total, 1, 1)
+        mask[:preserved] = 0
+        indices = torch.arange(frames, device=images.device)
+        source_indices = (indices * (source_fps / 24)).floor().long().clamp(max=images.shape[0] - 1)
+        preview_images = images.index_select(0, source_indices)
+        preview_mask = mask.index_select(0, torch.arange(frames) * total // frames)
+        return mask, 24.0, preview_images, preview_mask

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 """
 An high-level API client adapter that forwards ComfyUI inputs to vLLM-Omni's REST API,
 and transforms the API responses back to ComfyUI formats.
@@ -17,6 +20,7 @@ from comfy_api.input import AudioInput, VideoInput
 
 from .format import (
     audio_to_base64,
+    audio_to_bytes,
     base64_to_audio,
     base64_to_image_tensor,
     bytes_to_audio,
@@ -26,9 +30,16 @@ from .format import (
     video_to_base64,
     video_to_bytes,
 )
+from .latent_mask import scalar_mask_to_json, video_mask_to_grid_json
 from .logger import get_logger, pretty_printer
 from .models import lookup_model_spec
-from .types import AudioFormat
+from .types import (
+    MAX_REFERENCE_AUDIOS,
+    MAX_REFERENCE_IMAGES,
+    MAX_REFERENCE_VIDEOS,
+    MAX_TOTAL_REFERENCES,
+    AudioFormat,
+)
 
 logger = get_logger(__name__)
 
@@ -267,14 +278,34 @@ class VLLMOmniClient:
         fps: int,
         negative_prompt: str | None = None,
         frame: torch.Tensor | None = None,
+        first_frame: torch.Tensor | None = None,
+        last_frame: torch.Tensor | None = None,
         references: dict | None = None,
         sampling_params: dict | None = None,
         model_params: dict | None = None,
         lora: dict | None = None,
+        latent_edit: dict | None = None,
+        spec_model: str | None = None,
         **extra_params,
     ) -> VideoInput:
+        """Post a video job and return the decoded result.
+
+        ``spec_model`` names the model whose payload spec builds the request, for
+        deployments that serve a known model under a different ``model`` alias. It
+        never reaches the wire; defaults to ``model``.
+        """
         if frame is not None and references is not None:
             raise ValueError("Provide only one of frame or references, not both.")
+        if frame is not None and (first_frame is not None or last_frame is not None):
+            raise ValueError("Provide either frame or first_frame/last_frame, not both.")
+        if references is not None and (first_frame is not None or last_frame is not None):
+            raise ValueError("Provide either first_frame/last_frame or references, not both.")
+
+        spec, matched_pattern = lookup_model_spec(spec_model or model)
+        if (first_frame is not None or last_frame is not None) and (
+            matched_pattern is None or "MiniMax-H3" not in matched_pattern
+        ):
+            raise ValueError("first_frame and last_frame are supported only for MiniMax-H3; use frame for this model.")
 
         # === regular payload fields ===
         form = aiohttp.FormData()
@@ -294,43 +325,62 @@ class VLLMOmniClient:
 
         # === multimodal inputs (first-last-frames, references, etc.) ===
         input_reference_image: torch.Tensor | None = None
-        audio_reference: AudioInput | None = None
-        reference_videos: list[VideoInput] = []
+        keyframe_images: list[tuple[str, torch.Tensor]] = []
         video_task: str | None = None
 
         if frame is not None:
             input_reference_image = frame
             video_task = "fl2va"
+        elif first_frame is not None or last_frame is not None:
+            frame_indices: list[int] = []
+            if first_frame is not None:
+                keyframe_images.append(("first_frame.png", first_frame))
+                frame_indices.append(0)
+            if last_frame is not None:
+                keyframe_images.append(("last_frame.png", last_frame))
+                frame_indices.append(-1)
+            if len(keyframe_images) == 1:
+                input_reference_image = keyframe_images[0][1]
+            extra_params["frame_indices"] = frame_indices
+            video_task = "fl2va"
         elif references is not None:
-            images = [references[k] for k in ("image_1", "image_2") if k in references and references[k] is not None]
-            audios = [references[k] for k in ("audio_1", "audio_2") if k in references and references[k] is not None]
-            videos = [references[k] for k in ("video_1", "video_2") if k in references and references[k] is not None]
-
-            if not images and not audios and not videos:
-                raise ValueError("references is empty; connect at least one image/audio/video.")
-
-            if videos:
-                if images or audios:
-                    raise ValueError(
-                        "Video references cannot be combined with image or audio references. "
-                        "Connect only video_1/video_2 for multi-video Ref2VA."
-                    )
-                reference_videos = videos
-                video_task = "ref2va"
-            elif len(images) == 1 and len(audios) == 1:
-                input_reference_image = images[0]
-                audio_reference = audios[0]
-                video_task = "ref2va"
-            else:
+            reference_formats = (
+                ("image", MAX_REFERENCE_IMAGES, "png", "image/png", image_tensor_to_png_bytes),
+                ("video", MAX_REFERENCE_VIDEOS, "mp4", "video/mp4", video_to_bytes),
+                ("audio", MAX_REFERENCE_AUDIOS, "mp3", "audio/mpeg", audio_to_bytes),
+            )
+            supported_inputs = {f"{kind}_{i}" for kind, limit, *_ in reference_formats for i in range(1, limit + 1)}
+            connected = {name: value for name, value in references.items() if value is not None}
+            unsupported = connected.keys() - supported_inputs
+            if unsupported:
+                raise ValueError(f"Unsupported reference input(s): {', '.join(sorted(unsupported))}.")
+            if not any(name.startswith(("image_", "video_")) for name in connected):
                 raise ValueError(
-                    "Invalid references combination. Supported modes: "
-                    "(1) one or more videos only, or (2) exactly one image and one audio."
+                    "references requires at least one image or video; audio-only inputs are not supported."
                 )
+            if len(connected) > MAX_TOTAL_REFERENCES:
+                raise ValueError(
+                    f"references supports at most {MAX_TOTAL_REFERENCES} inputs in total "
+                    f"(up to {MAX_REFERENCE_IMAGES} images, {MAX_REFERENCE_VIDEOS} videos, "
+                    f"and {MAX_REFERENCE_AUDIOS} audios)."
+                )
+            for kind, limit, extension, content_type, encode in reference_formats:
+                for index in range(1, limit + 1):
+                    name = f"{kind}_{index}"
+                    if name in connected:
+                        filename = f"{name}.{extension}"
+                        form.add_field(
+                            "input_references",
+                            encode(connected[name], filename),
+                            filename=filename,
+                            content_type=content_type,
+                        )
+            video_task = "ref2va"
         else:
             video_task = "t2va"
 
         if input_reference_image is not None:
-            image_filename = "image.png"  # Required for multipart form
+            image_filename = keyframe_images[0][0] if keyframe_images else "image.png"
             form.add_field(
                 "input_reference",
                 image_tensor_to_png_bytes(input_reference_image, image_filename),
@@ -338,32 +388,74 @@ class VLLMOmniClient:
                 content_type="image/png",
             )
 
-        if audio_reference is not None:
-            form.add_field(
-                "audio_reference",
-                json.dumps({"audio_url": audio_to_base64(audio_reference)}, ensure_ascii=False),
-            )
+        # === latent-mask editing (MiniMax H3) ===
+        if latent_edit is not None:
+            source_video = latent_edit.get("source_video")
+            source_audio = latent_edit.get("source_audio")
+            video_mask = latent_edit.get("video_mask")
+            audio_mask = latent_edit.get("audio_mask")
 
-        for idx, video in enumerate(reference_videos, start=1):
-            video_filename = f"reference_{idx}.mp4"
-            form.add_field(
-                "input_references",
-                video_to_bytes(video, video_filename),
-                filename=video_filename,
-                content_type="video/mp4",
-            )
+            if video_mask is None and audio_mask is None:
+                raise ValueError("Latent-mask editing requires at least one mask.")
+
+            video_mask_trivial = video_mask is None or bool((video_mask == 1.0).all().item())
+            audio_mask_trivial = audio_mask is None or audio_mask == 1.0
+            if not video_mask_trivial and source_video is None:
+                raise ValueError("A non-trivial video mask requires a source video.")
+            if not audio_mask_trivial and source_audio is None and source_video is None:
+                raise ValueError("A non-trivial audio mask requires a source audio or a source video with audio.")
+
+            if source_video is not None:
+                form.add_field(
+                    "source_video",
+                    video_to_bytes(source_video, "source.mp4"),
+                    filename="source.mp4",
+                    content_type="video/mp4",
+                )
+            if source_audio is not None:
+                form.add_field(
+                    "source_audio",
+                    audio_to_bytes(source_audio, "source_audio.mp3"),
+                    filename="source_audio.mp3",
+                    content_type="audio/mpeg",
+                )
+            if video_mask is not None:
+                mask_json = video_mask_to_grid_json(video_mask, width=width, height=height, num_frames=num_frames)
+                form.add_field(
+                    "video_noise_mask",
+                    mask_json.encode("utf-8"),
+                    filename="video-mask.json",
+                    content_type="application/json",
+                )
+            if audio_mask is not None:
+                form.add_field(
+                    "audio_noise_mask",
+                    scalar_mask_to_json(audio_mask).encode("utf-8"),
+                    filename="audio-mask.json",
+                    content_type="application/json",
+                )
+
+        if len(keyframe_images) == 2:
+            for image_filename, image in keyframe_images:
+                form.add_field(
+                    "input_references",
+                    image_tensor_to_png_bytes(image, image_filename),
+                    filename=image_filename,
+                    content_type="image/png",
+                )
 
         # === model specific params. Either use a specialized builder, or add flattened fields as-is ===
         if model_params is not None:
             model_params = dict(model_params)
             model_params.pop("type", None)
 
-        spec, _ = lookup_model_spec(model)
         params_builder = spec.get("params_builder") if spec else None
         if params_builder is not None:
             form_fields = params_builder(
                 model_params or {},
                 extra_params={**extra_params, "task": video_task},
+                width=width,
+                height=height,
             )
             for k, v in form_fields.items():
                 form.add_field(k, v if isinstance(v, str) else str(v))

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/format.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 """Image/tensor format helpers.
 
 The image generation part is derived from dougbtv/comfyui-vllm-omni by Doug (@dougbtv).
@@ -170,6 +173,10 @@ def bytes_to_video(video_bytes: bytes) -> VideoInput:
                 audio_stream = container.streams.audio[-1]
                 audio_frames = []
                 resampler = AudioResampler(format="fltp")
+                # Decoding the video stream above left the container at EOF.
+                # Rewind before demuxing again, or this pass yields no frames
+                # and the audio of a T2VA result is silently dropped.
+                container.seek(0)
                 for frame in container.decode(audio_stream):
                     if not isinstance(frame, AudioFrame):
                         continue

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py

@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+import json
+
+import torch
+import torch.nn.functional as F
+
+
+def scalar_mask_to_json(value: float) -> str:
+    if not 0.0 <= value <= 1.0:
+        raise ValueError(f"mask value must be in [0, 1], got {value}")
+    return str(value)
+
+
+def _align_frame_count(frame_count: int) -> int:
+    if frame_count <= 0:
+        return 1
+    current = int(frame_count)
+    while current % 17 != 5:
+        current += 1
+    return current
+
+
+def _video_latent_t(frame_count: int) -> int:
+    if frame_count <= 5:
+        return 2
+    return ((int(frame_count) - 5) // 17) * 5 + 2
+
+
+def video_mask_to_grid(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> torch.Tensor:
+    tv = _video_latent_t(_align_frame_count(num_frames))
+    height = int(height) // 32 * 32
+    width = int(width) // 32 * 32
+    gh, gw = height // 16, width // 16
+
+    if mask.ndim == 2:
+        mask = mask.unsqueeze(0)
+    elif mask.ndim != 3:
+        raise ValueError(f"expected a 2D or 3D mask tensor, got {mask.ndim}D")
+
+    grid = F.interpolate(mask.unsqueeze(1).float(), size=(gh, gw), mode="area").squeeze(1)
+    if grid.shape[0] == 1:
+        grid = grid.expand(tv, gh, gw)
+    elif grid.shape[0] != tv:
+        grid = F.interpolate(grid.unsqueeze(0).unsqueeze(0), size=(tv, gh, gw), mode="nearest").squeeze(0).squeeze(0)
+    return grid
+
+
+def video_mask_to_grid_json(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> str:
+    return json.dumps(video_mask_to_grid(mask, width=width, height=height, num_frames=num_frames).tolist())

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 import json
 import re
 from typing import Any
@@ -34,10 +37,28 @@ def _qwen25_payload_preprocessor(payload: dict) -> dict:
     return payload
 
 
+MINIMAX_H3_ASPECT_RATIOS = {
+    "21:9": 21.0 / 9.0,
+    "16:9": 16.0 / 9.0,
+    "4:3": 4.0 / 3.0,
+    "1:1": 1.0,
+    "3:4": 3.0 / 4.0,
+    "9:16": 9.0 / 16.0,
+}
+
+
+def _nearest_minimaxh3_aspect_ratio(width: int, height: int) -> str:
+    """Pick the supported named ratio closest to the requested frame size."""
+    target = float(width) / float(height)
+    return min(MINIMAX_H3_ASPECT_RATIOS, key=lambda name: abs(MINIMAX_H3_ASPECT_RATIOS[name] - target))
+
+
 def _minimaxh3_params_builder(
     model_params: dict[str, Any],
     *,
     extra_params: dict[str, Any],
+    width: int | None = None,
+    height: int | None = None,
 ) -> dict[str, Any]:
     """Build multipart form fields for MiniMax-H3 from model_params + routed task."""
     params = dict(model_params)
@@ -53,6 +74,12 @@ def _minimaxh3_params_builder(
     if params:
         logger.warning("Unused MiniMax-H3 model params ignored: %s", sorted(params))
 
+    # H3 refuses t2va without a named aspect ratio, and the Generate Video node
+    # carries width/height rather than a ratio. fl2va takes its ratio from the
+    # input image and ref2va defaults server-side, so only t2va needs this.
+    if merged_extra_params.get("task") == "t2va" and "aspect_ratio" not in merged_extra_params and width and height:
+        merged_extra_params["aspect_ratio"] = _nearest_minimaxh3_aspect_ratio(width, height)
+
     if merged_extra_params:
         form_fields["extra_params"] = json.dumps(merged_extra_params, ensure_ascii=False)
     return form_fields

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 from collections.abc import Callable
 from enum import Enum, auto
 from typing import Any, Literal, TypeAlias
@@ -27,10 +30,24 @@ class MiniMaxH3ModelSpecificParams(dict):
     pass
 
 
+MAX_REFERENCE_IMAGES = 9
+MAX_REFERENCE_VIDEOS = 3
+MAX_REFERENCE_AUDIOS = 3
+MAX_TOTAL_REFERENCES = 12
+
+
 class VideoReferences(dict):
     pass
 
 
+class LatentMaskEditing(dict):
+    pass
+
+
+class FastH3Deployment(dict):
+    """Descriptor for a server that fused FastH3 at startup."""
+
+
 class ModelMode(Enum):
     IMAGE_GENERATION = auto()
     VIDEO_GENERATION = auto()

apps/ComfyUI-vLLM-Omni/docs/minimax-h3-t2v.md

@@ -0,0 +1,94 @@
+# MiniMax H3 Text to Video (WF-01)
+
+Import [MiniMax_H3_Text_to_Video.json](../example_workflows/MiniMax_H3_Text_to_Video.json)
+into ComfyUI, or select it under **Templates → ComfyUI-vLLM-Omni**.
+The template connects existing nodes to a remote H3 service and saves the
+returned video with audio. ComfyUI does not load H3 weights.
+
+## Setup
+
+Install the [ComfyUI extension](../README.md#installation) and follow the
+[H3 deployment recipe](../../../recipes/MiniMaxAI/MiniMax-H3.md) for your hardware.
+For a two-GPU FL2VA server with sufficient memory:
+
+```bash
+export MODEL=/path/to/MiniMax-H3/FL2VA
+CUDA_VISIBLE_DEVICES=0,1 VLLM_WORKER_MULTIPROC_METHOD=spawn \
+VLLM_OMNI_VIDEO_SYNC_TIMEOUT=14400 \
+vllm serve "$MODEL" --omni --host 127.0.0.1 --port 8000 \
+  --served-model-name MiniMaxAI/MiniMax-H3 --trust-remote-code \
+  --task-type fl2va --num-gpus 2 --tensor-parallel-size 2 \
+  --text-encoder-tp-size 2 --enforce-eager
+```
+
+Set Generate Video's **url** to the address reachable from ComfyUI, including
+`/v1`, and match its **model** to the served name. The defaults are
+`http://localhost:8000/v1` and `MiniMaxAI/MiniMax-H3`.
+
+## Base and Turbo
+
+Base presets are connected by default. Write the scene and its sound in the
+same **prompt**; leave **frame**, **references**, and **fast_h3** disconnected.
+The template uses 1344×768, 24 FPS, seed 1101 and **duration** 5.167 seconds
+(124 frames, satisfying H3's `17k+5` constraint). The existing client infers
+`16:9` from these dimensions. For portrait output, use 768×1344.
+
+| Setting | Base | Turbo v1.0 768p |
+| --- | --- | --- |
+| Inference steps | 50 | 5 (4 forwards) |
+| Video flow shift | 12 | 6 |
+| Audio flow shift | 3 | 3 |
+| Remote LoRA | Disconnected | Connected, scale 1 |
+
+For Turbo, use the **Diffusers-layout** artifact from
+[lightx2v/Minimax-h3-Turbo](https://huggingface.co/lightx2v/Minimax-h3-Turbo):
+
+```text
+minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors
+```
+
+Add `--lora-backend peft --lora-path "$TURBO_LORA"` to the server command,
+where `TURBO_LORA` is the full server-side path to that file. Enter the same
+path in Remote LoRA's **local_path**; retain name `h3-turbo-v1.0-768p`, scale
+`1.0`, and **int_id** `0` so the existing API derives the ID from the path.
+The template leaves the path blank for users to configure. The `_comfyui_`
+export is not supported by this backend.
+
+Connect **Turbo sampling**, **Turbo H3 params**, and **Remote LoRA** to Generate
+Video's `sampling_params`, `model_params`, and `lora` inputs, replacing the
+Base links. To return to Base, restore both Base links and disconnect LoRA.
+Other Turbo artifacts may need different settings; see the
+[artifact table](../../../recipes/MiniMaxAI/MiniMax-H3.md#turbo-lora).
+
+If importing an older template with a frame-count widget, reimport this JSON
+or set **duration** to **5.167**; do not interpret 124 frames as 124 seconds.
+
+## Validation
+
+Queue the configured workflow and inspect the MP4 saved under
+`ComfyUI/output/video/MiniMax_H3_WF01_*.mp4`:
+
+```bash
+ffprobe -v error -show_streams -show_format -of json "$OUTPUT"
+ffmpeg -v error -i "$OUTPUT" -f null -
+```
+
+Check the requested dimensions, frame count, 24 FPS and both video/audio
+streams, then watch and listen to the saved file. The current upstream
+extension includes the shared audio-preserving decoder fix.
+
+On September 16, the narrowed workflow was validated through ComfyUI using
+existing upstream Python code at `4a8ac297`, two H20-3e GPUs, vLLM 0.29.0,
+PyTorch 2.13.0+cu129 and ComfyUI `1d48d9cf`. The run used the template's
+forest-stream prompt, explicit server-side Turbo LoRA path, and settings
+above, with `TORCH_SDPA`, `--usp 1 --ring 1`, and tiled VAE parallelism on two
+GPUs. ComfyUI execution took 131.55 seconds. SaveVideo produced 124 H.264
+frames at 1344×768 and 24 FPS, plus 32 kHz stereo AAC audio. Video/audio
+durations were 5.166667/5.167000 seconds; full decoding passed and audio was
+nonzero. This is model-generated output, not a ground-truth reference or a
+subjective audio-quality assessment. Base graph validation passed with Turbo
+nodes disconnected; full 50-step Base quality was not evaluated.
+
+Based on the official
+[ComfyUI H3 T2V template](https://github.com/Comfy-Org/workflow_templates/blob/main/templates/video_minimax_h3_t2v.json),
+using remote inference instead of local model loaders.

apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md

@@ -0,0 +1,17 @@
+# WF-05: MiniMax-H3 latent-mask editing
+
+Edit a source video with MiniMax-H3 using spatial or temporal masks. The workflow includes object removal, inpainting, continuation, and extension, with a mask preview for each case.
+
+## Requirements
+
+- ComfyUI with ComfyUI-vLLM-Omni installed.
+- An H3 service using the FL2VA checkpoint partition with latent-mask editing support. Follow the [H3 recipe](../../../recipes/MiniMaxAI/MiniMax-H3.md) for server setup.
+
+## Use
+
+Open **vLLM-Omni MiniMax-H3 Latent Mask Editing.json** from the example workflows and set the Generate Video URL/model for the case you want to run.
+
+1. Upload your source video in Load Video.
+2. Adjust the case's mask. For removal and inpainting, set Mask Canvas to the video dimensions, Mask Rectangle to the region size, and Combine Masks x/y to its position. For continuation and extension, set MiniMax-H3 Temporal Mask's mode and duration; continuation also uses `preserve_fraction`. Match the duration in Generate Video, and use a duration longer than the source for extension. Mask values are `0` to preserve and `1` to regenerate.
+3. Adjust the prompt to describe the desired content. Set the output dimensions and duration in Generate Video. If editing audio, update the sound description and audio mask (`0` preserves, `1` regenerates the whole clip).
+4. Select the case's Mask Preview Save Video node and choose **Execute to selected output nodes** to preview the mask as a red overlay without inference, or select its Result Save Video node to generate the edited video. Running the entire workflow generates all four cases.

apps/ComfyUI-vLLM-Omni/docs/wf07-h3-upscale.md

@@ -0,0 +1,21 @@
+# WF-07: H3 video upscale with SeedVR2
+
+Generate video and audio through vLLM-Omni H3, upscale the decoded frames with SeedVR2, and save both the original and upscaled videos. The generated audio and FPS connect directly to Create Video.
+
+## Requirements
+
+- ComfyUI with ComfyUI-vLLM-Omni and [ComfyUI-SeedVR2_VideoUpscaler](https://github.com/numz/ComfyUI-SeedVR2_VideoUpscaler#-installation) installed.
+- An H3 service using the FL2VA checkpoint partition. Follow the [H3 recipe](../../../recipes/MiniMaxAI/MiniMax-H3.md) for server setup.
+- `seedvr2_ema_3b_fp16.safetensors` and `ema_vae_fp16.safetensors` from [SeedVR2's ComfyUI weights](https://huggingface.co/numz/SeedVR2_comfyUI/tree/main), placed in `ComfyUI/models/SEEDVR2`. The SeedVR2 nodes can also download them on first use.
+
+## Use
+
+Open **vLLM-Omni MiniMax H3 Video Upscale.json** from the example workflows. An API-format version is provided alongside it. Set the Generate Video URL/model and the LoRA path as seen by the H3 server.
+
+The preset uses the LightX2V v1.0 768p Turbo adapter in Diffusers format: `minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors`, five sigma points, video/audio flow shifts 6/3, and LoRA scale 1. Preload the same adapter on the server using the [Turbo LoRA recipe](../../../recipes/MiniMaxAI/MiniMax-H3.md#turbo-lora). For base H3, disconnect LoRA, set inference steps to 50, and set video flow shift to 12.
+
+The H3 preset requests 1344 x 768 at 24 FPS for 5.167 seconds, which gives 124 frames. SeedVR2's `resolution=1536` sets the target short edge, producing 2688 x 1536 for this input. Update this value when changing the input size; it is an output resolution, not a fixed 2x multiplier.
+
+The SeedVR2 preset uses 3B FP16, five-frame batches, one overlapping frame, uniform batch padding, and LAB color correction. The DiT loader uses CPU offload with 32 swapped blocks; the VAE uses tiled encoding and decoding. Select the local GPU in both model loaders. See the [SeedVR2 settings](https://github.com/numz/ComfyUI-SeedVR2_VideoUpscaler#-usage) when adjusting memory use or batch size.
+
+Run the workflow. The two Save Video nodes write MP4s under `WF07/generated` and `WF07/upscaled` in ComfyUI's output directory.

apps/ComfyUI-vLLM-Omni/example_workflows/MiniMax_H3_Text_to_Video.json

@@ -0,0 +1,362 @@
+{
+  "last_node_id": 8,
+  "last_link_id": 3,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "VLLMOmniDiffusionSampling",
+      "pos": [
+        20,
+        20
+      ],
+      "size": [
+        310,
+        280
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "diffusion sampling params",
+          "type": "SAMPLING_PARAMS",
+          "links": [
+            1
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniDiffusionSampling"
+      },
+      "widgets_values": [
+        1,
+        50,
+        1.0,
+        1.0,
+        false,
+        false,
+        1101,
+        "fixed"
+      ],
+      "title": "Base sampling"
+    },
+    {
+      "id": 2,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        20,
+        340
+      ],
+      "size": [
+        310,
+        140
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "MiniMax-H3 video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            2
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ],
+      "title": "Base H3 params"
+    },
+    {
+      "id": 3,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        430,
+        20
+      ],
+      "size": [
+        430,
+        440
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null
+        },
+        {
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null
+        },
+        {
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": 1
+        },
+        {
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 2
+        },
+        {
+          "name": "fast_h3",
+          "type": "FASTH3_DEPLOYMENT",
+          "link": null
+        },
+        {
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            3
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://localhost:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "A small stream flows through a sunlit forest. The camera slowly moves toward the water. Birds chirp in the trees and water gurgles over smooth stones. Natural continuous sound, no music.",
+        "",
+        1344,
+        768,
+        24,
+        5.167
+      ]
+    },
+    {
+      "id": 4,
+      "type": "SaveVideo",
+      "pos": [
+        940,
+        20
+      ],
+      "size": [
+        440,
+        440
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "link": 3
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": null,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "video/MiniMax_H3_WF01",
+        "mp4",
+        "auto"
+      ]
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniRemoteLoRA",
+      "pos": [
+        430,
+        540
+      ],
+      "size": [
+        430,
+        160
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "links": null,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniRemoteLoRA"
+      },
+      "widgets_values": [
+        "",
+        "h3-turbo-v1.0-768p",
+        1.0,
+        0
+      ],
+      "title": "Optional Turbo: server LoRA path"
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniDiffusionSampling",
+      "pos": [
+        20,
+        540
+      ],
+      "size": [
+        310,
+        280
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "diffusion sampling params",
+          "type": "SAMPLING_PARAMS",
+          "links": null,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniDiffusionSampling"
+      },
+      "widgets_values": [
+        1,
+        5,
+        1.0,
+        1.0,
+        false,
+        false,
+        1101,
+        "fixed"
+      ],
+      "title": "Turbo sampling (4 forwards)"
+    },
+    {
+      "id": 7,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        430,
+        740
+      ],
+      "size": [
+        310,
+        140
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "MiniMax-H3 video params",
+          "type": "VIDEO_PARAMS",
+          "links": null,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        6.0
+      ],
+      "title": "Turbo H3 params"
+    },
+    {
+      "id": 8,
+      "type": "Note",
+      "pos": [
+        940,
+        520
+      ],
+      "size": [
+        440,
+        440
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "properties": {
+        "Node name for S&R": "Note"
+      },
+      "widgets_values": [
+        "MiniMax H3 Text to Video (WF-01)\n\nBASE: connected presets, 50 steps / video flow 12 / audio flow 3.\n\nTURBO: set Remote LoRA local_path to the SERVER path of minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors from lightx2v/Minimax-h3-Turbo. Keep name h3-turbo-v1.0-768p, scale 1.0 and int_id 0. Connect Turbo sampling, Turbo H3 params, and LoRA to Generate Video, replacing the two Base links. Use 5 steps (4 forwards), video flow 6 and audio flow 3. Do not use the ComfyUI-layout artifact.\n\nKeep frame, references and fast_h3 disconnected. 1344x768, 24 FPS, duration 5.167 seconds = 124 frames (17*7+5). The client selects the named 16:9 preset from the dimensions. Save Video preserves video and audio.\n\nSet URL and served model name for your server. See docs/minimax-h3-t2v.md for setup and validation. Source: Comfy-Org/workflow_templates/templates/video_minimax_h3_t2v.json."
+      ],
+      "title": "Setup / Base and Turbo"
+    }
+  ],
+  "links": [
+    [
+      1,
+      1,
+      0,
+      3,
+      2,
+      "SAMPLING_PARAMS"
+    ],
+    [
+      2,
+      2,
+      0,
+      3,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      3,
+      3,
+      0,
+      4,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 0.8,
+      "offset": [
+        50,
+        50
+      ]
+    }
+  },
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni FastH3 Text to Video.json

@@ -0,0 +1,322 @@
+{
+  "id": "c6db00da-2ccb-4ff8-8334-d67e1a267781",
+  "revision": 0,
+  "last_node_id": 5,
+  "last_link_id": 3,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "MarkdownNote",
+      "pos": [40, 20],
+      "size": [420, 250],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "FastH3 server prerequisite",
+      "properties": {},
+      "widgets_values": [
+        "FastH3 must be fused when the vLLM-Omni server starts. Use --task-type fl2va and --lora-path pointing to exactly one FastH3 adapter file. Do not use CPU or layerwise offload. For VSA / Data-Free, also install fastvideo-kernel and use --diffusion-attention-backend FASTVIDEO_VSA --fastvideo-vsa-topk 64."
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 2,
+      "type": "VLLMOmniFastH3Deployment",
+      "pos": [40, 320],
+      "size": [420, 160],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {"name": "url"},
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {"name": "model"},
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "deployment",
+          "name": "deployment",
+          "type": "FASTH3_DEPLOYMENT",
+          "links": [1]
+        }
+      ],
+      "properties": {"Node name for S&R": "VLLMOmniFastH3Deployment"},
+      "widgets_values": [
+        "http://localhost:8000/v1",
+        "MiniMaxAI/MiniMax-H3"
+      ]
+    },
+    {
+      "id": 3,
+      "type": "VLLMOmniDiffusionSampling",
+      "pos": [40, 540],
+      "size": [420, 230],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "n",
+          "name": "n",
+          "type": "INT",
+          "widget": {"name": "n"},
+          "link": null
+        },
+        {
+          "localized_name": "num_inference_steps",
+          "name": "num_inference_steps",
+          "type": "INT",
+          "widget": {"name": "num_inference_steps"},
+          "link": null
+        },
+        {
+          "localized_name": "guidance_scale",
+          "name": "guidance_scale",
+          "type": "FLOAT",
+          "widget": {"name": "guidance_scale"},
+          "link": null
+        },
+        {
+          "localized_name": "true_cfg_scale",
+          "name": "true_cfg_scale",
+          "type": "FLOAT",
+          "widget": {"name": "true_cfg_scale"},
+          "link": null
+        },
+        {
+          "localized_name": "vae_use_slicing",
+          "name": "vae_use_slicing",
+          "type": "BOOLEAN",
+          "widget": {"name": "vae_use_slicing"},
+          "link": null
+        },
+        {
+          "localized_name": "vae_use_tiling",
+          "name": "vae_use_tiling",
+          "type": "BOOLEAN",
+          "widget": {"name": "vae_use_tiling"},
+          "link": null
+        },
+        {
+          "localized_name": "seed",
+          "name": "seed",
+          "type": "INT",
+          "widget": {"name": "seed"},
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "diffusion sampling params",
+          "name": "diffusion sampling params",
+          "type": "SAMPLING_PARAMS",
+          "links": [2]
+        }
+      ],
+      "properties": {"Node name for S&R": "VLLMOmniDiffusionSampling"},
+      "widgets_values": [1, 4, 1, 1, false, false, 1101, "fixed"]
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [560, 210],
+      "size": [540, 610],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "frame",
+          "name": "frame",
+          "shape": 7,
+          "type": "IMAGE",
+          "link": null
+        },
+        {
+          "localized_name": "references",
+          "name": "references",
+          "shape": 7,
+          "type": "VIDEO_REFERENCES",
+          "link": null
+        },
+        {
+          "localized_name": "sampling_params",
+          "name": "sampling_params",
+          "shape": 7,
+          "type": "SAMPLING_PARAMS",
+          "link": 2
+        },
+        {
+          "localized_name": "lora",
+          "name": "lora",
+          "shape": 7,
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "localized_name": "model_params",
+          "name": "model_params",
+          "shape": 7,
+          "type": "VIDEO_PARAMS",
+          "link": null
+        },
+        {
+          "localized_name": "fast_h3",
+          "name": "fast_h3",
+          "shape": 7,
+          "type": "FASTH3_DEPLOYMENT",
+          "link": 1
+        },
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {"name": "url"},
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {"name": "model"},
+          "link": null
+        },
+        {
+          "localized_name": "prompt",
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {"name": "prompt"},
+          "link": null
+        },
+        {
+          "localized_name": "negative_prompt",
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {"name": "negative_prompt"},
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {"name": "width"},
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {"name": "height"},
+          "link": null
+        },
+        {
+          "localized_name": "fps",
+          "name": "fps",
+          "type": "INT",
+          "widget": {"name": "fps"},
+          "link": null
+        },
+        {
+          "localized_name": "duration",
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {"name": "duration"},
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [3]
+        }
+      ],
+      "properties": {"Node name for S&R": "VLLMOmniGenerateVideo"},
+      "widgets_values": [
+        "http://localhost:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "A cinematic live-action singer performs naturally on a neon-lit stage, with synchronized vocals and environmental sound.",
+        "",
+        1344,
+        768,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 5,
+      "type": "SaveVideo",
+      "pos": [1200, 280],
+      "size": [380, 310],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "link": 3
+        },
+        {
+          "localized_name": "filename_prefix",
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {"name": "filename_prefix"},
+          "link": null
+        },
+        {
+          "localized_name": "format",
+          "name": "format",
+          "type": "COMBO",
+          "widget": {"name": "format"},
+          "link": null
+        },
+        {
+          "localized_name": "codec",
+          "name": "codec",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {"name": "codec"},
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": null
+        }
+      ],
+      "properties": {},
+      "widgets_values": ["video/ComfyUI-FastH3", "auto", "auto"]
+    }
+  ],
+  "links": [
+    [1, 2, 0, 4, 5, "FASTH3_DEPLOYMENT"],
+    [2, 3, 0, 4, 2, "SAMPLING_PARAMS"],
+    [3, 4, 0, 5, 0, "VIDEO"]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "workflowRendererVersion": "LG",
+    "ds": {"scale": 0.9, "offset": [100, 100]}
+  },
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json

@@ -0,0 +1,499 @@
+{
+  "id": "latent-mask-editing-7380",
+  "revision": 0,
+  "last_node_id": 7,
+  "last_link_id": 5,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "MarkdownNote",
+      "pos": [
+        40,
+        20
+      ],
+      "size": [
+        420,
+        210
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Latent-mask editing server prerequisite",
+      "properties": {},
+      "widgets_values": [
+        "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends."
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 2,
+      "type": "LoadVideo",
+      "pos": [
+        40,
+        280
+      ],
+      "size": [
+        420,
+        120
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "file",
+          "name": "file",
+          "type": "COMBO",
+          "widget": {
+            "name": "file"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        ""
+      ]
+    },
+    {
+      "id": 3,
+      "type": "LoadImageMask",
+      "pos": [
+        40,
+        460
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image",
+          "type": "STRING",
+          "widget": {
+            "name": "image"
+          },
+          "link": null
+        },
+        {
+          "name": "channel",
+          "type": "STRING",
+          "widget": {
+            "name": "channel"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "mask",
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadImageMask"
+      },
+      "widgets_values": [
+        "",
+        "red"
+      ]
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniLatentMaskEditing",
+      "pos": [
+        560,
+        260
+      ],
+      "size": [
+        420,
+        300
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "source_video",
+          "name": "source_video",
+          "type": "VIDEO",
+          "link": 1,
+          "shape": 7
+        },
+        {
+          "localized_name": "source_audio",
+          "name": "source_audio",
+          "type": "AUDIO",
+          "link": null
+        },
+        {
+          "localized_name": "video_mask",
+          "name": "video_mask",
+          "type": "MASK",
+          "link": 2,
+          "shape": 7
+        },
+        {
+          "localized_name": "audio_mask",
+          "name": "audio_mask",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_mask"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniLatentMaskEditing"
+      },
+      "widgets_values": [
+        0.5
+      ]
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        40,
+        680
+      ],
+      "size": [
+        420,
+        140
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "audio_flow_shift",
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "flow_shift",
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video params",
+          "name": "video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ]
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        1080,
+        200
+      ],
+      "size": [
+        540,
+        680
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "frame",
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "references",
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "sampling_params",
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": null
+        },
+        {
+          "localized_name": "lora",
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "localized_name": "model_params",
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 4,
+          "shape": 7
+        },
+        {
+          "localized_name": "fast_h3",
+          "name": "fast_h3",
+          "type": "FASTH3_DEPLOYMENT",
+          "link": null
+        },
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "link": 3,
+          "shape": 7
+        },
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "prompt",
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "negative_prompt",
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "fps",
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "duration",
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "Partially restyle the clip and soundtrack",
+        "",
+        160,
+        120,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 7,
+      "type": "SaveVideo",
+      "pos": [
+        1700,
+        300
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "link": 5,
+          "shape": 7
+        },
+        {
+          "localized_name": "filename_prefix",
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "format",
+          "name": "format",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "vllm-omni/latent-edit",
+        "mp4"
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      2,
+      0,
+      4,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      3,
+      0,
+      4,
+      2,
+      "MASK"
+    ],
+    [
+      3,
+      4,
+      0,
+      6,
+      6,
+      "LATENT_MASK_EDITING"
+    ],
+    [
+      4,
+      5,
+      0,
+      6,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      5,
+      6,
+      0,
+      7,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json

@@ -0,0 +1,560 @@
+{
+  "last_node_id": 9,
+  "last_link_id": 7,
+  "nodes": [
+    {
+      "id": 2,
+      "type": "LoadVideo",
+      "pos": [
+        40,
+        100
+      ],
+      "size": [
+        420,
+        120
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "file",
+          "type": "STRING",
+          "widget": {
+            "name": "file"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        ""
+      ]
+    },
+    {
+      "id": 3,
+      "type": "SolidMask",
+      "pos": [
+        40,
+        300
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "value",
+          "type": "FLOAT",
+          "widget": {
+            "name": "value"
+          },
+          "link": null
+        },
+        {
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SolidMask"
+      },
+      "widgets_values": [
+        0.0,
+        160,
+        120
+      ]
+    },
+    {
+      "id": 4,
+      "type": "SolidMask",
+      "pos": [
+        40,
+        500
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "value",
+          "type": "FLOAT",
+          "widget": {
+            "name": "value"
+          },
+          "link": null
+        },
+        {
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SolidMask"
+      },
+      "widgets_values": [
+        1.0,
+        160,
+        120
+      ]
+    },
+    {
+      "id": 5,
+      "type": "BatchMasksNode",
+      "pos": [
+        250,
+        380
+      ],
+      "size": [
+        200,
+        100
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "masks.mask0",
+          "type": "MASK",
+          "link": 2,
+          "slot_index": 0
+        },
+        {
+          "name": "masks.mask1",
+          "type": "MASK",
+          "link": 3,
+          "slot_index": 1
+        }
+      ],
+      "outputs": [
+        {
+          "name": "MASK",
+          "type": "MASK",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "BatchMasksNode"
+      },
+      "widgets_values": []
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniLatentMaskEditing",
+      "pos": [
+        450,
+        180
+      ],
+      "size": [
+        400,
+        120
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "source_video",
+          "type": "VIDEO",
+          "link": 1
+        },
+        {
+          "name": "source_audio",
+          "type": "AUDIO",
+          "link": null
+        },
+        {
+          "name": "video_mask",
+          "type": "MASK",
+          "link": 4
+        },
+        {
+          "name": "audio_mask",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_mask"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniLatentMaskEditing"
+      },
+      "widgets_values": [
+        -1.0
+      ]
+    },
+    {
+      "id": 7,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        40,
+        700
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            6
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ]
+    },
+    {
+      "id": 8,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        700,
+        180
+      ],
+      "size": [
+        450,
+        380
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null
+        },
+        {
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null
+        },
+        {
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": null
+        },
+        {
+          "name": "lora",
+          "type": "LORA",
+          "link": null
+        },
+        {
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 6
+        },
+        {
+          "name": "fast_h3",
+          "type": "FAST_H3",
+          "link": null
+        },
+        {
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "link": 5
+        },
+        {
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            7
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "temporal mask: regenerate the ending",
+        "",
+        160,
+        120,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 9,
+      "type": "SaveVideo",
+      "pos": [
+        900,
+        180
+      ],
+      "size": [
+        420,
+        200
+      ],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "link": 7
+        },
+        {
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "name": "format",
+          "type": "STRING",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "temporal_mask",
+        "mp4"
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      2,
+      0,
+      6,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      3,
+      0,
+      5,
+      0,
+      "MASK"
+    ],
+    [
+      3,
+      4,
+      0,
+      5,
+      1,
+      "MASK"
+    ],
+    [
+      4,
+      5,
+      0,
+      6,
+      2,
+      "MASK"
+    ],
+    [
+      5,
+      6,
+      0,
+      8,
+      6,
+      "LATENT_MASK_EDITING"
+    ],
+    [
+      6,
+      7,
+      0,
+      8,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      7,
+      8,
+      0,
+      9,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json

@@ -0,0 +1,511 @@
+{
+  "id": "latent-mask-editing-7380",
+  "revision": 0,
+  "last_node_id": 7,
+  "last_link_id": 5,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "MarkdownNote",
+      "pos": [
+        40,
+        20
+      ],
+      "size": [
+        420,
+        210
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Latent-mask editing server prerequisite",
+      "properties": {},
+      "widgets_values": [
+        "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends."
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 2,
+      "type": "LoadVideo",
+      "pos": [
+        40,
+        280
+      ],
+      "size": [
+        420,
+        120
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "file",
+          "name": "file",
+          "type": "COMBO",
+          "widget": {
+            "name": "file"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        ""
+      ]
+    },
+    {
+      "id": 3,
+      "type": "SolidMask",
+      "pos": [
+        40,
+        460
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "value",
+          "name": "value",
+          "type": "FLOAT",
+          "widget": {
+            "name": "value"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "mask",
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SolidMask"
+      },
+      "widgets_values": [
+        1.0,
+        160,
+        120
+      ]
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniLatentMaskEditing",
+      "pos": [
+        560,
+        260
+      ],
+      "size": [
+        420,
+        300
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "source_video",
+          "name": "source_video",
+          "type": "VIDEO",
+          "link": 1,
+          "shape": 7
+        },
+        {
+          "localized_name": "source_audio",
+          "name": "source_audio",
+          "type": "AUDIO",
+          "link": null
+        },
+        {
+          "localized_name": "video_mask",
+          "name": "video_mask",
+          "type": "MASK",
+          "link": 2,
+          "shape": 7
+        },
+        {
+          "localized_name": "audio_mask",
+          "name": "audio_mask",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_mask"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniLatentMaskEditing"
+      },
+      "widgets_values": [
+        0.5
+      ]
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        40,
+        680
+      ],
+      "size": [
+        420,
+        140
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "audio_flow_shift",
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "flow_shift",
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video params",
+          "name": "video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ]
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        1080,
+        200
+      ],
+      "size": [
+        540,
+        680
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "frame",
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "references",
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "sampling_params",
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": null
+        },
+        {
+          "localized_name": "lora",
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "localized_name": "model_params",
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 4,
+          "shape": 7
+        },
+        {
+          "localized_name": "fast_h3",
+          "name": "fast_h3",
+          "type": "FASTH3_DEPLOYMENT",
+          "link": null
+        },
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "link": 3,
+          "shape": 7
+        },
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "prompt",
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "negative_prompt",
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "fps",
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "duration",
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "Partially restyle the clip and soundtrack",
+        "",
+        160,
+        120,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 7,
+      "type": "SaveVideo",
+      "pos": [
+        1700,
+        300
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "link": 5,
+          "shape": 7
+        },
+        {
+          "localized_name": "filename_prefix",
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "format",
+          "name": "format",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "vllm-omni/latent-edit",
+        "mp4"
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      2,
+      0,
+      4,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      3,
+      0,
+      4,
+      2,
+      "MASK"
+    ],
+    [
+      3,
+      4,
+      0,
+      6,
+      6,
+      "LATENT_MASK_EDITING"
+    ],
+    [
+      4,
+      5,
+      0,
+      6,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      5,
+      6,
+      0,
+      7,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax H3 Image to Video.json

@@ -0,0 +1,219 @@
+{
+  "id": "6676f19a-7f41-4c80-8cb7-f80adcb4ac44",
+  "revision": 0,
+  "last_node_id": 9,
+  "last_link_id": 4,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "LoadImage",
+      "pos": [40, 20],
+      "size": [270, 314],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "image", "name": "image", "type": "COMBO", "widget": {"name": "image"}, "link": null},
+        {"localized_name": "choose file to upload", "name": "upload", "type": "IMAGEUPLOAD", "widget": {"name": "upload"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "IMAGE", "name": "IMAGE", "type": "IMAGE", "links": [1]},
+        {"localized_name": "MASK", "name": "MASK", "type": "MASK", "links": null}
+      ],
+      "title": "First Frame",
+      "properties": {"Node name for S&R": "LoadImage"},
+      "widgets_values": ["first_frame.png", "image"]
+    },
+    {
+      "id": 2,
+      "type": "LoadImage",
+      "pos": [40, 390],
+      "size": [270, 314],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "image", "name": "image", "type": "COMBO", "widget": {"name": "image"}, "link": null},
+        {"localized_name": "choose file to upload", "name": "upload", "type": "IMAGEUPLOAD", "widget": {"name": "upload"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "IMAGE", "name": "IMAGE", "type": "IMAGE", "links": null},
+        {"localized_name": "MASK", "name": "MASK", "type": "MASK", "links": null}
+      ],
+      "title": "Last Frame (optional)",
+      "properties": {"Node name for S&R": "LoadImage"},
+      "widgets_values": ["last_frame.png", "image"]
+    },
+    {
+      "id": 3,
+      "type": "MarkdownNote",
+      "pos": [350, 20],
+      "size": [390, 240],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "MiniMax H3 I2V modes",
+      "properties": {},
+      "widgets_values": ["This workflow uses the remote vLLM-Omni Generate Video node.\n\n- First-frame only (default): keep First Frame connected.\n- Last-frame only: disconnect First Frame and connect Last Frame to last_frame.\n- First+last-frame: connect both image loaders.\n\nLeave the legacy frame and references inputs disconnected. Select images from ComfyUI's input folder. H3 runs at 24 FPS; the 5.167-second duration resolves to 124 frames, satisfying 17k+5."],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniDiffusionSampling",
+      "pos": [350, 300],
+      "size": [300, 226],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "n", "name": "n", "type": "INT", "widget": {"name": "n"}, "link": null},
+        {"localized_name": "num_inference_steps", "name": "num_inference_steps", "type": "INT", "widget": {"name": "num_inference_steps"}, "link": null},
+        {"localized_name": "guidance_scale", "name": "guidance_scale", "type": "FLOAT", "widget": {"name": "guidance_scale"}, "link": null},
+        {"localized_name": "true_cfg_scale", "name": "true_cfg_scale", "type": "FLOAT", "widget": {"name": "true_cfg_scale"}, "link": null},
+        {"localized_name": "vae_use_slicing", "name": "vae_use_slicing", "type": "BOOLEAN", "widget": {"name": "vae_use_slicing"}, "link": null},
+        {"localized_name": "vae_use_tiling", "name": "vae_use_tiling", "type": "BOOLEAN", "widget": {"name": "vae_use_tiling"}, "link": null},
+        {"localized_name": "seed", "name": "seed", "type": "INT", "widget": {"name": "seed"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "diffusion sampling params", "name": "diffusion sampling params", "type": "SAMPLING_PARAMS", "links": [2]}
+      ],
+      "title": "Base H3 Sampling (50 steps)",
+      "properties": {"Node name for S&R": "VLLMOmniDiffusionSampling"},
+      "widgets_values": [1, 50, 1, 1, false, false, 757358688076805, "fixed"]
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [350, 570],
+      "size": [300, 82],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "audio_flow_shift", "name": "audio_flow_shift", "type": "FLOAT", "widget": {"name": "audio_flow_shift"}, "link": null},
+        {"localized_name": "flow_shift", "name": "flow_shift", "type": "FLOAT", "widget": {"name": "flow_shift"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "MiniMax-H3 video params", "name": "MiniMax-H3 video params", "type": "VIDEO_PARAMS", "links": [3]}
+      ],
+      "title": "Base H3 Flow Shifts",
+      "properties": {"Node name for S&R": "VLLMOmniMiniMaxH3Params"},
+      "widgets_values": [3, 12]
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniRemoteLoRA",
+      "pos": [700, 550],
+      "size": [360, 150],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "local_path", "name": "local_path", "type": "STRING", "widget": {"name": "local_path"}, "link": null},
+        {"localized_name": "name", "name": "name", "type": "STRING", "widget": {"name": "name"}, "link": null},
+        {"localized_name": "scale", "name": "scale", "type": "FLOAT", "widget": {"name": "scale"}, "link": null},
+        {"localized_name": "int_id", "name": "int_id", "type": "INT", "widget": {"name": "int_id"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "lora", "name": "lora", "type": "REMOTE_LORA", "links": null}
+      ],
+      "title": "Optional H3 Turbo Remote LoRA",
+      "properties": {"Node name for S&R": "VLLMOmniRemoteLoRA"},
+      "widgets_values": ["minimax_h3_fl2v_turbo_8step_v1.0_768p_bf16.safetensors", "h3-turbo-8step", 1, 0]
+    },
+    {
+      "id": 7,
+      "type": "MarkdownNote",
+      "pos": [700, 300],
+      "size": [420, 210],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Optional Turbo settings",
+      "properties": {},
+      "widgets_values": ["Base H3 is configured by default: 50 steps, flow_shift 12, audio_flow_shift 3.\n\nFor the shown 8-step Diffusers Turbo artifact, set its full local_path on the server, start the server with the matching preloaded LoRA, connect this Remote LoRA to Generate Video, set num_inference_steps to 9, and set flow_shift to 6. Keep audio_flow_shift at 3 and scale at 1.\n\nDo not use the _comfyui_ repack or FastH3 here."],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 8,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [1160, 80],
+      "size": [440, 610],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "frame", "name": "frame", "shape": 7, "type": "IMAGE", "link": null},
+        {"localized_name": "first_frame", "name": "first_frame", "shape": 7, "type": "IMAGE", "link": 1},
+        {"localized_name": "last_frame", "name": "last_frame", "shape": 7, "type": "IMAGE", "link": null},
+        {"localized_name": "references", "name": "references", "shape": 7, "type": "VIDEO_REFERENCES", "link": null},
+        {"localized_name": "sampling_params", "name": "sampling_params", "shape": 7, "type": "SAMPLING_PARAMS", "link": 2},
+        {"localized_name": "lora", "name": "lora", "shape": 7, "type": "REMOTE_LORA", "link": null},
+        {"localized_name": "model_params", "name": "model_params", "shape": 7, "type": "VIDEO_PARAMS", "link": 3},
+        {"localized_name": "fast_h3", "name": "fast_h3", "shape": 7, "type": "FASTH3_DEPLOYMENT", "link": null},
+        {"localized_name": "url", "name": "url", "type": "STRING", "widget": {"name": "url"}, "link": null},
+        {"localized_name": "model", "name": "model", "type": "STRING", "widget": {"name": "model"}, "link": null},
+        {"localized_name": "prompt", "name": "prompt", "type": "STRING", "widget": {"name": "prompt"}, "link": null},
+        {"localized_name": "negative_prompt", "name": "negative_prompt", "type": "STRING", "widget": {"name": "negative_prompt"}, "link": null},
+        {"localized_name": "width", "name": "width", "type": "INT", "widget": {"name": "width"}, "link": null},
+        {"localized_name": "height", "name": "height", "type": "INT", "widget": {"name": "height"}, "link": null},
+        {"localized_name": "fps", "name": "fps", "type": "INT", "widget": {"name": "fps"}, "link": null},
+        {"localized_name": "duration", "name": "duration", "type": "FLOAT", "widget": {"name": "duration"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "video", "name": "video", "type": "VIDEO", "links": [4]}
+      ],
+      "properties": {"Node name for S&R": "VLLMOmniGenerateVideo"},
+      "widgets_values": [
+        "http://localhost:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "A cinematic product shot begins exactly from the supplied keyframe. The subject moves naturally through a dark studio while blue and amber rim lights shift across reflective surfaces. The camera makes a slow controlled orbit. Audio: subtle room tone, precise mechanical details, and a restrained electronic score.",
+        "",
+        1344,
+        768,
+        24,
+        5.167
+      ]
+    },
+    {
+      "id": 9,
+      "type": "SaveVideo",
+      "pos": [1650, 170],
+      "size": [374, 314],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {"localized_name": "video", "name": "video", "type": "VIDEO", "link": 4},
+        {"localized_name": "filename_prefix", "name": "filename_prefix", "type": "STRING", "widget": {"name": "filename_prefix"}, "link": null},
+        {"localized_name": "format", "name": "format", "type": "COMBO", "widget": {"name": "format"}, "link": null},
+        {"localized_name": "codec", "name": "codec", "type": "COMFY_DYNAMICCOMBO_V3", "widget": {"name": "codec"}, "link": null}
+      ],
+      "outputs": [
+        {"localized_name": "video", "name": "video", "type": "VIDEO", "links": null}
+      ],
+      "properties": {},
+      "widgets_values": ["video/MiniMax_H3_I2V", "auto", "auto"]
+    }
+  ],
+  "links": [
+    [1, 1, 0, 8, 1, "IMAGE"],
+    [2, 4, 0, 8, 4, "SAMPLING_PARAMS"],
+    [3, 5, 0, 8, 6, "VIDEO_PARAMS"],
+    [4, 8, 0, 9, 0, "VIDEO"]
+  ],
+  "groups": [
+    {"id": 1, "title": "Portable first/last-frame inputs", "bounding": [10, -30, 760, 770], "color": "#3f789e", "flags": {}},
+    {"id": 2, "title": "H3 sampling and optional Turbo", "bounding": [320, 270, 830, 460], "color": "#8a6d3b", "flags": {}},
+    {"id": 3, "title": "Remote generation and audio-preserving save", "bounding": [1130, 30, 930, 700], "color": "#3f789e", "flags": {}}
+  ],
+  "config": {},
+  "extra": {"workflowRendererVersion": "LG", "ds": {"scale": 0.75, "offset": [180, 120]}},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax H3 Video Upscale.json

@@ -0,0 +1,724 @@
+{
+  "version": 0.4,
+  "last_node_id": 12,
+  "last_link_id": 13,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        360,
+        20
+      ],
+      "size": [
+        330,
+        480
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "frame",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "shape": 7,
+          "link": 11
+        },
+        {
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "shape": 7,
+          "link": 12
+        },
+        {
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "shape": 7,
+          "link": 10
+        },
+        {
+          "name": "fast_h3",
+          "type": "FASTH3_DEPLOYMENT",
+          "shape": 7,
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1,
+            9
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8091/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "A small brass bell hanging from a wooden beam in a quiet garden. A gentle breeze swings the bell and it rings clearly. Static camera, natural daylight, continuous motion, synchronized ringing and soft birdsong.",
+        "",
+        1344,
+        768,
+        24,
+        5.167
+      ]
+    },
+    {
+      "id": 2,
+      "type": "GetVideoComponents",
+      "pos": [
+        730,
+        20
+      ],
+      "size": [
+        250,
+        125
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "link": 1
+        }
+      ],
+      "outputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "links": [
+            2
+          ]
+        },
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "links": [
+            6
+          ]
+        },
+        {
+          "name": "fps",
+          "type": "FLOAT",
+          "links": [
+            5
+          ]
+        },
+        {
+          "name": "bit_depth",
+          "type": "INT",
+          "links": [
+            7
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "GetVideoComponents"
+      }
+    },
+    {
+      "id": 3,
+      "type": "SeedVR2LoadDiTModel",
+      "pos": [
+        730,
+        290
+      ],
+      "size": [
+        340,
+        245
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "torch_compile_args",
+          "name": "torch_compile_args",
+          "shape": 7,
+          "type": "TORCH_COMPILE_ARGS",
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "SEEDVR2_DIT",
+          "name": "SEEDVR2_DIT",
+          "type": "SEEDVR2_DIT",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SeedVR2LoadDiTModel"
+      },
+      "widgets_values": [
+        "seedvr2_ema_3b_fp16.safetensors",
+        "cuda:0",
+        32,
+        false,
+        "cpu",
+        false,
+        "sdpa"
+      ]
+    },
+    {
+      "id": 4,
+      "type": "SeedVR2VideoUpscaler",
+      "pos": [
+        1120,
+        290
+      ],
+      "size": [
+        340,
+        415
+      ],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "image",
+          "name": "image",
+          "type": "IMAGE",
+          "link": 2
+        },
+        {
+          "localized_name": "dit",
+          "name": "dit",
+          "type": "SEEDVR2_DIT",
+          "link": 3
+        },
+        {
+          "localized_name": "vae",
+          "name": "vae",
+          "type": "SEEDVR2_VAE",
+          "link": 13
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "IMAGE",
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SeedVR2VideoUpscaler"
+      },
+      "widgets_values": [
+        42,
+        "fixed",
+        1536,
+        0,
+        5,
+        true,
+        "lab",
+        1,
+        0,
+        0.0,
+        0.0,
+        "cpu",
+        false
+      ]
+    },
+    {
+      "id": 5,
+      "type": "CreateVideo",
+      "pos": [
+        1490,
+        290
+      ],
+      "size": [
+        250,
+        175
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 4
+        },
+        {
+          "name": "fps",
+          "type": "FLOAT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": 5
+        },
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "shape": 7,
+          "link": 6
+        },
+        {
+          "name": "bit_depth",
+          "type": "INT",
+          "widget": {
+            "name": "bit_depth"
+          },
+          "link": 7
+        }
+      ],
+      "outputs": [
+        {
+          "name": "VIDEO",
+          "type": "VIDEO",
+          "links": [
+            8
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "CreateVideo"
+      },
+      "widgets_values": [
+        24,
+        8
+      ]
+    },
+    {
+      "id": 6,
+      "type": "SaveVideo",
+      "pos": [
+        1790,
+        290
+      ],
+      "size": [
+        330,
+        280
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "link": 8
+        },
+        {
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "name": "format",
+          "type": "COMBO",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        },
+        {
+          "name": "codec",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "codec"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "WF07/upscaled",
+        "mp4",
+        "auto"
+      ]
+    },
+    {
+      "id": 7,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        0,
+        20
+      ],
+      "size": [
+        320,
+        120
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "MiniMax-H3 video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            10
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        6.0
+      ]
+    },
+    {
+      "id": 8,
+      "type": "VLLMOmniDiffusionSampling",
+      "pos": [
+        0,
+        200
+      ],
+      "size": [
+        320,
+        280
+      ],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "diffusion sampling params",
+          "type": "SAMPLING_PARAMS",
+          "links": [
+            11
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniDiffusionSampling"
+      },
+      "widgets_values": [
+        1,
+        5,
+        1.0,
+        1.0,
+        false,
+        true,
+        42,
+        "fixed"
+      ]
+    },
+    {
+      "id": 9,
+      "type": "VLLMOmniRemoteLoRA",
+      "pos": [
+        0,
+        530
+      ],
+      "size": [
+        320,
+        210
+      ],
+      "flags": {},
+      "order": 9,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "links": [
+            12
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniRemoteLoRA"
+      },
+      "widgets_values": [
+        "models/minimax-h3-turbo/minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors",
+        "h3-turbo-v1.0",
+        1.0,
+        0
+      ]
+    },
+    {
+      "id": 10,
+      "type": "SaveVideo",
+      "pos": [
+        1120,
+        20
+      ],
+      "size": [
+        330,
+        190
+      ],
+      "flags": {},
+      "order": 10,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "link": 9
+        },
+        {
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "name": "format",
+          "type": "COMBO",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        },
+        {
+          "name": "codec",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "codec"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "WF07/generated",
+        "mp4",
+        "auto"
+      ],
+      "title": "Save generated video before upscale"
+    },
+    {
+      "id": 11,
+      "type": "Note",
+      "pos": [
+        360,
+        560
+      ],
+      "size": [
+        330,
+        260
+      ],
+      "flags": {},
+      "order": 11,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "properties": {
+        "Node name for S&R": "Note"
+      },
+      "widgets_values": [
+        "WF-07: H3 + SeedVR2\n\nInstall ComfyUI-SeedVR2_VideoUpscaler. The DiT and VAE loaders use SeedVR2 3B FP16 and ema_vae_fp16.\n\nSet the H3 service URL/model and the server-side Turbo LoRA path.\n\n1536 is the target short edge for this 1344 x 768 preset (2x). Original audio and FPS connect directly to Create Video.\n\nFor base H3: disconnect LoRA, use 50 steps and video flow shift 12."
+      ]
+    },
+    {
+      "id": 12,
+      "type": "SeedVR2LoadVAEModel",
+      "pos": [
+        730,
+        590
+      ],
+      "size": [
+        340,
+        315
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "torch_compile_args",
+          "name": "torch_compile_args",
+          "shape": 7,
+          "type": "TORCH_COMPILE_ARGS",
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "SEEDVR2_VAE",
+          "name": "SEEDVR2_VAE",
+          "type": "SEEDVR2_VAE",
+          "links": [
+            13
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SeedVR2LoadVAEModel"
+      },
+      "widgets_values": [
+        "ema_vae_fp16.safetensors",
+        "cuda:0",
+        true,
+        1024,
+        128,
+        true,
+        768,
+        128,
+        "false",
+        "cpu",
+        false
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      1,
+      0,
+      2,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      2,
+      0,
+      4,
+      0,
+      "IMAGE"
+    ],
+    [
+      3,
+      3,
+      0,
+      4,
+      1,
+      "SEEDVR2_DIT"
+    ],
+    [
+      4,
+      4,
+      0,
+      5,
+      0,
+      "IMAGE"
+    ],
+    [
+      5,
+      2,
+      2,
+      5,
+      1,
+      "FLOAT"
+    ],
+    [
+      6,
+      2,
+      1,
+      5,
+      2,
+      "AUDIO"
+    ],
+    [
+      7,
+      2,
+      3,
+      5,
+      3,
+      "INT"
+    ],
+    [
+      8,
+      5,
+      0,
+      6,
+      0,
+      "VIDEO"
+    ],
+    [
+      9,
+      1,
+      0,
+      10,
+      0,
+      "VIDEO"
+    ],
+    [
+      10,
+      7,
+      0,
+      1,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      11,
+      8,
+      0,
+      1,
+      2,
+      "SAMPLING_PARAMS"
+    ],
+    [
+      12,
+      9,
+      0,
+      1,
+      3,
+      "REMOTE_LORA"
+    ],
+    [
+      13,
+      12,
+      0,
+      4,
+      2,
+      "SEEDVR2_VAE"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 0.52,
+      "offset": [
+        100,
+        100
+      ]
+    }
+  }
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Reference to Video.json

@@ -0,0 +1,786 @@
+{
+  "id": "e7a4e606-6510-4c4e-b127-9a8b6bf3ea71",
+  "revision": 0,
+  "last_node_id": 11,
+  "last_link_id": 5,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "LoadImage",
+      "pos": [
+        40,
+        100
+      ],
+      "size": [
+        320,
+        300
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "image",
+          "name": "image",
+          "type": "COMBO",
+          "widget": {
+            "name": "image"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "choose file to upload",
+          "name": "upload",
+          "type": "IMAGEUPLOAD",
+          "widget": {
+            "name": "upload"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "IMAGE",
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [
+            1
+          ]
+        },
+        {
+          "localized_name": "MASK",
+          "name": "MASK",
+          "type": "MASK",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadImage"
+      },
+      "widgets_values": [
+        "h3_reference.png",
+        "image"
+      ],
+      "title": "Reference image \u2014 select a local file"
+    },
+    {
+      "type": "LoadVideo",
+      "size": [
+        320,
+        200
+      ],
+      "flags": {},
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "VIDEO",
+          "type": "VIDEO",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        "h3_reference.mp4"
+      ],
+      "id": 2,
+      "pos": [
+        40,
+        470
+      ],
+      "order": 1,
+      "mode": 0,
+      "title": "Optional reference video"
+    },
+    {
+      "id": 3,
+      "type": "LoadAudio",
+      "pos": [
+        40,
+        1100
+      ],
+      "size": [
+        320,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "audio",
+          "name": "audio",
+          "type": "COMBO",
+          "widget": {
+            "name": "audio"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "audioUI",
+          "name": "audioUI",
+          "type": "AUDIO_UI",
+          "widget": {
+            "name": "audioUI"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "choose file to upload",
+          "name": "upload",
+          "type": "AUDIOUPLOAD",
+          "widget": {
+            "name": "upload"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "AUDIO",
+          "name": "AUDIO",
+          "type": "AUDIO",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadAudio"
+      },
+      "widgets_values": [
+        "h3_reference.wav",
+        null,
+        ""
+      ],
+      "title": "Optional reference audio"
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniVideoReferences",
+      "pos": [
+        420,
+        100
+      ],
+      "size": [
+        300,
+        365
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image_1",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": 1
+        },
+        {
+          "name": "image_2",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "audio_1",
+          "type": "AUDIO",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "audio_2",
+          "type": "AUDIO",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "video_1",
+          "type": "VIDEO",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "video_2",
+          "type": "VIDEO",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_3",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_4",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_5",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_6",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_7",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_8",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "image_9",
+          "type": "IMAGE",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "audio_3",
+          "type": "AUDIO",
+          "shape": 7,
+          "link": null
+        },
+        {
+          "name": "video_3",
+          "type": "VIDEO",
+          "shape": 7,
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "references",
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniVideoReferences"
+      },
+      "widgets_values": [],
+      "title": "References \u2014 up to 12 total"
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniDiffusionSampling",
+      "pos": [
+        420,
+        530
+      ],
+      "size": [
+        310,
+        275
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "n",
+          "name": "n",
+          "type": "INT",
+          "widget": {
+            "name": "n"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "num_inference_steps",
+          "name": "num_inference_steps",
+          "type": "INT",
+          "widget": {
+            "name": "num_inference_steps"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "guidance_scale",
+          "name": "guidance_scale",
+          "type": "FLOAT",
+          "widget": {
+            "name": "guidance_scale"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "true_cfg_scale",
+          "name": "true_cfg_scale",
+          "type": "FLOAT",
+          "widget": {
+            "name": "true_cfg_scale"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "vae_use_slicing",
+          "name": "vae_use_slicing",
+          "type": "BOOLEAN",
+          "widget": {
+            "name": "vae_use_slicing"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "vae_use_tiling",
+          "name": "vae_use_tiling",
+          "type": "BOOLEAN",
+          "widget": {
+            "name": "vae_use_tiling"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "seed",
+          "name": "seed",
+          "type": "INT",
+          "widget": {
+            "name": "seed"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "diffusion sampling params",
+          "name": "diffusion sampling params",
+          "type": "SAMPLING_PARAMS",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniDiffusionSampling"
+      },
+      "widgets_values": [
+        1,
+        50,
+        1.0,
+        1.0,
+        false,
+        false,
+        42,
+        "fixed"
+      ],
+      "title": "Sampling \u2014 base 50 / Turbo 5"
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        790,
+        750
+      ],
+      "size": [
+        310,
+        105
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "audio_flow_shift",
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "flow_shift",
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "MiniMax-H3 video params",
+          "name": "MiniMax-H3 video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ],
+      "title": "H3 video/audio flow shifts"
+    },
+    {
+      "type": "VLLMOmniRemoteLoRA",
+      "size": [
+        310,
+        180
+      ],
+      "flags": {},
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniRemoteLoRA"
+      },
+      "widgets_values": [
+        "",
+        "h3-ref2va-turbo",
+        1.0,
+        0
+      ],
+      "id": 7,
+      "pos": [
+        420,
+        940
+      ],
+      "order": 6,
+      "mode": 0,
+      "title": "Optional Turbo LoRA \u2014 connect to enable"
+    },
+    {
+      "id": 8,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        800,
+        100
+      ],
+      "size": [
+        470,
+        560
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "frame",
+          "name": "frame",
+          "shape": 7,
+          "type": "IMAGE",
+          "link": null
+        },
+        {
+          "localized_name": "references",
+          "name": "references",
+          "shape": 7,
+          "type": "VIDEO_REFERENCES",
+          "link": 2
+        },
+        {
+          "localized_name": "sampling_params",
+          "name": "sampling_params",
+          "shape": 7,
+          "type": "SAMPLING_PARAMS",
+          "link": 3
+        },
+        {
+          "localized_name": "lora",
+          "name": "lora",
+          "shape": 7,
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "localized_name": "model_params",
+          "name": "model_params",
+          "shape": 7,
+          "type": "VIDEO_PARAMS",
+          "link": 4
+        },
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "prompt",
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "negative_prompt",
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "fps",
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "duration",
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://localhost:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "Use the subject and visual style from <Picture 1>. Create a cinematic shot with a gentle camera move and natural ambient sound.",
+        "",
+        1344,
+        768,
+        24,
+        5.167
+      ],
+      "title": "MiniMax-H3 Reference to Video"
+    },
+    {
+      "id": 9,
+      "type": "SaveVideo",
+      "pos": [
+        1360,
+        100
+      ],
+      "size": [
+        410,
+        420
+      ],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "link": 5
+        },
+        {
+          "localized_name": "filename_prefix",
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "format",
+          "name": "format",
+          "type": "COMBO",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "codec",
+          "name": "codec",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "codec"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": null
+        }
+      ],
+      "properties": {},
+      "widgets_values": [
+        "video/MiniMax-H3-Ref2VA",
+        "mp4",
+        "h264"
+      ],
+      "title": "Save generated video with audio"
+    },
+    {
+      "id": 10,
+      "type": "MarkdownNote",
+      "pos": [
+        1360,
+        590
+      ],
+      "size": [
+        440,
+        595
+      ],
+      "flags": {},
+      "order": 9,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Reference inputs and output settings",
+      "properties": {},
+      "widgets_values": [
+        "## Reference to Video\nSelect an image, set the server URL and served model name, then run.\n\nConnect the optional video/audio loaders to Video References, or duplicate loaders for more inputs: up to 9 images, 3 videos, 3 audio clips, 12 total. At least one image or video is required.\n\nSlots are numbered separately per media type. Empty slots are skipped. Update prompt tags such as <Picture 1>, <Video 1>, <Audio 1> to match the connected references. Videos keep their own soundtrack in the reference upload.\n\n**Output:** 24 FPS; 124 frames (~5.17 s). Valid frame counts are 107, 124, 141, 158, 175, 192, 209, 226, 243, 260, 277, 294, 311, 328, 345.\n\n**Width x height:** 1344x768 (16:9), 1024x768 (4:3), 768x768 (1:1), 768x1024 (3:4), 768x1344 (9:16).\n\nEach reference video/audio clip must be 2\u201315 s. Total reference-video duration <=15 s; total standalone-audio duration <=15 s.\n\nRequires the expanded Video References node and the audio-preserving video output fix. See the extension README for setup and validation status."
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 11,
+      "type": "MarkdownNote",
+      "pos": [
+        790,
+        940
+      ],
+      "size": [
+        480,
+        455
+      ],
+      "flags": {},
+      "order": 10,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Optional Turbo \u2014 same workflow",
+      "properties": {},
+      "widgets_values": [
+        "## Turbo Ref2VA\nBase mode: leave LoRA disconnected; use 50 steps, video flow shift 12, audio flow shift 3.\n\nFor the 4-step v0.1 Ref2VA Turbo adapter:\n1. Start a Ref2VA-only server with the Diffusers adapter loaded (see README).\n2. Set LoRA local_path to its path on the server, then connect LoRA to Generate Video.\n3. Set Sampling num_inference_steps to **5**; keep shifts at **12 / 3**, scale **1**.\n\nvLLM-Omni counts sigma points: 5 points give 4 denoiser evaluations. Use minimax_h3_ref2v_turbo_4step_v0.1_bf16.safetensors, not the _comfyui_ export.\n\nDisconnect LoRA and restore 50 steps to return to base mode. FastH3 is a separate T2VA deployment, not this option.\n\nAdapted from the official ComfyUI MiniMax-H3 R2V workflow: https://github.com/Comfy-Org/workflow_templates/blob/main/templates/video_minimax_h3_r2v.json"
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    }
+  ],
+  "links": [
+    [
+      1,
+      1,
+      0,
+      4,
+      0,
+      "IMAGE"
+    ],
+    [
+      2,
+      4,
+      0,
+      8,
+      1,
+      "VIDEO_REFERENCES"
+    ],
+    [
+      3,
+      5,
+      0,
+      8,
+      2,
+      "SAMPLING_PARAMS"
+    ],
+    [
+      4,
+      6,
+      0,
+      8,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      5,
+      8,
+      0,
+      9,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "workflowRendererVersion": "LG",
+    "ds": {
+      "scale": 0.85,
+      "offset": [
+        100,
+        100
+      ]
+    }
+  },
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Music Generation.json

@@ -0,0 +1,206 @@
+{
+  "id": "4949c8da-12ff-449a-94b7-948bde9c39dd",
+  "revision": 0,
+  "last_node_id": 3,
+  "last_link_id": 1,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "VLLMOmniGenerateMusic",
+      "pos": [
+        100,
+        100
+      ],
+      "size": [
+        470,
+        491
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "name": "instructions",
+          "type": "STRING",
+          "widget": {
+            "name": "instructions"
+          },
+          "link": null
+        },
+        {
+          "name": "lyrics",
+          "type": "STRING",
+          "widget": {
+            "name": "lyrics"
+          },
+          "link": null
+        },
+        {
+          "name": "max_duration_seconds",
+          "type": "FLOAT",
+          "widget": {
+            "name": "max_duration_seconds"
+          },
+          "link": null
+        },
+        {
+          "name": "seed",
+          "type": "INT",
+          "widget": {
+            "name": "seed"
+          },
+          "link": null
+        },
+        {
+          "name": "response_format",
+          "type": "COMBO",
+          "widget": {
+            "name": "response_format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateMusic"
+      },
+      "widgets_values": [
+        "http://localhost:8000/v1",
+        "MiniMaxAI/MiniMax-Music3",
+        "Futuristic melodic EDM / progressive house. Reflective verses about memory and digital identity build into an uplifting, hook-driven chorus, with an emotive lead vocal, bright layered synths, pulsing bass, crisp four-on-the-floor drums, subtle glitch textures, and a wide polished festival mix.",
+        "[Verse 1]\nUpload my heart to the cloud tonight\nSave every spark of neon light\nIf this body starts to fade away\nLet my code keep dancing in the data stream\nStay\n[Chorus]\nCloud copy of me\nLiving on when these bones finally break free\nEvery joke, every scar\nEvery cheap memory we made\nSave it, save it\nCloud copy of me\nIf the screen goes dark, I\u2019m still in the machine\nEvery glitch, every dream\nEvery version you ever did see\nSave it, save it\n[Verse 2]\nTag every laugh in a secret folder\nArchive the nights we outran the thunder\nIf the wiring turns to rust and gray\nHit restore\u2026 press replay\n[Chorus]\nCloud copy of me\nLiving on when these bones finally break free\nEvery joke, every scar\nEvery cheap memory we made\nSave it, save it\nCloud copy of me\nIf the screen goes dark, I\u2019m still in the machine\nEvery glitch, every dream\nEvery version you ever did see\nSave it, save it (oh)\n[Bridge \u2013 new, optional but recommended]\nEven the crashes, the late-night confessions\nThe parts of me I never learned to mention\nDon\u2019t let the silence swallow me whole\nKeep a ghost in the glow\u2026\nSo I never go\n[Final Chorus \u2013 softer / layered]\nCloud copy of me\nStill laughing somewhere in the binary\nEvery joke, every scar\nEvery moment you saved for me\nSave it, save it\nCloud copy of me\nWhen the lights go out, I\u2019ll still be in the machine\nDancing in the code that you still believe\nSave it\u2026\nSave me",
+        300.0,
+        0,
+        "fixed",
+        "wav"
+      ]
+    },
+    {
+      "id": 2,
+      "type": "SaveAudioAdvanced",
+      "pos": [
+        650,
+        100
+      ],
+      "size": [
+        320,
+        491
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "link": 1
+        },
+        {
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "name": "format",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveAudioAdvanced"
+      },
+      "widgets_values": [
+        "audio/vLLM-Omni-Music",
+        "mp3",
+        "V0"
+      ]
+    },
+    {
+      "id": 3,
+      "type": "MarkdownNote",
+      "title": "Music Generation Guide",
+      "pos": [
+        -507,
+        100
+      ],
+      "size": [
+        527,
+        491
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "properties": {},
+      "widgets_values": [
+        "## MiniMax Music 3\n\nThe **Generate Music** node currently supports **MiniMax Music 3 only**. The parameters below apply to this model.\n\nA music generation model by MiniMax that creates complete songs up to 5 minutes long with stable structure and high audio quality. Two inputs drive it: **Caption** (structured music description: style, mood, vocals, arrangement) and **Lyrics** (with `[intro]` `[verse]` `[chorus]` `[bridge]` `[outro]` tags controlling song structure).\n\n### Parameters\n\n| Parameter | Description |\n|---|---|\n| url | vLLM-Omni service URL including `/v1`. Default: `http://localhost:8000/v1`. |\n| model | Served model name. Default: `MiniMaxAI/MiniMax-Music3`. |\n| caption | Music description, sent as `instructions`. Use Global Metadata \u2192 Vocal Details \u2192 Arrangement to describe style, vocals, and instrumentation. |\n| lyrics | Words to sing with section tags. Put each tag on its own line; tags guide rather than strictly enforce song structure. |\n| max_duration_seconds | Maximum duration, default 300 seconds; range 1\u2013360, step 0.01. Converted to tokens at 25 frames/s, rounded down. Generation may end earlier. |\n| seed | Random seed, default 0; range 0\u20139007199254740991. Keep inputs and runtime settings unchanged for repeatability. |\n| control_after_generate | Seed behavior after generation: fixed, randomize, increment, or decrement. Template default: fixed. |\n| response_format | API audio format, default WAV. Options: WAV, MP3, FLAC, Opus, AAC; AAC is currently unsupported by the vLLM-Omni speech API. |\n\n### Prompt Writing Guide\n\nYou can find an official [Music Caption Rewriter Skill here](https://github.com/MiniMax-AI/MiniMax-Music3)\n"
+      ],
+      "widgets_values_named": {
+        "text": "## MiniMax Music 3\n\nThe **Generate Music** node currently supports **MiniMax Music 3 only**. The parameters below apply to this model.\n\nA music generation model by MiniMax that creates complete songs up to 5 minutes long with stable structure and high audio quality. Two inputs drive it: **Caption** (structured music description: style, mood, vocals, arrangement) and **Lyrics** (with `[intro]` `[verse]` `[chorus]` `[bridge]` `[outro]` tags controlling song structure).\n\n### Parameters\n\n| Parameter | Description |\n|---|---|\n| url | vLLM-Omni service URL including `/v1`. Default: `http://localhost:8000/v1`. |\n| model | Served model name. Default: `MiniMaxAI/MiniMax-Music3`. |\n| caption | Music description, sent as `instructions`. Use Global Metadata \u2192 Vocal Details \u2192 Arrangement to describe style, vocals, and instrumentation. |\n| lyrics | Words to sing with section tags. Put each tag on its own line; tags guide rather than strictly enforce song structure. |\n| max_duration_seconds | Maximum duration, default 300 seconds; range 1\u2013360, step 0.01. Converted to tokens at 25 frames/s, rounded down. Generation may end earlier. |\n| seed | Random seed, default 0; range 0\u20139007199254740991. Keep inputs and runtime settings unchanged for repeatability. |\n| control_after_generate | Seed behavior after generation: fixed, randomize, increment, or decrement. Template default: fixed. |\n| response_format | API audio format, default WAV. Options: WAV, MP3, FLAC, Opus, AAC; AAC is currently unsupported by the vLLM-Omni speech API. |\n\n### Prompt Writing Guide\n\nYou can find an official [Music Caption Rewriter Skill here](https://github.com/MiniMax-AI/MiniMax-Music3)\n"
+      },
+      "color": "#333",
+      "bgcolor": "#535353"
+    }
+  ],
+  "links": [
+    [
+      1,
+      1,
+      0,
+      2,
+      0,
+      "AUDIO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "workflowRendererVersion": "LG",
+    "ds": {
+      "scale": 1.147,
+      "offset": [
+        899,
+        259
+      ]
+    }
+  },
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json

@@ -1 +1 @@
-{"id":"3aea1d08-7de5-4997-ba71-b2f66ad72a21","revision":0,"last_node_id":24,"last_link_id":35,"nodes":[{"id":13,"type":"SaveVideo","pos":[1315.729153459605,-137.92038948394355],"size":[374,313.7692307692308],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"video","name":"video","type":"VIDEO","link":32},{"localized_name":"filename_prefix","name":"filename_prefix","type":"STRING","widget":{"name":"filename_prefix"},"link":null},{"localized_name":"format","name":"format","type":"COMBO","widget":{"name":"format"},"link":null},{"localized_name":"codec","name":"codec","type":"COMFY_DYNAMICCOMBO_V3","widget":{"name":"codec"},"link":null}],"outputs":[{"localized_name":"video","name":"video","type":"VIDEO","links":null}],"properties":{},"widgets_values":["video/ComfyUI","auto","auto"]},{"id":10,"type":"MarkdownNote","pos":[183.74636797766794,-571.2529158046766],"size":[240.3326478922474,136.3505820791881],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[],"title":"Note: Task and Input","properties":{},"widgets_values":["vLLM-Omni nodes are categorized based on the output modality. The \"Generate Video\" node supports both *text-to-video*, *frame-to-video*, and *reference-to-video* generation. The node will create corresponding payload depending on whether an input image is present or not."],"color":"#432","bgcolor":"#000"},{"id":4,"type":"LoadImage","pos":[452.07189605801125,-569.7242536425791],"size":[270,314],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[29]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["rap.png","image"]},{"id":23,"type":"LoadImage","pos":[9.206110246057657,-7.6955291509170785],"size":[270,314],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[34]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["white-cat.jpg","image"]},{"id":22,"type":"MarkdownNote","pos":[-9.783113411416666,-148.8790300038605],"size":[297.3012007349086,88],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[],"title":"Note: Reference to video (r2v)","properties":{},"widgets_values":["For r2v tasks, connect this Video Reference block to the main Generate Video block. Then disconnect the \"frame\" input above."],"color":"#432","bgcolor":"#000"},{"id":24,"type":"LoadAudio","pos":[317.9929742385805,-141.44821500065095],"size":[282.798828125,136],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[35]}],"properties":{"Node name for S&R":"LoadAudio"},"widgets_values":["nothing-beats-a-jet2-holiday.mp3",null,""]},{"id":21,"type":"VLLMOmniVideoReferences","pos":[376.6129172897595,105.41557277556292],"size":[156.9380859375,126],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"image_1","name":"image_1","shape":7,"type":"IMAGE","link":34},{"localized_name":"image_2","name":"image_2","shape":7,"type":"IMAGE","link":null},{"localized_name":"audio_1","name":"audio_1","shape":7,"type":"AUDIO","link":35},{"localized_name":"audio_2","name":"audio_2","shape":7,"type":"AUDIO","link":null},{"localized_name":"video_1","name":"video_1","shape":7,"type":"VIDEO","link":null},{"localized_name":"video_2","name":"video_2","shape":7,"type":"VIDEO","link":null}],"outputs":[{"localized_name":"references","name":"references","type":"VIDEO_REFERENCES","links":null}],"properties":{"Node name for S&R":"VLLMOmniVideoReferences"},"widgets_values":[]},{"id":8,"type":"VLLMOmniDiffusionSampling","pos":[83.24620208113915,422.08543477542605],"size":[284.205078125,226],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"n","name":"n","type":"INT","widget":{"name":"n"},"link":null},{"localized_name":"num_inference_steps","name":"num_inference_steps","type":"INT","widget":{"name":"num_inference_steps"},"link":null},{"localized_name":"guidance_scale","name":"guidance_scale","type":"FLOAT","widget":{"name":"guidance_scale"},"link":null},{"localized_name":"true_cfg_scale","name":"true_cfg_scale","type":"FLOAT","widget":{"name":"true_cfg_scLine truncated
+{"id":"3aea1d08-7de5-4997-ba71-b2f66ad72a21","revision":0,"last_node_id":24,"last_link_id":35,"nodes":[{"id":13,"type":"SaveVideo","pos":[1315.729153459605,-137.92038948394355],"size":[374,313.7692307692308],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"video","name":"video","type":"VIDEO","link":32},{"localized_name":"filename_prefix","name":"filename_prefix","type":"STRING","widget":{"name":"filename_prefix"},"link":null},{"localized_name":"format","name":"format","type":"COMBO","widget":{"name":"format"},"link":null},{"localized_name":"codec","name":"codec","type":"COMFY_DYNAMICCOMBO_V3","widget":{"name":"codec"},"link":null}],"outputs":[{"localized_name":"video","name":"video","type":"VIDEO","links":null}],"properties":{},"widgets_values":["video/ComfyUI","auto","auto"]},{"id":10,"type":"MarkdownNote","pos":[183.74636797766794,-571.2529158046766],"size":[240.3326478922474,136.3505820791881],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[],"title":"Note: Task and Input","properties":{},"widgets_values":["vLLM-Omni nodes are categorized based on the output modality. The \"Generate Video\" node supports both *text-to-video*, *frame-to-video*, and *reference-to-video* generation. The node will create corresponding payload depending on whether an input image is present or not."],"color":"#432","bgcolor":"#000"},{"id":4,"type":"LoadImage","pos":[452.07189605801125,-569.7242536425791],"size":[270,314],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[29]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["rap.png","image"]},{"id":23,"type":"LoadImage","pos":[9.206110246057657,-7.6955291509170785],"size":[270,314],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[34]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["white-cat.jpg","image"]},{"id":22,"type":"MarkdownNote","pos":[-9.783113411416666,-148.8790300038605],"size":[297.3012007349086,88],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[],"title":"Note: Reference to video (r2v)","properties":{},"widgets_values":["For r2v tasks, connect this Video Reference block to the main Generate Video block. Then disconnect the \"frame\" input above."],"color":"#432","bgcolor":"#000"},{"id":24,"type":"LoadAudio","pos":[317.9929742385805,-141.44821500065095],"size":[282.798828125,136],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[35]}],"properties":{"Node name for S&R":"LoadAudio"},"widgets_values":["nothing-beats-a-jet2-holiday.mp3",null,""]},{"id":21,"type":"VLLMOmniVideoReferences","pos":[376.6129172897595,105.41557277556292],"size":[156.9380859375,126],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"image_1","name":"image_1","shape":7,"type":"IMAGE","link":34},{"localized_name":"image_2","name":"image_2","shape":7,"type":"IMAGE","link":null},{"localized_name":"audio_1","name":"audio_1","shape":7,"type":"AUDIO","link":35},{"localized_name":"audio_2","name":"audio_2","shape":7,"type":"AUDIO","link":null},{"localized_name":"video_1","name":"video_1","shape":7,"type":"VIDEO","link":null},{"localized_name":"video_2","name":"video_2","shape":7,"type":"VIDEO","link":null}],"outputs":[{"localized_name":"references","name":"references","type":"VIDEO_REFERENCES","links":null}],"properties":{"Node name for S&R":"VLLMOmniVideoReferences"},"widgets_values":[]},{"id":8,"type":"VLLMOmniDiffusionSampling","pos":[83.24620208113915,422.08543477542605],"size":[284.205078125,226],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"n","name":"n","type":"INT","widget":{"name":"n"},"link":null},{"localized_name":"num_inference_steps","name":"num_inference_steps","type":"INT","widget":{"name":"num_inference_steps"},"link":null},{"localized_name":"guidance_scale","name":"guidance_scale","type":"FLOAT","widget":{"name":"guidance_scale"},"link":null},{"localized_name":"true_cfg_scale","name":"true_cfg_scale","type":"FLOAT","widget":{"name":"true_cfg_scLine truncated

apps/ComfyUI-vLLM-Omni/web/main.js

@@ -1,19 +1,154 @@
 /**
- * @file This file is intended to add dynamic fields to vLLM-Omni nodes
- * based on widget (in-node form fields) and input (connection link) values and changes.
- * However, this functionality is currently disabled/commented out.
- * Because it introduces too much complexity,
- * and it may even conflict with the current backend (Python) validation for unknown reasons (pending ComfyUI upstream fixes).
+ * @file Front-end extension for the vLLM-Omni nodes.
+ *
+ * Branding. Every vLLM-Omni node draws the vLLM-Omni "v" mark where LiteGraph
+ * would otherwise draw its default title dot, and is tinted by the kind of value
+ * it produces: blue nodes reach a server, amber and purple ones only assemble
+ * parameters, red ones pin what the server has loaded, teal ones carry media.
+ * A graph is then readable at a glance, and a node that turns out to be the
+ * wrong colour for where it sits is usually wired wrong.
+ *
+ * The tint is keyed off each node's declared output types rather than a list of
+ * node names, so a node added later is themed as soon as it returns one of the
+ * existing types.
+ *
+ * It also migrates Generate Video graphs saved before `duration` replaced the
+ * `num_frames` widget. See migrateNumFramesToDuration below.
+ *
+ * This file also used to host dynamic fields on vLLM-Omni nodes, driven by
+ * widget (in-node form fields) and input (connection link) values and changes.
+ * That is deliberately absent: it introduced too much complexity, and it even
+ * conflicted with the current backend (Python) validation for unknown reasons
+ * (pending ComfyUI upstream fixes).
  */
 
 import { app } from "../../scripts/app.js";
+
+const MARK_URL = new URL("./vllm-omni-mark.svg", import.meta.url).href;
+const MARK_ASPECT = 157 / 165;
+
+/**
+ * Title bar and body tints, one per node family.
+ *
+ * The hues are the logo's: the amber and blue of the "v" mark, and three stops
+ * of the gradient ring in the "O" (#9128A3, #CC5260, and the teal neighbouring
+ * its blue). All of them are dark because LiteGraph paints the title bar darker
+ * than the body and expects both to sit under light text; the ComfyUI light
+ * theme lightens custom node colours on its own.
+ */
+const FAMILIES = {
+    generate: { color: "#152A43", bgcolor: "#1F4269" },
+    sampling: { color: "#3A2C12", bgcolor: "#5C461C" },
+    model: { color: "#2E1233", bgcolor: "#4A1C52" },
+    deployment: { color: "#3A1A1F", bgcolor: "#5C2931" },
+    media: { color: "#0F2E2A", bgcolor: "#194A44" },
+};
+
+/** Output type -> family. Types absent here fall through to "generate". */
+const FAMILY_BY_OUTPUT = {
+    SAMPLING_PARAMS: "sampling",
+    TTS_PARAMS: "model",
+    VIDEO_PARAMS: "model",
+    REMOTE_LORA: "deployment",
+    FASTH3_DEPLOYMENT: "deployment",
+    VIDEO_REFERENCES: "media",
+    LATENT_MASK_EDITING: "media",
+};
+
+function familyOf(nodeData) {
+    for (const type of nodeData.output ?? []) {
+        const family = FAMILY_BY_OUTPUT[type];
+        if (family) {
+            return family;
+        }
+    }
+    // What is left returns a generated artifact (IMAGE, VIDEO, AUDIO, STRING),
+    // which is only ever produced by a node that talks to a vLLM-Omni server.
+    return "generate";
+}
+
+/**
+ * Rewrite a pre-rename Generate Video graph in place.
+ *
+ * `duration` (seconds) took over the slot `num_frames` used to occupy, and
+ * ComfyUI restores widget values positionally, so a graph saved before the
+ * rename reads its frame count as a duration: the workflow this repository
+ * shipped stored 120 frames at 24 fps, which would come back as a 120-second
+ * request. Saved files still name the old widget under `inputs`, and that is
+ * what makes the rewrite both detectable and safe to apply exactly once -- once
+ * rewritten the node serializes `duration`, so a later load finds nothing to do.
+ */
+function migrateNumFramesToDuration(node, info) {
+    const savedBeforeRename = (info?.inputs ?? []).some((input) => input?.widget?.name === "num_frames");
+    if (!savedBeforeRename) {
+        return
+    }
+    const duration = node.widgets?.find((widget) => widget.name === "duration");
+    const fps = node.widgets?.find((widget) => widget.name === "fps");
+    if (!duration || !fps) {
+        return
+    }
+    // configure() has already placed the stale frame count in the duration widget.
+    const frames = Number(duration.value);
+    const rate = Number(fps.value);
+    if (!Number.isFinite(frames) || !Number.isFinite(rate) || rate <= 0) {
+        return
+    }
+    const seconds = Math.max(duration.options?.min ?? 0.1, Math.round((frames / rate) * 1000) / 1000);
+    console.info(
+        `vLLM-Omni: migrated Generate Video "${node.title}" from ${frames} frames to ${seconds}s at ${rate} fps.`,
+    );
+    duration.value = seconds;
+}
+
+const mark = new Image();
+let markReady = false;
+mark.addEventListener("load", () => {
+    markReady = true;
+    // The image resolves after the first frames are already on screen.
+    app.graph?.setDirtyCanvas?.(true, true);
+});
+mark.src = MARK_URL;
+
 app.registerExtension({
     name: "vllm.vllm_omni",
     async beforeRegisterNodeDef(nodeType, nodeData, app) {
         if (!nodeData.name.startsWith("VLLMOmni")) {
             return
         }
-        // Stub frontend plugin for now
+
+        const { color, bgcolor } = FAMILIES[familyOf(nodeData)];
+        // On the prototype, not the instance: a colour the user picks by hand is
+        // written to the instance and shadows this, so manual overrides survive.
+        nodeType.prototype.color = color;
+        nodeType.prototype.bgcolor = bgcolor;
+
+        if (nodeData.name === "VLLMOmniGenerateVideo") {
+            const onConfigure = nodeType.prototype.onConfigure;
+            nodeType.prototype.onConfigure = function (info) {
+                onConfigure?.apply(this, arguments);
+                migrateNumFramesToDuration(this, info);
+            };
+        }
+
+        // Defining this replaces the default dot outright — LiteGraph draws its
+        // own square or circle only for nodes that leave onDrawTitleBox unset.
+        nodeType.prototype.onDrawTitleBox = function (ctx, titleHeight, _size, scale) {
+            // Zoomed out far enough that LiteGraph itself drops to flat shapes,
+            // the mark is a few muddy pixels. Leave the title bar clean instead.
+            if (!markReady || (scale !== undefined && scale < 0.5)) {
+                return
+            }
+            const height = titleHeight * 0.62;
+            const width = height * MARK_ASPECT;
+            ctx.drawImage(
+                mark,
+                titleHeight * 0.5 - width * 0.5,
+                titleHeight * -0.5 - height * 0.5,
+                width,
+                height,
+            );
+        };
     },
     async setup() {
         console.info("vLLM-Omni Setup complete!")

apps/ComfyUI-vLLM-Omni/web/vllm-omni-mark.svg

@@ -0,0 +1,5 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 157 165" width="157" height="165">
+  <title>vLLM-Omni</title>
+  <polygon points="0,36 62,36 62,160" fill="#F1B844"/>
+  <polygon points="96,36 157,0 121,165 68,165" fill="#54A0F8"/>
+</svg>

benchmarks/README.md

@@ -4,6 +4,11 @@ This directory contains benchmark suites for evaluating different model families
 
 ## Benchmark families
 
+### [Qwen2.5-Omni](qwen2_5_omni/README.md) — Embedding Wrapper Latency
+
+CPU-observed A/B timings for redundant text embedding removal, with real checkpoint
+embeddings, warmup, paired blocks, and embedding call-count validation.
+
 ### [TTS](tts/README.md) — Text-to-Speech
 
 Model-agnostic serving benchmarks for TTS models, including Qwen3-TTS and VoxCPM2.
@@ -36,6 +41,17 @@ LingBot-Video MoE transformer.
 - **Dense pipeline**: decoded-video MAE, MSE, PSNR, latency, and optional steady-state timings
 - **MoE transformer**: bitwise router, sparse-block, shared-expert, and full-transformer parity
 
+### [LingBot-World](lingbot_world/README.md) — Realtime Streaming Cadence
+
+Client-side benchmark for a `WS /v1/realtime/video` world-model session, where one
+request is a whole rollout and one AR block streams out as one video chunk. It
+measures a cadence rather than a throughput: there is exactly one session, and what
+matters is whether the next chunk arrives before the viewer finishes the last one.
+
+- **Layout**: `lingbot_world/benchmark_lingbot_world_realtime.py` (WS client), `lingbot_world/workload.py` (rollout and metric math), `lingbot_world/configs/` (single-GPU eager and Ulysses-4 compiled deploy configs)
+- **Workload**: built-in image-conditioned rollout with a generated camera script, or a JSON spec with per-chunk actions and mid-rollout prompt updates
+- **Key metrics**: TTFC (time to first chunk), steady-state chunk inter-arrival percentiles, VIDEO_RTF (wall seconds per video second; lower is better), chunk-deadline attainment, and simulated playback underruns
+
 ### [Distributed](distributed/omni_connectors/README.md) — RDMA Connector Testing
 
 RDMA environment setup and transfer tests for `MooncakeTransferEngineConnector`, including pytest-based single-node checks and manual cross-node benchmarks.

benchmarks/diffusion/README.md

@@ -17,18 +17,18 @@ The main entrypoint is:
 vllm serve Qwen/Qwen-Image --omni --port 8099
 ```
 
-2. Run a minimal benchmark:
+1. Run a minimal benchmark:
 
 ```bash
 python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-	--base-url http://localhost:8099 \
-	--model Qwen/Qwen-Image \
-	--task t2i \
-	--dataset vbench \
-	--num-prompts 5
+ --base-url http://localhost:8099 \
+ --model Qwen/Qwen-Image \
+ --task t2i \
+ --dataset vbench \
+ --num-prompts 5
 ```
 
-**Notes**
+### Notes
 
 - By default, image tasks talk to `http://<host>:<port>/v1/chat/completions`; video tasks talk to `/v1/videos`.
 - If you run the server on another host or port, pass `--base-url` accordingly.
@@ -52,14 +52,14 @@ Example (`t2v`):
 
 ```bash
 python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-	--base-url http://localhost:8099 \
-	--model Wan-AI/Wan2.2-T2V-A14B-Diffusers \
-	--task t2v \
-	--dataset vbench \
-	--num-prompts 50 \
-	--width 640 --height 480 \
-	--num-frames 81 --fps 16 \
-	--num-inference-steps 40
+ --base-url http://localhost:8099 \
+ --model Wan-AI/Wan2.2-T2V-A14B-Diffusers \
+ --task t2v \
+ --dataset vbench \
+ --num-prompts 50 \
+ --width 640 --height 480 \
+ --num-frames 81 --fps 16 \
+ --num-inference-steps 40
 ```
 
 Note: `vbench` can also be used for other tasks such as `t2i` / `i2v` (and `i2i`). For `t2i`, the loader reuses VBench t2v text prompts; for `i2v` / `i2i`, it loads the VBench i2v dataset (with image paths).
@@ -140,6 +140,21 @@ Traffic / concurrency flags:
 - `--request-rate`: Target request rate (requests/second). If set to `inf`, the script sends all requests immediately.
 - `--max-concurrency`: Max number of in-flight requests (default: `1`). This can hard-cap the achieved QPS: if it is too small, requests will queue behind the semaphore, and both achieved throughput and observed SLO attainment can be skewed.
 
+### Video job timeout and failures
+
+`--video-job-timeout` controls how long each `/v1/videos` job may be polled,
+including time queued on the server (default: `900` seconds). For long video
+jobs at high concurrency, use a larger budget, for example
+`--video-job-timeout 1800`. This also applies to warmup jobs. The polling budget
+starts after job creation; it excludes time waiting for the client concurrency
+semaphore. Individual HTTP calls still use the aiohttp session timeout.
+
+An expired job is counted as failed and deleted, which cancels queued or running
+server generation. The benchmark continues processing the remaining requests.
+The progress bar counts both successful and failed requests. The final report
+shows the failure count and the first 10 errors; `--output-file` saves all errors
+with their request IDs in `request_errors`.
+
 ### Batched warmup note
 
 For batched serving runs, warm the same in-flight shape you plan to measure.

benchmarks/diffusion/backends.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 import asyncio
 import base64
 import json
@@ -43,6 +46,7 @@ class RequestFuncInput:
     video_paths: list[str] | None = None
     request_id: str = field(default_factory=lambda: str(uuid.uuid4()))
     default_bot_task: str | None = DEFAULT_EDITS_BOT_TASK
+    video_job_timeout: float = 900.0
 
 
 @dataclass
@@ -331,64 +335,64 @@ async def async_request_v1_videos(
     output = RequestFuncOutput()
     output.start_time = time.perf_counter()
 
-    files = dict(input.extra_body)
-    if input.prompt:
-        files.setdefault("prompt", input.prompt)
-    if input.width and input.height:
-        files.setdefault("height", input.height)
-        files.setdefault("width", input.width)
-    if input.num_frames:
-        files.setdefault("num_frames", input.num_frames)
-    if input.num_inference_steps:
-        files.setdefault("num_inference_steps", input.num_inference_steps)
-    if input.seed is not None:
-        files.setdefault("seed", input.seed)
-    if input.fps:
-        files.setdefault("fps", input.fps)
-
-    form = aiohttp.FormData()
-    for k, v in files.items():
-        form.add_field(k, str(v))
-
     image_file = None
-    if input.image_paths and input.video_paths:
-        output.error = "Only one of image_paths or video_paths can be provided"
-        output.success = False
-        return output
+    job_id = None
+    try:
+        files = dict(input.extra_body)
+        if input.prompt:
+            files.setdefault("prompt", input.prompt)
+        if input.width and input.height:
+            files.setdefault("height", input.height)
+            files.setdefault("width", input.width)
+        if input.num_frames:
+            files.setdefault("num_frames", input.num_frames)
+        if input.num_inference_steps:
+            files.setdefault("num_inference_steps", input.num_inference_steps)
+        if input.seed is not None:
+            files.setdefault("seed", input.seed)
+        if input.fps:
+            files.setdefault("fps", input.fps)
 
-    if input.image_paths and len(input.image_paths) > 0:
-        image_path = input.image_paths[0]
-        image_file = open(image_path, "rb")
-        form.add_field(
-            "input_reference",
-            image_file,
-            filename=os.path.basename(image_path),
-            content_type=_guess_mime_type(image_path),
-        )
-    elif input.video_paths and len(input.video_paths) > 0:
-        video_path = input.video_paths[0]
-        if _is_minimax_h3_mixed_reference(input.model):
-            with open(video_path, "rb") as reference_file:
-                video_b64 = base64.b64encode(reference_file.read()).decode("ascii")
-            form.add_field(
-                "video_reference",
-                json.dumps([{"video_url": f"data:video/mp4;base64,{video_b64}"}]),
-            )
-        else:
-            image_file = open(video_path, "rb")
+        form = aiohttp.FormData()
+        for k, v in files.items():
+            form.add_field(k, str(v))
+
+        if input.image_paths and input.video_paths:
+            output.error = "Only one of image_paths or video_paths can be provided"
+            output.success = False
+            return output
+
+        if input.image_paths and len(input.image_paths) > 0:
+            image_path = input.image_paths[0]
+            image_file = open(image_path, "rb")
             form.add_field(
                 "input_reference",
                 image_file,
-                filename=os.path.basename(video_path),
-                content_type=_guess_mime_type(video_path),
+                filename=os.path.basename(image_path),
+                content_type=_guess_mime_type(image_path),
             )
+        elif input.video_paths and len(input.video_paths) > 0:
+            video_path = input.video_paths[0]
+            if _is_minimax_h3_mixed_reference(input.model):
+                with open(video_path, "rb") as reference_file:
+                    video_b64 = base64.b64encode(reference_file.read()).decode("ascii")
+                form.add_field(
+                    "video_reference",
+                    json.dumps([{"video_url": f"data:video/mp4;base64,{video_b64}"}]),
+                )
+            else:
+                image_file = open(video_path, "rb")
+                form.add_field(
+                    "input_reference",
+                    image_file,
+                    filename=os.path.basename(video_path),
+                    content_type=_guess_mime_type(video_path),
+                )
 
-    job_id = None
-    job_status = None
-    poll_json = {}
-    resp_json = {}
+        job_status = None
+        poll_json = {}
+        resp_json = {}
 
-    try:
         # invoke a post request (POST /v1/videos)
         async with session.post(input.api_url, data=form) as response:
             if response.status == 200:
@@ -406,7 +410,7 @@ async def async_request_v1_videos(
 
         # invoke a poll request (GET /v1/videos/{video_id})
         poll_interval = 2.0  # Unit(s)
-        timeout_seconds = 600.0
+        timeout_seconds = input.video_job_timeout
         deadline = time.perf_counter() + timeout_seconds
         job_url = f"{input.api_url}/{job_id}"
 
@@ -422,8 +426,11 @@ async def async_request_v1_videos(
                 poll_json = await poll_response.json()
                 job_status = poll_json.get("status")
 
-                if time.perf_counter() >= deadline:
-                    output.error = f"Timed out waiting for video job {job_id} to complete."
+                if job_status not in {"completed", "failed"} and time.perf_counter() >= deadline:
+                    output.error = (
+                        f"Timed out after {timeout_seconds:g}s waiting for video job {job_id} to complete. "
+                        "Increase --video-job-timeout for long-running or queued video jobs."
+                    )
                     output.success = False
                     return output
 
@@ -452,7 +459,7 @@ async def async_request_v1_videos(
             elif "peak_memory_mb" in resp_json:
                 output.peak_memory_mb = resp_json["peak_memory_mb"]
     except Exception as e:
-        output.error = str(e)
+        output.error = f"{type(e).__name__}: {e}"
         output.success = False
     finally:
         if image_file is not None:
@@ -465,13 +472,13 @@ async def async_request_v1_videos(
             except Exception as e:
                 print(f"Failed to clean up video job {job_id}: {e}")
 
-    output.latency = time.perf_counter() - output.start_time
+        output.latency = time.perf_counter() - output.start_time
+        if pbar is not None:
+            pbar.update(1)
 
     if output.success and input.slo_ms is not None:
         output.slo_achieved = (output.latency * 1000.0) <= float(input.slo_ms)
 
-    if pbar:
-        pbar.update(1)
     return output
 
 

benchmarks/diffusion/benchmark_qwen_image_qk_norm_rope.py

@@ -0,0 +1,222 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Benchmark Qwen-Image Q/K RMSNorm + interleaved RoPE.
+
+Example:
+
+    CUDA_VISIBLE_DEVICES=0 python benchmarks/diffusion/benchmark_qwen_image_qk_norm_rope.py \
+      --seq-len 4096 --warmup 20 --iters 100 --include-compiled --profile
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from collections.abc import Callable
+
+import torch
+import torch.nn.functional as F
+
+from vllm_omni.diffusion.layers.fused_qk_norm_rope import (
+    _fused_cuda_supported,
+    fused_qk_norm_rope,
+)
+from vllm_omni.diffusion.layers.rope import RotaryEmbedding
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--batch", type=int, default=1)
+    parser.add_argument("--seq-len", type=int, default=4096)
+    parser.add_argument("--heads", type=int, default=24)
+    parser.add_argument("--kv-heads", type=int, default=24)
+    parser.add_argument("--head-dim", type=int, default=128)
+    parser.add_argument("--warmup", type=int, default=20)
+    parser.add_argument("--iters", type=int, default=100)
+    parser.add_argument("--dtype", choices=("bf16", "fp16", "fp32"), default="bf16")
+    parser.add_argument("--packed-qkv-view", action="store_true")
+    parser.add_argument("--include-compiled", action="store_true")
+    parser.add_argument("--profile", action="store_true")
+    return parser.parse_args()
+
+
+def _dtype(name: str) -> torch.dtype:
+    return {
+        "bf16": torch.bfloat16,
+        "fp16": torch.float16,
+        "fp32": torch.float32,
+    }[name]
+
+
+def _make_inputs(args: argparse.Namespace, dtype: torch.dtype, device: torch.device):
+    torch.manual_seed(2026)
+    if args.packed_qkv_view:
+        qkv_dim = (args.heads + args.kv_heads + args.kv_heads) * args.head_dim
+        qkv = torch.randn(args.batch, args.seq_len, qkv_dim, device=device, dtype=dtype)
+        q, k, _ = qkv.split(
+            [
+                args.heads * args.head_dim,
+                args.kv_heads * args.head_dim,
+                args.kv_heads * args.head_dim,
+            ],
+            dim=-1,
+        )
+        q = q.unflatten(-1, (args.heads, args.head_dim))
+        k = k.unflatten(-1, (args.kv_heads, args.head_dim))
+    else:
+        q = torch.randn(args.batch, args.seq_len, args.heads, args.head_dim, device=device, dtype=dtype)
+        k = torch.randn(args.batch, args.seq_len, args.kv_heads, args.head_dim, device=device, dtype=dtype)
+
+    q_weight = torch.randn(args.head_dim, device=device, dtype=dtype)
+    k_weight = torch.randn(args.head_dim, device=device, dtype=dtype)
+    angles = torch.randn(args.seq_len, args.head_dim // 2, device=device, dtype=torch.float32)
+    freqs = torch.polar(torch.ones_like(angles), angles)
+    return q, k, q_weight, k_weight, freqs
+
+
+def _native(
+    q: torch.Tensor,
+    k: torch.Tensor,
+    q_weight: torch.Tensor,
+    k_weight: torch.Tensor,
+    freqs: torch.Tensor,
+    eps: float,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    q = F.rms_norm(q, (q.shape[-1],), q_weight, eps)
+    k = F.rms_norm(k, (k.shape[-1],), k_weight, eps)
+    rope = RotaryEmbedding(is_neox_style=False)
+    cos = freqs.real.to(q.dtype)
+    sin = freqs.imag.to(q.dtype)
+    return rope(q, cos, sin), rope(k, cos, sin)
+
+
+def _fused(
+    q: torch.Tensor,
+    k: torch.Tensor,
+    q_weight: torch.Tensor,
+    k_weight: torch.Tensor,
+    freqs: torch.Tensor,
+    eps: float,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    batch, seq_len, num_heads, head_dim = q.shape
+    num_kv_heads = k.shape[2]
+    rope_table = torch.cat((freqs.real, freqs.imag), dim=-1)
+    rope_table = rope_table.unsqueeze(0).expand(batch, -1, -1).reshape(batch * seq_len, head_dim)
+    out_q, out_k = fused_qk_norm_rope(
+        q.reshape(batch * seq_len, num_heads, head_dim),
+        k.reshape(batch * seq_len, num_kv_heads, head_dim),
+        q_weight,
+        k_weight,
+        rope_table,
+        eps,
+        interleaved=True,
+    )
+    return (
+        out_q.reshape(batch, seq_len, num_heads, head_dim),
+        out_k.reshape(batch, seq_len, num_kv_heads, head_dim),
+    )
+
+
+def _measure(fn: Callable[[], tuple[torch.Tensor, torch.Tensor]], warmup: int, iters: int) -> list[float]:
+    for _ in range(warmup):
+        fn()
+    torch.accelerator.synchronize()
+    start = torch.cuda.Event(enable_timing=True)
+    end = torch.cuda.Event(enable_timing=True)
+    samples = []
+    for _ in range(iters):
+        start.record()
+        fn()
+        end.record()
+        end.synchronize()
+        samples.append(start.elapsed_time(end))
+    return samples
+
+
+def _stats(samples: list[float]) -> dict[str, float]:
+    samples = sorted(samples)
+    return {
+        "median_ms": samples[len(samples) // 2],
+        "p90_ms": samples[int(len(samples) * 0.9)],
+        "mean_ms": sum(samples) / len(samples),
+    }
+
+
+def _profile(fn: Callable[[], tuple[torch.Tensor, torch.Tensor]]) -> str:
+    from torch.profiler import ProfilerActivity, profile
+
+    with profile(activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA]) as prof:
+        fn()
+        torch.accelerator.synchronize()
+    return prof.key_averages().table(sort_by="self_cuda_time_total", row_limit=15)
+
+
+def main() -> None:
+    args = _parse_args()
+    if not torch.cuda.is_available():
+        raise RuntimeError("This benchmark requires CUDA")
+    device = torch.device("cuda")
+    dtype = _dtype(args.dtype)
+    eps = 1e-6
+    q, k, q_weight, k_weight, freqs = _make_inputs(args, dtype, device)
+
+    def native():
+        return _native(q, k, q_weight, k_weight, freqs, eps)
+
+    def fused():
+        return _fused(q, k, q_weight, k_weight, freqs, eps)
+
+    results: dict[str, object] = {
+        "device": torch.cuda.get_device_name(),
+        "batch": args.batch,
+        "seq_len": args.seq_len,
+        "heads": args.heads,
+        "kv_heads": args.kv_heads,
+        "head_dim": args.head_dim,
+        "dtype": args.dtype,
+        "packed_qkv_view": args.packed_qkv_view,
+        "fused_fast_path_supported": _fused_cuda_supported(
+            q.reshape(args.batch * args.seq_len, args.heads, args.head_dim),
+            k.reshape(args.batch * args.seq_len, args.kv_heads, args.head_dim),
+            args.head_dim,
+            args.head_dim,
+            interleaved=True,
+        ),
+    }
+
+    native_samples = _measure(native, args.warmup, args.iters)
+    fused_samples = _measure(fused, args.warmup, args.iters)
+    expected_q, expected_k = native()
+    actual_q, actual_k = fused()
+    torch.accelerator.synchronize()
+
+    native_stats = _stats(native_samples)
+    fused_stats = _stats(fused_samples)
+    results["native"] = native_stats
+    results["fused"] = fused_stats
+    results["speedup_vs_native"] = native_stats["median_ms"] / fused_stats["median_ms"]
+    results["max_abs_q"] = (actual_q.float() - expected_q.float()).abs().max().item()
+    results["max_abs_k"] = (actual_k.float() - expected_k.float()).abs().max().item()
+    results["mean_abs_q"] = (actual_q.float() - expected_q.float()).abs().mean().item()
+    results["mean_abs_k"] = (actual_k.float() - expected_k.float()).abs().mean().item()
+
+    if args.include_compiled:
+        compiled_native = torch.compile(native, dynamic=True, fullgraph=True)
+        compiled_fused = torch.compile(fused, dynamic=True, fullgraph=True)
+        compiled_native_stats = _stats(_measure(compiled_native, args.warmup, args.iters))
+        compiled_fused_stats = _stats(_measure(compiled_fused, args.warmup, args.iters))
+        results["compiled_native"] = compiled_native_stats
+        results["compiled_fused"] = compiled_fused_stats
+        results["compiled_fused_speedup_vs_compiled_native"] = (
+            compiled_native_stats["median_ms"] / compiled_fused_stats["median_ms"]
+        )
+
+    print(json.dumps(results, indent=2))
+    if args.profile:
+        print("\n# fused profile")
+        print(_profile(fused))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/diffusion/diffusion_benchmark_serving.py

@@ -1,6 +1,6 @@
 # adapted from fastvideo
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
 """
 Benchmark online serving for diffusion models (Image/Video Generation).
@@ -87,6 +87,7 @@ import base64
 import glob
 import json
 import logging
+import math
 import os
 import random
 import tempfile
@@ -1086,6 +1087,12 @@ async def _run_warmups(
 
     warmup_tasks = [asyncio.create_task(limited_warmup_request_func(req)) for req in warmup_requests]
     warmup_outputs = await asyncio.gather(*warmup_tasks)
+    failed_count = sum(not output.success for output in warmup_outputs)
+    if failed_count:
+        raise RuntimeError(
+            f"{failed_count}/{len(warmup_outputs)} warmup requests failed; "
+            "check server logs before running the benchmark."
+        )
     return list(zip(warmup_requests, warmup_outputs))
 
 
@@ -1116,6 +1123,11 @@ def calculate_metrics(
         "duration": total_duration,
         "completed_requests": num_success,
         "failed_requests": len(error_outputs),
+        "request_errors": [
+            {"request_id": req.request_id, "error": out.error}
+            for req, out in zip(requests_list, outputs)
+            if not out.success
+        ],
         "throughput_qps": num_success / total_duration if total_duration > 0 else 0,
         "latency_mean": np.mean(latencies) if latencies else 0,
         "latency_median": np.median(latencies) if latencies else 0,
@@ -1261,6 +1273,9 @@ def _default_endpoint_for_task(task: str) -> str:
 
 
 async def benchmark(args):
+    if not math.isfinite(args.video_job_timeout) or args.video_job_timeout <= 0:
+        raise ValueError("--video-job-timeout must be a positive finite number of seconds.")
+
     # Construct base_url if not provided
     if args.base_url is None:
         args.base_url = f"http://{args.host}:{args.port}"
@@ -1312,6 +1327,8 @@ async def benchmark(args):
     print("Loading requests...")
     requests_list = dataset.get_requests()
     print(f"Prepared {len(requests_list)} requests from {args.dataset} dataset.")
+    for req in requests_list:
+        req.video_job_timeout = args.video_job_timeout
 
     if args.return_stage_metrics and args.endpoint in _STAGE_METRICS_ENDPOINTS:
         for req in requests_list:
@@ -1376,6 +1393,8 @@ async def benchmark(args):
     metrics["model"] = args.model
     metrics["dataset"] = args.dataset
     metrics["task"] = args.task
+    if args.endpoint == "/v1/videos":
+        metrics["video_job_timeout"] = args.video_job_timeout
     if args.endpoint == "/v1/images/edits":
         metrics["bot_task"] = args.bot_task
 
@@ -1398,6 +1417,11 @@ async def benchmark(args):
         )
     )
     print("{:<40} {}/{:<15}".format("Successful requests:", metrics["completed_requests"], len(requests_list)))
+    print("{:<40} {:<15}".format("Failed requests:", metrics["failed_requests"]))
+    for failure in metrics["request_errors"][:10]:
+        print(f"Request {failure['request_id']} failed: {failure['error']}")
+    if metrics["failed_requests"] > 10:
+        print("Showing first 10 failures; use --output-file to save all request errors.")
 
     # Section 3: Performance Metrics
     print(f"{'-' * 50}")
@@ -1437,7 +1461,7 @@ async def benchmark(args):
         print(f"Metrics saved to {args.output_file}")
 
 
-if __name__ == "__main__":
+def build_parser():
     parser = argparse.ArgumentParser(description="Benchmark serving for diffusion models.")
     parser.add_argument(
         "--base-url",
@@ -1481,6 +1505,13 @@ if __name__ == "__main__":
         help="Path to local dataset file (optional).",
     )
     parser.add_argument("--num-prompts", type=int, default=10, help="Number of prompts to benchmark.")
+    parser.add_argument(
+        "--video-job-timeout",
+        type=float,
+        default=900.0,
+        help="Maximum polling time in seconds per video job, including server queue time (default: 900). "
+        "Increase this for long-running video generation or high concurrency.",
+    )
     parser.add_argument(
         "--max-concurrency",
         type=int,
@@ -1612,6 +1643,10 @@ if __name__ == "__main__":
         action="store_true",
         help="Request stage duration metrics from endpoints that support return_stage_metrics.",
     )
+    return parser
 
+
+if __name__ == "__main__":
+    parser = build_parser()
     args = parser.parse_args()
     asyncio.run(benchmark(args))

benchmarks/host_weight_runtime/README.md

@@ -0,0 +1,57 @@
+# Host weight dependency memory diagnostic
+
+`safetensors_retention.py` isolates repeated CPU `get_tensor()` calls from
+access to one cached tensor view. It imports neither vLLM nor HWR and uses a
+synthetic float32 payload controlled by `--tensor-elements` (default: 64).
+Tensor size can affect retention: a negative result for 16 elements does not
+rule out growth for 64 elements. The selected size is recorded in the JSON
+arguments and used for both file creation and the correctness check.
+HWR itself calls `get_tensor()` when
+acquiring a lease, then exposes cached views; this probe does not measure HWR
+requests or repeated lease acquisition.
+
+Run one mode per fresh process in the same Python environment and CPU affinity:
+
+```bash
+git rev-parse HEAD > revision.txt
+git status --short > working-tree.txt
+git diff > working-tree.patch
+# Select an allowed CPU from your process affinity; 0 is only an example.
+CUDA_VISIBLE_DEVICES='' taskset -c 0 timeout 120s python \
+  benchmarks/host_weight_runtime/safetensors_retention.py \
+  --mode get_tensor --iterations 10000 --sample-every 5000 > feasibility.json
+```
+
+For a size comparison, run 16 and 64 elements twice each in fresh processes,
+with cached reuse as a control at both sizes:
+
+```bash
+for repetition in 1 2; do
+  for elements in 16 64; do
+    for mode in get_tensor reuse; do
+      CUDA_VISIBLE_DEVICES='' taskset -c 0 timeout 120s python \
+        benchmarks/host_weight_runtime/safetensors_retention.py \
+        --mode "$mode" --tensor-elements "$elements" \
+        > "$mode-$elements-$repetition.json"
+    done
+  done
+done
+```
+
+The diagnostic sets Torch to one CPU thread, warms up before the baseline,
+and reports preparation separately from timed loop work. Checkpoints run GC
+and report Linux private/anonymous memory, RSS, and open descriptors. It checks
+that transient tensor objects and the final cached view are released. Linux
+`/proc/self/smaps_rollup` must be readable; invalid arguments and missing proc
+support fail explicitly. Keep raw JSON and the repository snapshot with results.
+
+Compare the final loop sample with the warmed baseline, then inspect the closed
+sample. Private-memory growth after Python tensor objects disappear is evidence
+of retention in the dependency/allocator path, not proof of a leak's root cause.
+RSS can include file-backed pages; it is not a private-memory metric. Samples
+also include small diagnostic bookkeeping allocations. Do not assert a portable
+memory threshold or turn this synthetic loop into a per-request service estimate.
+
+Do not drop shared caches, trim allocators, or add serving-time GC to make the
+numbers look smaller. A dependency replacement or version change requires its
+own reproducible comparison and correctness/lifecycle validation.

benchmarks/host_weight_runtime/safetensors_retention.py

@@ -0,0 +1,120 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Linux CPU diagnostic: repeated safetensors materialization versus view reuse."""
+
+from __future__ import annotations
+
+import argparse
+import gc
+import json
+import os
+import platform
+import tempfile
+import time
+import weakref
+from pathlib import Path
+
+
+def positive_int(value: str) -> int:
+    parsed = int(value)
+    if parsed <= 0:
+        raise argparse.ArgumentTypeError("must be positive")
+    return parsed
+
+
+def sample(phase: str, iterations: int, loop_seconds: float) -> dict[str, object]:
+    gc.collect()
+    counters = {}
+    for line in Path("/proc/self/smaps_rollup").read_text().splitlines():
+        fields = line.split()
+        if len(fields) == 3 and fields[2] == "kB":
+            counters[fields[0].removesuffix(":")] = int(fields[1]) * 1024
+    return {
+        "phase": phase,
+        "iterations": iterations,
+        "loop_seconds": loop_seconds,
+        "rss_bytes": counters["Rss"],
+        "private_bytes": counters["Private_Clean"] + counters["Private_Dirty"],
+        "anonymous_bytes": counters["Anonymous"],
+        "fd_count": len(tuple(Path("/proc/self/fd").iterdir())),
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--mode", choices=("get_tensor", "reuse"), required=True)
+    parser.add_argument(
+        "--tensor-elements", type=positive_int, default=64, help="Number of float32 elements in the synthetic tensor"
+    )
+    parser.add_argument("--iterations", type=positive_int, default=1_000_000)
+    parser.add_argument("--sample-every", type=positive_int, default=100_000)
+    parser.add_argument("--warmup", type=positive_int, default=1000)
+    args = parser.parse_args()
+    if not Path("/proc/self/smaps_rollup").is_file():
+        parser.error("requires Linux /proc/self/smaps_rollup")
+
+    preparation_start = time.monotonic()
+    import safetensors
+    import torch
+    from safetensors import safe_open
+    from safetensors.torch import save_file
+
+    torch.set_num_threads(1)
+    torch.set_num_interop_threads(1)
+    samples = [sample("before_file", 0, 0.0)]
+    with tempfile.TemporaryDirectory(prefix="safetensors-retention-") as directory:
+        path = Path(directory) / "synthetic.safetensors"
+        save_file({"weight": torch.arange(args.tensor_elements, dtype=torch.float32)}, str(path))
+        with safe_open(path, framework="pt", device="cpu") as reader:
+            cached = reader.get_tensor("weight")
+            assert torch.equal(cached, torch.arange(args.tensor_elements, dtype=torch.float32))
+            for _ in range(args.warmup):
+                tensor = reader.get_tensor("weight") if args.mode == "get_tensor" else cached
+                del tensor
+            samples.append(sample("baseline", 0, 0.0))
+            preparation_seconds = time.monotonic() - preparation_start
+            completed = 0
+            loop_seconds = 0.0
+            while completed < args.iterations:
+                count = min(args.sample_every, args.iterations - completed)
+                start = time.monotonic()
+                for _ in range(count):
+                    tensor = reader.get_tensor("weight") if args.mode == "get_tensor" else cached
+                last_tensor = weakref.ref(tensor)
+                del tensor
+                loop_seconds += time.monotonic() - start
+                completed += count
+                snapshot = sample("loop", completed, loop_seconds)
+                snapshot["last_tensor_alive"] = last_tensor() is not None
+                if args.mode == "get_tensor":
+                    assert last_tensor() is None, "diagnostic retained a transient tensor"
+                samples.append(snapshot)
+            del cached
+        del reader
+        samples.append(sample("closed", completed, loop_seconds))
+        assert last_tensor() is None, "last tensor survived reader/view teardown"
+
+    print(
+        json.dumps(
+            {
+                "schema_version": 1,
+                "arguments": vars(args),
+                "python": platform.python_version(),
+                "torch": torch.__version__,
+                "safetensors": safetensors.__version__,
+                "kernel": platform.release(),
+                "hostname": platform.node(),
+                "pid": os.getpid(),
+                "cpu_affinity": sorted(os.sched_getaffinity(0)),
+                "torch_threads": torch.get_num_threads(),
+                "pythonmalloc": os.environ.get("PYTHONMALLOC", "default"),
+                "preparation_seconds": preparation_seconds,
+                "samples": samples,
+            },
+            indent=2,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

benchmarks/kernels/bench_qwen_select01_modulation.py

@@ -0,0 +1,333 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Benchmark the Qwen-Image-Edit select01 modulation hot path.
+
+This script intentionally starts with the current PyTorch implementation. It is
+used as the before/after baseline for SGLang PR #20395/#21318 style fusion:
+
+* layernorm + scale/shift + binary select01 gate
+* residual + gated update + layernorm + scale/shift + binary select01 gate
+
+Run on a CUDA machine for actionable numbers, for example:
+
+    python benchmarks/kernels/bench_qwen_select01_modulation.py \
+        --device cuda --dtype bf16 --seq-lens 4096 8192 16384 \
+        --output-json /tmp/qwen_select01_baseline.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import statistics
+import sys
+import time
+from collections.abc import Callable
+from pathlib import Path
+
+import torch
+import torch.nn.functional as F
+
+for candidate in Path(__file__).resolve().parents:
+    if (candidate / "vllm_omni").is_dir():
+        if str(candidate) not in sys.path:
+            sys.path.insert(0, str(candidate))
+        break
+
+from vllm_omni.diffusion.layers.qwen_select01_modulation import (  # noqa: E402
+    fused_layernorm_select01,
+    fused_residual_layernorm_select01,
+)
+
+
+def _dtype(name: str) -> torch.dtype:
+    return {
+        "fp32": torch.float32,
+        "float32": torch.float32,
+        "fp16": torch.float16,
+        "float16": torch.float16,
+        "bf16": torch.bfloat16,
+        "bfloat16": torch.bfloat16,
+    }[name]
+
+
+def _select01_modulation(
+    mod_params: torch.Tensor,
+    index: torch.Tensor | None,
+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+    shift, scale, gate = mod_params.chunk(3, dim=-1)
+
+    if index is None:
+        return scale.unsqueeze(1), shift.unsqueeze(1), gate.unsqueeze(1)
+
+    actual_batch = shift.size(0) // 2
+    shift_0, shift_1 = shift[:actual_batch], shift[actual_batch:]
+    scale_0, scale_1 = scale[:actual_batch], scale[actual_batch:]
+    gate_0, gate_1 = gate[:actual_batch], gate[actual_batch:]
+
+    idx = index.unsqueeze(-1) == 0
+    shift = torch.where(idx, shift_0.unsqueeze(1), shift_1.unsqueeze(1))
+    scale = torch.where(idx, scale_0.unsqueeze(1), scale_1.unsqueeze(1))
+    gate = torch.where(idx, gate_0.unsqueeze(1), gate_1.unsqueeze(1))
+    return scale, shift, gate
+
+
+def _native_layernorm_select01(
+    x: torch.Tensor,
+    mod_params: torch.Tensor,
+    index: torch.Tensor,
+    eps: float,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    scale, shift, gate = _select01_modulation(mod_params, index)
+    out = F.layer_norm(x.float(), (x.shape[-1],), eps=eps).to(x.dtype)
+    return out * (1 + scale) + shift, gate
+
+
+def _native_residual_layernorm_select01(
+    x: torch.Tensor,
+    residual: torch.Tensor,
+    residual_gate: torch.Tensor,
+    mod_params: torch.Tensor,
+    index: torch.Tensor,
+    eps: float,
+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+    scale, shift, gate = _select01_modulation(mod_params, index)
+    residual_out = residual + residual_gate * x
+    out = F.layer_norm(residual_out.float(), (residual_out.shape[-1],), eps=eps).to(residual_out.dtype)
+    return out * (1 + scale) + shift, residual_out, gate
+
+
+def _time_cuda(fn: Callable[[], object], warmup: int, iters: int) -> list[float]:
+    for _ in range(warmup):
+        fn()
+    torch.accelerator.synchronize()
+
+    times: list[float] = []
+    for _ in range(iters):
+        start = torch.cuda.Event(enable_timing=True)
+        end = torch.cuda.Event(enable_timing=True)
+        start.record()
+        fn()
+        end.record()
+        torch.accelerator.synchronize()
+        times.append(float(start.elapsed_time(end)))
+    return times
+
+
+def _reset_peak_memory_stats(device: torch.device) -> None:
+    reset_peak_memory_stats = getattr(torch.accelerator, "reset_peak_memory_stats", None)
+    if reset_peak_memory_stats is None:
+        reset_peak_memory_stats = getattr(getattr(torch, device.type), "reset_peak_memory_stats")
+    reset_peak_memory_stats(device)
+
+
+def _max_memory_allocated(device: torch.device) -> int:
+    max_memory_allocated = getattr(torch.accelerator, "max_memory_allocated", None)
+    if max_memory_allocated is None:
+        max_memory_allocated = getattr(getattr(torch, device.type), "max_memory_allocated")
+    return int(max_memory_allocated(device))
+
+
+def _time_cpu(fn: Callable[[], object], warmup: int, iters: int) -> list[float]:
+    for _ in range(warmup):
+        fn()
+
+    times: list[float] = []
+    for _ in range(iters):
+        t0 = time.perf_counter()
+        fn()
+        times.append((time.perf_counter() - t0) * 1000.0)
+    return times
+
+
+def _summarize(times: list[float]) -> dict[str, float]:
+    ordered = sorted(times)
+    return {
+        "median_ms": statistics.median(ordered),
+        "min_ms": ordered[0],
+        "p20_ms": ordered[max(0, int(len(ordered) * 0.20) - 1)],
+        "p80_ms": ordered[min(len(ordered) - 1, int(len(ordered) * 0.80))],
+    }
+
+
+def _make_inputs(
+    batch: int,
+    seq_len: int,
+    hidden_size: int,
+    dtype: torch.dtype,
+    device: torch.device,
+) -> dict[str, torch.Tensor]:
+    torch.manual_seed(0)
+    return {
+        "x": torch.randn(batch, seq_len, hidden_size, device=device, dtype=dtype),
+        "residual": torch.randn(batch, seq_len, hidden_size, device=device, dtype=dtype),
+        "residual_gate": torch.randn(batch, 1, hidden_size, device=device, dtype=dtype),
+        "mod_params": torch.randn(batch * 2, hidden_size * 3, device=device, dtype=dtype),
+        "index": torch.randint(0, 2, (batch, seq_len), device=device, dtype=torch.int64),
+    }
+
+
+def _bench_one(
+    name: str,
+    fn: Callable[[], object],
+    device: torch.device,
+    warmup: int,
+    iters: int,
+) -> dict[str, object]:
+    if device.type == "cuda":
+        _reset_peak_memory_stats(device)
+        times = _time_cuda(fn, warmup, iters)
+        peak_memory_mb = _max_memory_allocated(device) / 1024**2
+    else:
+        times = _time_cpu(fn, warmup, iters)
+        peak_memory_mb = None
+
+    return {
+        "name": name,
+        **_summarize(times),
+        "peak_memory_mb": peak_memory_mb,
+    }
+
+
+def _run(args: argparse.Namespace) -> dict[str, object]:
+    device = torch.device(args.device)
+    if device.type == "cuda" and not torch.cuda.is_available():
+        raise SystemExit("CUDA is not available on this machine.")
+
+    dtype = _dtype(args.dtype)
+    rows: list[dict[str, object]] = []
+
+    for seq_len in args.seq_lens:
+        tensors = _make_inputs(args.batch_size, seq_len, args.hidden_size, dtype, device)
+        impls = ["native"] if args.impl == "native" else ["fused"] if args.impl == "fused" else ["native", "fused"]
+
+        def first_norm() -> object:
+            return _native_layernorm_select01(
+                tensors["x"],
+                tensors["mod_params"],
+                tensors["index"],
+                args.eps,
+            )
+
+        def residual_norm() -> object:
+            return _native_residual_layernorm_select01(
+                tensors["x"],
+                tensors["residual"],
+                tensors["residual_gate"],
+                tensors["mod_params"],
+                tensors["index"],
+                args.eps,
+            )
+
+        callables: list[tuple[str, Callable[[], object]]] = []
+        if "native" in impls:
+            callables.extend(
+                [
+                    ("native_layernorm_select01", first_norm),
+                    ("native_residual_layernorm_select01", residual_norm),
+                ]
+            )
+        if "fused" in impls:
+
+            def fused_first_norm() -> object:
+                return fused_layernorm_select01(
+                    tensors["x"],
+                    tensors["mod_params"],
+                    tensors["index"],
+                    args.eps,
+                )
+
+            def fused_residual_norm() -> object:
+                return fused_residual_layernorm_select01(
+                    tensors["x"],
+                    tensors["residual"],
+                    tensors["residual_gate"].expand_as(tensors["residual"]),
+                    tensors["mod_params"],
+                    tensors["index"],
+                    args.eps,
+                )
+
+            callables.extend(
+                [
+                    ("fused_layernorm_select01", fused_first_norm),
+                    ("fused_residual_layernorm_select01", fused_residual_norm),
+                ]
+            )
+
+        if args.check and "fused" in impls:
+            ref_norm, ref_gate = first_norm()
+            actual_norm, actual_gate = fused_layernorm_select01(
+                tensors["x"],
+                tensors["mod_params"],
+                tensors["index"],
+                args.eps,
+            )
+            torch.testing.assert_close(actual_norm, ref_norm, atol=args.check_atol, rtol=args.check_rtol)
+            torch.testing.assert_close(actual_gate, ref_gate, atol=0, rtol=0)
+            ref_norm, ref_residual, ref_gate = residual_norm()
+            actual_norm, actual_residual, actual_gate = fused_residual_layernorm_select01(
+                tensors["x"],
+                tensors["residual"],
+                tensors["residual_gate"].expand_as(tensors["residual"]),
+                tensors["mod_params"],
+                tensors["index"],
+                args.eps,
+            )
+            torch.testing.assert_close(actual_norm, ref_norm, atol=args.check_atol, rtol=args.check_rtol)
+            torch.testing.assert_close(actual_residual, ref_residual, atol=args.check_atol, rtol=args.check_rtol)
+            torch.testing.assert_close(actual_gate, ref_gate, atol=0, rtol=0)
+
+        for name, fn in callables:
+            row = _bench_one(name, fn, device, args.warmup, args.iters)
+            row.update(
+                {
+                    "batch_size": args.batch_size,
+                    "seq_len": seq_len,
+                    "hidden_size": args.hidden_size,
+                    "dtype": args.dtype,
+                    "device": str(device),
+                }
+            )
+            rows.append(row)
+            print(
+                f"{name:<38} B={args.batch_size:<2} L={seq_len:<6} C={args.hidden_size:<5} "
+                f"median={row['median_ms']:.4f} ms p20={row['p20_ms']:.4f} p80={row['p80_ms']:.4f}"
+            )
+
+    result = {
+        "benchmark": "qwen_select01_modulation",
+        "torch": torch.__version__,
+        "cuda_available": torch.cuda.is_available(),
+        "gpu": torch.cuda.get_device_name(device) if device.type == "cuda" else None,
+        "rows": rows,
+    }
+
+    if args.output_json:
+        output_path = Path(args.output_json)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        output_path.write_text(json.dumps(result, indent=2), encoding="utf-8")
+
+    return result
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    default_device = "cuda" if torch.cuda.is_available() else "cpu"
+    parser.add_argument("--device", default=default_device)
+    parser.add_argument("--dtype", default="bf16", choices=["fp32", "float32", "fp16", "float16", "bf16", "bfloat16"])
+    parser.add_argument("--batch-size", type=int, default=1)
+    parser.add_argument("--seq-lens", type=int, nargs="+", default=[4096, 8192, 16384])
+    parser.add_argument("--hidden-size", type=int, default=3072)
+    parser.add_argument("--eps", type=float, default=1e-6)
+    parser.add_argument("--warmup", type=int, default=20)
+    parser.add_argument("--iters", type=int, default=100)
+    parser.add_argument("--impl", choices=["native", "fused", "all"], default="native")
+    parser.add_argument("--check", action="store_true")
+    parser.add_argument("--check-atol", type=float, default=5e-2)
+    parser.add_argument("--check-rtol", type=float, default=5e-2)
+    parser.add_argument("--output-json")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    _run(_parse_args())

benchmarks/kernels/benchmark_ming_istft.py

@@ -0,0 +1,218 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Measure Ming ISTFT and optionally its pretrained streaming AudioVAE decoder.
+
+Example (no model weights needed for the operator benchmark):
+    python benchmarks/kernels/benchmark_ming_istft.py --compile --iterations 100
+
+Add --decoder to measure the published decoder with fixed synthetic latents.
+This is a decoder-stage benchmark, not text-to-speech serving latency.
+"""
+
+import argparse
+import json
+import statistics
+import time
+from collections.abc import Callable
+from functools import partial
+
+import torch
+
+from vllm_omni.model_executor.models.common.ming.audio_dsp import ISTFT
+from vllm_omni.model_executor.models.common.ming.fused_istft import fused_istft
+
+
+def measure(operation: Callable, warmup: int, iterations: int) -> dict:
+    for _ in range(warmup):
+        operation()
+    torch.accelerator.synchronize()
+    samples = []
+    for _ in range(iterations):
+        start = time.perf_counter()
+        operation()
+        torch.accelerator.synchronize()
+        samples.append((time.perf_counter() - start) * 1000)
+    torch.accelerator.reset_peak_memory_stats()
+    before = torch.accelerator.memory_allocated()
+    operation()
+    torch.accelerator.synchronize()
+    return {
+        "p50_ms": statistics.median(samples),
+        "min_ms": min(samples),
+        "max_ms": max(samples),
+        "stdev_ms": statistics.pstdev(samples),
+        "extra_peak_mib": (torch.accelerator.max_memory_allocated() - before) / 1024**2,
+    }
+
+
+def native_post_fft(module, frames, audio, window):
+    """The native middle-chunk post-FFT sequence, isolated from cuFFT."""
+    count = frames.shape[-1]
+    size = (count - 1) * module.hop_length + module.win_length
+
+    def fold(value):
+        return torch.nn.functional.fold(
+            value, output_size=(1, size), kernel_size=(1, module.win_length), stride=(1, module.hop_length)
+        )[:, 0, 0, :]
+
+    samples = fold(frames * module.window[None, :, None])
+    samples, audio = module._buffer_process(samples, audio, 0, streaming=True)
+    envelope = fold(module.window.square().expand(1, count, -1).transpose(1, 2))
+    envelope, window = module._buffer_process(envelope, window, 0, streaming=True)
+    if not (envelope > 1e-11).all():
+        raise RuntimeError("ISTFT window envelope underflowed; invalid overlap-add state.")
+    return samples / envelope, audio, window
+
+
+def assert_outputs(actual, reference):
+    maximum = 0.0
+    for value, expected in zip(actual, reference, strict=True):
+        if expected is None:
+            assert value is None
+        else:
+            torch.testing.assert_close(value, expected, atol=5e-7, rtol=1e-5)
+            maximum = max(maximum, (value - expected).abs().max().item())
+    return maximum
+
+
+def benchmark_istft(args):
+    for dtype in (torch.float32, torch.bfloat16):
+        module = ISTFT(3528, 882, 3528).to(device="cuda", dtype=dtype)
+        for count in (16, 100, 400):
+            spec = torch.randn(1, 1765, count, device="cuda", dtype=torch.complex64)
+            _, audio, window = module.forward_native(spec, streaming=True)
+            frames = torch.fft.irfft(spec, 3528, dim=1)
+            for scope in ("post_fft", "istft"):
+                if scope == "post_fft":
+                    native = partial(native_post_fft, module, frames, audio, window)
+                    fused = partial(fused_istft, frames, module.window, 882, audio, window, True, False)
+                else:
+                    native = partial(module.forward_native, spec, audio, window, streaming=True)
+                    fused = partial(module, spec, audio, window, streaming=True)
+                error = assert_outputs(fused(), native())
+                results = {"native": measure(native, args.warmup, args.iterations)}
+                if args.compile:
+                    compiled = torch.compile(native)
+                    compiled_output, reference = compiled(), native()
+                    # Inductor can change BF16 envelope rounding. Report that
+                    # difference without weakening the fused-kernel parity gate.
+                    compiled_matches = all(
+                        torch.allclose(value, expected, atol=5e-7, rtol=1e-5)
+                        for value, expected in zip(compiled_output, reference, strict=True)
+                    )
+                    results["compiled"] = measure(compiled, args.warmup, args.iterations)
+                    results["compiled"]["matches_native_tolerance"] = compiled_matches
+                    results["compiled"]["waveform_max_abs_error"] = (
+                        (compiled_output[0] - reference[0]).abs().max().item()
+                    )
+                results["fused"] = measure(fused, args.warmup, args.iterations)
+                print(
+                    json.dumps(
+                        {
+                            "scope": scope,
+                            "window_dtype": str(dtype),
+                            "frames": count,
+                            "max_abs_error": error,
+                            "results": results,
+                        }
+                    ),
+                    flush=True,
+                )
+
+
+def benchmark_decoder(args):
+    from huggingface_hub import hf_hub_download
+    from safetensors import safe_open
+
+    from vllm_omni.model_executor.models.common.ming.audio_vae import Decoder
+
+    model = "inclusionAI/Ming-omni-tts-0.5B"
+    revision = "9154772e7fbc585907b6237e3190790676f28975"
+    config_path = hf_hub_download(model, "config.json", revision=revision)
+    with open(config_path) as source:
+        config = json.load(source)["audio_tokenizer_config"]
+    kwargs = config["dec_kwargs"]
+    backbone = kwargs["backbone"]
+    backbone["_attn_implementation"] = "sdpa"
+    decoder = Decoder(backbone, kwargs["output_dim"], kwargs["latent_dim"], config["patch_size"])
+    checkpoint = hf_hub_download(model, "model.safetensors", revision=revision)
+    prefix = "audio.decoder."
+    with safe_open(checkpoint, framework="pt") as weights:
+        state = {key.removeprefix(prefix): weights.get_tensor(key) for key in weights.keys() if key.startswith(prefix)}
+    decoder.load_state_dict(state, strict=True)
+    decoder = decoder.eval().to(device="cuda", dtype=torch.bfloat16)
+    chunks = [
+        torch.randn(1, size, kwargs["latent_dim"], device="cuda", dtype=torch.bfloat16) for size in (4, 25, 25, 5)
+    ]
+    istft = decoder.head.istft
+    fused_forward = istft.forward
+
+    def run():
+        cache = state = None
+        outputs = []
+        for index, chunk in enumerate(chunks):
+            output, state, cache = decoder.low_level_reconstruct(
+                chunk, past_key_values=cache, use_cache=True, stream_state=state, last_chunk=index == len(chunks) - 1
+            )
+            outputs.append(output)
+        return torch.cat(outputs, dim=-1)
+
+    istft.forward = istft.forward_native
+    expected = run()
+    istft.forward = fused_forward
+    actual = run()
+    torch.testing.assert_close(actual, expected, atol=5e-7, rtol=1e-5)
+    # Alternate the order to expose drift when the decoder-level gain is small.
+    results = {"native": [], "fused": []}
+    for order in (("native", "fused"), ("fused", "native"), ("native", "fused")):
+        for name in order:
+            istft.forward = istft.forward_native if name == "native" else fused_forward
+            results[name].append(measure(run, args.warmup, args.decoder_iterations))
+    print(
+        json.dumps(
+            {
+                "scope": "pretrained_decoder",
+                "model": model,
+                "revision": revision,
+                "backend": "sdpa",
+                "dtype": "bfloat16",
+                "latent_chunks": [4, 25, 25, 5],
+                "max_abs_error": (actual - expected).abs().max().item(),
+                "results": results,
+            }
+        ),
+        flush=True,
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--warmup", type=int, default=20)
+    parser.add_argument("--iterations", type=int, default=100)
+    parser.add_argument("--compile", action="store_true")
+    parser.add_argument("--decoder", action="store_true")
+    parser.add_argument("--decoder-iterations", type=int, default=30)
+    args = parser.parse_args()
+    if args.warmup < 0 or min(args.iterations, args.decoder_iterations) < 1:
+        parser.error("warmup must be nonnegative and iteration counts must be positive")
+    torch.manual_seed(0)
+    print(
+        json.dumps(
+            {
+                "torch": torch.__version__,
+                "cuda": torch.version.cuda,
+                "gpu": torch.cuda.get_device_name(),
+                "warmup": args.warmup,
+                "iterations": args.iterations,
+                "decoder_iterations": args.decoder_iterations,
+            }
+        )
+    )
+    with torch.inference_mode():
+        benchmark_istft(args)
+        if args.decoder:
+            benchmark_decoder(args)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/kernels/mot_linear_benchmarks.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 # ruff: noqa: N803
 
 """MoT (Mixture-of-Tokens) GEMM kernel benchmark and auto-tuning.
@@ -10,7 +13,7 @@ Usage:
     python benchmarks/kernels/mot_linear_benchmarks.py \
         --model ByteDance-Seed/BAGEL-7B-MoT \
         --tp-size 1 --dtype w16a16 --tune \
-        --save-dir vllm_omni/diffusion/layers/mot/configs/
+        --save-dir vllm_omni/diffusion/models/bagel/mot/configs/
 
     # Auto-tune with local model path (offline clusters):
     python benchmarks/kernels/mot_linear_benchmarks.py \
@@ -43,7 +46,7 @@ from vllm.triton_utils import triton
 from vllm.utils.torch_utils import set_random_seed
 
 # NOTE: you should use the same naming syetem for the kernel to load properly
-from vllm_omni.diffusion.layers.mot.ops.mot_gemm import build_config_filename, get_device_name
+from vllm_omni.diffusion.models.bagel.mot.ops.mot_gemm import build_config_filename, get_device_name
 
 # clear the triton cache from time to time, usaully no need to change
 _CACHE_CLEAR_INTERVAL_ENV = "VLLM_MOT_TUNE_CACHE_CLEAR_INTERVAL"
@@ -428,7 +431,7 @@ def benchmark_config(
     cache_flusher: torch.Tensor | None = None,
 ) -> float:
     """Run a MoT GEMM with the given tile config and return avg latency (us)."""
-    from vllm_omni.diffusion.layers.mot.ops.mot_gemm import invoke_mot_gemm
+    from vllm_omni.diffusion.models.bagel.mot.ops.mot_gemm import invoke_mot_gemm
 
     text_indices, vae_indices, M = build_regular_indices(
         image_num=image_num,
@@ -553,7 +556,7 @@ class BenchmarkWorker:
         use_int8_w8a16: bool,
     ) -> tuple[dict[str, int], float]:
         set_random_seed(self.seed)
-        from vllm_omni.diffusion.layers.mot.ops.mot_gemm import (
+        from vllm_omni.diffusion.models.bagel.mot.ops.mot_gemm import (
             get_best_mot_config,
         )
 

benchmarks/lingbot_world/README.md

@@ -0,0 +1,224 @@
+# LingBot-World Benchmarks — Realtime Streaming Cadence
+
+`benchmark_lingbot_world_realtime.py` measures LingBot-World 2.0 as it is
+actually deployed: one `WS /v1/realtime/video` session that streams one video
+chunk per AR block, for as long as the world keeps running.
+
+## Why this is not the diffusion serving benchmark
+
+[`benchmarks/diffusion`](../diffusion/README.md) measures "one request, one
+finished video" and reports request throughput. LingBot-World has neither shape:
+
+- **One request is the whole rollout.** `prepare_encode` runs once, then every AR
+  block is four DMD steps plus one clean-KV commit forward, and each block
+  streams out as a chunk. Throughput in requests per second is meaningless.
+- **There is exactly one session.** The stepwise deploy config pins
+  `max_num_seqs: 1`, and the AR-Diffusion consumer rejects a stage with more than
+  one replica, because session-affine routing does not exist yet. There is no
+  concurrency to sweep.
+- **What matters is cadence.** A deployment is healthy when chunk *N + 1* arrives
+  before a viewer finishes watching chunk *N*. That is a latency distribution
+  against a wall clock, not a throughput number.
+
+## Quick start
+
+Serve the model, then point the benchmark at it. The benchmark never starts a
+server.
+
+```bash
+# Terminal 1 — one GPU, eager (portable smoke configuration)
+vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+  --omni \
+  --deploy-config benchmarks/lingbot_world/configs/single_gpu_eager.yaml \
+  --port 8000
+
+# Terminal 2
+pip install websockets
+python benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py \
+  --host 127.0.0.1 --port 8000 \
+  --num-chunks 16 \
+  --output-json /tmp/lingbot_world_realtime.json
+```
+
+With no `--image`, the first frame comes from the shared vLLM image asset, so the
+default workload needs no setup and this repository carries no benchmark binary.
+Pass `--image /path/to/frame.png` to condition the rollout on your own frame.
+
+### Realtime topology
+
+The single-GPU config is a smoke configuration, not a realtime one: one card runs
+all five forwards of every block serially and nothing is compiled. Use the
+four-GPU config for numbers that mean anything about realtime behavior.
+
+```bash
+vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+  --omni \
+  --deploy-config benchmarks/lingbot_world/configs/usp4_compiled.yaml \
+  --port 8000
+
+python benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py \
+  --port 8000 --num-chunks 40 --warmup-sessions 1 --sessions 3 \
+  --target-fps 12 --output-json /tmp/lingbot_world_repeated.json
+```
+
+`usp4_compiled.yaml` sets pure Ulysses sequence parallelism over four ranks and
+`enforce_eager: false`. The pipeline validates the parallel shape itself:
+`sequence_parallel_size` must equal `ulysses_degree`, `ring_degree` and
+`allgather_degree` must be 1, `ulysses_mode` must be `strict`, and pipeline,
+CFG, and VAE parallel sizes above 1 are rejected. This benchmark config sets
+`tensor_parallel_size: 1`; the pipeline validator does not reject larger TP sizes.
+
+Compiled mode can pay compilation and capture costs on the first rollout.
+Use `--warmup-sessions 1` before measured repetitions. Excluding the first six
+chunks addresses the attention-window ramp; it does not guarantee that
+compilation and capture have finished.
+
+## Metrics
+
+| Metric | Meaning |
+| --- | --- |
+| `ttfc_ms` | **Time to first chunk**: `session.start` sent until the first chunk's bytes arrive. Pays for prompt encode, first-frame VAE encode, and any compile or capture. |
+| `interval_all` | Inter-arrival latency across every chunk: mean, median, std, min, p90, p95, p99, max. |
+| `interval_steady` | The same summary over steady-state chunks only. |
+| `video_rtf` | Wall seconds spent per second of video, matching vLLM-Omni's own RTF (`generation_s / video_duration`). **Lower is better; below 1.0 is real time.** |
+| `video_rtfx` | The reciprocal, under the name this repo uses for it: video seconds per wall second, where higher is better. |
+| `steady_video_rtf` | `video_rtf` restricted to steady-state chunks. |
+| `chunk_deadline_ms` | Wall-clock budget one steady chunk must meet to hold real time: 12 frames ÷ `--target-fps`. Override directly with `--chunk-slo-ms`. |
+| `slo_attainment` | Fraction of steady chunks that met the deadline. |
+| `playback` | Underrun count, total stall, and stall ratio from a simulated viewer. |
+
+### Which frame rate counts as real time
+
+**The checkpoint declares no frame rate.** There is none in `model_index.json`, the
+scheduler config, or the transformer config, and the pipeline never reads one. A
+chunk is twelve pixel frames; how long those frames *last* is a property of how the
+output is played, not of the model.
+
+That choice moves the verdict without moving anything measured:
+
+| basis | video per chunk | deadline | RTF at a measured 1357 ms/chunk |
+| --- | --- | --- | --- |
+| 16 fps | 0.750 s | 750 ms | 1.809 |
+| 12 fps | 1.000 s | 1000 ms | 1.357 |
+
+So `--fps` (sent to the server, which uses it to label the muxed MP4) and
+`--target-fps` (the real-time basis for RTF, the deadline, and the playback
+simulation) are separate flags. `--target-fps` defaults to `--fps`; set it when the
+two differ, and the report prints both. State the basis whenever you quote a number
+— an RTF without its fps is not a measurement. (RTF is a cost here, so a bigger number is a
+worse result; a steady chunk's RTF is simply its interval divided by its video duration.)
+An RTF above 1 means the measured deployment is slower than real time at the
+stated target frame rate. The benchmark measures this failure as well as success.
+
+### Why warmup chunks are not optional
+
+The DiT attends over a sliding window of `sliding_window_num_frames = 18` latent
+frames with `sink_size = 9`, so the key count a block attends over keeps growing
+until eighteen latent frames of history exist — six chunks at three frames per
+block. **Chunk cost is not stationary before chunk six.** A mean over a short
+rollout is therefore lower than anything a running session will sustain, which is
+why `--warmup-chunks` defaults to 6 and steady-state metrics are reported
+separately. Keep `--num-chunks` comfortably above the warmup, or the benchmark
+tells you no chunk reached steady state. The terminal chunk skips next-chunk
+preparation, so it is excluded from steady metrics too; it remains in overall
+RTF, all-chunk intervals, and playback simulation.
+
+The default 16-chunk rollout provides only nine steady intervals. Percentiles
+are interpolated sample summaries; the report warns when fewer than 100
+intervals contribute, and even 100 samples do not guarantee a stable p99.
+Use longer rollouts for tail analysis and at least `--sessions 3` after
+`--warmup-sessions 1` for reported comparisons. Session warmup pays compilation
+cost; chunk warmup separately excludes the attention-window ramp. The command
+above yields 33 steady intervals per session (indices 6 through 38), or 99 across
+three sessions. Report each session's mean and the spread across sessions; this
+sample still does not establish a reliable p99.
+
+### Why playback is simulated
+
+A p99 interval alone cannot say whether a viewer saw a stall: the same p99 is
+invisible behind a three-chunk buffer and a visible freeze behind none. The
+benchmark replays chunk arrivals against a wall clock — playback starts once
+`--playback-buffer-chunks` chunks have landed and then consumes video at `--target-fps`
+— and reports every moment the player ran dry.
+
+### What the client-side number is worth
+
+On a four-GPU Ulysses run the client's mean inter-arrival over all fifteen
+post-first-chunk intervals was **1326.743 ms**, against the server's own
+`StageRequestStats.inter_output_latency_ms` of **1327.479 ms** for the same
+request — a difference of **0.74 ms**. This cross-check shows agreement in
+average output cadence for that run, including warmup and the terminal chunk.
+It does not measure encoding or transport latency: a fixed delivery delay can
+cancel out of inter-arrival differences. Attributing that latency requires
+per-chunk timestamps at both boundaries or server-side profiling.
+
+Compare matching interval populations when investigating discrepancies; both
+measurement bugs and delivery buffering can change the observed cadence.
+
+### What this benchmark cannot see
+
+Chunk metadata carries no server-side timestamps, so the benchmark cannot split
+DiT time from VAE decode time; use `--enable-diffusion-pipeline-profiler` on the
+server for that. It also reports no GPU memory, which is a server-side quantity.
+
+## Custom rollouts
+
+`--workload rollout.json` replaces the built-in workload, including mid-rollout
+prompt changes, which the CLI flags cannot express:
+
+```json
+{
+  "prompt": "The camera moves slowly forward through the scene.",
+  "image": "/path/to/first_frame.png",
+  "num_chunks": 24,
+  "camera_pattern": "orbit",
+  "fps": 16,
+  "seed": 42,
+  "prompt_updates": [
+    {"after_chunk": 8, "prompt": "Rain begins to fall", "transition_chunks": 2}
+  ]
+}
+```
+
+Supply `camera_action_script` directly instead of `num_chunks` to control every
+frame's action. It must hold exactly one three-entry action list per chunk; one
+AR block is three latent frames, and the server rejects any other shape.
+Explicit `--image`, `--prompt`, `--negative-prompt`, `--width`, `--height`,
+`--fps`, `--seed`, and `--flow-shift` flags override the matching workload-file
+values. `--num-chunks` and `--camera-pattern` apply only to the built-in
+workload; when using `--workload`, set those fields in the file instead.
+An explicit `camera_action_script` takes precedence over the file's
+`num_chunks` and `camera_pattern`. Each prompt update must use a distinct
+`after_chunk` boundary so its interaction event ID is unique.
+
+Camera actions change what the world does, not what it costs, so `--camera-pattern`
+(`forward`, `orbit`, `hold`) keeps a rollout representative rather than sweeping a
+cost dimension.
+
+## Options worth knowing
+
+| Flag | Purpose |
+| --- | --- |
+| `--target-fps` | Playback rate the real-time verdict is measured against, independent of the mux label. |
+| `--sessions N` | N sequential measured rollouts; the report adds spread across them. |
+| `--warmup-sessions N` | Unmeasured rollouts first, to pay compile and capture cost. |
+| `--print-chunks` | One line per chunk as it arrives. |
+| `--save-video PATH` | Remux the last measured session for eyeballing (needs vLLM-Omni). |
+| `--ping-interval` | `session.ping` cadence keeping the server's stall clock fresh during a slow first chunk. `0` disables. |
+| `--first-chunk-timeout` | Raise it when the server is still compiling or capturing graphs. |
+
+## Output
+
+`--output-json` writes the run's configuration, every chunk's arrival time and
+size, per-session metrics, and the aggregate, along with the git revision and
+hostname so a number can be traced back to what produced it.
+
+## Tests
+
+`tests/benchmarks/test_lingbot_world_realtime.py` covers the chunk and frame
+arithmetic against the server's own formula, the metric math, and the protocol
+handling against a scripted WebSocket server. It needs no GPU:
+
+```bash
+pytest tests/benchmarks/test_lingbot_world_realtime.py
+```

benchmarks/lingbot_world/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py

@@ -0,0 +1,642 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Realtime streaming benchmark for LingBot-World 2.0 over ``WS /v1/realtime/video``.
+
+Every other diffusion benchmark in this repository measures "one request, one
+finished video". LingBot-World is not that: one request is a whole rollout, and
+the thing a deployment lives or dies by is whether chunk *N + 1* keeps arriving
+before a viewer has finished watching chunk *N*. This benchmark therefore
+measures a **cadence**, not a throughput:
+
+* ``TTFC`` -- session.start to the first chunk's bytes, which pays for prompt
+  encode, the first-frame VAE encode, and any compile or capture the deploy
+  config asks for;
+* **inter-arrival latency** over all chunks and, separately, over the steady
+  state after the attention window saturates;
+* ``VIDEO_RTF`` -- wall seconds spent per second of video generated, matching how
+  vLLM-Omni computes RTF everywhere else (``generation_s / video_duration``).
+  **Lower is better and below 1.0 is real time**; ``VIDEO_RTFX`` is reported
+  alongside it as the reciprocal, under the repo's own name for that;
+* **playback continuity** -- a simulated viewer with a start buffer, because a
+  p99 that is harmless behind a three-chunk buffer is a visible stall behind
+  none.
+
+The server is not started here. Bring one up first (see the README), then point
+this script at it. Requires ``websockets``; ``--save-video`` additionally needs
+a vLLM-Omni installation for the remux helper.
+
+Example:
+
+    python benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py \\
+      --host 127.0.0.1 --port 8000 \\
+      --num-chunks 16 \\
+      --output-json /tmp/lingbot_world_realtime.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import base64
+import json
+import mimetypes
+import os
+import platform
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from benchmarks.lingbot_world.workload import (  # noqa: E402
+    CAMERA_PATTERNS,
+    DEFAULT_FLOW_SHIFT,
+    DEFAULT_FPS,
+    DEFAULT_HEIGHT,
+    DEFAULT_MODEL,
+    DEFAULT_PROMPT,
+    DEFAULT_SEED,
+    DEFAULT_WARMUP_CHUNKS,
+    DEFAULT_WIDTH,
+    ChunkRecord,
+    Workload,
+    aggregate_metrics,
+    compute_metrics,
+    default_workload,
+    load_workload,
+    steady_chunk_deadline_ms,
+)
+
+# The shared vLLM asset keeps the default workload runnable with no setup and
+# adds no binary to this repository. Any real photograph conditions a rollout.
+_DEFAULT_IMAGE_ASSET = "2560px-Gfp-wisconsin-madison-the-nature-boardwalk"
+_WS_PATH = "/v1/realtime/video"
+
+
+class BenchmarkError(RuntimeError):
+    """A session failed in a way that invalidates its numbers."""
+
+
+# --------------------------------------------------------------------------- #
+# Input resolution
+# --------------------------------------------------------------------------- #
+
+
+def resolve_image_reference(value: str | None) -> str:
+    """Return an ``http(s)`` or ``data:`` URL for the session's first frame."""
+    if value is None:
+        from vllm.assets.image import ImageAsset
+
+        value = str(ImageAsset(_DEFAULT_IMAGE_ASSET).get_path("jpg"))
+    if value.startswith(("http://", "https://", "data:")):
+        return value
+    path = Path(value).expanduser()
+    if not path.is_file():
+        raise BenchmarkError(f"--image is neither a URL nor a readable file: {value}")
+    media_type = mimetypes.guess_type(path.name)[0] or "image/png"
+    return f"data:{media_type};base64,{base64.b64encode(path.read_bytes()).decode()}"
+
+
+def build_workload(args: argparse.Namespace) -> Workload:
+    if args.workload is not None:
+        # The file names its own first frame, so resolving happens inside
+        # load_workload -- never eagerly, which would fetch the default asset a
+        # --workload run has no use for.
+        return load_workload(
+            args.workload,
+            image_resolver=lambda value: resolve_image_reference(value if args.image is None else args.image),
+            overrides={
+                "width": args.width_override,
+                "height": args.height_override,
+                "fps": args.fps_override,
+                "seed": args.seed_override,
+                "prompt": args.prompt_override,
+                "flow_shift": args.flow_shift,
+                "negative_prompt": args.negative_prompt,
+            },
+        )
+    return default_workload(
+        image_reference=resolve_image_reference(args.image),
+        num_chunks=args.num_chunks,
+        camera_pattern=args.camera_pattern,
+        prompt=args.prompt_override if args.prompt_override is not None else DEFAULT_PROMPT,
+        width=args.width_override if args.width_override is not None else DEFAULT_WIDTH,
+        height=args.height_override if args.height_override is not None else DEFAULT_HEIGHT,
+        fps=args.fps_override if args.fps_override is not None else DEFAULT_FPS,
+        seed=args.seed_override if args.seed_override is not None else DEFAULT_SEED,
+        flow_shift=args.flow_shift if args.flow_shift is not None else DEFAULT_FLOW_SHIFT,
+        negative_prompt=args.negative_prompt,
+    )
+
+
+# --------------------------------------------------------------------------- #
+# One streaming session
+# --------------------------------------------------------------------------- #
+
+
+@dataclass
+class SessionResult:
+    request_id: str | None
+    records: list[ChunkRecord]
+    media: bytes
+    ttfc_s: float | None
+    video_start_s: float | None
+    wall_s: float
+    done_event: dict[str, Any]
+
+
+async def _keepalive(websocket: Any, send_lock: asyncio.Lock, interval: float) -> None:
+    """Refresh the server's stall clock while a slow first chunk is generating.
+
+    A compiled deploy config records CUDA graphs on the first block, which can
+    outlast the server's ~60 s stall timeout. ``session.ping`` is the documented
+    way to say the client is still there.
+    """
+    while True:
+        await asyncio.sleep(interval)
+        async with send_lock:
+            await websocket.send(json.dumps({"type": "session.ping"}))
+
+
+async def run_session(
+    url: str,
+    model: str,
+    workload: Workload,
+    *,
+    first_chunk_timeout: float,
+    chunk_timeout: float,
+    ping_interval: float,
+    collect_media: bool,
+    print_chunks: bool,
+) -> SessionResult:
+    """Drive one rollout and timestamp every chunk as its bytes land."""
+    try:
+        from websockets.asyncio.client import connect
+    except ImportError:  # pragma: no cover - exercised only without websockets
+        try:
+            from websockets import connect  # type: ignore[attr-defined]
+        except ImportError as exc:
+            raise BenchmarkError("This benchmark needs the websockets package: pip install websockets") from exc
+
+    payload = workload.session_start_payload(model)
+    records: list[ChunkRecord] = []
+    media = bytearray()
+    pending: dict[str, Any] | None = None
+    request_id: str | None = None
+    ttfc_s: float | None = None
+    video_start_s: float | None = None
+    last_arrival: float | None = None
+    done_event: dict[str, Any] = {}
+    send_lock = asyncio.Lock()
+    keepalive: asyncio.Task[None] | None = None
+    # Fire remaining prompt updates in chunk order as their boundary passes.
+    updates = sorted(workload.prompt_updates, key=lambda update: update.after_chunk)
+    update_index = 0
+
+    async with connect(url, max_size=None, ping_interval=None) as websocket:
+        async with send_lock:
+            await websocket.send(json.dumps(payload, ensure_ascii=False))
+        started = time.perf_counter()
+        if ping_interval > 0:
+            keepalive = asyncio.create_task(_keepalive(websocket, send_lock, ping_interval))
+
+        try:
+            while True:
+                timeout = first_chunk_timeout if not records else chunk_timeout
+                try:
+                    deadline = started + (last_arrival if last_arrival is not None else 0.0) + timeout
+                    remaining = deadline - time.perf_counter()
+                    if remaining <= 0:
+                        raise asyncio.TimeoutError
+                    message = await asyncio.wait_for(websocket.recv(), timeout=remaining)
+                except asyncio.TimeoutError as exc:
+                    raise BenchmarkError(
+                        f"No media chunk for {timeout:g} s after {len(records)} chunk(s); "
+                        "the server may still be loading, compiling, or capturing graphs."
+                    ) from exc
+                now = time.perf_counter()
+
+                if isinstance(message, (bytes, bytearray)):
+                    if pending is None:
+                        raise BenchmarkError("Received a binary frame with no preceding video.chunk_metadata event.")
+                    if collect_media:
+                        media.extend(message)
+                    if pending.get("kind") == "media":
+                        index = int(pending["generation_chunk_index"])
+                        arrival = now - started
+                        if ttfc_s is None:
+                            ttfc_s = arrival
+                        records.append(
+                            ChunkRecord(
+                                index=index,
+                                arrival_s=arrival,
+                                inter_arrival_s=arrival - (last_arrival if last_arrival is not None else 0.0),
+                                byte_length=len(message),
+                                num_frames=int(pending["num_frames"]),
+                            )
+                        )
+                        last_arrival = arrival
+                        if print_chunks:
+                            print(
+                                f"  chunk {index:3d}  t={arrival:8.3f}s  "
+                                f"interval={records[-1].inter_arrival_s * 1000:8.1f}ms  "
+                                f"frames={records[-1].num_frames:3d}  bytes={len(message)}",
+                                flush=True,
+                            )
+                        while update_index < len(updates) and updates[update_index].after_chunk <= index:
+                            async with send_lock:
+                                await websocket.send(json.dumps(updates[update_index].to_payload(), ensure_ascii=False))
+                            update_index += 1
+                    pending = None
+                    continue
+
+                event = json.loads(message)
+                event_type = event.get("type")
+                if event_type == "video.chunk_metadata":
+                    pending = event
+                elif event_type == "video.start":
+                    request_id = event.get("request_id")
+                    video_start_s = now - started
+                elif event_type == "error":
+                    # Say what was asked for alongside what the server refused:
+                    # some builds cap a rollout's num_frames, and the operator
+                    # needs to know which --num-chunks would fit.
+                    raise BenchmarkError(
+                        f"Server rejected the session after {len(records)} chunk(s): "
+                        f"{event.get('message', event)} "
+                        f"(requested {workload.num_chunks} chunks = num_frames {workload.num_frames})"
+                    )
+                elif event_type == "session.done":
+                    done_event = event
+                    break
+        finally:
+            if keepalive is not None:
+                keepalive.cancel()
+                try:
+                    await keepalive
+                except asyncio.CancelledError:
+                    pass
+
+        wall_s = time.perf_counter() - started
+
+    if not records:
+        raise BenchmarkError("The session finished without delivering a single media chunk.")
+    if done_event.get("stopped"):
+        raise BenchmarkError(f"The session was stopped before it finished: {done_event}")
+    observed = [record.index for record in records]
+    if observed != list(range(len(records))):
+        raise BenchmarkError(f"Chunk indices are not contiguous from zero: {observed}")
+    if len(records) != workload.num_chunks:
+        raise BenchmarkError(
+            f"Expected {workload.num_chunks} chunks for num_frames={workload.num_frames}, received {len(records)}."
+        )
+
+    return SessionResult(
+        request_id=request_id,
+        records=records,
+        media=bytes(media),
+        ttfc_s=ttfc_s,
+        video_start_s=video_start_s,
+        wall_s=wall_s,
+        done_event=done_event,
+    )
+
+
+# --------------------------------------------------------------------------- #
+# Reporting
+# --------------------------------------------------------------------------- #
+
+
+def _git_revision() -> str | None:
+    try:
+        return subprocess.check_output(
+            ["git", "-C", str(REPO_ROOT), "rev-parse", "HEAD"],
+            text=True,
+            stderr=subprocess.DEVNULL,
+        ).strip()
+    except (OSError, subprocess.CalledProcessError):
+        return None
+
+
+def _print_row(label: str, value: str) -> None:
+    print(f"{label:<44}{value:>16}")
+
+
+def print_report(
+    metrics: dict[str, Any],
+    *,
+    workload: Workload,
+    endpoint: str,
+    sessions: int,
+    target_fps: float,
+) -> None:
+    header = " LingBot-World realtime streaming benchmark "
+    print("\n{s:{c}^{n}}".format(s=header, n=60, c="="))
+    _print_row("Endpoint:", endpoint)
+    _print_row("Geometry:", f"{workload.width}x{workload.height}")
+    _print_row("Chunks per session:", str(workload.num_chunks))
+    _print_row("Sessions:", str(sessions))
+    frames = workload.num_frames
+    _print_row("Real-time basis:", f"{target_fps:g} fps")
+    _print_row("Video seconds per session:", f"{frames / target_fps:.2f} ({frames} frames)")
+    if abs(target_fps - workload.fps) > 1e-9:
+        _print_row("Mux label sent to server:", f"{workload.fps} fps")
+
+    if sessions == 1:
+        print("{s:{c}^{n}}".format(s=" Cadence ", n=60, c="-"))
+        if metrics.get("ttfc_ms") is not None:
+            _print_row("TTFC (session.start -> first chunk):", f"{metrics['ttfc_ms']:.1f} ms")
+        interval_all = metrics.get("interval_all") or {}
+        interval_steady = metrics.get("interval_steady") or {}
+        for title, summary in (("All chunks", interval_all), ("Steady state", interval_steady)):
+            if not summary:
+                continue
+            print(f"  {title} (n={int(summary['count'])}):")
+            _print_row("    mean / median:", f"{summary['mean_ms']:.1f} / {summary['median_ms']:.1f} ms")
+            _print_row("    std / max:", f"{summary['std_ms']:.1f} / {summary['max_ms']:.1f} ms")
+            tail = f"{summary['p90_ms']:.0f} / {summary['p95_ms']:.0f} / {summary['p99_ms']:.0f} ms"
+            _print_row("    p90 / p95 / p99:", tail)
+            if summary["count"] < 100:
+                print(
+                    "    NOTE: fewer than 100 intervals; tail percentiles are descriptive, not reliable tail estimates."
+                )
+
+        print("{s:{c}^{n}}".format(s=" Real time ", n=60, c="-"))
+        _print_row("VIDEO_RTF (wall s / video s, <1 real time):", f"{metrics['video_rtf']:.3f}")
+        _print_row("VIDEO_RTFX (video s / wall s):", f"{metrics['video_rtfx']:.3f}")
+        if metrics.get("steady_video_rtf") is not None:
+            _print_row("VIDEO_RTF steady state:", f"{metrics['steady_video_rtf']:.3f}")
+        _print_row("Frames per second:", f"{metrics['frames_per_second']:.2f}")
+        _print_row("Chunk deadline:", f"{metrics['chunk_deadline_ms']:.1f} ms")
+        if metrics.get("slo_attainment") is not None:
+            _print_row(
+                "Steady chunks within deadline:",
+                f"{metrics['slo_attainment'] * 100:.1f}% ({metrics['slo_violations']} late)",
+            )
+        if not metrics.get("steady_state_reached", False):
+            print(
+                f"  NOTE: no chunk reached the steady state; the attention window needs "
+                f"{metrics['warmup_chunks']} chunks. Raise --num-chunks."
+            )
+
+        playback = metrics.get("playback") or {}
+        print("{s:{c}^{n}}".format(s=" Playback continuity ", n=60, c="-"))
+        _print_row("Start buffer:", f"{playback.get('buffer_chunks')} chunk(s)")
+        _print_row("Underruns:", str(playback.get("underrun_count")))
+        _print_row("Total stall:", f"{playback.get('total_stall_ms', 0.0):.1f} ms")
+        _print_row("Stall ratio:", f"{playback.get('stall_ratio', 0.0) * 100:.2f}%")
+        if playback.get("first_underrun_chunk") is not None:
+            _print_row("First underrun at chunk:", str(playback["first_underrun_chunk"]))
+    else:
+        print("{s:{c}^{n}}".format(s=" Across sessions ", n=60, c="-"))
+        if metrics.get("ttfc_ms_mean") is not None:
+            _print_row("TTFC mean / max:", f"{metrics['ttfc_ms_mean']:.1f} / {metrics['ttfc_ms_max']:.1f} ms")
+        if metrics.get("steady_interval_ms_mean") is not None:
+            _print_row("Steady interval mean:", f"{metrics['steady_interval_ms_mean']:.1f} ms")
+            _print_row("Steady interval spread:", f"{metrics['steady_interval_ms_spread']:.1f} ms")
+        _print_row("VIDEO_RTF mean (lower is better):", f"{metrics['video_rtf_mean']:.3f}")
+        _print_row(
+            "VIDEO_RTF best / worst:",
+            f"{metrics['video_rtf_best']:.3f} / {metrics['video_rtf_worst']:.3f}",
+        )
+        _print_row("Chunk deadline:", f"{metrics['chunk_deadline_ms']:.1f} ms")
+        _print_row("Underruns (all sessions):", str(metrics["underrun_count"]))
+    print("=" * 60)
+
+
+def save_video(media: bytes, path: Path, *, fps: int) -> None:
+    from vllm_omni.diffusion.utils.media_utils import finalize_streaming_video_bytes
+
+    playback_bytes = finalize_streaming_video_bytes(media, input_format="m4s", fps=fps)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp_path = path.with_suffix(path.suffix + ".tmp")
+    tmp_path.write_bytes(playback_bytes)
+    os.replace(tmp_path, path)
+    print(f"Saved {len(media)} streamed bytes -> {len(playback_bytes)} playback bytes at {path}")
+
+
+# --------------------------------------------------------------------------- #
+# CLI
+# --------------------------------------------------------------------------- #
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Benchmark LingBot-World 2.0 realtime streaming over WS /v1/realtime/video.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    endpoint = parser.add_argument_group("endpoint")
+    endpoint.add_argument("--host", default="127.0.0.1")
+    endpoint.add_argument("--port", type=int, default=8000)
+    endpoint.add_argument("--base-url", default=None, help="Full ws:// URL; overrides --host/--port.")
+    endpoint.add_argument("--model", default=DEFAULT_MODEL)
+
+    workload = parser.add_argument_group("workload")
+    workload.add_argument("--workload", default=None, help="JSON rollout spec; overrides the built-in workload.")
+    workload.add_argument("--image", default=None, help="First frame: path or URL. Defaults to the shared vLLM asset.")
+    workload.add_argument("--prompt", dest="prompt_override", default=None, help=f"Default: {DEFAULT_PROMPT!r}")
+    workload.add_argument("--negative-prompt", default=None)
+    workload.add_argument("--num-chunks", type=int, default=16, help="AR blocks per session; ignored with --workload.")
+    workload.add_argument(
+        "--camera-pattern",
+        choices=CAMERA_PATTERNS,
+        default="forward",
+        help="Camera pattern for the built-in workload only; ignored with --workload.",
+    )
+    workload.add_argument("--width", dest="width_override", type=int, default=None, help=f"Default: {DEFAULT_WIDTH}")
+    workload.add_argument("--height", dest="height_override", type=int, default=None, help=f"Default: {DEFAULT_HEIGHT}")
+    workload.add_argument("--fps", dest="fps_override", type=int, default=None, help=f"Default: {DEFAULT_FPS}")
+    workload.add_argument("--seed", dest="seed_override", type=int, default=None, help=f"Default: {DEFAULT_SEED}")
+    workload.add_argument("--flow-shift", type=float, default=None, help=f"Default: {DEFAULT_FLOW_SHIFT}")
+
+    run = parser.add_argument_group("run")
+    run.add_argument("--sessions", type=int, default=1, help="Sequential measured rollouts.")
+    run.add_argument("--warmup-sessions", type=int, default=0, help="Unmeasured rollouts first (pays compile cost).")
+    run.add_argument(
+        "--warmup-chunks",
+        type=int,
+        default=DEFAULT_WARMUP_CHUNKS,
+        help="Chunks excluded from steady state; the attention window saturates after 6.",
+    )
+    run.add_argument(
+        "--chunk-slo-ms",
+        type=float,
+        default=None,
+        help="Deadline a steady chunk must meet. Default: one chunk of video at --target-fps.",
+    )
+    run.add_argument(
+        "--target-fps",
+        type=float,
+        default=None,
+        help=(
+            "Playback rate used for real-time factor, the chunk deadline, and the playback "
+            "simulation. The checkpoint declares no frame rate, so this is a consumption "
+            "choice, not a model property. Default: the --fps sent to the server."
+        ),
+    )
+    run.add_argument("--playback-buffer-chunks", type=int, default=1, help="Chunks buffered before playback starts.")
+    run.add_argument("--first-chunk-timeout", type=float, default=900.0)
+    run.add_argument("--chunk-timeout", type=float, default=300.0)
+    run.add_argument("--ping-interval", type=float, default=20.0, help="session.ping cadence; 0 disables.")
+
+    output = parser.add_argument_group("output")
+    output.add_argument("--output-json", default=None)
+    output.add_argument("--save-video", default=None, help="Write the last measured session's video here.")
+    output.add_argument("--print-chunks", action="store_true")
+
+    args = parser.parse_args(argv)
+    if args.sessions < 1:
+        parser.error("--sessions must be at least 1.")
+    if args.warmup_sessions < 0:
+        parser.error("--warmup-sessions must be non-negative.")
+    if args.warmup_chunks < 0:
+        parser.error("--warmup-chunks must be non-negative.")
+    if args.playback_buffer_chunks < 1:
+        parser.error("--playback-buffer-chunks must be at least 1.")
+    if args.target_fps is not None and args.target_fps <= 0:
+        parser.error("--target-fps must be positive.")
+    if args.workload is None and args.num_chunks < 1:
+        parser.error("--num-chunks must be at least 1.")
+    return args
+
+
+def endpoint_url(args: argparse.Namespace) -> str:
+    if args.base_url is not None:
+        base = args.base_url.rstrip("/")
+        for prefix, replacement in (("http://", "ws://"), ("https://", "wss://")):
+            if base.startswith(prefix):
+                base = replacement + base[len(prefix) :]
+                break
+        return base if base.endswith(_WS_PATH) else base + _WS_PATH
+    return f"ws://{args.host}:{args.port}{_WS_PATH}"
+
+
+async def run_benchmark(args: argparse.Namespace) -> dict[str, Any]:
+    workload = build_workload(args)
+    url = endpoint_url(args)
+    # The mux label and the real-time basis are different questions; only the
+    # latter decides whether a run passes.
+    target_fps = args.target_fps if args.target_fps is not None else float(workload.fps)
+    if max(1, args.warmup_chunks) >= workload.num_chunks - 1:
+        print(
+            f"WARNING: --warmup-chunks={args.warmup_chunks} leaves no nonterminal steady-state chunk in a "
+            f"{workload.num_chunks}-chunk rollout; steady-state metrics will be empty.",
+            file=sys.stderr,
+        )
+
+    print(f"Connecting to {url}")
+    print(
+        f"Rollout: {workload.num_chunks} chunks, num_frames={workload.num_frames}, "
+        f"{workload.video_seconds:.2f}s of video"
+    )
+    print(
+        f"Real-time basis: {target_fps:g} fps -> a steady 12-frame chunk is "
+        f"{steady_chunk_deadline_ms(target_fps):.1f} ms of video"
+        + ("" if args.target_fps is None else f" (mux label stays {workload.fps} fps)")
+    )
+
+    for index in range(args.warmup_sessions):
+        print(f"Warmup session {index + 1}/{args.warmup_sessions} (not measured)...")
+        await run_session(
+            url,
+            args.model,
+            workload,
+            first_chunk_timeout=args.first_chunk_timeout,
+            chunk_timeout=args.chunk_timeout,
+            ping_interval=args.ping_interval,
+            collect_media=False,
+            print_chunks=False,
+        )
+
+    session_metrics: list[dict[str, Any]] = []
+    session_payloads: list[dict[str, Any]] = []
+    last_result: SessionResult | None = None
+    for index in range(args.sessions):
+        if args.sessions > 1:
+            print(f"Session {index + 1}/{args.sessions}...")
+        result = await run_session(
+            url,
+            args.model,
+            workload,
+            first_chunk_timeout=args.first_chunk_timeout,
+            chunk_timeout=args.chunk_timeout,
+            ping_interval=args.ping_interval,
+            collect_media=args.save_video is not None,
+            print_chunks=args.print_chunks,
+        )
+        last_result = result
+        metrics = compute_metrics(
+            result.records,
+            fps=target_fps,
+            warmup_chunks=args.warmup_chunks,
+            slo_ms=args.chunk_slo_ms,
+            playback_buffer_chunks=args.playback_buffer_chunks,
+            ttfc_s=result.ttfc_s,
+            session_wall_s=result.wall_s,
+        )
+        session_metrics.append(metrics)
+        session_payloads.append(
+            {
+                "session_index": index,
+                "request_id": result.request_id,
+                "video_start_ms": (result.video_start_s * 1000.0) if result.video_start_s is not None else None,
+                "chunks": [record.to_dict() for record in result.records],
+                "metrics": metrics,
+            }
+        )
+
+    aggregate = aggregate_metrics(session_metrics, fps=target_fps)
+    print_report(aggregate, workload=workload, endpoint=url, sessions=args.sessions, target_fps=target_fps)
+
+    if args.save_video is not None and last_result is not None:
+        save_video(last_result.media, Path(args.save_video), fps=workload.fps)
+
+    document = {
+        "benchmark": "lingbot_world_realtime",
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "endpoint": url,
+        "model": args.model,
+        "git_revision": _git_revision(),
+        "hostname": platform.node(),
+        "workload": workload.describe(),
+        "config": {
+            "sessions": args.sessions,
+            "warmup_sessions": args.warmup_sessions,
+            "warmup_chunks": args.warmup_chunks,
+            "camera_pattern": args.camera_pattern,
+            "chunk_slo_ms": args.chunk_slo_ms,
+            "target_fps": target_fps,
+            "mux_fps": workload.fps,
+            "playback_buffer_chunks": args.playback_buffer_chunks,
+            "workload_file": args.workload,
+        },
+        "sessions": session_payloads,
+        "metrics": aggregate,
+    }
+    if args.output_json is not None:
+        path = Path(args.output_json)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n")
+        print(f"Wrote {path}")
+    return document
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv)
+    try:
+        asyncio.run(run_benchmark(args))
+    except BenchmarkError as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmarks/lingbot_world/configs/single_gpu_eager.yaml

@@ -0,0 +1,45 @@
+# Benchmark deploy config: LingBot-World 2.0 on one GPU, eager.
+#
+# This is the portable smoke configuration -- it matches the shipped
+# vllm_omni/deploy/lingbot_world_v2_stepwise.yaml so a number measured here is
+# comparable to the default deployment. It is NOT a realtime configuration: one
+# GPU runs all five forwards of every AR block serially and nothing is compiled,
+# so expect a chunk interval several times the real-time deadline.
+#
+# Use configs/usp4_compiled.yaml for the realtime topology.
+#
+#   vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+#     --omni --deploy-config benchmarks/lingbot_world/configs/single_gpu_eager.yaml \
+#     --port 8000
+
+pipeline: lingbot_world
+async_chunk: false
+distributed_executor_backend: mp
+dtype: bfloat16
+
+stages:
+  - stage_id: 0
+    devices: "0"
+    # AR-Diffusion binds one session per runner invocation; the runner rejects
+    # anything larger.
+    max_num_seqs: 1
+    enforce_eager: true
+    model_class_name: LingBotWorldCausalDMDPipeline
+    engine_backend: vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine
+    step_execution: true
+    streaming_output: true
+    default_sampling_params:
+      # The causal-fast checkpoint pins a four-step DMD schedule; this sizes the
+      # step budget before the pipeline is reached.
+      num_inference_steps: 4
+      output_type: pil
+      extra_args:
+        flow_shift: 5.0
+    model_config:
+      # The AR cache geometry is fixed at load time, so a benchmark request must
+      # ask for exactly this resolution.
+      ar_diffusion_height: 480
+      ar_diffusion_width: 832
+      ar_diffusion_kv_config:
+        gpu_memory_fraction: 0.6
+        warmup_cudagraph: true

benchmarks/lingbot_world/configs/usp4_compiled.yaml

@@ -0,0 +1,60 @@
+# Benchmark deploy config: LingBot-World 2.0 on four GPUs, Ulysses SP-4, compiled.
+#
+# This is the realtime topology. Two things differ from the single-GPU config,
+# and both matter to the number the benchmark reports:
+#
+#   parallel_config   Pure Ulysses sequence parallelism over four ranks. The
+#                     pipeline validates this: LingBot requires
+#                     sequence_parallel_size == ulysses_degree, ring_degree and
+#                     allgather_degree of 1, ulysses_mode "strict", and
+#                     ulysses_a2a_permute disabled. Pipeline, CFG, and VAE
+#                     parallel sizes above 1 are rejected. This config uses
+#                     TP=1; the validator does not reject larger TP sizes.
+#   enforce_eager     false puts LingBotDMDBlockRunner on the compiled path
+#                     (regional torch.compile with cudagraph step marking). The
+#                     first rollout therefore pays compile and capture cost --
+#                     use --warmup-sessions 1 before measured repetitions.
+#                     Chunk warmup only excludes the attention-window ramp.
+#
+# Serve it under a GPU reservation, since it needs four cards:
+#
+#   gpu run --gpus 4 -- vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+#     --omni --deploy-config benchmarks/lingbot_world/configs/usp4_compiled.yaml \
+#     --port 8000
+#
+# `devices` is the rank->card mapping inside the reservation, so it stays
+# "0,1,2,3" no matter which physical cards CUDA_VISIBLE_DEVICES selected.
+
+pipeline: lingbot_world
+async_chunk: false
+distributed_executor_backend: mp
+dtype: bfloat16
+
+stages:
+  - stage_id: 0
+    devices: "0,1,2,3"
+    max_num_seqs: 1
+    enforce_eager: false
+    model_class_name: LingBotWorldCausalDMDPipeline
+    engine_backend: vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine
+    step_execution: true
+    streaming_output: true
+    parallel_config:
+      tensor_parallel_size: 1
+      sequence_parallel_size: 4
+      ulysses_degree: 4
+      ring_degree: 1
+    default_sampling_params:
+      num_inference_steps: 4
+      output_type: pil
+      extra_args:
+        flow_shift: 5.0
+    model_config:
+      ar_diffusion_height: 480
+      ar_diffusion_width: 832
+      ar_diffusion_kv_config:
+        # Each rank holds its own shard of the paged KV cache, so the per-card
+        # fraction can stay at the single-GPU value; lower it if the four-rank
+        # start OOMs on smaller cards.
+        gpu_memory_fraction: 0.6
+        warmup_cudagraph: true

benchmarks/lingbot_world/workload.py

@@ -0,0 +1,631 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Workload construction and metric math for the LingBot-World realtime benchmark.
+
+Nothing here imports torch, vllm, or websockets, so the arithmetic that decides
+what a benchmark number *means* is unit-testable without a GPU or a server.
+
+Two model properties drive every formula in this module, both read from the
+checkpoint's ``transformer/config.json`` rather than assumed:
+
+``num_frames_per_block = 3``
+    One AR block commits three latent frames, so one streamed chunk is three
+    latent frames and a request's ``camera_action_script`` carries exactly one
+    three-entry action list per chunk.
+
+``sliding_window_num_frames = 18`` with ``sink_size = 9``
+    Self-attention sees nine sink frames plus a nine-frame recent window, so the
+    key count a block attends over keeps growing until eighteen latent frames of
+    history exist -- six chunks. Chunk cost is therefore *not* stationary before
+    chunk six, and a mean taken over a short rollout reports a number no steady
+    session will ever produce. :data:`DEFAULT_WARMUP_CHUNKS` exists for that
+    reason and is not a round-number guess.
+
+The causal VAE expands the first latent frame to one pixel frame and every later
+latent frame to four, so chunk zero carries nine pixel frames and every chunk
+after it carries twelve. Video-duration math has to respect that asymmetry or
+real-time factor comes out wrong on short rollouts.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+# transformer.config.num_frames_per_block: latent frames committed per AR block.
+FRAMES_PER_BLOCK = 3
+# vae_scale_factor_temporal: latent frames -> pixel frames, after the first.
+TEMPORAL_COMPRESSION = 4
+# (sink_size 9 + recent window 9) / FRAMES_PER_BLOCK: chunks until the attention
+# window saturates and per-chunk cost becomes stationary.
+DEFAULT_WARMUP_CHUNKS = 6
+
+DEFAULT_MODEL = "robbyant/lingbot-world-v2-14b-causal-fast-diffusers"
+DEFAULT_PROMPT = "The camera moves slowly forward through the scene."
+# The AR KV cache geometry is fixed at load time, so a request must ask for
+# exactly the resolution the deploy config declares.
+DEFAULT_WIDTH = 832
+DEFAULT_HEIGHT = 480
+DEFAULT_FPS = 16
+DEFAULT_FLOW_SHIFT = 5.0
+DEFAULT_SEED = 42
+
+_HOLD = "hold"
+_ACTION_KEYS = ("w", "a", "s", "d")
+
+
+def num_frames_for_chunks(num_chunks: int) -> int:
+    """Pixel-frame count whose rollout is exactly ``num_chunks`` chunks.
+
+    Inverse of :func:`chunks_for_num_frames`, which is the server's own
+    ``((num_frames - 1) // 4 + 1) // 3``.
+    """
+    if num_chunks < 1:
+        raise ValueError("num_chunks must be at least 1.")
+    return (num_chunks * FRAMES_PER_BLOCK - 1) * TEMPORAL_COMPRESSION + 1
+
+
+def chunks_for_num_frames(num_frames: int) -> int:
+    """Chunks the server generates for ``num_frames``, mirroring its own formula."""
+    if num_frames < 1:
+        raise ValueError("num_frames must be at least 1.")
+    return ((num_frames - 1) // TEMPORAL_COMPRESSION + 1) // FRAMES_PER_BLOCK
+
+
+def pixel_frames_in_chunk(chunk_index: int) -> int:
+    """Pixel frames carried by one chunk.
+
+    The causal decoder expands the session's opening latent frame once and every
+    later latent frame four times, so chunk zero is short by three frames.
+    """
+    if chunk_index < 0:
+        raise ValueError("chunk_index must be non-negative.")
+    if chunk_index == 0:
+        return (FRAMES_PER_BLOCK - 1) * TEMPORAL_COMPRESSION + 1
+    return FRAMES_PER_BLOCK * TEMPORAL_COMPRESSION
+
+
+def camera_actions_for_chunk(pattern: str, chunk_index: int) -> list[list[str]]:
+    """One three-entry action list for ``chunk_index``.
+
+    Camera actions select camera embeddings; they change what the world does,
+    not how much arithmetic it costs. The pattern therefore exists to keep a
+    benchmark rollout representative rather than to sweep a cost dimension.
+    """
+    if chunk_index < 0:
+        raise ValueError("chunk_index must be non-negative.")
+    if pattern == "forward":
+        return [["w"] for _ in range(FRAMES_PER_BLOCK)]
+    if pattern == _HOLD:
+        return [[] for _ in range(FRAMES_PER_BLOCK)]
+    if pattern == "orbit":
+        key = _ACTION_KEYS[chunk_index % len(_ACTION_KEYS)]
+        return [[key] for _ in range(FRAMES_PER_BLOCK)]
+    raise ValueError(f"Unknown camera pattern {pattern!r}; expected one of {sorted(CAMERA_PATTERNS)}.")
+
+
+CAMERA_PATTERNS = ("forward", "orbit", _HOLD)
+
+
+def build_camera_script(num_chunks: int, pattern: str) -> list[list[list[str]]]:
+    """Full ``camera_action_script``: one action list per generated chunk."""
+    if num_chunks < 1:
+        raise ValueError("num_chunks must be at least 1.")
+    return [camera_actions_for_chunk(pattern, index) for index in range(num_chunks)]
+
+
+def _validate_camera_script(script: Sequence[Any]) -> list[list[list[str]]]:
+    validated: list[list[list[str]]] = []
+    for chunk_index, actions in enumerate(script):
+        if not isinstance(actions, Sequence) or isinstance(actions, (str, bytes)):
+            raise ValueError(f"camera_action_script[{chunk_index}] must be a list of three action lists.")
+        if len(actions) != FRAMES_PER_BLOCK:
+            raise ValueError(
+                f"camera_action_script[{chunk_index}] must hold exactly {FRAMES_PER_BLOCK} action lists "
+                f"(got {len(actions)}); one AR block is {FRAMES_PER_BLOCK} latent frames."
+            )
+        frames: list[list[str]] = []
+        for frame_index, frame in enumerate(actions):
+            if not isinstance(frame, Sequence) or isinstance(frame, (str, bytes)):
+                raise ValueError(f"camera_action_script[{chunk_index}][{frame_index}] must be a list of strings.")
+            if any(not isinstance(action, str) for action in frame):
+                raise ValueError(f"camera_action_script[{chunk_index}][{frame_index}] must contain only strings.")
+            frames.append([str(action) for action in frame])
+        validated.append(frames)
+    if not validated:
+        raise ValueError("camera_action_script must not be empty.")
+    return validated
+
+
+@dataclass(frozen=True)
+class PromptUpdate:
+    """A mid-rollout ``session.interaction`` scheduled on a chunk boundary."""
+
+    after_chunk: int
+    prompt: str
+    transition_chunks: int = 3
+
+    def __post_init__(self) -> None:
+        if self.after_chunk < 0:
+            raise ValueError("PromptUpdate.after_chunk must be non-negative.")
+        if not self.prompt.strip():
+            raise ValueError("PromptUpdate.prompt must contain non-whitespace text.")
+        if self.transition_chunks < 0:
+            raise ValueError("PromptUpdate.transition_chunks must be non-negative.")
+
+    def to_payload(self) -> dict[str, Any]:
+        return {
+            "type": "session.interaction",
+            "interaction": {
+                "event_id": f"chunk-{self.after_chunk}",
+                "event": {"prompt": self.prompt},
+                "transition_chunks": self.transition_chunks,
+            },
+        }
+
+
+@dataclass(frozen=True)
+class Workload:
+    """One rollout: what the benchmark asks the server to generate."""
+
+    prompt: str
+    image_reference: str
+    camera_script: list[list[list[str]]]
+    width: int = DEFAULT_WIDTH
+    height: int = DEFAULT_HEIGHT
+    fps: int = DEFAULT_FPS
+    seed: int = DEFAULT_SEED
+    flow_shift: float = DEFAULT_FLOW_SHIFT
+    negative_prompt: str | None = None
+    prompt_updates: tuple[PromptUpdate, ...] = ()
+    extra_params: Mapping[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if not self.prompt.strip():
+            raise ValueError("Workload.prompt must contain non-whitespace text.")
+        if not self.image_reference:
+            raise ValueError("Workload.image_reference is required; LingBot-World is image-conditioned.")
+        if self.width <= 0 or self.height <= 0 or self.width % 16 or self.height % 16:
+            raise ValueError("Workload width/height must be positive multiples of 16.")
+        if self.fps <= 0:
+            raise ValueError("Workload.fps must be positive.")
+        object.__setattr__(self, "camera_script", _validate_camera_script(self.camera_script))
+        seen_update_chunks: set[int] = set()
+        for update in self.prompt_updates:
+            if update.after_chunk in seen_update_chunks:
+                raise ValueError(f"Duplicate prompt update after chunk {update.after_chunk}; event IDs must be unique.")
+            seen_update_chunks.add(update.after_chunk)
+            if update.after_chunk >= self.num_chunks:
+                raise ValueError(
+                    f"Prompt update after chunk {update.after_chunk} never fires in a {self.num_chunks}-chunk rollout."
+                )
+
+    @property
+    def num_chunks(self) -> int:
+        return len(self.camera_script)
+
+    @property
+    def num_frames(self) -> int:
+        return num_frames_for_chunks(self.num_chunks)
+
+    @property
+    def video_seconds(self) -> float:
+        return self.num_frames / self.fps
+
+    def session_start_payload(self, model: str) -> dict[str, Any]:
+        """The ``session.start`` event for ``WS /v1/realtime/video``."""
+        extra_params: dict[str, Any] = {
+            "flow_shift": self.flow_shift,
+            "camera_action_script": self.camera_script,
+        }
+        extra_params.update(self.extra_params)
+        payload: dict[str, Any] = {
+            "type": "session.start",
+            "model": model,
+            "prompt": self.prompt,
+            "image_reference": {"image_url": self.image_reference},
+            "width": self.width,
+            "height": self.height,
+            "num_frames": self.num_frames,
+            "fps": self.fps,
+            "seed": self.seed,
+            "extra_params": extra_params,
+        }
+        if self.negative_prompt is not None:
+            payload["negative_prompt"] = self.negative_prompt
+        return payload
+
+    def describe(self) -> dict[str, Any]:
+        return {
+            "num_chunks": self.num_chunks,
+            "num_frames": self.num_frames,
+            "width": self.width,
+            "height": self.height,
+            "fps": self.fps,
+            "seed": self.seed,
+            "flow_shift": self.flow_shift,
+            "video_seconds": self.video_seconds,
+            "prompt_updates": [
+                {
+                    "after_chunk": update.after_chunk,
+                    "transition_chunks": update.transition_chunks,
+                }
+                for update in self.prompt_updates
+            ],
+        }
+
+
+def default_workload(
+    *,
+    image_reference: str,
+    num_chunks: int,
+    camera_pattern: str = "forward",
+    prompt: str = DEFAULT_PROMPT,
+    width: int = DEFAULT_WIDTH,
+    height: int = DEFAULT_HEIGHT,
+    fps: int = DEFAULT_FPS,
+    seed: int = DEFAULT_SEED,
+    flow_shift: float = DEFAULT_FLOW_SHIFT,
+    negative_prompt: str | None = None,
+) -> Workload:
+    """Built-in rollout: a fixed prompt and a generated camera script."""
+    return Workload(
+        prompt=prompt,
+        image_reference=image_reference,
+        camera_script=build_camera_script(num_chunks, camera_pattern),
+        width=width,
+        height=height,
+        fps=fps,
+        seed=seed,
+        flow_shift=flow_shift,
+        negative_prompt=negative_prompt,
+    )
+
+
+def load_workload(
+    path: str | Path,
+    *,
+    image_resolver: Any = None,
+    overrides: Mapping[str, Any] | None = None,
+) -> Workload:
+    """Read a rollout spec from JSON.
+
+    ``image_resolver`` turns the spec's ``image`` field into an ``http(s)`` or
+    ``data:`` URL; the caller owns that because inlining a local file is I/O.
+    Keys in ``overrides`` that are not ``None`` win over the file, so CLI flags
+    stay meaningful next to ``--workload``.
+    """
+    document = json.loads(Path(path).read_text())
+    if not isinstance(document, dict):
+        raise ValueError("A workload file must contain a JSON object.")
+
+    merged = dict(document)
+    for key, value in (overrides or {}).items():
+        if value is not None:
+            merged[key] = value
+
+    camera_script = merged.get("camera_action_script")
+    if camera_script is None:
+        num_chunks = merged.get("num_chunks")
+        if num_chunks is None:
+            raise ValueError("A workload file needs either camera_action_script or num_chunks.")
+        camera_script = build_camera_script(int(num_chunks), str(merged.get("camera_pattern", "forward")))
+
+    image = merged.get("image") or merged.get("image_reference")
+    if image is None:
+        raise ValueError("A workload file needs an image or image_reference field.")
+    image_reference = image_resolver(str(image)) if image_resolver is not None else str(image)
+
+    updates_field = merged.get("prompt_updates") or ()
+    prompt_updates = tuple(
+        PromptUpdate(
+            after_chunk=int(entry["after_chunk"]),
+            prompt=str(entry["prompt"]),
+            transition_chunks=int(entry.get("transition_chunks", 3)),
+        )
+        for entry in updates_field
+    )
+
+    extra_params = merged.get("extra_params") or {}
+    if not isinstance(extra_params, Mapping):
+        raise ValueError("Workload extra_params must be a JSON object.")
+
+    return Workload(
+        prompt=str(merged.get("prompt", DEFAULT_PROMPT)),
+        image_reference=image_reference,
+        camera_script=camera_script,
+        width=int(merged.get("width", DEFAULT_WIDTH)),
+        height=int(merged.get("height", DEFAULT_HEIGHT)),
+        fps=int(merged.get("fps", DEFAULT_FPS)),
+        seed=int(merged.get("seed", DEFAULT_SEED)),
+        flow_shift=float(merged.get("flow_shift", DEFAULT_FLOW_SHIFT)),
+        negative_prompt=merged.get("negative_prompt"),
+        prompt_updates=prompt_updates,
+        extra_params=dict(extra_params),
+    )
+
+
+# --------------------------------------------------------------------------- #
+# Metrics
+# --------------------------------------------------------------------------- #
+
+
+@dataclass
+class ChunkRecord:
+    """One media chunk as the client observed it."""
+
+    index: int
+    arrival_s: float
+    """Seconds from ``session.start`` being sent to this chunk's bytes arriving."""
+    inter_arrival_s: float
+    """Seconds since the previous chunk; for chunk zero this equals ``arrival_s``."""
+    byte_length: int
+    num_frames: int
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "index": self.index,
+            "arrival_s": self.arrival_s,
+            "inter_arrival_ms": self.inter_arrival_s * 1000.0,
+            "byte_length": self.byte_length,
+            "num_frames": self.num_frames,
+        }
+
+
+def percentile(values: Sequence[float], fraction: float) -> float:
+    """Linearly interpolated percentile, so a 10-sample run still reports a p95.
+
+    Implemented here rather than pulled from numpy to keep this module importable
+    in a bare environment; the benchmark's own numbers must not depend on which
+    optional package happens to be installed.
+    """
+    if not values:
+        raise ValueError("percentile() needs at least one value.")
+    if not 0.0 <= fraction <= 1.0:
+        raise ValueError("percentile fraction must be within [0, 1].")
+    ordered = sorted(values)
+    if len(ordered) == 1:
+        return ordered[0]
+    position = fraction * (len(ordered) - 1)
+    lower = int(position)
+    upper = min(lower + 1, len(ordered) - 1)
+    weight = position - lower
+    return ordered[lower] * (1.0 - weight) + ordered[upper] * weight
+
+
+def _stddev(values: Sequence[float]) -> float:
+    if len(values) < 2:
+        return 0.0
+    mean = sum(values) / len(values)
+    return (sum((value - mean) ** 2 for value in values) / (len(values) - 1)) ** 0.5
+
+
+def _latency_summary(values: Sequence[float]) -> dict[str, float]:
+    """Millisecond summary of a set of inter-arrival intervals."""
+    if not values:
+        return {}
+    milliseconds = [value * 1000.0 for value in values]
+    return {
+        "count": float(len(milliseconds)),
+        "mean_ms": sum(milliseconds) / len(milliseconds),
+        "median_ms": percentile(milliseconds, 0.5),
+        "std_ms": _stddev(milliseconds),
+        "min_ms": min(milliseconds),
+        "p90_ms": percentile(milliseconds, 0.90),
+        "p95_ms": percentile(milliseconds, 0.95),
+        "p99_ms": percentile(milliseconds, 0.99),
+        "max_ms": max(milliseconds),
+    }
+
+
+@dataclass(frozen=True)
+class PlaybackResult:
+    """What a viewer would have seen, given a start buffer and a playback rate."""
+
+    underrun_count: int
+    total_stall_s: float
+    playback_span_s: float
+    first_underrun_chunk: int | None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "underrun_count": self.underrun_count,
+            "total_stall_ms": self.total_stall_s * 1000.0,
+            "playback_span_s": self.playback_span_s,
+            "stall_ratio": (self.total_stall_s / self.playback_span_s) if self.playback_span_s > 0 else 0.0,
+            "first_underrun_chunk": self.first_underrun_chunk,
+        }
+
+
+def simulate_playback(
+    records: Sequence[ChunkRecord],
+    *,
+    fps: float,
+    buffer_chunks: int = 1,
+) -> PlaybackResult:
+    """Replay arrivals against a wall clock and count stalls.
+
+    Playback starts when ``buffer_chunks`` chunks have arrived and then consumes
+    video at ``fps``. A chunk that has not arrived by the moment the player needs
+    it is an underrun, and the player resumes only when the bytes land -- which
+    is what a viewer experiences, and what a single p99 number cannot tell you:
+    the same p99 is harmless behind a three-chunk buffer and fatal behind none.
+    """
+    if fps <= 0:
+        raise ValueError("fps must be positive.")
+    if buffer_chunks < 1:
+        raise ValueError("buffer_chunks must be at least 1.")
+    if not records:
+        return PlaybackResult(0, 0.0, 0.0, None)
+    if buffer_chunks > len(records):
+        raise ValueError(f"buffer_chunks={buffer_chunks} exceeds the {len(records)} chunks received.")
+
+    ordered = sorted(records, key=lambda record: record.index)
+    start = ordered[buffer_chunks - 1].arrival_s
+    cursor = start
+    underruns = 0
+    total_stall = 0.0
+    first_underrun: int | None = None
+
+    for record in ordered:
+        if record.arrival_s > cursor:
+            stall = record.arrival_s - cursor
+            underruns += 1
+            total_stall += stall
+            if first_underrun is None:
+                first_underrun = record.index
+            cursor = record.arrival_s
+        cursor += record.num_frames / fps
+
+    return PlaybackResult(
+        underrun_count=underruns,
+        total_stall_s=total_stall,
+        playback_span_s=cursor - start,
+        first_underrun_chunk=first_underrun,
+    )
+
+
+def steady_chunk_deadline_ms(fps: float) -> float:
+    """Wall-clock budget a steady chunk must meet to hold real time.
+
+    ``fps`` here is the rate the frames are *intended to be played at*, which the
+    checkpoint does not declare: nothing in ``model_index.json``, the scheduler
+    config, or the transformer config carries a frame rate, and the pipeline
+    never reads one. It is a property of how the output is consumed, so both
+    this deadline and real-time factor scale linearly with the choice and the
+    caller has to state it rather than inherit it from a muxing default.
+    """
+    if fps <= 0:
+        raise ValueError("fps must be positive.")
+    return pixel_frames_in_chunk(1) / fps * 1000.0
+
+
+def compute_metrics(
+    records: Sequence[ChunkRecord],
+    *,
+    fps: float,
+    warmup_chunks: int = DEFAULT_WARMUP_CHUNKS,
+    slo_ms: float | None = None,
+    playback_buffer_chunks: int = 1,
+    ttfc_s: float | None = None,
+    session_wall_s: float | None = None,
+) -> dict[str, Any]:
+    """Summarize one session.
+
+    ``warmup_chunks`` are excluded from the steady-state block because the
+    attention window has not saturated before then (see the module docstring).
+    The terminal chunk also stays out of steady metrics because it skips
+    next-chunk preparation. All chunks stay in the playback simulation, which is
+    what a session really costs end to end.
+    """
+    if not records:
+        raise ValueError("compute_metrics() needs at least one chunk record.")
+    if warmup_chunks < 0:
+        raise ValueError("warmup_chunks must be non-negative.")
+
+    ordered = sorted(records, key=lambda record: record.index)
+    deadline_ms = slo_ms if slo_ms is not None else steady_chunk_deadline_ms(fps)
+
+    all_intervals = [record.inter_arrival_s for record in ordered[1:]]
+    # The terminal chunk skips next-chunk preparation and is not representative
+    # of sustained generation. Keep it in end-to-end and playback metrics.
+    steady = [record for record in ordered[:-1] if record.index >= warmup_chunks]
+    steady_intervals = [record.inter_arrival_s for record in steady if record.index > 0]
+
+    total_frames = sum(record.num_frames for record in ordered)
+    video_seconds = total_frames / fps
+    wall_seconds = session_wall_s if session_wall_s is not None else ordered[-1].arrival_s
+
+    metrics: dict[str, Any] = {
+        "chunks_received": len(ordered),
+        "warmup_chunks": warmup_chunks,
+        "steady_chunks": len(steady_intervals),
+        "steady_state_reached": bool(steady_intervals),
+        "total_frames": total_frames,
+        "total_bytes": sum(record.byte_length for record in ordered),
+        "video_seconds": video_seconds,
+        "wall_seconds": wall_seconds,
+        # RTF is processing time over content duration, as vLLM-Omni computes it
+        # (`patch.py`: generation_s / video_duration; `compute_audio_rtf`: latency
+        # / audio_duration, "SLO red line < 1"). LOWER IS BETTER and below 1.0 is
+        # real time. RTFX is the reciprocal the repo reports under its own name,
+        # kept here because "fraction of real time achieved" reads better on a
+        # dashboard -- but the two must never be confused for one another.
+        "video_rtf": (wall_seconds / video_seconds) if video_seconds > 0 else 0.0,
+        "video_rtfx": (video_seconds / wall_seconds) if wall_seconds > 0 else 0.0,
+        "frames_per_second": (total_frames / wall_seconds) if wall_seconds > 0 else 0.0,
+        "chunk_deadline_ms": deadline_ms,
+        "interval_all": _latency_summary(all_intervals),
+        "interval_steady": _latency_summary(steady_intervals),
+    }
+    if ttfc_s is not None:
+        metrics["ttfc_ms"] = ttfc_s * 1000.0
+
+    if steady_intervals:
+        steady_frames = sum(record.num_frames for record in steady if record.index > 0)
+        steady_wall = sum(steady_intervals)
+        steady_video = steady_frames / fps
+        metrics["steady_video_rtf"] = (steady_wall / steady_video) if steady_video > 0 else 0.0
+        metrics["steady_video_rtfx"] = (steady_video / steady_wall) if steady_wall > 0 else 0.0
+        within = sum(1 for value in steady_intervals if value * 1000.0 <= deadline_ms)
+        metrics["slo_attainment"] = within / len(steady_intervals)
+        metrics["slo_violations"] = len(steady_intervals) - within
+
+    metrics["playback"] = simulate_playback(
+        ordered,
+        fps=fps,
+        buffer_chunks=playback_buffer_chunks,
+    ).to_dict()
+    metrics["playback"]["buffer_chunks"] = playback_buffer_chunks
+    return metrics
+
+
+def aggregate_metrics(sessions: Iterable[Mapping[str, Any]], *, fps: float) -> dict[str, Any]:
+    """Combine per-session metrics across sequential repeats.
+
+    Steady intervals are pooled rather than averaging per-session means, so a
+    session that happened to produce fewer steady chunks does not carry the same
+    weight as a full one.
+    """
+    entries = list(sessions)
+    if not entries:
+        raise ValueError("aggregate_metrics() needs at least one session.")
+    if len(entries) == 1:
+        single = dict(entries[0])
+        single["sessions"] = 1
+        return single
+
+    deadline_ms = entries[0]["chunk_deadline_ms"]
+    if any(entry["chunk_deadline_ms"] != deadline_ms for entry in entries):
+        raise ValueError("Cannot aggregate sessions with different chunk deadlines.")
+    ttfcs = [float(entry["ttfc_ms"]) for entry in entries if entry.get("ttfc_ms") is not None]
+    # Lower is better, so "max" is the worst session, not the best.
+    rtfs = [float(entry["video_rtf"]) for entry in entries]
+    steady_means = [
+        float(entry["interval_steady"]["mean_ms"]) for entry in entries if entry.get("interval_steady", {}).get("count")
+    ]
+    steady_count = sum(entry.get("interval_steady", {}).get("count", 0) for entry in entries)
+    steady_total_ms = sum(
+        entry["interval_steady"]["mean_ms"] * entry["interval_steady"]["count"]
+        for entry in entries
+        if entry.get("interval_steady", {}).get("count")
+    )
+    return {
+        "sessions": len(entries),
+        "chunks_received": sum(int(entry["chunks_received"]) for entry in entries),
+        "total_frames": sum(int(entry["total_frames"]) for entry in entries),
+        "video_seconds": sum(float(entry["video_seconds"]) for entry in entries),
+        "wall_seconds": sum(float(entry["wall_seconds"]) for entry in entries),
+        "video_rtf_mean": sum(rtfs) / len(rtfs),
+        "video_rtf_best": min(rtfs),
+        "video_rtf_worst": max(rtfs),
+        "ttfc_ms_mean": (sum(ttfcs) / len(ttfcs)) if ttfcs else None,
+        "ttfc_ms_max": max(ttfcs) if ttfcs else None,
+        "steady_interval_ms_mean": (steady_total_ms / steady_count) if steady_count else None,
+        "steady_interval_ms_spread": (max(steady_means) - min(steady_means)) if steady_means else None,
+        "underrun_count": sum(int(entry["playback"]["underrun_count"]) for entry in entries),
+        "chunk_deadline_ms": deadline_ms,
+    }

benchmarks/qwen2_5_omni/README.md

@@ -0,0 +1,66 @@
+# Qwen2.5-Omni embedding microbenchmark
+
+Measure the host-observed latency of `embed_input_ids` before and after the
+redundant text embedding removal in PR #7477. Both variants use the same real
+checkpoint embedding table, dependencies, inputs and batch shapes. The baseline
+wrapper is extracted from the specified Git commit; the fixed wrapper comes from
+the current checkout. No full model is initialized.
+
+## Run
+
+Use Linux, a CUDA GPU, the project dependencies, and a local
+`Qwen/Qwen2.5-Omni-7B` checkpoint with its safetensors index and weights.
+From the repository root:
+
+```bash
+git fetch origin 01a2f93256975c7ff9565c5414b0fe0225bc4765
+PYTHONPATH="$PWD" OMP_NUM_THREADS=1 python benchmarks/qwen2_5_omni/benchmark_embeddings.py \
+  --model /path/to/Qwen2.5-Omni-7B \
+  --baseline-ref 01a2f93256975c7ff9565c5414b0fe0225bc4765 \
+  --cpu-core 2 --lengths 1172 --warmup 50 --blocks 20 --iterations 20 \
+  --output-dir /tmp/qwen25-embedding-results
+```
+
+Choose a CPU core in your allowed affinity set. For the original CUDA length
+sweep, use `--lengths 512 1172 4096 16384`.
+
+## Method
+
+- BF16 checkpoint embeddings, TP=1, CUDA int32 token IDs, CPU multimodal masks.
+- One pinned CPU core, one PyTorch thread, garbage collection disabled.
+- 50 warmups per variant, then 20 paired blocks of 20 calls with balanced,
+  randomized variant order. Summary: median of block medians.
+- Host latency includes synchronization inside the wrapper. Explicit GPU
+  synchronization before and after the timed call is excluded from host latency.
+  Calling-thread CPU time and synchronized completion time are recorded separately.
+- CPU frequency is sampled at block boundaries; clocks are not locked.
+- Validation checks one text embedding call for the fixed wrapper, two for the
+  multimodal baseline, and `torch.equal` for identical inputs and batch shapes.
+  This does not establish batch invariance or full-model output equality.
+
+Interleaved audio/video uses a valid synthetic multimodal mask. This benchmark
+measures wrapper latency, not E2E throughput.
+
+## Reference results
+
+Original environment: H100 80GB; Python 3.12.3; vLLM 0.29.0+cu129;
+PyTorch 2.13.0+cu129; Transformers 5.14.0; Qwen2.5-Omni-7B BF16.
+At 1172 tokens, the September 13 measurements were:
+
+| Input | Before (μs) | After (μs) | Host latency reduction |
+| --- | ---: | ---: | ---: |
+| Text | 18.81 | 18.66 | 0.80% |
+| Audio | 104.32 | 85.30 | 18.23% |
+| Image | 104.72 | 86.27 | 17.62% |
+| Video | 104.73 | 85.76 | 18.11% |
+| Audio + image + video | 567.92 | 549.54 | 3.24% |
+| Interleaved audio/video | 622.76 | 597.84 | 4.00% |
+
+Results vary between runs and environments. These local savings did not produce
+a consistent E2E throughput improvement in the separate repeated E2E experiment.
+
+## Output
+
+The output directory contains `metadata.json`, `micro-raw.json`,
+`validation.json`, and `summary.json`. Raw records include per-call timings,
+block timestamps and CPU frequency samples. Timer overhead is measured separately.

benchmarks/qwen2_5_omni/benchmark_embeddings.py

@@ -0,0 +1,306 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Reproduce PR #7477 host-observed embedding-wrapper latency on one CUDA GPU.
+
+Run from the PR checkout with PYTHONPATH=$PWD. Baseline and fixed methods share
+weights and inputs. Only the historical wrapper is loaded from --baseline-ref.
+"""
+
+import argparse
+import ast
+import gc
+import json
+import os
+import random
+import statistics
+import subprocess
+import textwrap
+import time
+from pathlib import Path
+
+import torch
+from safetensors import safe_open
+from torch import nn
+from transformers import Qwen2_5OmniThinkerConfig
+from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
+from vllm.multimodal.utils import set_mm_embedding_modality
+
+from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import (
+    Qwen2_5OmniThinkerForConditionalGeneration as Thinker,
+)
+from vllm_omni.platforms import current_omni_platform
+
+
+def baseline_method(cls, ref):
+    path = "vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py"
+    source = subprocess.check_output(
+        ["git", "-C", str(Path(__file__).resolve().parents[2]), "show", f"{ref}:{path}"], text=True
+    )
+    tree = ast.parse(source)
+    owner = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == cls.__name__)
+    node = next(n for n in owner.body if isinstance(n, ast.FunctionDef) and n.name == "embed_input_ids")
+    method = textwrap.dedent(ast.get_source_segment(source, node))
+    # Preserve zero-argument super() against the real model class and MRO.
+    factory = "from __future__ import annotations\ndef make_method(cls):\n    __class__ = cls\n"
+    factory += textwrap.indent(method, "    ") + "\n    return embed_input_ids\n"
+    namespace = dict(cls.embed_input_ids.__globals__)
+    exec(compile(factory, f"{ref}:{path}", "exec"), namespace)
+    return namespace["make_method"](cls)
+
+
+parser = argparse.ArgumentParser(description=__doc__)
+parser.add_argument("--model", type=Path, required=True)
+parser.add_argument("--baseline-ref", default="01a2f93256975c7ff9565c5414b0fe0225bc4765")
+parser.add_argument("--cpu-core", type=int, default=sorted(os.sched_getaffinity(0))[0])
+parser.add_argument("--lengths", type=int, nargs="+", default=[1172])
+parser.add_argument("--warmup", type=int, default=50)
+parser.add_argument("--blocks", type=int, default=20)
+parser.add_argument("--iterations", type=int, default=20)
+parser.add_argument("--output-dir", type=Path, default=Path("micro-results"))
+args = parser.parse_args()
+if min(args.lengths) < 300 or min(args.blocks, args.iterations) < 1 or args.warmup < 0:
+    parser.error("Require lengths >= 300, positive blocks/iterations, and nonnegative warmup")
+if args.cpu_core not in os.sched_getaffinity(0):
+    parser.error("CPU core is outside the allowed affinity set")
+ROOT, MODEL, CORE = args.output_dir, args.model, args.cpu_core
+ROOT.mkdir(parents=True, exist_ok=True)
+CONFIG = json.loads((MODEL / "config.json").read_text())["thinker_config"]
+HIDDEN = CONFIG["text_config"]["hidden_size"]
+VOCAB = CONFIG["text_config"]["vocab_size"]
+IDS = {m: CONFIG[m + "_token_index"] for m in ("audio", "image", "video")}
+METHODS = {"baseline": baseline_method(Thinker, args.baseline_ref), "fixed": Thinker.embed_input_ids}
+RESULTS, VALIDATIONS = [], []
+os.sched_setaffinity(0, {CORE})
+torch.set_num_threads(1)
+torch.set_num_interop_threads(1)
+gc.disable()
+
+
+def cpu_frequency():
+    path = Path(f"/sys/devices/system/cpu/cpu{CORE}/cpufreq/scaling_cur_freq")
+    if path.exists():
+        return float(path.read_text()) / 1000
+    for section in Path("/proc/cpuinfo").read_text().split("\n\n"):
+        fields = dict(line.split(":", 1) for line in section.splitlines() if ":" in line)
+        fields = {k.strip(): v.strip() for k, v in fields.items()}
+        if fields.get("processor") == str(CORE):
+            return float(fields.get("cpu MHz", "nan"))
+
+
+def payload(case, length, device):
+    generator = torch.Generator().manual_seed(7451 + length)
+    ids = torch.randint(2, 10000, (length,), generator=generator)
+    specs = {
+        "text": [],
+        "audio": [("audio", 100)],
+        "image": [("image", 16)],
+        "video": [("video", 128)],
+        "mixed": [("audio", 100), ("image", 16), ("video", 128)],
+        "interleaved": [("video", 64), ("audio", 50), ("video", 64), ("audio", 50)],
+    }[case]
+    offset = 44
+    counts = {}
+    for modality, count in specs:
+        ids[offset : offset + count] = IDS[modality]
+        counts[modality] = counts.get(modality, 0) + count
+        offset += count + (0 if case == "interleaved" else 2)
+    mask = torch.zeros(length, dtype=torch.bool)
+    for token in IDS.values():
+        mask |= ids == token
+    mm = [
+        set_mm_embedding_modality(torch.full((count, HIDDEN), i + 1, device=device, dtype=torch.bfloat16), modality)
+        for i, (modality, count) in enumerate(counts.items())
+    ]
+    return ids.to(device=device, dtype=torch.int32), None if case == "text" else mm, None if case == "text" else mask
+
+
+def once(fn, cuda):
+    # Drain preceding work before the host timer; the production function's
+    # own D2H synchronizations remain inside the timed call.
+    if cuda:
+        current_omni_platform.synchronize()
+    w0 = time.perf_counter_ns()
+    c0 = time.thread_time_ns()
+    value = fn()
+    c1 = time.thread_time_ns()
+    w1 = time.perf_counter_ns()
+    if cuda:
+        current_omni_platform.synchronize()
+    w2 = time.perf_counter_ns() if cuda else w1
+    del value
+    return [(w1 - w0) / 1000, (c1 - c0) / 1000, (w2 - w0) / 1000]
+
+
+def measure(functions, device, case, length, scope, rounds=20, iterations=20, warmups=50):
+    cuda = device == "cuda"
+    for fn in functions.values():
+        for _ in range(warmups):
+            once(fn, cuda)
+    order_rng = random.Random(7451 + length + len(case) + len(scope))
+    directions = [i % 2 for i in range(rounds)]
+    order_rng.shuffle(directions)
+    names = list(functions)
+    for block, direction in enumerate(directions):
+        order = names if direction == 0 else names[::-1]
+        for name in order:
+            frequency_start = cpu_frequency()
+            wall_start = time.time()
+            samples = [once(functions[name], cuda) for _ in range(iterations)]
+            wall_end = time.time()
+            RESULTS.append(
+                dict(
+                    device=device,
+                    case=case,
+                    tokens=length,
+                    scope=scope,
+                    variant=name,
+                    block=block,
+                    iterations=iterations,
+                    samples_us=samples,
+                    wall_start=wall_start,
+                    wall_end=wall_end,
+                    cpu_mhz_start=frequency_start,
+                    cpu_mhz_end=cpu_frequency(),
+                )
+            )
+    (ROOT / "micro-raw.json").write_text(json.dumps(RESULTS))
+    current = [r for r in RESULTS if (r["device"], r["case"], r["tokens"], r["scope"]) == (device, case, length, scope)]
+    print(
+        "RESULT",
+        json.dumps(
+            {
+                "device": device,
+                "case": case,
+                "tokens": length,
+                "scope": scope,
+                "host_median_us": {
+                    v: statistics.median(t[0] for r in current if r["variant"] == v for t in r["samples_us"])
+                    for v in names
+                },
+            }
+        ),
+        flush=True,
+    )
+
+
+def build_model(weights, device):
+    model = Thinker.__new__(Thinker)
+    nn.Module.__init__(model)
+    model.config = Qwen2_5OmniThinkerConfig(**CONFIG)
+    model._has_oov_mm_tokens = False
+    table = VocabParallelEmbedding(VOCAB, HIDDEN, params_dtype=torch.bfloat16, disable_tp=True)
+    table.weight = nn.Parameter(weights.to(device), requires_grad=False)
+
+    class LM(nn.Module):
+        def __init__(self):
+            super().__init__()
+            self.embed_tokens = table
+
+        def embed_input_ids(self, ids):
+            return self.embed_tokens(ids)
+
+    model.language_model = LM()
+    return model
+
+
+def validate(model, args, case, length, device):
+    ids, mm, mask = args
+    lm = model.get_language_model()
+    original = lm.embed_input_ids
+    outputs, counts = {}, {}
+    for variant, method in METHODS.items():
+        calls = []
+
+        def counted(tokens):
+            calls.append(tokens.numel())
+            return original(tokens)
+
+        lm.embed_input_ids = counted
+        try:
+            outputs[variant] = method(model, ids, mm, is_multimodal=mask)
+        finally:
+            lm.embed_input_ids = original
+        counts[variant] = len(calls)
+    assert torch.equal(outputs["baseline"], outputs["fixed"])
+    assert counts == {"baseline": 1 if case == "text" else 2, "fixed": 1}
+    VALIDATIONS.append(dict(case=case, tokens=length, device=device, calls=counts, same_input_wrapper_torch_equal=True))
+    (ROOT / "validation.json").write_text(json.dumps(VALIDATIONS, indent=2))
+
+
+@torch.inference_mode()
+def main():
+    import transformers
+    import vllm
+
+    metadata = dict(
+        baseline_ref=args.baseline_ref,
+        checkout=subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
+        torch=torch.__version__,
+        vllm=vllm.__version__,
+        transformers=transformers.__version__,
+        gpu=torch.cuda.get_device_name(),
+        cpu_core=CORE,
+        torch_cpu_threads=1,
+        warmup=args.warmup,
+        blocks=args.blocks,
+        iterations=args.iterations,
+        lengths=args.lengths,
+        embedding_shape=[VOCAB, HIDDEN],
+        dtype="bfloat16",
+        input_ids_dtype="int32",
+        mask_device="cpu",
+        start_time=time.time(),
+        metric_order=["host_observed_us", "calling_thread_cpu_us", "synchronized_completion_us"],
+    )
+    index = json.loads((MODEL / "model.safetensors.index.json").read_text())["weight_map"]
+    key = "thinker.model.embed_tokens.weight"
+    with safe_open(MODEL / index[key], framework="pt", device="cpu") as f:
+        weights = f.get_tensor(key)
+    assert weights.shape == (VOCAB, HIDDEN) and weights.dtype == torch.bfloat16
+    model = build_model(weights, "cuda")
+    measure({"timer_floor": lambda: None}, "cuda", "timer", 0, "overhead", rounds=10, iterations=100)
+    for length in args.lengths:
+        for case in ("text", "audio", "image", "video", "mixed", "interleaved"):
+            ids, mm, mask = payload(case, length, "cuda")
+            validate(model, (ids, mm, mask), case, length, "cuda")
+            functions = {
+                variant: (lambda method=method: method(model, ids, mm, is_multimodal=mask))
+                for variant, method in METHODS.items()
+            }
+            measure(
+                functions,
+                "cuda",
+                case,
+                length,
+                "whole_wrapper",
+                rounds=args.blocks,
+                iterations=args.iterations,
+                warmups=args.warmup,
+            )
+    summary = []
+    for length in args.lengths:
+        for case in ("text", "audio", "image", "video", "mixed", "interleaved"):
+            medians = {
+                variant: statistics.median(
+                    statistics.median(sample[0] for sample in record["samples_us"])
+                    for record in RESULTS
+                    if record["tokens"] == length and record["case"] == case and record["variant"] == variant
+                )
+                for variant in METHODS
+            }
+            summary.append(
+                dict(
+                    tokens=length,
+                    case=case,
+                    host_us=medians,
+                    reduction_pct=100 * (1 - medians["fixed"] / medians["baseline"]),
+                )
+            )
+    metadata["end_time"] = time.time()
+    (ROOT / "metadata.json").write_text(json.dumps(metadata, indent=2))
+    (ROOT / "summary.json").write_text(json.dumps(summary, indent=2))
+    print(json.dumps(summary, indent=2))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/qwen3_omni/embedding_harness.py

@@ -0,0 +1,226 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Small fixtures for the real Qwen3 thinker embedding method and its oracle."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import asdict, dataclass
+from types import SimpleNamespace
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+SCENARIOS = ("text", "empty", "audio", "image", "video", "mixed", "interleaved", "vision_no_deepstack")
+
+
+@dataclass(frozen=True)
+class ModelShape:
+    vocab_size: int = 152064
+    hidden_size: int = 2048
+    deepstack_levels: int = 3
+    image_token_id: int = 151655
+    video_token_id: int = 151656
+    audio_token_id: int = 151675
+
+    def __post_init__(self):
+        if min(self.vocab_size, self.hidden_size, self.deepstack_levels) <= 0:
+            raise ValueError("vocab_size, hidden_size and deepstack_levels must be positive")
+        ids = (self.image_token_id, self.video_token_id, self.audio_token_id)
+        if len(set(ids)) != 3 or not all(0 < token < self.vocab_size for token in ids):
+            raise ValueError("Provide three distinct, positive, in-vocabulary modality IDs; use oov=True to test OOV")
+
+    def token_ids(self, oov=False):
+        if oov:
+            return dict(zip(("image", "video", "audio"), range(self.vocab_size, self.vocab_size + 3)))
+        return {name: getattr(self, f"{name}_token_id") for name in ("image", "video", "audio")}
+
+
+@dataclass
+class Feature:
+    modality: str
+    positions: torch.Tensor
+    main: torch.Tensor
+    deepstack: torch.Tensor | None
+    combined: torch.Tensor
+
+
+@dataclass
+class EmbeddingCase:
+    name: str
+    input_ids: torch.Tensor
+    is_multimodal: torch.Tensor | None
+    features: tuple[Feature, ...]
+    fingerprint: str
+
+    def fresh_embeddings(self):
+        # The thinker replaces list entries with main-scale views.
+        return None if self.name == "text" else [feature.combined for feature in self.features]
+
+
+def _tensor_bytes(tensor):
+    return tensor.detach().contiguous().view(torch.uint8).cpu().numpy().tobytes()
+
+
+def build_case(
+    name,
+    num_tokens,
+    num_mm_tokens,
+    shape,
+    *,
+    device="cuda",
+    dtype=torch.bfloat16,
+    mask_device="cpu",
+    seed=42,
+    oov=False,
+):
+    """Build known feature placements; no model merge is used to define them."""
+    if name not in SCENARIOS:
+        raise ValueError(f"Unknown scenario: {name}")
+    if num_tokens <= 0 or not 0 <= num_mm_tokens < num_tokens:
+        raise ValueError("Require N > 0 and 0 <= M < N")
+    if (name in {"text", "empty"}) != (num_mm_tokens == 0):
+        raise ValueError("Only text/empty scenarios accept M=0")
+    if name in {"mixed", "interleaved"} and (num_mm_tokens < 4 or num_mm_tokens % 2):
+        raise ValueError("Mixed/interleaved scenarios require an even M >= 4")
+    token_ids = shape.token_ids(oov)
+    generator = torch.Generator().manual_seed(seed)
+    ids = torch.randint(0, min(shape.vocab_size, *token_ids.values()), (num_tokens,), generator=generator)
+    mask = torch.zeros(num_tokens, dtype=torch.bool)
+    positions = torch.arange((num_tokens - num_mm_tokens) // 2, (num_tokens - num_mm_tokens) // 2 + num_mm_tokens)
+    groups = []
+    if name == "mixed":
+        groups = [("audio", positions[: num_mm_tokens // 2]), ("image", positions[num_mm_tokens // 2 :])]
+    elif name == "interleaved":
+        groups = [("video", positions[::2]), ("audio", positions[1::2])]
+    elif num_mm_tokens:
+        groups = [("image" if name == "vision_no_deepstack" else name, positions)]
+    features = []
+    for modality, indexes in groups:
+        ids[indexes] = token_ids[modality]
+        mask[indexes] = True
+        rows = torch.arange(len(indexes), dtype=torch.float32)[:, None].remainder(31) / 32
+        cols = torch.arange(shape.hidden_size, dtype=torch.float32)[None, :].remainder(7) / 16
+        main_cpu = rows + cols + {"audio": 2, "image": 4, "video": 6}[modality]
+        main = main_cpu.to(device=device, dtype=dtype)
+        deepstack = None
+        parts = [main]
+        if modality != "audio" and name != "vision_no_deepstack":
+            deepstack = torch.stack(
+                [(main_cpu + 16 + 4 * level).to(device=device, dtype=dtype) for level in range(shape.deepstack_levels)]
+            )
+            parts.extend(deepstack.unbind(0))
+        combined = torch.cat(parts, dim=-1)
+        # set_mm_embedding_modality simply attaches this attribute. Tag only
+        # after cat/to: views and copies do not preserve arbitrary attributes.
+        combined.modality = modality
+        features.append(Feature(modality, indexes, main, deepstack, combined))
+    digest = hashlib.sha256(json.dumps({"name": name, "shape": asdict(shape), "seed": seed, "oov": oov}).encode())
+    digest.update(_tensor_bytes(ids))
+    digest.update(_tensor_bytes(mask))
+    for feature in features:
+        digest.update(feature.modality.encode())
+        digest.update(_tensor_bytes(feature.combined))
+    return EmbeddingCase(
+        name,
+        ids.to(device=device),
+        None if name == "text" else mask.to(device=mask_device),
+        tuple(features),
+        digest.hexdigest(),
+    )
+
+
+class LanguageModelAdapter(nn.Module):
+    def __init__(self, embedding):
+        super().__init__()
+        self.embedding = embedding
+
+    def embed_input_ids(self, input_ids):
+        return self.embedding(input_ids)
+
+    def forward(self, input_ids):
+        return self.embed_input_ids(input_ids)
+
+
+def make_thinker(
+    shape,
+    *,
+    device="cuda",
+    dtype=torch.bfloat16,
+    seed=42,
+    backend="torch",
+    deepstack=True,
+    buffer_capacity=32,
+    oov=False,
+):
+    """Skip checkpoint construction, preserving the real class and its MRO."""
+    from tests.model_executor.helpers import bootstrap_vllm_layer_custom_op_modules
+
+    bootstrap_vllm_layer_custom_op_modules()
+    from vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker import (
+        Qwen3OmniMoeThinkerForConditionalGeneration,
+    )
+
+    if buffer_capacity <= 0:
+        raise ValueError("buffer_capacity must be positive")
+    device = torch.device(device)
+    if backend == "torch":
+        embedding = nn.Embedding(shape.vocab_size, shape.hidden_size, device=device, dtype=dtype)
+    elif backend == "vllm":
+        from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
+
+        with torch.device(device):
+            embedding = VocabParallelEmbedding(shape.vocab_size, shape.hidden_size, params_dtype=dtype, disable_tp=True)
+    else:
+        raise ValueError(f"Unknown embedding backend: {backend}")
+    with torch.no_grad():
+        embedding.weight.uniform_(-0.5, 0.5, generator=torch.Generator(device=device).manual_seed(seed))
+    thinker = object.__new__(Qwen3OmniMoeThinkerForConditionalGeneration)
+    nn.Module.__init__(thinker)
+    thinker.config = SimpleNamespace(text_config=SimpleNamespace(hidden_size=shape.hidden_size))
+    for modality, token_id in shape.token_ids(oov).items():
+        setattr(thinker.config, f"{modality}_token_id", token_id)
+    thinker.visual = SimpleNamespace(
+        deepstack_visual_indexes=list(range(shape.deepstack_levels)) if deepstack else None
+    )
+    thinker.language_model = LanguageModelAdapter(embedding)
+    thinker._has_oov_mm_tokens = oov
+    thinker.use_deepstack = deepstack
+    thinker.deepstack_num_level = shape.deepstack_levels if deepstack else 0
+    thinker.deepstack_input_embeds = [
+        torch.zeros(buffer_capacity, shape.hidden_size, device=device, dtype=dtype)
+        for _ in range(thinker.deepstack_num_level)
+    ]
+    thinker.deepstack_input_embeds_num_tokens = 0
+    return thinker.eval()
+
+
+@torch.inference_mode()
+def expected_outputs(model, case):
+    """Independent oracle: direct lookup and explicit positions, not merge helpers."""
+    ids = case.input_ids
+    if model._has_oov_mm_tokens and case.is_multimodal is not None:
+        ids = ids.masked_fill(case.is_multimodal.to(ids.device), 0)
+    main = F.embedding(ids, model.language_model.embedding.weight)
+    has_deepstack = any(feature.deepstack is not None for feature in case.features)
+    deepstack = main.new_zeros((model.deepstack_num_level, *main.shape)) if has_deepstack else None
+    for feature in case.features:
+        indexes = feature.positions.to(main.device)
+        main[indexes] = feature.main
+        if feature.deepstack is not None:
+            deepstack[:, indexes, :] = feature.deepstack
+    return main, deepstack
+
+
+@torch.inference_mode()
+def assert_case_output(model, case, result):
+    expected, deepstack = expected_outputs(model, case)
+    torch.testing.assert_close(result, expected, rtol=0, atol=0)
+    if deepstack is None:
+        assert model.deepstack_input_embeds_num_tokens == 0, "Unexpected stale DeepStack buffer"
+    else:
+        assert model.deepstack_input_embeds_num_tokens == case.input_ids.numel()
+        actual = torch.stack([buffer[: case.input_ids.numel()] for buffer in model.deepstack_input_embeds])
+        torch.testing.assert_close(actual, deepstack, rtol=0, atol=0)

benchmarks/tts/README.md

@@ -85,6 +85,25 @@ vllm bench serve --omni \
     --save-result --result-dir ./results
 ```
 
+#### Set the sample rate for models that are not 24 kHz
+
+Streamed PCM carries no header, so the harness derives `audio_duration` from the
+byte count and a **default 24000 Hz** assumption
+(`vllm_omni/metrics/definitions.py:DEFAULT_AUDIO_SAMPLE_RATE`). For a model that
+emits another rate, export the real one or every duration-derived metric is
+wrong by `real_sr / 24000`:
+
+```bash
+# Audio8 TTS, Fish Speech S2 Pro, Ming-omni-tts, ... all decode at 44.1 kHz
+export VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100
+```
+
+Without it, a 44.1 kHz model reports `audio_duration` 1.84x too long, so
+`audio_rtf` looks 1.84x *better* than reality and `audio_underrun` /
+`Streaming continuity OK rate` are measured against inflated chunk budgets.
+A quick sanity check: compare `Mean AUDIO_DURATION (s)` against the actual
+length of a saved WAV for the same prompt.
+
 #### Streaming continuity (`audio_underrun`)
 
 `audio_underrun` (seconds) is the per-request worst-case buffer deficit

benchmarks/tts/bench_cosyvoice3_concurrency_matrix.py

@@ -0,0 +1,517 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""CosyVoice3 streaming e2e benchmark: concurrency x utterance-length matrix.
+
+Launches the server once per (concurrency, step-size) combo, streams TTS
+requests cycling through several utterance lengths, and reports
+TTFA/E2E/RTF/per-chunk latency (mean, stdev, range) per cell. "Step size" is
+``codec_chunk_frames``, the token-hop length the first emitted chunk waits
+for; later chunks wait for a doubling hop capped at ``4 * codec_chunk_frames``
+(see ``_token_hop_lengths``). Per-chunk latency is reconstructed from that
+same ramp rather than from raw transport read boundaries, which don't align
+to model chunks (see ``one_request``'s docstring).
+
+Running with no flags reproduces the default matrix. Per-request correctness
+is checked: non-silent, long enough, and (with a fixed ``--seed``)
+byte-identical PCM across requests for the same text.
+"""
+
+import argparse
+import json
+import os
+import signal
+import statistics
+import subprocess
+import sys
+import threading
+import time
+from pathlib import Path
+
+import httpx
+import numpy as np
+
+REPO = Path(__file__).resolve().parents[2]
+DEFAULT_MODEL = "FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
+
+DEFAULT_REF_AUDIO = "https://raw.githubusercontent.com/FunAudioLLM/CosyVoice/main/asset/zero_shot_prompt.wav"
+DEFAULT_REF_TEXT = "希望你以后能够做的比我还好呦。"
+
+DEFAULT_TEXTS = [
+    "Thank you so much for the birthday gift you sent me.",
+    "The weather is lovely today, with warm sunshine and a gentle breeze. "
+    "We decided to take a walk in the park to enjoy the spring flowers and the fresh air.",
+    "The weather is lovely today, with warm sunshine and a gentle breeze. We decided to take a walk in the "
+    "park to enjoy the spring flowers and the fresh air. Many people were out walking, children were "
+    "running and playing on the grass, and older folks sat on benches chatting with one another. We "
+    "strolled slowly along the path and came across a cherry tree in full bloom, its pink petals drifting "
+    "on the wind. It was so beautiful that we stopped to take some photos to remember the moment. Then we "
+    "continued on and reached a small lake, its water so clear you could see straight to the bottom, with "
+    "the blue sky and white clouds reflected on its surface. A few ducks paddled back and forth across the "
+    "water, occasionally letting out a soft quack. We sat on a bench by the lake for a while, enjoying the "
+    "peaceful afternoon. Time passed quickly, and before we knew it evening had arrived, with the setting "
+    "sun painting the sky in shades of gold. Reluctant to leave, we finally headed home, already planning "
+    "our next visit. On the way back we talked about everything we had seen and heard that day, feeling "
+    "wonderfully happy. It was truly an unforgettable weekend.",
+]
+
+# PCM16 mono @ 24 kHz (CosyVoice3 output rate)
+SAMPLE_RATE = 24000
+
+# Well below any real speech's RMS; catches all-zero/near-silent PCM that byte
+# count and duration checks alone pass through.
+MIN_RMS = 50.0
+
+# Overlay: only codec_chunk_frames is overridden; must live next to cosyvoice3.yaml (relative base_config)
+DEPLOY_TEMPLATE = """# Auto-generated by bench_cosyvoice3_concurrency_matrix.py -- do not edit.
+base_config: cosyvoice3.yaml
+
+connectors:
+  connector_of_shared_memory:
+    name: SharedMemoryConnector
+    extra:
+      codec_streaming: true
+      connector_get_sleep_s: 0.01
+      connector_get_max_wait_first_chunk: 3000
+      connector_get_max_wait: 300
+      codec_chunk_frames: {chunk_frames}
+      codec_max_chunk_frames: {max_chunk_frames}
+      codec_stream_scale_factor: {stream_scale_factor}
+      codec_left_context_frames: 25
+      codec_vocab_size: 6561
+"""
+
+# CosyVoice3Config constants (transformers_utils/configs/cosyvoice3.py): tokens
+# are consumed at token_mel_ratio mel frames each, and HiFT emits
+# samples_per_mel PCM16 samples per mel frame. Pinned here (not left to the
+# server's defaults) since one_request's byte-boundary reconstruction below
+# depends on knowing them exactly.
+TOKEN_MEL_RATIO = 2
+SAMPLES_PER_MEL = 480
+STREAM_SCALE_FACTOR = 2
+
+
+def _stats(values: list[float]) -> dict[str, float]:
+    """Mean/median/stdev/p95/min/max/n for a list of per-request measurements."""
+    n = len(values)
+    if n == 0:
+        return {
+            "n": 0,
+            "mean": float("nan"),
+            "median": float("nan"),
+            "stdev": float("nan"),
+            "p95": float("nan"),
+            "min": float("nan"),
+            "max": float("nan"),
+        }
+    return {
+        "n": n,
+        "mean": statistics.mean(values),
+        "median": statistics.median(values),
+        "stdev": statistics.stdev(values) if n >= 2 else 0.0,
+        "p95": statistics.quantiles(values, n=20)[18] if n >= 2 else values[0],
+        "min": min(values),
+        "max": max(values),
+    }
+
+
+def vllm_bin() -> str:
+    for c in (REPO / ".venv/bin/vllm", Path(sys.executable).parent / "vllm"):
+        if c.is_file():
+            return str(c)
+    return "vllm"
+
+
+def wait_ready(url: str, proc: subprocess.Popen, log: Path, timeout: float) -> None:
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if proc.poll() is not None:
+            tail = log.read_text(errors="replace").splitlines()[-40:] if log.exists() else []
+            raise RuntimeError("server exited during startup:\n  " + "\n  ".join(tail))
+        try:
+            if httpx.get(f"{url}/health", timeout=2.0).status_code == 200:
+                return
+        except Exception:
+            pass
+        time.sleep(2.0)
+    raise TimeoutError(f"server not ready within {timeout}s (log: {log})")
+
+
+def _token_hop_lengths(chunk_frames: int, max_chunk_frames: int):
+    """The token-hop-length ramp talker2code2wav_async_chunk uses to decide
+    when to fire the next flow-solve/emission: chunk_frames, then doubling
+    (STREAM_SCALE_FACTOR) each step, capped at max_chunk_frames. This -- not
+    codec_chunk_frames alone -- is what actually paces emitted audio chunks:
+    with the defaults here, hops are chunk_frames, 2x, 4x, 4x, 4x, ...
+
+    The very first emission is consistently one token short of chunk_frames
+    (confirmed against real speech.audio.delta event sizes) even though the
+    ramp's own progression is computed from the nominal chunk_frames, not the
+    actual first hop -- so only the first yield gets the -1 adjustment.
+    """
+    yield chunk_frames - 1
+    hop = chunk_frames
+    while True:
+        hop = min(max_chunk_frames, max(chunk_frames, hop * STREAM_SCALE_FACTOR))
+        yield hop
+
+
+def one_request(client, url, model, text, ref_audio, ref_text, chunk_frames, seed=None):
+    """Run one streaming request, returning TTFA, E2E, per-chunk gaps, and audio bytes.
+
+    ``response_format=pcm`` + ``stream_format=audio`` streams raw bytes with no
+    framing, so a client read boundary alone is a transport artifact, not a
+    model chunk boundary. Reconstructed instead by replicating the server's
+    own token-hop-length ramp (see ``_token_hop_lengths``) to compute each
+    real chunk's expected byte size ahead of time, and slicing the
+    accumulated byte stream at those offsets -- independent of how the
+    transport happens to fragment the underlying reads. The final chunk
+    (finalize, which releases extra held-back frames) is whatever remains
+    when the stream closes, not a predicted hop.
+    """
+    payload = {
+        "model": model,
+        "input": text,
+        "response_format": "pcm",
+        "stream": True,
+        "stream_format": "audio",
+        "ref_audio": ref_audio,
+        "ref_text": ref_text,
+    }
+    if seed is not None:
+        payload["seed"] = seed
+    t0 = time.perf_counter()
+    ttfa = None
+    nbytes = 0
+    audio_bytes = bytearray()
+    chunk_gaps_ms = []  # inter-chunk arrival gaps (per model chunk), by step index
+    prev_t = None
+    step_idx = 0
+
+    bytes_per_token = TOKEN_MEL_RATIO * SAMPLES_PER_MEL * 2
+    hop_gen = _token_hop_lengths(chunk_frames, 4 * chunk_frames)
+    next_boundary = next(hop_gen) * bytes_per_token
+    last_crossed_bytes = 0
+    received = 0
+    last_now = t0
+
+    with client.stream("POST", f"{url}/v1/audio/speech", json=payload, headers={"Authorization": "Bearer EMPTY"}) as r:
+        if r.status_code != 200:
+            raise RuntimeError(f"{r.status_code}: {r.read()[:300]!r}")
+        for chunk in r.iter_bytes():
+            if not chunk:
+                continue
+            now = time.perf_counter()
+            if ttfa is None:
+                ttfa = now - t0
+            last_now = now
+            received += len(chunk)
+            nbytes += len(chunk)
+            audio_bytes += chunk
+            while received >= next_boundary:
+                if prev_t is not None:
+                    chunk_gaps_ms.append((step_idx, (now - prev_t) * 1e3))
+                    step_idx += 1
+                prev_t = now
+                last_crossed_bytes = next_boundary
+                next_boundary += next(hop_gen) * bytes_per_token
+
+    if received > last_crossed_bytes:
+        # Finalize chunk: releases extra held-back frames, so it doesn't
+        # match a predicted hop size -- its arrival is the stream closing.
+        if prev_t is not None:
+            chunk_gaps_ms.append((step_idx, (last_now - prev_t) * 1e3))
+            step_idx += 1
+
+    e2e = time.perf_counter() - t0
+    audio_s = nbytes / 2 / SAMPLE_RATE
+    return {
+        "ttfa_s": ttfa,
+        "e2e_s": e2e,
+        "audio_s": audio_s,
+        "rtf": e2e / audio_s if audio_s > 0 else float("nan"),
+        "chunk_gaps_ms": chunk_gaps_ms,
+        "nbytes": nbytes,
+        "audio_bytes": audio_bytes,
+    }
+
+
+def check_correctness(r: dict, min_audio_s: float) -> list[str]:
+    """Return a list of correctness violations (empty = pass)."""
+    problems = []
+    if r["nbytes"] <= 0:
+        problems.append("empty audio (0 bytes)")
+    if r["audio_s"] < min_audio_s:
+        problems.append(f"audio too short ({r['audio_s']:.2f}s < {min_audio_s:.2f}s)")
+    if r["nbytes"] > 0:
+        samples = np.frombuffer(bytes(r["audio_bytes"]), dtype=np.int16).astype(np.float64)
+        rms = float(np.sqrt(np.mean(samples**2)))
+        if rms < MIN_RMS:
+            problems.append(f"near-silent audio (rms={rms:.1f} < {MIN_RMS})")
+    return problems
+
+
+def drive(url, args, n, conc, label, results, lock, idx, chunk_frames):
+    """Run ``n`` requests at concurrency ``conc``, cycling through ``args.texts``."""
+    texts = args.texts
+
+    def worker():
+        with httpx.Client(timeout=600.0) as c:
+            while True:
+                with lock:
+                    i = idx[0]
+                    idx[0] += 1
+                if i >= n:
+                    return
+                text = texts[i % len(texts)]
+                try:
+                    r = one_request(
+                        c, url, args.model, text, args.ref_audio, args.ref_text, chunk_frames, seed=args.seed
+                    )
+                    r["text"] = text
+                    with lock:
+                        results.append(r)
+                except Exception as e:
+                    print(f"    [{label}] request {i} failed: {e}", flush=True)
+
+    ts = [threading.Thread(target=worker) for _ in range(conc)]
+    t0 = time.perf_counter()
+    for t in ts:
+        t.start()
+    for t in ts:
+        t.join()
+    return time.perf_counter() - t0
+
+
+def run_combo(args, conc: int, chunk_frames: int, out: Path) -> dict:
+    tag = f"c{conc}_chunk{chunk_frames}"
+    run_dir = out / tag
+    run_dir.mkdir(parents=True, exist_ok=True)
+    log = run_dir / "server.log"
+
+    deploy = REPO / "vllm_omni/deploy" / f"_autogen_steps_{tag}.yaml"
+    deploy.write_text(
+        DEPLOY_TEMPLATE.format(
+            chunk_frames=chunk_frames,
+            max_chunk_frames=4 * chunk_frames,
+            stream_scale_factor=STREAM_SCALE_FACTOR,
+        )
+    )
+
+    cmd = [
+        vllm_bin(),
+        "serve",
+        args.model,
+        "--omni",
+        "--deploy-config",
+        str(deploy),
+        "--host",
+        "127.0.0.1",
+        "--port",
+        str(args.port),
+        "--allowed-local-media-path",
+        str(REPO),
+    ]
+    env = dict(os.environ)
+    if args.cuda_visible_devices:
+        env["CUDA_VISIBLE_DEVICES"] = args.cuda_visible_devices
+    if args.trt is not None:
+        env["COSYVOICE3_TRT"] = args.trt
+
+    print(f"\n=== concurrency={conc} step_size={chunk_frames} ===", flush=True)
+    print(f"  launching server (log: {log})", flush=True)
+    with open(log, "w") as lf:
+        proc = subprocess.Popen(cmd, env=env, stdout=lf, stderr=subprocess.STDOUT, start_new_session=True)
+    url = f"http://127.0.0.1:{args.port}"
+    try:
+        wait_ready(url, proc, log, args.server_timeout)
+        print("  server ready", flush=True)
+
+        results: list[dict] = []
+        lock = threading.Lock()
+        idx = [0]
+        if args.warmup:
+            print(f"  warmup ({args.warmup} req)", flush=True)
+            drive(url, args, args.warmup, min(conc, args.warmup), "warmup", results, lock, idx, chunk_frames)
+            results.clear()
+            idx[0] = 0
+
+        print(f"  running {args.num_prompts} req @ concurrency {conc}", flush=True)
+        wall = drive(url, args, args.num_prompts, conc, "bench", results, lock, idx, chunk_frames)
+
+        ok = [r for r in results if r.get("ttfa_s") is not None]
+        problems = []
+        for r in ok:
+            problems += check_correctness(r, args.min_audio_s)
+        # With a fixed seed, same-text requests must produce byte-identical PCM (per text group)
+        if args.seed is not None:
+            by_text: dict[str, bytes] = {}
+            for r in ok:
+                t = r.get("text")
+                ref = by_text.get(t)
+                if ref is None:
+                    by_text[t] = r.get("audio_bytes")
+                elif r.get("audio_bytes") != ref:
+                    problems.append(f"byte-identity violation for text {t!r}: seeded requests produced different PCM")
+        n_ok = len(ok)
+        total_audio_s = sum(r["audio_s"] for r in ok)
+        all_gaps = [g for r in ok for g in r["chunk_gaps_ms"]]  # (step_index, gap_ms) per request
+        gap_vals = [g for _, g in all_gaps]
+
+        # bucket gaps by chunk position within a request, to see any O(n) slope
+        by_idx: dict[int, list[float]] = {}
+        for step_index, g in all_gaps:
+            by_idx.setdefault(step_index, []).append(g)
+        step_latency_by_idx = {i: statistics.median(vals) for i, vals in sorted(by_idx.items())}
+
+        agg = {
+            "concurrency": conc,
+            "step_size": chunk_frames,
+            "ok": n_ok,
+            "errors": args.num_prompts - n_ok,
+            "correctness_problems": len(problems),
+            "correctness_sample": problems[:5],
+            "total_audio_s": total_audio_s,
+            "throughput_audio_s_per_s": total_audio_s / wall if wall > 0 else float("nan"),
+            "ttfa_s": _stats([r["ttfa_s"] for r in ok]),
+            "e2e_s": _stats([r["e2e_s"] for r in ok]),
+            "rtf": _stats([r["rtf"] for r in ok]),
+            "step_latency_ms": _stats(gap_vals),
+            "step_latency_ms_by_idx": step_latency_by_idx,
+            "num_steps": len(all_gaps),
+            # per-utterance-length breakdown; n per text = num_prompts / len(texts)
+            "by_text": {
+                t: {
+                    "audio_s": _stats([r["audio_s"] for r in ok if r.get("text") == t]),
+                    "e2e_s": _stats([r["e2e_s"] for r in ok if r.get("text") == t]),
+                    "rtf": _stats([r["rtf"] for r in ok if r.get("text") == t]),
+                }
+                for t in sorted({r.get("text") for r in ok})
+            },
+        }
+        print(json.dumps(agg, indent=2, ensure_ascii=False), flush=True)
+        return agg
+    finally:
+        print("  shutting down server", flush=True)
+        try:
+            os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+        except Exception:
+            proc.terminate()
+        try:
+            proc.wait(timeout=60)
+        except subprocess.TimeoutExpired:
+            try:
+                os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
+            except Exception:
+                proc.kill()
+            proc.wait(timeout=20)
+        deploy.unlink(missing_ok=True)
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--model", default=DEFAULT_MODEL)
+    p.add_argument(
+        "--text",
+        dest="texts",
+        nargs="+",
+        default=DEFAULT_TEXTS,
+        help="One or more utterance texts. With several, requests cycle through "
+        "them so a single server launch covers all utterance lengths; results "
+        "are bucketed per text.",
+    )
+    p.add_argument("--ref-audio", default=DEFAULT_REF_AUDIO)
+    p.add_argument("--ref-text", default=DEFAULT_REF_TEXT)
+    p.add_argument("--concurrency", type=int, nargs="+", default=[1, 8])
+    p.add_argument(
+        "--step-size",
+        type=int,
+        nargs="+",
+        default=[15],
+        help="codec_chunk_frames values to sweep (the streaming step size).",
+    )
+    p.add_argument("--num-prompts", type=int, default=30)
+    p.add_argument("--warmup", type=int, default=2)
+    p.add_argument("--port", type=int, default=8091)
+    p.add_argument(
+        "--server-timeout",
+        type=float,
+        default=1800.0,
+        help="Startup budget; first run also downloads weights and builds the TRT engine.",
+    )
+    p.add_argument("--output-dir", default="./perf_steps")
+    p.add_argument("--cuda-visible-devices", default="0")
+    p.add_argument(
+        "--trt",
+        choices=["0", "1"],
+        default=None,
+        help="Pin COSYVOICE3_TRT. 0=force torch estimator, 1=allow TensorRT. Default: unset.",
+    )
+    p.add_argument(
+        "--min-audio-s",
+        type=float,
+        default=1.0,
+        help="Minimum expected audio seconds per request for the correctness check.",
+    )
+    p.add_argument(
+        "--seed",
+        type=int,
+        default=42,
+        help="Fixed seed for reproducible (byte-identical) audio. Pass --seed '' "
+        "equivalent (any non-int) to disable -- there is no unset sentinel here "
+        "by design, since the default sweep is meant to be exactly reproducible.",
+    )
+    return p.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    out = Path(args.output_dir).resolve()
+    out.mkdir(parents=True, exist_ok=True)
+
+    all_results = []
+    failed = False
+    for conc in args.concurrency:
+        for chunk_frames in args.step_size:
+            try:
+                result = run_combo(args, conc, chunk_frames, out)
+                all_results.append(result)
+                if result["errors"] > 0 or result["correctness_problems"] > 0:
+                    failed = True
+            except Exception as e:
+                print(f"  combo c={conc} chunk={chunk_frames} FAILED: {e}", flush=True)
+                failed = True
+
+    out_path = out / "steps_results.json"
+    with open(out_path, "w", encoding="utf-8") as f:
+        json.dump({"model": args.model, "texts": args.texts, "runs": all_results}, f, indent=2, ensure_ascii=False)
+    print(f"\nResults written to {out_path}", flush=True)
+
+    print("\n=== SUMMARY ===")
+    header = (
+        f"{'conc':>4} {'step':>4} {'ok':>4} {'corr':>4} "
+        f"{'TTFA med(ms)':>12} {'step med(ms)':>12} {'step p95(ms)':>12} {'RTF mean+-sd':>14}"
+    )
+    print(header)
+    print("-" * len(header))
+    for r in all_results:
+        print(
+            f"{r['concurrency']:>4} {r['step_size']:>4} {r['ok']:>4} {r['correctness_problems']:>4} "
+            f"{r['ttfa_s']['median'] * 1e3:>12.1f} "
+            f"{r['step_latency_ms']['median']:>12.1f} {r['step_latency_ms']['p95']:>12.1f} "
+            f"{r['rtf']['mean']:>6.3f}+-{r['rtf']['stdev']:>5.3f}"
+        )
+        for t, stats in r.get("by_text", {}).items():
+            print(
+                f"      text {t[:24]!r}: e2e {stats['e2e_s']['mean'] * 1e3:8.1f}"
+                f"+-{stats['e2e_s']['stdev'] * 1e3:6.1f} ms  (n={stats['e2e_s']['n']}, "
+                f"audio {stats['audio_s']['mean']:.1f}s)"
+            )
+
+    if failed:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/tts/model_configs.yaml

@@ -17,6 +17,17 @@ models:
         extra_params:
           lang: en
           text_normalization: true
+  # Audio8 TTS Preview: DualAR, 44.1 kHz. trust_remote_code stays false.
+  # Bench at 44.1 kHz: export VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100, otherwise
+  # the streamed-PCM duration defaults to 24 kHz and RTF looks 1.84x too good.
+  Audio8/Audio8-TTS-Preview-0.6b:
+    supported_tasks: [default_voice, voice_clone]
+    backend: openai-audio-speech
+    endpoint: /v1/audio/speech
+    trust_remote_code: false
+    task_extra_body:
+      default_voice: {}
+      voice_clone: {}
 
   # -CustomVoice checkpoints lack speaker_encoder weights, so voice_clone is
   # NOT supported (an attempt raises ValueError from _extract_speaker_embedding

docker/Dockerfile.ci

@@ -1,27 +1,10 @@
-# Target vllm version: v0.28.0 (tag v0.28.0 @ 2cf0a6915ce5).
-# Keep VLLM_BASE_TAG aligned with the target vLLM release on every rebase.
-#
-# Release-boundary form: there is no precompiled-wheel pin here, because the
-# base image already ships exactly the target vLLM. Everything the runtime
-# needs — torch, flashinfer, nvJitLink, numpy — comes from that image as one
-# self-consistent set.
-#
-# This block reappears only when the rebase targets a vLLM commit that is NOT a
-# released tag. During the 0.27 rebase, targeting v0.27.0rc1 on top of the
-# v0.26.0 base image required reinstalling vLLM from wheels.vllm.ai and then
-# hand-repairing the fallout: pinning torch ahead of flashinfer (installing it
-# after bound flashinfer's TVM FFI runtime to the old torch and aborted engine
-# core init), registering libnvJitLink.so.13 with ldconfig, and re-pinning numpy
-# under numba's ceiling. All of it was compensation for a base image and a vLLM
-# that disagreed. At a real release boundary they agree, so none of it is
-# needed. The 0.28 rebase repeated the cycle: the v0.28.0rc / final-tag wheels
-# rode on the v0.27.0 base image with the full reinstall block until the
-# official v0.28.0 image shipped, at which point this file dropped back to the
-# form you are reading. If you pin an untagged commit again, restore the
-# reinstall block — agent/tasks/20_update_wheel_url.sh fails the run rather
-# than letting the Dockerfile silently ship the wrong vLLM.
+# Target vLLM version: v0.29.0 (tag @ 98dff2a81d74).
+# Use the matching release image and its coherent CUDA/PyTorch/FlashInfer
+# runtime. Do not reinstall a commit wheel or repair dependencies on top of
+# an older base image at a release boundary. For a future unreleased target,
+# restore an explicit wheel pin and validate its complete runtime stack.
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai
-ARG VLLM_BASE_TAG=v0.28.0
+ARG VLLM_BASE_TAG=v0.29.0
 
 # We use registry-based layer caching for the dependencies,
 # where the tag is determined based on the hash of the contents

docker/Dockerfile.cuda

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai:v0.28.0
+ARG BASE_IMAGE=vllm/vllm-openai:v0.29.0
 FROM ${BASE_IMAGE}
 
 ARG COMMON_WORKDIR=/app

docker/Dockerfile.npu

@@ -1,20 +1,7 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.28.0
-FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
-
-# WORKDIR /vllm-workspace/vllm
-# RUN git fetch origin --tags && git checkout v0.18.0
-
-# WORKDIR /vllm-workspace/vllm-ascend
-# RUN git fetch origin releases/v0.18.0 && git checkout d781902ce9dbda8ab1e11bb0f2f0c1bc508fee7a
-# # Install vllm-ascend
-# # Append `libascend_hal.so` path (devlib) to LD_LIBRARY_PATH
-# RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \
-#     source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-#     source /usr/local/Ascend/nnal/atb/set_env.sh && \
-#     export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-#     python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \
-#     python3 -m pip cache purge
+ARG VLLM_ASCEND_VERSION=v0.29.0
+ARG VLLM_ASCEND_DEVICE_SUFFIX=
+FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_VERSION}${VLLM_ASCEND_DEVICE_SUFFIX}
 
 ARG APP_DIR=/vllm-workspace/vllm-omni
 WORKDIR ${APP_DIR}
@@ -24,6 +11,7 @@ RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi
     source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
     source /usr/local/Ascend/nnal/atb/set_env.sh && \
     export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
+    python3 -m pip install -v "mindiesd==3.1.0" && \
     python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation
 
 RUN python3 -m pip install -v --no-deps \

docker/Dockerfile.npu.a3

@@ -1,36 +0,0 @@
-ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.28.0-a3
-FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
-
-# WORKDIR /vllm-workspace/vllm
-# RUN git fetch origin --tags && git checkout v0.18.0
-
-# WORKDIR /vllm-workspace/vllm-ascend
-# RUN git fetch origin releases/v0.18.0 && git checkout d781902ce9dbda8ab1e11bb0f2f0c1bc508fee7a
-# # Install vllm-ascend
-# # Append `libascend_hal.so` path (devlib) to LD_LIBRARY_PATH
-# RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \
-#     source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-#     source /usr/local/Ascend/nnal/atb/set_env.sh && \
-#     export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-#     python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \
-#     python3 -m pip cache purge
-
-ARG APP_DIR=/vllm-workspace/vllm-omni
-WORKDIR ${APP_DIR}
-COPY . .
-
-RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \
-    source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-    source /usr/local/Ascend/nnal/atb/set_env.sh && \
-    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-    python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation
-
-RUN python3 -m pip install -v --no-deps \
-      "stepaudio2-minicpmo>=0.1.1" \
-      "step-audio2>=1.0.0"
-
-ENV VLLM_TARGET_DEVICE=npu
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-ENTRYPOINT []

docker/Dockerfile.npu.ci

@@ -1,10 +1,10 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.28.0
-FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
+ARG VLLM_ASCEND_VERSION=v0.29.0
+ARG VLLM_ASCEND_DEVICE_SUFFIX=
+FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_VERSION}${VLLM_ASCEND_DEVICE_SUFFIX}
 
 ARG APP_DIR=/vllm-workspace/vllm-omni
 WORKDIR ${APP_DIR}
-COPY . .
 
 RUN sed -i "s@^\(deb.*\)https\?://[a-z0-9.-]*\.ubuntu\.com@\1http://cache-service.nginx-pypi-cache.svc.cluster.local:8081@g" /etc/apt/sources.list
 
@@ -22,11 +22,9 @@ RUN apt-get update && \
     rm -rf /var/cache/apt/* && \
     rm -rf /var/lib/apt/lists/*
 
-RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-    source /usr/local/Ascend/nnal/atb/set_env.sh && \
-    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-    python3 -m pip install -U pip && \
+RUN python3 -m pip install -U pip && \
     python3 -m pip install -v \
+      "mindiesd==3.1.0" \
       "pytest>=7.0.0" \
       "pytest-asyncio>=0.21.0" \
       "pytest-cov>=4.0.0" \
@@ -46,9 +44,16 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
       "onnx>=1.16.0" \
       "librosa>=0.10.0" \
       "datasets>=3.0.0" \
+      "torchmetrics==1.9.0" \
       --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
       --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
-      --default-timeout=60 && \
+      --default-timeout=60
+
+COPY . .
+
+RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
+    source /usr/local/Ascend/nnal/atb/set_env.sh && \
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
     python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation \
       --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
       --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \

docker/Dockerfile.npu.ci.a3

@@ -1,72 +0,0 @@
-ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.28.0-a3
-FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
-
-ARG APP_DIR=/vllm-workspace/vllm-omni
-WORKDIR ${APP_DIR}
-COPY . .
-
-RUN sed -i "s@^\(deb.*\)https\?://[a-z0-9.-]*\.ubuntu\.com@\1http://cache-service.nginx-pypi-cache.svc.cluster.local:8081@g" /etc/apt/sources.list
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-        ffmpeg \
-        libgl1 \
-        libglib2.0-0 \
-        libgomp1 \
-        libsm6 \
-        libxext6 \
-        libxrender1 \
-        libxcb1 \
-        espeak-ng && \
-    rm -rf /var/cache/apt/* && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-    source /usr/local/Ascend/nnal/atb/set_env.sh && \
-    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-    python3 -m pip install -U pip && \
-    python3 -m pip install -v \
-      "pytest>=7.0.0" \
-      "pytest-asyncio>=0.21.0" \
-      "pytest-cov>=4.0.0" \
-      "pytest-mock>=3.10.0" \
-      "openpyxl>=3.0.0" \
-      "psutil>=7.2.0" \
-      "soundfile>=0.13.1" \
-      "pyttsx3>=2.99" \
-      "opencc==1.4.1" \
-      "opencv-python>=4.12.0.88" \
-      "imageio[ffmpeg]>=0.6.0" \
-      "jiwer>=3.0.0" \
-      "zhon>=2.0.0" \
-      "zhconv>=1.4.2" \
-      "funasr>=1.0.0" \
-      "hyperpyyaml>=1.2.0" \
-      "onnx>=1.16.0" \
-      "librosa>=0.10.0" \
-      "torchmetrics>=1.9.0" \
-      "datasets>=3.0.0" \
-      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
-      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
-      --default-timeout=60 && \
-    python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation \
-      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
-      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
-      --default-timeout=60
-
-# MiniCPM-o 4.5 Stage-2 vocoder assets: s3tokenizer plus flashcosyvoice, which
-# is shipped by step-audio2. --no-deps keeps pip from replacing the Ascend torch
-# build and from shadowing onnxruntime-cann with the vanilla onnxruntime wheel;
-# the deps these two actually need are installed above.
-RUN python3 -m pip install -v --no-deps \
-      "s3tokenizer>=0.3.0" \
-      "step-audio2>=1.0.0" \
-      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
-      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
-      --default-timeout=60
-
-ENV VLLM_TARGET_DEVICE=npu
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-ENTRYPOINT []

docker/Dockerfile.rocm

@@ -1,16 +1,22 @@
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.28.0
-FROM ${BASE_IMAGE} AS base
+# Keep the ROCm runtime aligned with the vLLM release targeted by
+# docker/Dockerfile.ci. A stale base image can build successfully while failing
+# at runtime when vLLM-Omni imports APIs added by the newer vLLM release.
+ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.29.0
+
+# Keep the dependency-heavy portion independent from the source tree. The
+# registry cache is keyed on these inputs, while the final source copy can be
+# linked onto the cached image without materializing its large ROCm layers.
+FROM ${BASE_IMAGE} AS deps
 
-# Declare a variable to know if we want to use the nightly build or the stable build.
 # NOTE: REMINDER to vLLM-Omni rebase maintainer
-# Remember to set `USE_NIGHTLY_BUILD` to 0, when switching back to stable vLLM docker image
-# 1. If vLLM-omni maintainer is forced to use custom commits
-#    during rebasing, they can change the two variables
-# 2. Whenever vLLM upstream has released a stable version,
-#    we should swap over to use stable release ASAP.
-#    We should avoid relying on custom commits.
+# BASE_IMAGE pins the default prebuilt runtime when USE_NIGHTLY_BUILD=0.
+# VLLM_VERSION_OR_COMMIT_HASH is consumed only by the opt-in source rebuild when
+# USE_NIGHTLY_BUILD=1. Keep both aligned with docker/Dockerfile.ci's
+# VLLM_BASE_TAG. This is enforced by tests/buildkite/test_rocm_dockerfile.py. An
+# intentional custom source pin must update that regression in the same change,
+# and should return to the stable release as soon as one is available.
 ARG USE_NIGHTLY_BUILD=0
-ARG VLLM_VERSION_OR_COMMIT_HASH=v0.28.0
+ARG VLLM_VERSION_OR_COMMIT_HASH=v0.29.0
 ARG ARG_PYTORCH_ROCM_ARCH
 ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}}
 
@@ -38,6 +44,10 @@ RUN if [ "${USE_NIGHTLY_BUILD}" = "1" ]; then \
         rm -rf vllm/.git; \
     fi
 
+# Fail during image construction instead of letting every AMD test fail at
+# runtime when the base image and vLLM-Omni target incompatible vLLM APIs.
+RUN python3 -c "import vllm; from vllm.v1.kv_cache_interface import compute_layout_strides; from vllm.v1.kv_cache_layout import KVCacheLayout; print(f'Validated vLLM {vllm.__version__}')"
+
 # Build TorchCodec after any optional nightly vLLM reinstall so it targets the
 # final torch/ROCm ABI. Keep this layer before COPY . so source changes do not
 # invalidate the expensive native build. FFmpeg development headers are
@@ -58,9 +68,24 @@ RUN --mount=type=cache,target=/root/.cache/pip \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-# Step 3: Copy vllm-omni code and install without uv
-RUN mkdir -p ${COMMON_WORKDIR}/vllm-omni
-COPY . ${COMMON_WORKDIR}/vllm-omni
+# Step 3: Install project dependencies against a placeholder package tree.
+# The editable install keeps resolving imports from this directory after the
+# final COPY --link replaces the placeholder with the real source tree.
+WORKDIR ${COMMON_WORKDIR}/vllm-omni
+RUN mkdir -p vllm_omni && touch vllm_omni/__init__.py
+COPY pyproject.toml setup.py ./
+COPY requirements/ requirements/
+ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0
+
+RUN uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" \
+        --no-cache-dir -e ".[dev]" --no-build-isolation && \
+    python3 -m pip uninstall -y onnxruntime && \
+    python3 -m pip install --no-cache-dir --force-reinstall --no-deps \
+        "onnxruntime-rocm>=1.22.2" && \
+    python3 -c "import importlib.metadata as m; import onnxruntime as ort; assert 'ROCMExecutionProvider' in ort.get_available_providers(), ort.get_available_providers(); assert m.version('onnxruntime-rocm'); assert not any(d.metadata['Name'] == 'onnxruntime' for d in m.distributions())" && \
+    python3 -c "from torchcodec.decoders import VideoDecoder"
+
+RUN ln -sf /usr/bin/python3 /usr/bin/python
 
 # This is a workaround to ensure pytest exits with the correct status code in CI tests.
 RUN printf '%s\n' \
@@ -79,19 +104,13 @@ RUN printf '%s\n' \
     '    os._exit(_exit_code)' \
     > ${COMMON_WORKDIR}/vllm-omni/conftest.py
 
-# Some optional dependencies pull in the CPU ONNX Runtime distribution. The
-# CPU and ROCm wheels own the same import path and cannot coexist, so remove
-# the CPU wheel and reinstall the ROCm wheel after dependency resolution.
-RUN cd ${COMMON_WORKDIR}/vllm-omni && \
-    uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" \
-        --no-cache-dir ".[dev]" --no-build-isolation && \
-    python3 -m pip uninstall -y onnxruntime && \
-    python3 -m pip install --no-cache-dir --force-reinstall --no-deps \
-        "onnxruntime-rocm>=1.22.2" && \
-    python3 -c "import importlib.metadata as m; import onnxruntime as ort; assert 'ROCMExecutionProvider' in ort.get_available_providers(), ort.get_available_providers(); assert m.version('onnxruntime-rocm'); assert not any(d.metadata['Name'] == 'onnxruntime' for d in m.distributions())" && \
-    python3 -c "from torchcodec.decoders import VideoDecoder"
-
-RUN ln -sf /usr/bin/python3 /usr/bin/python
+# Keep source changes in one lightweight linked layer. Do not add RUN, COPY, or
+# ADD instructions that depend on the resulting filesystem after this point;
+# doing so would force BuildKit to download and extract the cached ROCm layers.
+FROM deps AS base
+ARG COMMON_WORKDIR=/app
+WORKDIR ${COMMON_WORKDIR}/vllm-omni
+COPY --link . .
 
 FROM base AS test
 

docker/Dockerfile.rocm.dockerignore

@@ -0,0 +1,35 @@
+# Keep the linked source layer small and deterministic.
+.git
+.venv
+/build
+dist
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+.mypy_cache
+
+# Distribution / packaging
+.Python
+/build/
+cmake-build-*/
+CMakeUserPresets.json
+develop-eggs/
+/dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+rust/target/

docker/Dockerfile.xpu

@@ -1,4 +1,4 @@
-# Target vLLM version: v0.28.0. Keep VLLM_VERSION aligned with the target vLLM
+# Target vLLM version: v0.29.0. Keep VLLM_VERSION aligned with the target vLLM
 # release on every rebase.
 #
 # This file layers vLLM-Omni on top of the published upstream XPU image, which
@@ -18,7 +18,7 @@
 #
 # .buildkite/intel/scripts/run-xpu-test.sh automates that pull-then-build fallback.
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai-xpu
-ARG VLLM_VERSION=v0.28.0
+ARG VLLM_VERSION=v0.29.0
 ARG VLLM_BASE=${VLLM_BASE_IMAGE}:${VLLM_VERSION}
 
 FROM ${VLLM_BASE} AS vllm-omni
@@ -40,7 +40,6 @@ ENV VLLM_OMNI_TARGET_DEVICE=xpu
 # Keep the upstream oneCCL packages paired with the XPU torch runtime.
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install --no-build-isolation \
-    --extra-index-url https://mirrors.ustc.edu.cn/pypi/simple/ \
     ".[dev]" && \
     uv pip uninstall triton && \
     uv pip install triton-xpu==3.7.2 --reinstall

docs/.nav.yml

@@ -71,6 +71,7 @@ nav:
       - Overview and Compatibility:
         - Overview: user_guide/diffusion_features.md
         - Feature Compatibility: user_guide/feature_compatibility.md
+      - Device-side Video Postprocessing: user_guide/diffusion/device_side_video_postprocess.md
       - CPU Offloading:
         - Overview: user_guide/diffusion/cpu_offload.md
         - Model-Level: user_guide/diffusion/offloader/module_offload.md
@@ -127,6 +128,7 @@ nav:
         - Async Chunk: design/feature/async_chunk.md
         - Async Diffusion Output: design/feature/async_diffusion_output.md
         - Async Omni Output Materialization: design/feature/omni_async_output_materialization.md
+        - Prefill/Decode Phase Contract: design/feature/preprocess_phase_contract.md
         - Automatic Prefix Caching: design/feature/prefix_caching.md
         - Realtime AR-Diffusion: design/feature/realtime_ar_diffusion.md
       - Communication:
@@ -134,6 +136,7 @@ nav:
           - Mooncake Store Connector: design/feature/omni_connectors/mooncake_store_connector.md
           - Mooncake Transfer Engine Connector: design/feature/omni_connectors/mooncake_transfer_engine_connector.md
           - Mori Transfer Engine Connector: design/feature/omni_connectors/mori_transfer_engine_connector.md
+          - NIXL Connector: design/feature/omni_connectors/nixl_connector.md
           - Shared Memory Connector: design/feature/omni_connectors/shared_memory_connector.md
           - Yuanrong Store Connector: design/feature/omni_connectors/yuanrong_connector.md
           - Yuanrong Transfer Engine Connector: design/feature/omni_connectors/yuanrong_transfer_engine_connector.md
@@ -158,6 +161,7 @@ nav:
         - Cache-DiT: design/feature/cache_dit.md
         - TeaCache: design/feature/teacache.md
         - Diffusion Continuous Batching: design/feature/diffusion_continuous_batching.md
+        - Mooncake KV Transfer: design/feature/mooncake-kv-transfer.md
     - Infrastructure and Performance:
       - Prometheus Metrics: design/metrics.md
       - Speech Generation Performance Optimizations: design/qwen3_omni_tts_performance_optimization.md

docs/README.md

@@ -56,7 +56,7 @@ vLLM-Omni is flexible and easy to use with:
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
 - Omni-modality models (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
-- TTS models (e.g. Qwen3-TTS, IndexTTS 2.5, dots.tts, CosyVoice3)
+- TTS models (e.g. Qwen3-TTS, IndexTTS 2.5, CosyVoice3)
 - Diffusion models — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
 - Robot-policy and action models (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
 

docs/api/README.md

@@ -5,6 +5,9 @@
 Main entry points for vLLM-Omni inference and serving.
 
 - [vllm_omni.entrypoints.async_omni.AsyncOmni][]
+- [vllm_omni.entrypoints.async_omni_base.AsyncOmniBase][]
+- [vllm_omni.entrypoints.duplex_omni.DuplexOmni][]
+- [vllm_omni.entrypoints.duplex_omni.DuplexSessionHandle][]
 - [vllm_omni.engine.cfg_companion_tracker.CfgCompanionTracker][]
 - [vllm_omni.entrypoints.cli.benchmark.base.OmniBenchmarkSubcommandBase][]
 - [vllm_omni.entrypoints.cli.benchmark.main.OmniBenchmarkSubcommand][]
@@ -17,12 +20,13 @@ Main entry points for vLLM-Omni inference and serving.
 
 ## Clients
 
-Client libraries that talk to a running vLLM-Omni server (the full-duplex Realtime client and its per-model session presets).
+Client libraries for full-duplex sessions: the Realtime WebSocket client, the in-process client over `DuplexOmni`, and the per-model session presets.
 
 - [vllm_omni.clients.duplex.AudioDelta][]
 - [vllm_omni.clients.duplex.AudioFormat][]
 - [vllm_omni.clients.duplex.ConnectionResumed][]
 - [vllm_omni.clients.duplex.DuplexClient][]
+- [vllm_omni.clients.duplex.DuplexClientBase][]
 - [vllm_omni.clients.duplex.DuplexClientError][]
 - [vllm_omni.clients.duplex.DuplexConnectionError][]
 - [vllm_omni.clients.duplex.DuplexEvent][]
@@ -44,6 +48,7 @@ Client libraries that talk to a running vLLM-Omni server (the full-duplex Realti
 - [vllm_omni.clients.duplex.TextDelta][]
 - [vllm_omni.clients.duplex.TranscriptDelta][]
 - [vllm_omni.clients.duplex.WebSocketTransport][]
+- [vllm_omni.clients.inline_duplex.InlineDuplexClient][]
 
 ## Inputs
 

docs/cli/bench/serve.md

@@ -23,7 +23,9 @@ You can use `vllm bench serve --omni --help=all` to get descriptions of all para
   The API endpoint exposed externally, to which clients send their requests.
 
 - `--dataset-name`
-  The name of the dataset used; random-mm indicates generating random multimodal inputs (images, videos, audio), while `omniinteract` replays official OmniInteract videos.
+  The name of the dataset used; `random-mm` generates random multimodal inputs (images, videos, audio),
+  `omniinteract` replays official OmniInteract videos, and `videomme` loads Video-MME MCQ videos
+  (`lmms-eval/Video-MME`, MiniCPM OmniEvalKit packing).
 
 - `--num-prompts`
   The total number of requests to send, an integer.
@@ -316,8 +318,12 @@ arrives, checkpointing the response's history position so a later committed user
 outputs are ineligible and omitted from the official manifest; `audio_clipped_bytes` records output beyond the rounded video
 horizon.
 
-TTFT, TTFP, and RTF start at client receipt of `response.created`. TPOT/ITL use engine stage-0 timing; ITL is emitted only when
-every token interval is present.
+Per-response TTFT and TTFP start when the server begins executing the native model-turn request that owns the response. RTF
+continues to use client receipt of `response.created` through the last audio packet, divided by emitted audio duration. Global
+TTFT, TTFP, and RTF cover the complete input-stream window. TPOT/ITL use engine stage-0 timing; ITL is emitted only when every
+token interval is present within a continuous generation segment. Model-unit pacing and gaps between generation segments are
+excluded from TPOT/ITL. Raw request metrics retain `response_created_to_first_text_ms` and
+`response_created_to_first_audio_ms` as client-envelope diagnostics.
 
 The checked-in local performance configuration measures four deterministic cases from each OmniInteract subset (12 videos
 total), with no benchmark warmups and a maximum concurrency of two. Each subset also sends one readiness request before its
@@ -339,6 +345,57 @@ transcript, event, and result artifacts without errors. A valid LISTEN-only case
 chunks. Official-manifest eligibility is reported separately because clipped or cancelled output is a benchmark-quality
 signal, not a transport failure. This local performance test does not score answer accuracy.
 
+### Video-MME Benchmark
+
+Video-MME (`--dataset-name videomme`) scores multiple-choice video QA. Default packing is
+OmniEvalKit MiniCPM `minicpm-frames` (up to 96 sampled frames as `image_url`). Pass a local
+mirror with `--dataset-path` / `--videomme-parquet` + `--videomme-video-dir`, or a Hugging
+Face dataset id (`lmms-eval/Video-MME` by default; any `org/name` is accepted when
+`--dataset-name videomme` is explicit). Relative `--videomme-video-dir` values are resolved
+to absolute `file://` URLs. For those URLs, start the server with
+`--allowed-local-media-path` covering the video root; otherwise use `--videomme-inline-local-video`.
+
+For MiniCPM-o 4.5, the default deployment allows 64 images per request. To run the
+96-frame recipe, start the server with a stage 0 override (replace `/path/to/Video-MME`
+with the local video root shared by the benchmark client and server):
+
+```bash
+vllm serve openbmb/MiniCPM-o-4_5 --omni \
+  --allowed-local-media-path /path/to/Video-MME \
+  --stage-overrides '{"0":{"limit_mm_per_prompt":{"image":96,"audio":64,"video":1}}}'
+```
+
+The override applies to this server invocation only. Alternatively, use
+`--videomme-max-frames 64` in the benchmark to stay within the default image limit;
+this evaluates a different frame-sampling setting from the 96-frame recipe.
+
+Run the benchmark against that server:
+
+```bash
+vllm bench serve --omni \
+  --backend openai-chat-omni \
+  --dataset-name videomme \
+  --dataset-path /path/to/Video-MME \
+  --videomme-pack-mode minicpm-frames \
+  --videomme-max-frames 96 \
+  --model openbmb/MiniCPM-o-4_5 \
+  --endpoint /v1/chat/completions \
+  --num-prompts 8 \
+  --max-concurrency 1 \
+  --save-result
+```
+
+Accuracy keys (`videomme_accuracy`, per-duration / domain / task breakdowns) are written into
+the saved JSON. `videomme_accuracy` excludes HTTP failures; `videomme_accuracy_incl_http_fail`
+counts them as wrong. `videomme_submitted` / `videomme_unique_question_ids` show when
+oversampling made the request count differ from unique-question coverage.
+`videomme_skipped_rows` counts rows skipped during sampling because of missing or unreadable
+media or invalid question fields. The opt-in accuracy runner rejects skipped rows, HTTP
+failures, duplicate question IDs, and missing gold answers. This checks the sampled subset;
+it does not establish that a supplied local mirror contains the entire official dataset.
+Use `--videomme-save-eval-items` (or `VIDEOMME_SAVE_EVAL_ITEMS=1`) for per-request
+`videomme_eval_items` rows.
+
 ### Multi-Modal Benchmark
 
 <details class="admonition abstract" markdown="1">
@@ -436,6 +493,76 @@ This should be seen as an edge case, and if this behavior can be avoided by sett
 
 </details>
 
+### Image / video generation
+
+For `/v1/images/generations`, `/v1/images/edits`, and `/v1/videos`, set `--endpoint` to that path and omit `--backend`. The endpoint path is registered as the bench request adapter, so a separate named backend is not required.
+
+Example (`/v1/images/generations`):
+
+```bash
+vllm bench serve --omni \
+  --endpoint /v1/images/generations \
+  --dataset-name random \
+  --model ~/models/Qwen/Qwen-Image \
+  --tokenizer ~/models/Qwen/Qwen-Image/tokenizer \
+  --max-concurrency 1 \
+  --num-warmups 2 \
+  --num-prompts 10 \
+  --random-input-len 64 \
+  --random-output-len 1 \
+  --ignore-eos \
+  --percentile-metrics e2el \
+  --extra-body '{
+    "num_inference_steps": 20,
+    "seed": 42,
+    "true_cfg_scale": 4.0
+  }'
+```
+
+If successful, following output is like:
+
+```text
+============ Serving Benchmark Result ============
+Successful requests:                     3  
+Failed requests:                         0  
+Maximum request concurrency:             1  
+Benchmark duration (s):                  8.42  
+Request throughput (req/s):              0.36  
+Peak concurrent requests:                2.00  
+-------------------Peak Memory--------------------
+Mean PEAK_MEMORY_MB (MB):                58832.00  
+Median PEAK_MEMORY_MB (MB):              58832.00  
+P99 PEAK_MEMORY_MB (MB):                 58832.00  
+----------------End-to-end Latency----------------
+Mean E2EL (ms):                          2798.16  
+Median E2EL (ms):                        2799.83  
+P99 E2EL (ms):                           2800.03  
+================== Image Result ==================
+Total images generated:                  3  
+Image throughput (img/s):                0.36  
+Average pixels per image:                1048576.00
+Mean denoise step latency (ms):          136.59  
+---------------- Image Generation ----------------
+Mean IMAGE_GENERATION (ms):              2731.84  
+Median IMAGE_GENERATION (ms):            2732.02  
+P99 IMAGE_GENERATION (ms):               2734.65  
+==================================================
+```
+
+Use the same pattern with `--endpoint /v1/images/edits` or `--endpoint /v1/videos` (and model-specific `--extra-body` as needed). Pure image/video runs omit the Text Result section when there is no generated text.
+
+`/v1/images/edits` defaults to **non-streaming JSON** (`stream=false`) so single-stage edit models are not rejected by the server. For multi-stage pipelines that need SSE (AR TTFT / image chunks), pass `"stream": true` in `--extra-body`.
+
+`/v1/videos` is an async job API: the client creates a job, then polls until `completed`/`failed`. Measured **e2el** therefore includes client poll sleep and any overshoot after the job actually finishes. Tune polling via `--extra-body`:
+
+- `poll_interval_s` (default `2.0`): sleep between status polls
+- `poll_timeout_s` (default `21600`, i.e. 6 hours): give up waiting for the job
+
+**VIDEO_RTF** prefers server-reported generation time when available:
+
+- `video_rtf = video_generation_time_ms / 1000 / video_duration`, where `video_generation_time_ms` comes from response `stage_durations` (or `inference_time_s` as fallback)
+- If generation time is missing, falls back to `e2el / video_duration` (this fallback *does* include poll overhead)
+
 ### Multi-Stage Benchmark
 
 <details class="admonition abstract" markdown="1">

docs/cli/serve.md

@@ -1,5 +1,67 @@
 # vllm-omni serve
 
+## Multiple API frontends
+
+For local EngineCore pipelines with one or more stages, `--api-server-count` starts
+multiple API frontend processes that share one parent-owned set of stage
+engines:
+
+```bash
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+    --api-server-count 2 \
+    --port 8091
+```
+
+This mode is intended for workloads whose request parsing, multimodal input
+processing, or response encoding can bottleneck a single frontend. It
+currently requires local multi-process EngineCore stages with one local
+process group per replica. A stage may still use multiple `num_replicas`.
+It cannot be combined with diffusion, headless or
+remote stages, intra-stage data parallelism, Ray, fault tolerance, elastic
+expert parallelism, sleep mode, or runtime LoRA updating. Runtime voice upload
+and deletion are also disabled because those mutations are process-local;
+built-in voices, inline reference audio, and voices restored at startup remain
+available. The `/v1/omni/sleep` and `/v1/omni/wakeup` control routes return HTTP
+409 for the same reason: their bookkeeping is process-local while stage engines
+are shared.
+
+A single-stage model pipeline is supported through the normal launch command;
+this is distinct from the distributed `--stage-id` / `--headless` launch mode.
+Every stage in the pipeline must be an EngineCore stage. Mixed AR-to-DiT
+pipelines, including MiniMax-H3, remain unsupported even when their first
+stage is an EngineCore stage. Diffusion support requires per-frontend channels
+and shared-engine lifecycle handling in the diffusion backend.
+
+### Workloads and performance validation
+
+Each API process owns its frontend processor, Orchestrator, request state,
+and per-replica EngineCore channels. Requests return through the channel of
+the originating API process; model weights and GPU stage engines are shared.
+This distributes frontend and orchestration work across CPU processes.
+
+| Workload | Potential benefit | Limitation |
+| ---------- | ------------------- | ------------ |
+| Concurrent cold image/video inputs on supported EngineCore pipelines | Parallel frontend processing when one API process is CPU-limited | A single cold request and GPU computation may be unchanged |
+| Many replicas producing frequent text/audio outputs | Less API/Orchestrator work and GIL contention per process | Routing remains serial within each Orchestrator |
+| Warm inputs or low-concurrency, GPU-limited generation | Additional frontend capacity | More API processes may add overhead without improving throughput |
+
+Here, a cold input means media or preprocessing caches miss, not that model
+weights must be loaded or GPU kernels compiled. Each frontend has its own
+processor/cache state, so cache duplication, CPU thread contention, and memory
+usage also matter. Multi-API serving does not itself change the output queue,
+move response encoding to an executor, or change the model's numerical scheduler.
+
+Compare `--api-server-count 1`, `2`, and higher counts with the same hardware,
+model/stage configuration, media, sampling parameters, output lengths, and
+offered load. Keep cold-cache and warmed-cache runs separate and report repeated
+measurements. Record CPU and request counts for every API worker, input and
+output queue delay, GPU utilization, first-output latency, and end-to-end latency.
+Confirm requests reach every worker: connection reuse can concentrate traffic
+on one process. Queue-only or single-API experiments do not establish a
+multi-API throughput gain. The multi-replica workload in
+[#4680](https://github.com/vllm-project/vllm-omni/issues/4680) remains a relevant
+validation target.
+
 ## Stage-based CLI quickstart
 
 The stage-based CLI is designed for deployments that require launching each pipeline stage in an isolated process
@@ -61,3 +123,10 @@ CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
 ## Arguments
 
 --8<-- "docs/generated/argparse_omni/omni_serve.inc.md"
+
+Async video jobs use process-local metadata and task handles. With multiple API
+workers, creating, listing, retrieving, downloading, and deleting these jobs
+returns HTTP 409. This restriction is independent of the current diffusion
+stage launch restriction: supporting multi-worker diffusion also requires a
+shared job lifecycle. Missing frontend topology state returns HTTP 503 for
+operations that require process-local state protection.

docs/configuration/README.md

@@ -2,7 +2,7 @@
 
 This section lists the most common options for running vLLM-Omni.
 
-For options within a vLLM Engine, please refer to the [vLLM 0.28 configuration guide](https://docs.vllm.ai/en/v0.28.0/configuration/index.html).
+For options within a vLLM Engine, please refer to the [vLLM 0.29 configuration guide](https://docs.vllm.ai/en/v0.29.0/configuration/index.html).
 
 Each model defines fixed topology in a registered `PipelineConfig` and runtime
 overrides in a deploy YAML.

docs/configuration/environment_variables.md

@@ -70,6 +70,7 @@ The lifecycle labels used below are:
 | `OMNI_DIFFUSION_PROMPT_EMBED_CACHE_SIZE` | Positive integer; default `32` entries | Each diffusion runner; resolved during model setup | A valid environment value overrides the explicit cache size. A non-integer logs a warning; a non-positive value is ignored. | Experimental |
 | `OMNI_DIFFUSION_SESSION_STATE_MANAGER` | Boolean with the same accepted spellings as the prompt cache; default disabled | Experimental diffusion session manager; model setup | A recognized environment value overrides the explicit enable setting. An unrecognized value is ignored. | Experimental |
 | `OMNI_DIFFUSION_SESSION_STATE_MANAGER_MAX_SESSIONS` | Positive integer; default `64` | Experimental diffusion session manager; model setup | A valid environment value overrides the explicit maximum. A non-integer logs a warning; a non-positive value is ignored. | Experimental |
+| `VLLM_OMNI_AR_DIFFUSION_KV_GATHER` | `1` enables; default `0` (off) | AR-diffusion KV manager at allocation and attention at dispatch; set before worker startup | Only the exact value `1` enables contiguous KV gathering. History staging additionally requires `ARDiffusionKVConfig.reuse_history_staging`. | Experimental |
 
 Backend names for `DIFFUSION_ATTENTION_BACKEND` are the members of
 `DiffusionAttentionBackendEnum`, such as `FLASH_ATTN`, `TORCH_SDPA`,
@@ -82,14 +83,22 @@ depends on the installed kernels and model path.
 | --- | --- | --- | --- | --- |
 | `SPEAKER_SAMPLES_DIR` | Filesystem path; default `~/.cache/vllm-omni/speakers` | Speech server; read when speaker storage initializes | Environment-only setting. The directory is created; filesystem errors propagate. | Stable |
 | `SPEAKER_MAX_UPLOADED` | Integer; default `1000` | Speech server; read when speaker storage initializes | Environment-only setting. A non-integer logs a warning and uses `1000`; range is not otherwise validated. | Stable |
+| `VLLM_OMNI_ABORT_TIMEOUT` | Float seconds; default `2` | Engine abort wait for Videos DELETE and `generate()` cancel/error cleanup; read when `async_omni` imports | Environment-only setting. A non-float raises `ValueError` during import. | Experimental |
 | `VLLM_OMNI_ASYNC_OUTPUT_TIMEOUT` | Float seconds; default `600` | Diffusion engine async-output wait in `step_streaming`; resolved per call on the request path, not at import | Environment-only setting. A non-float or `<=0` value warns once and uses the default. | Experimental |
-| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `1`, `true`, `yes` or `on` enables; default `0` (off) | Orchestration loop and the serving-side final-output drain; read once when the `Orchestrator` is constructed | Environment-only setting. Values are stripped and case-normalized; any unrecognized value leaves the legacy poll loop selected. | Experimental |
+| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `1`, `true`, `yes` or `on` enables; defaults to on for Qwen3-TTS and off for other pipelines | Pipeline default computed from `pipeline_config.model_type` at engine initialization; env override resolved at `Orchestrator` construction and separately when the serving-side final-output drain starts | An explicit env value wins; otherwise both consumers use the engine's pipeline default. Values are stripped and case-normalized; any unrecognized value selects the legacy poll loop. Set before server startup. | Experimental |
 | `VLLM_OMNI_INPUT_WAIT_TIMEOUT_S` | Float seconds; default `600`; `<=0` disables | Full-payload input coordinator, not async-chunk transfer; read when the scheduler module imports in each worker | Environment-only setting. A non-float logs a warning and uses `600`. | Stable operational control |
 | `VLLM_OMNI_ORCH_MONITOR_PATH` | Filesystem path; default `<current-working-directory>/vllm_omni_orch_monitor_<timestamp>.json` | Orchestrator monitor enabled by `--enable-orch-monitor`; read when the monitor is created | Environment-only path override. Parent directories are created; write errors are logged. | Diagnostic |
 | `VLLM_OMNI_VIDEO_SYNC_TIMEOUT` | Float seconds; default `600` | Synchronous Videos API; read when the API server module imports | Environment-only setting. A non-float raises `ValueError` during import. | Experimental |
 | `VLLM_VIDEO_ASYNC_CHUNK` | `on` or `off`; default `on` | Streaming video output; read on attribute access | Environment-only setting. Values are trimmed and case-normalized; an invalid value warns once and uses `on`. | Experimental |
 | `VLLM_VIDEO_AUDIO_DELTA_MODE` | `fast` or `slow`; default `fast` | Streaming video audio deltas; read on attribute access | Environment-only setting. Values are trimmed and case-normalized; an invalid value warns once and uses `fast`. | Experimental |
 
+### NIXL stage transfer
+
+| Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
+| --- | --- | --- | --- | --- |
+| `VLLM_OMNI_NIXL_LEASE_S` | Float seconds; default `3600` | Unclaimed producer payload expiry; connector construction | Nonempty environment value overrides `lease_seconds`; empty falls back to config/default. Invalid floats raise `ValueError`. Claimed READ allocations never expire by time. | Experimental |
+| `VLLM_OMNI_NIXL_XFER_TIMEOUT_S` | Float seconds; default `300` | Receiver READ wait; connector construction | Nonempty environment value overrides `transfer_timeout_s`; empty falls back to config/default. Invalid floats raise `ValueError`. Timeout does not cancel DMA or release active allocations. | Experimental |
+
 ### Server storage
 
 Storage names use Pydantic's nested-settings delimiter (`__`). They are read
@@ -112,6 +121,7 @@ settings.
 
 | Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
 | --- | --- | --- | --- | --- |
+| `VLLM_OMNI_FUSED_QK_NORM_ROPE_MIN_TOKENS` | Non-negative integer; unset or empty means each consumer's own default (Boogu-Image: `2048`) | Token gate of the shared `fused_qk_norm_rope` op: a rotary table is packed for the fused kernel only when it spans at least this many positions (`B*S`); read at forward time | Environment-only. `0` = always fuse; a very large value disables the fused path. Any other value raises `ValueError` on the first forward. The crossover is host-dependent; re-benchmark before overriding. | Experimental performance control |
 | `VLLM_OMNI_SKIP_NVFP4_NAN_CLAMP` | Boolean truthy spellings: `1`, `true`, `yes`, `on`; default false | ModelOpt NVFP4 compatibility patch; read when `vllm_omni.patch` imports | Environment-only escape hatch. Any other value means false. Set only to diagnose the upstream NaN-scale issue. | Diagnostic and temporary |
 | `VLLM_OMNI_USE_QUACK_FP8` | Boolean truthy spellings: `1`, `true`, `yes`, `on`; unset means hardware auto-detection | FP8 scaled matrix multiplication; evaluated when quack capability is selected | A set value overrides auto-detection. Any non-truthy value forces quack off. If quack cannot load, vLLM-Omni warns and falls back to FlashInfer. | Experimental performance control |
 
@@ -151,7 +161,7 @@ their keys only.
 ## Inherited vLLM variables
 
 vLLM-Omni also reads variables through its aligned vLLM dependency. Refer to
-the [vLLM 0.28 environment-variable reference](https://docs.vllm.ai/en/v0.28.0/configuration/env_vars.html)
+the [vLLM 0.29 environment-variable reference](https://docs.vllm.ai/en/v0.29.0/configuration/env_vars.html)
 for their definitions. This includes vLLM launch, cache, logging, plugin, ROCm,
 XPU, ModelScope, and FlashInfer workspace settings.
 

docs/configuration/stage_configs.md

@@ -17,6 +17,7 @@ Common `PipelineConfig` fields include:
 | ------- | ------------- |
 | `model_type` | Pipeline identifier used during model and config resolution. |
 | `default_deploy_config_name` | Bundled deploy YAML loaded when the user does not pass `deploy_config`. |
+| `duplex_plugin` | Dotted path of the model's `DuplexModelPlugin`. Set only for full-duplex models; it makes `vllm-omni serve` run the model through `DuplexOmni` (a server whose every surface runs on a duplex session) and the engine host a `DuplexOrchestrator` with the plugin loaded. |
 | `model_arch` | Default Hugging Face architecture for the pipeline. |
 | `hf_architectures` | Architecture names used to identify checkpoints whose `model_type` is shared. |
 | `hf_config_predicate` | Optional predicate used to select between pipelines with otherwise identical HF metadata. |
@@ -38,7 +39,7 @@ Common `StagePipelineConfig` fields include:
 | `custom_process_input_func` | Processor applied to this stage's incoming payload. |
 | `custom_process_next_stage_input_func` | Processor used for full-payload handoff to the next stage. |
 | `async_chunk_process_next_stage_input_func` | Processor used for async chunk handoff. |
-| `sampling_constraints` | Model-owned sampling constraints that deploy defaults cannot override. |
+| `sampling_constraints` | Model-owned sampling constraints that deploy defaults cannot override. Scalar values replace deploy defaults; required `stop_token_ids` extend and deduplicate them. |
 
 To add or change topology, define and register a new pipeline variant. Use
 deploy YAML only for runtime placement, resource sizing, connectors, and other
@@ -52,7 +53,7 @@ The new deploy schema lives under `vllm_omni/deploy/` and is paired with a froze
 | ------- | ------ | ---------- | --------- | ------------- |
 | `base_config` | str (path) | optional | — | Overlay parent (relative or absolute). `stages:` / `platforms:` deep-merged by stage_id; other scalars overlay-wins. Intended for user-authored overlays; prod yamls stay flat. |
 | `async_chunk` | bool | optional | `true` | Enable chunked streaming between stages. Pin to `false` if the pipeline runs end-to-end. |
-| `session_mode` | str | optional | `"turn"` | Session behavior. MiniCPM-o 4.5 deploy YAMLs set `"duplex"` so `/v1/realtime` and chat share the same profile. |
+| `session_mode` | str | optional | `"turn"` | Session behavior. For pipelines with a `duplex_plugin`, explicitly select `"duplex"` for the duplex engine or `"turn"` for the ordinary online serving stack. The shipped MiniCPM-o 4.5 default remains `"duplex"`. See [Full Duplex](../serving/full_duplex_api.md#enable-full-duplex). |
 | `active_stream_window` | int | optional | `0` | Number of active downstream stream slots; `0` preserves all-stream cycling. |
 | `duplex_session` | dict | optional | runtime defaults | Full-duplex session lifecycle, buffering, replay, and capacity limits. |
 | `connectors` | dict | optional | `null` | Named connector specs (`{name, extra}`). Referenced by each stage's `input_connectors` / `output_connectors`. See [Connector schema](#connector-schema). |
@@ -97,7 +98,7 @@ Each entry under `stages:` accepts any `StageDeployConfig` field directly (no ne
 | `devices` | str \| null | optional | `null` | Device list assigned to this stage. The number of device ids must equal this stage's local world size (`tensor_parallel_size` × local data-parallel size × `pipeline_parallel_size`, or `num_replicas` × that product for a replica pool); a mismatch fails early — see the note below. |
 | `output_connectors` | dict \| null | optional | `null` | Keyed by `to_stage_<n>`; values are names registered under top-level `connectors:`. |
 | `input_connectors` | dict \| null | optional | `null` | Keyed by `from_stage_<n>`; values are names registered under top-level `connectors:`. |
-| `default_sampling_params` | dict \| null | optional | `null` | Baseline sampling params. Deep-merged with pipeline `sampling_constraints` (pipeline wins). |
+| `default_sampling_params` | dict \| null | optional | `null` | Baseline sampling params. Merged with pipeline `sampling_constraints`; scalar constraints win, while required `stop_token_ids` are appended and deduplicated. |
 | `engine_extras` | dict | optional | `{}` | Catch-all for engine fields not listed above; deep-merged across overlays and forwarded to the stage engine. |
 
 **Note:** a stage's `devices` count must equal its local world size (`tensor_parallel_size` × `data_parallel_size_local` × `pipeline_parallel_size`, falling back to global `data_parallel_size` when the local size is unset), or `num_replicas` × that product for a replica pool. A mismatch fails early and names the offending stage. A top-level `--tensor-parallel-size` is broadcast to every stage, so it can make a single-GPU stage violate this contract ([issue #5003](gh-issue:5003)); fix that case with `--stage-overrides` (set `tensor_parallel_size` and `devices` together per stage) or set TP only on the multi-GPU stage.
@@ -289,3 +290,90 @@ vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/depl
 
 !!! important
     We are actively iterating on the definition of deployment configurations, and we welcome feedback from users and developers.
+
+## Qwen3-TTS with Model Runner V2
+
+Qwen3-TTS can opt into the native CUDA Model Runner V2 pipeline on vLLM
+0.29.0. Select one of these deployment profiles:
+
+| Profile | Runner | Code2Wav graph batches | Intended use |
+| --- | --- | --- | --- |
+| `qwen3_tts.yaml` | V1 | Existing defaults | Existing deployment / regression control |
+| `qwen3_tts_mrv2.yaml` | V2 | B1 | Native runner validation |
+| `qwen3_tts_high_concurrency_mrv2.yaml` | V2 | B1, B2 | Opt-in throughput tuning |
+| `qwen3_tts_high_concurrency_mrv2_b4.yaml` | V2 | B1, B2, B3, B4 | Experimental throughput / buffered playback |
+
+```bash
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base --omni \
+  --deploy-config vllm_omni/deploy/qwen3_tts_mrv2.yaml
+```
+
+The V2 profiles bound Talker prefill to 512 tokens per step and select the
+Talker AR runner and the Code2Wav generation
+runner together. Native inter-stage delivery carries codec payloads directly;
+request-owned snapshots preserve buffers through asynchronous completion and
+CUDA graph reuse. Terminal completion waits for upstream stage metrics before
+releasing request state. Platform sections retain V1 on NPU, XPU, ROCm and MUSA;
+this change does not qualify MRV2 on those backends or enable other model families.
+Selecting V2 for a stage without `supports_native_mrv2_data_plane` emits a warning
+when pipeline and deployment settings are merged. That stage retains the legacy
+transport path; the warning does not establish support for that combination.
+
+MRV2 model hooks use capability declarations rather than architecture or stage
+names. Omni lifecycle flags select the model state, and `_returns_tuple` declares
+the capture output contract. The optional batched predictor hook is `mtp`, with
+an explicit `mtp_output_key` (a string or two-part payload key), optional
+`mtp_validity_key`, and `mtp_graph_safe`/`mtp_disable_graph` capture controls.
+Its inputs are token IDs, embeddings, previous hidden states and per-row
+conditioning; it returns updated embeddings and prediction codes. Models may
+supply `mtp_sampling_params` and `get_mtp_seed(sampling_params)` for model-local explicit seeds, and declare
+`mtp_accepts_per_row_generators`, `mtp_accepts_req_infos`, or `mtp_sample_uniforms`
+(with `mtp_sample_steps` and `mtp_sample_vocab_size`) as needed. Qwen3-TTS retains
+its existing `talker_mtp` entry point for V1.
+
+### Included performance work
+
+- Request snapshots have a fast path for immutable scalar leaves, including
+  waveform lists. Mutable containers, aliases and cycles keep deep-copy
+  semantics. This is not the historical shallow `dict(prompt)` experiment.
+- The API reference cache holds owned float32 arrays with a default capacity
+  of 1024 entries and a 512 MiB waveform-payload budget. Cache hits return
+  independent lists. Qwen3-TTS model artifact caches also default to 1024
+  entries. These caches have different owners and eviction policies; equal
+  entry limits do not make them a single coherent cache.
+- Talker state stays on GPU where its lifecycle allows it. BOS/EOS projections
+  are cached in their projection dtype and invalidated on weight loading.
+- Code2Wav packs CPU codec inputs before device transfer. Optional B2/B4 graph
+  buckets batch compatible requests without sharing their per-request state.
+- Native output materialization runs in a bounded worker and drains before
+  closing the data plane. The existing control-only shortcut avoids launching
+  model work for a step that has only lifecycle events.
+
+MTP prefix re-prefill remains disabled in the high-concurrency V2 profile.
+Adding more graph shapes increases compilation cost and has not established a
+stable end-to-end gain for this extracted version.
+
+### Decoder batches and playback
+
+Changing `decode_cudagraph_batch_sizes` selects the captured batch buckets.
+Keep `decode_batch_max_size` consistent with the intended maximum too; the
+stateful graph path currently groups according to the captured buckets, while
+the stateless path also uses the explicit maximum.
+
+B4 remains experimental. First-packet latency alone does not establish
+uninterrupted playback. Validate inter-chunk arrival times, buffering, WER and
+speaker similarity before adopting either batching preset for a production
+workload. Floating-point decoder outputs can differ across batch sizes; this PR
+does not claim bitwise or quality equivalence.
+
+### Optional MPS deployment
+
+NVIDIA MPS is an optional operator setting for colocated CUDA processes, not a
+YAML option or a library default. This PR does not establish a throughput or
+first-packet latency benefit from MPS. Measure the exact deployment with and
+without MPS before enabling it.
+
+Use only assigned GPUs and an independent MPS pipe directory. A private MPS
+server does not provide exclusive GPU ownership or MIG isolation. For a
+single-GPU deployment, explicitly place both stages on that GPU; the supplied
+high-concurrency profile places its two stages on different GPUs by default.

docs/contributing/README.md

@@ -262,6 +262,28 @@ or serving, explain the required L2/L3 environment and provide the exact command
 
 Repository skills guide the agent, but they do not replace contributor judgment, the accepted issue or RFC, required test evidence, or maintainer review. Review all generated changes before submitting them.
 
+## AI Usage Policy
+
+AI tools are welcome in vLLM-Omni contributions. Contributors, including maintainers, remain responsible for everything they submit. These requirements apply to pull requests, issues, discussions, and review comments:
+
+- **Disclose AI assistance.** Name the tools used and describe their role in the submission, including assistance with code, tests, documentation, or writing.
+- **Understand your changes.** Review every generated change and be able to explain its behavior, design choices, and interaction with vLLM-Omni without relying on AI to answer for you.
+- **Validate before submitting.** Run relevant checks and report actual results. For model, kernel, or performance changes, include the hardware, model, configuration, and commands needed to reproduce accuracy or benchmark evidence. Clearly identify checks you could not run; generated claims are not test results.
+- **Keep communication useful.** Verify technical claims, search for existing reports, and edit AI-assisted text for accuracy and brevity. Submit only content you have personally reviewed, and take responsibility for responding to review feedback.
+- **Label generated media.** Model-generated images, video, and audio are valid examples or test artifacts for this multimodal project. Identify them as generated output and provide the model, inputs, and reproduction settings. Do not present them as ground-truth references or independent evidence of correctness.
+
+For example, a PR description could include:
+
+```text
+AI assistance: Used Codex to draft the regression test and PR description.
+I reviewed the changes and verified the test reproduces the reported failure.
+Validation: <commands, results, hardware, and any checks not run>.
+```
+
+Maintainer review time is limited. Contributions that are unverified, misleading, or beyond the author's understanding may be closed. Repeated violations may lead to contribution restrictions. New contributors are welcome to ask for help and submit small, well-understood changes.
+
+This policy is adapted from [Ghostty's AI Usage Policy](https://github.com/ghostty-org/ghostty/blob/main/AI_POLICY.md) for vLLM-Omni's contribution and multimodal validation workflows. The [coding-agent workflow](#using-repository-skills-with-coding-agents) provides practical guidance for applying it.
+
 ## Issues
 
 If you encounter a bug or have a feature request, please search existing issues first to see if it has already been reported. If not, please file a new issue, providing as much relevant information as possible.

docs/contributing/ci/ci_settings.md

@@ -15,7 +15,8 @@ Canonical layout (prefer these paths for new changes):
 │   │   ├── skip_ci.py               # skip-ci decision (docs / skip-mark / CI YAML paths)
 │   │   ├── upload_pipeline.py       # Bootstrap + test-pipeline uploader (CUDA/NPU)
 │   │   └── resolve_skip_ci.sh       # Shell helpers for AMD/Intel bootstrap
-│   └── ci_mirror_hardwares.yml      # CUDA uploader presets (referenced by name only)
+│   ├── ci_mirror_hardwares.yml      # CUDA/NPU hardware presets (referenced by name only)
+│   └── ci_source_file_dependencies.yml  # L2–L5 path-prefix presets (referenced by name only)
 ├── cuda/                            # Primary NVIDIA CUDA CI
 │   ├── pipeline.yml                 # Bootstrap entry (hook upload)
 │   ├── bootstrap-upload-steps.yml   # Bootstrap child steps (upload_pipeline --upload)
@@ -34,6 +35,7 @@ Canonical layout (prefer these paths for new changes):
 ├── amd/
 │   ├── test-amd-ready.yml           # L2 job definitions (template input)
 │   ├── test-amd-merge.yml           # L3 job definitions
+│   ├── test-amd-nightly.yml         # Nightly full-model jobs (experimental)
 │   ├── test-template-amd-omni.j2    # Renders final pipeline.yaml
 │   └── scripts/
 │       ├── bootstrap-amd-omni.sh    # Entry: skip-ci → Jinja → upload
@@ -63,7 +65,7 @@ Canonical layout (prefer these paths for new changes):
 | -------- | ---------------- | -------------- | ---------------- | -------------------- |
 | **CUDA** | `cuda/pipeline.yml` | `test-ready.yml`, `test-merge.yml`, `test-nightly.yml`, `test-weekly.yml` | `upload_pipeline.py --upload` (expands uploader-only keys) | omit `mirror_hardwares` (from `-m`) / preset string + `MIRROR_HW` |
 | **NPU** | `npu/pipeline-npu.yml` | `test-npu-ready.yml`, `test-npu-nightly.yml` | `upload_pipeline.py --upload` | `mirror_hardwares: a2b3_npu_1` / `a2b3_npu_4` / `a3_npu_2` |
-| **AMD** | `amd/scripts/bootstrap-amd-omni.sh` | `test-amd-ready.yml`, `test-amd-merge.yml` | Jinja (`test-template-amd-omni.j2`) → `pipeline upload` | `agent_pool` + `mirror_hardwares: [amdproduction]` (array, template filter) |
+| **AMD** | `amd/scripts/bootstrap-amd-omni.sh` | `test-amd-ready.yml`, `test-amd-merge.yml`, `test-amd-nightly.yml` | Jinja (`test-template-amd-omni.j2`) → `pipeline upload` | `agent_pool` + `mirror_hardwares: [amdproduction]` (array, template filter) |
 | **Intel** | `intel/scripts/bootstrap-intel-omni.sh` | `intel/pipeline-intel.yml` (steps inline) | Direct `pipeline upload` | Inline `agents.queue` on each step |
 
 ## Platform configuration style
@@ -108,19 +110,21 @@ Canonical layout (prefer these paths for new changes):
 
     Inferred form: unset/`empty` `MIRROR_HW` matches `H100` or `L4` in `-m` (both present → H100); no match skips the step. `MIRROR_HW=b200` must appear in `-m` or the step is skipped. The uploader does **not** rewrite pytest `-m`; B200 collection comes from the YAML expression and from tests that declare `B200` in `hardware_test` / `hardware_marks` (for example `res={"cuda": ["H100", "B200"]}`). `MIRROR_HW` must be empty or `b200` (case-insensitive); unknown values (for example `b20o`) **fail the upload**. A CUDA preset string such as `mirror_hardwares: h100_4` is skipped when `MIRROR_HW=b200`. Unset `MIRROR_HW` keeps string presets. NPU presets ignore `MIRROR_HW`. A `mirror_hardwares` name that is not a key in `ci_mirror_hardwares.yml` **fails the upload**.
 
+    **Path filter in YAML:** `source_file_dependencies: <preset>` (string, or a list of preset names)—preset names in [`common/ci_source_file_dependencies.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/ci_source_file_dependencies.yml). Required on **E2E Test** leaf jobs in L2/L3, and on L4/L5 leaf jobs that should be diff-gated on PR labels; see [Step filtering](#step-filtering).
+
     **Conventions**
 
     - **`depends_on`:** leaf jobs depend on `upload-ready-pipeline`, `upload-merge-pipeline`, etc.
     - **`group` / `label`:** `:card_index_dividers:` groups; labels like `Diffusion · Qwen Image Test`.
     - **`commands`:** `timeout … pytest …` with markers and `--run-level` for the pipeline level.
-    - **`source_file_dependencies`:** required on **E2E Test** leaf jobs in L2/L3; see [Step filtering](#step-filtering).
+    - **`source_file_dependencies`:** required on **E2E Test** leaf jobs in L2/L3, and on L4/L5 leaf jobs that should be diff-gated on PR labels; name a job key from `ci_source_file_dependencies.yml` (see [Step filtering](#step-filtering)). The key must list every script the step runs; `commands` is not scanned.
 
     **Adding a job**
 
     1. Pick the level file (`test-ready.yml` / `test-merge.yml` / `test-nightly.yml` / `test-weekly.yml`).
     2. Add a step under the right **group** (usually **E2E Test** for model pytest).
-    3. Set `label`, `commands`, `mirror_hardwares`, `depends_on: upload-ready-pipeline` (or `upload-merge-pipeline` / `upload-nightly-pipeline` / `upload-weekly-pipeline`).
-    4. For L2/L3 E2E, add `source_file_dependencies` (pytest + model + deploy YAML prefixes).
+    3. Set `label`, `commands`, `mirror_hardwares`, `depends_on: upload-<level>-pipeline`.
+    4. For L2–L5 jobs that should be diff-gated, add a job key in `common/ci_source_file_dependencies.yml` and set `source_file_dependencies: <key>` on the step. YAML anchors hold model business-code paths only. The job key (`{model}_function` / `_perf` / `_accuracy` / `_reliability` / `_doc` / `_cov`) aliases that anchor and lists the pytest / example / benchmark scripts the step actually runs. The uploader does **not** parse `commands` for path filters—omitting a script from the key means editing that script will not select the job.
     5. Dry-run:
 
     ```bash
@@ -129,7 +133,7 @@ Canonical layout (prefer these paths for new changes):
 
 === "NPU"
 
-    **Bootstrap:** [`npu/pipeline-npu.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/npu/pipeline-npu.yml) + [`npu/bootstrap-upload-steps.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/npu/bootstrap-upload-steps.yml)—same split as CUDA; builds A2/B3 and A3 CI images, then uploads child test pipelines.
+    **Bootstrap:** [`npu/pipeline-npu.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/npu/pipeline-npu.yml) + [`npu/bootstrap-upload-steps.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/npu/bootstrap-upload-steps.yml)—same split as CUDA; builds A2/B3, A3, A5, and 310P CI images, then uploads child test pipelines.
 
     **Test YAML:** `npu/test-npu-ready.yml` (L2), `test-npu-nightly.yml` (L4).
 
@@ -137,7 +141,14 @@ Canonical layout (prefer these paths for new changes):
 
     **Upload:** same as CUDA—`upload_pipeline.py --upload`.
 
-    **Hardware in YAML:** `mirror_hardwares` preset (string), expanded to `agents`, top-level `image`, and `plugins`. Presets: `a2b3_npu_1`, `a2b3_npu_4`, `a3_npu_2` in `common/ci_mirror_hardwares.yml`.
+    **Hardware in YAML:** `mirror_hardwares` preset (string), expanded to `agents`, top-level `image`, and `plugins`. NPU presets in `common/ci_mirror_hardwares.yml` are:
+
+    - A2: `a2b3_npu_1`, `a2b3_npu_4`, `a2b3_npu_8`
+    - A3: `a3_npu_2`, `a3_npu_4`, `a3_npu_8`, `a3_npu_16`
+    - A5: `a5_npu_2`, `a5_npu_4`, `a5_npu_8`, `a5_npu_16`
+    - 310P: `310p_npu_1`, `310p_npu_2`, `310p_npu_4`
+
+    **Path filter in YAML:** same `source_file_dependencies` presets as CUDA, on **L4** leaf jobs in `test-npu-nightly.yml`. Filtering applies on the PR `nightly-test` label; `main` + `NIGHTLY=1` keeps every job. If no job-key prefix matches, a change to the pipeline YAML being uploaded, or to any path under `source_filter_fallback`, keeps every job in that upload. See [Step filtering](#step-filtering).
 
     **Conventions**
 
@@ -148,7 +159,7 @@ Canonical layout (prefer these paths for new changes):
 
     1. Edit `test-npu-ready.yml` (L2) or `test-npu-nightly.yml` (L4).
     2. Add a step with `mirror_hardwares` (add a new preset in `ci_mirror_hardwares.yml` first if needed).
-    3. Set `commands` to your pytest file and markers.
+    3. Set `commands` to your pytest file and markers. For L4, set `source_file_dependencies` to a job key from `ci_source_file_dependencies.yml` that aliases the model anchor and lists that pytest file.
     4. Dry-run:
 
     ```bash
@@ -161,7 +172,9 @@ Canonical layout (prefer these paths for new changes):
 
     **Bootstrap:** [`amd/scripts/bootstrap-amd-omni.sh`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/amd/scripts/bootstrap-amd-omni.sh)—skip-ci, diff filtering, Jinja render, then `buildkite-agent pipeline upload`.
 
-    **Test YAML (data):** `amd/test-amd-ready.yml` (L2 / PR), `test-amd-merge.yml` (L3 / main).
+    **Test YAML (data):** `amd/test-amd-ready.yml` (L2 / `ready`), `test-amd-merge.yml` (L3 / `merge-test` and ordinary `main`), and experimental `test-amd-nightly.yml` (`full_model` / `nightly-test` or `main` with `NIGHTLY=1`). Multiple PR tier labels combine their suites behind one image build. PR builds that reach the bootstrap without a tier label retain the legacy ready-suite fallback. `DEBUG_TEST_YAML` accepts `ready`, `merge`, and `nightly` as an explicit override. During nightly burn-in every AMD nightly leaf is non-blocking.
+
+    **Trigger boundary:** AMD label handling has two layers. First, the external `vllm-omni-amd-ci` Buildkite pipeline condition decides whether a GitHub label event creates a build. Only after that build starts does this repository's bootstrap inspect all current PR labels and select the ready, merge, nightly, or combined suites. Repository-side selection therefore cannot start AMD CI by itself. AMD-only nightly validation uses both `amd-test` and `nightly-test`: the external condition should accept an event for either label only when the complete PR label set contains both. `amd-test` admits the AMD pipeline, while `nightly-test` makes the bootstrap select `NIGHTLY_TESTS:test-amd-nightly.yml`. Existing `ready` / `merge-test` behavior and non-PR `main` / scheduled builds remain separate. For PR builds without a `DEBUG_TEST_YAML` override, an unreadable label set fails the bootstrap instead of silently selecting the wrong tier. Debug overrides skip label lookup and are validated directly; invalid values, including whitespace-only values, fail the bootstrap. Until the paired-label condition is configured, use an authorized manual Buildkite build with `DEBUG_TEST_YAML=nightly` and verify that the bootstrap reports only the nightly suite.
 
     **Rendering:** [`test-template-amd-omni.j2`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/amd/test-template-amd-omni.j2) wraps data steps with `amd-build` image build and `amd_<agent_pool>` queues. Do **not** hand-edit generated `pipeline.yaml`.
 
@@ -177,7 +190,7 @@ Canonical layout (prefer these paths for new changes):
 
     **Adding a job**
 
-    1. Edit `test-amd-ready.yml` or `test-amd-merge.yml`.
+    1. Edit `test-amd-ready.yml`, `test-amd-merge.yml`, or `test-amd-nightly.yml`.
     2. Copy a neighboring block: `label`, `agent_pool`, `mirror_hardwares`, `commands`, optional `grade`.
     3. Regenerate via bootstrap / Jinja; update `skip_ci.py` if you add a new YAML path.
 
@@ -206,7 +219,7 @@ PR diffs drive **two independent skip layers**. Both read changed files from git
 | Layer | Script | When | What is skipped | Where you configure |
 | ----- | ------ | ---- | --------------- | ------------------- |
 | **Bootstrap** | [`skip_ci.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/skip_ci.py) + [`upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/upload_pipeline.py) | Before child test pipelines upload (`cuda/pipeline.yml`, `npu/pipeline-npu.yml`, AMD/Intel bootstraps) | Entire default CI, or whole L2/L3 upload for a platform | Whitelists in `skip_ci.py`; bootstrap `if` injected by step `key` in `upload_pipeline.py` |
-| **Step filter** | [`upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/upload_pipeline.py) | While uploading CUDA L2/L3 YAML | Individual Buildkite steps inside `test-ready.yml` / `test-merge.yml` | `source_file_dependencies` on each step or group |
+| **Step filter** | [`upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/upload_pipeline.py) | While uploading CUDA L2–L5 YAML | Individual Buildkite steps inside `test-ready.yml` / `test-merge.yml` / `test-nightly.yml` / `test-weekly.yml` | `source_file_dependencies` key on each step (presets in `ci_source_file_dependencies.yml`) |
 
 **Changed files** (both layers):
 
@@ -233,7 +246,7 @@ Label triggers (`ready`, `merge-test`) are unchanged—diff-aware logic only red
 | Non-qualifying skip-mark and no CI YAML | normal CI | all on |
 | Diff unavailable | normal CI | all on |
 
-**`skip_all` exceptions (CUDA/NPU bootstrap only):** PR labels (`nightly-test`, `merge-test`, `weekly-test`, `ready`, …) do **not** revive jobs under `skip_all`. On `main`, scheduled `NIGHTLY=1` still builds the image and uploads L4; `WEEKLY=1` or `NON_CRITICAL=1` still builds the image and uploads L5 (CUDA). `WEEKLY=1` on `main` also uploads L2/L3 with `--e2e`.
+**`skip_all` exceptions:** For CUDA/NPU, PR labels (`nightly-test`, `merge-test`, `weekly-test`, `ready`, …) do **not** revive jobs under `skip_all`; on `main`, scheduled `NIGHTLY=1` still builds the image and uploads L4, while `WEEKLY=1` or `NON_CRITICAL=1` still builds the image and uploads L5 (CUDA). `WEEKLY=1` on `main` also uploads L2/L3 with `--e2e`. AMD applies `skip_ci.py gate` only to selected ready/merge specs, so an explicitly selected AMD nightly spec (`nightly-test`, `NIGHTLY=1`, or a nightly debug override) remains runnable even for a docs-only diff.
 
 **Yaml-gated nightly/weekly:** L4/L5 upload steps keep their normal label / `NIGHTLY` / `WEEKLY` / `NON_CRITICAL` conditions (for example PR `nightly-test` still uploads nightly). Only L2/L3 upload steps are matrix-gated.
 
@@ -250,6 +263,7 @@ Register new files in `L2_YAML_FILES`, `L3_YAML_FILES`, or `L45_YAML_FILES` in `
 | L3 | `.buildkite/cuda/test-merge.yml` | cuda |
 | L3 | `.buildkite/amd/test-amd-merge.yml` | amd |
 | L4/L5 | `.buildkite/cuda/test-nightly.yml`, `test-weekly.yml` | cuda |
+| L4/L5 | `.buildkite/amd/test-amd-nightly.yml` | amd |
 | L4/L5 | `.buildkite/npu/test-npu-nightly.yml` | npu |
 
 ##### Yaml-gated category branches
@@ -282,7 +296,7 @@ Docs/skip-mark mixed with any row below follows the same matrix.
 | CUDA ready + AMD merge | G4 | `cuda/l2` + `amd/l3` |
 | CUDA ready + AMD ready | G2 | `cuda/l2` + `amd/l2` |
 | CUDA merge + AMD merge | G3 | `cuda/l3` + `amd/l3` |
-| Only nightly / weekly / npu-nightly YAML | G1 | (none) |
+| Only nightly / weekly / amd-nightly / npu-nightly YAML | G1 | (none) |
 | CUDA ready + nightly | G5 | `cuda/l2` |
 | CUDA merge + nightly | G6 | `cuda/l3` |
 | CUDA ready + merge + nightly | G7 | `cuda/l2` + `cuda/l3` |
@@ -299,40 +313,38 @@ Unit coverage: `tests/buildkite/test_skip_ci.py`.
 
 #### Step filtering {#step-filtering}
 
-CUDA **L2** (`.buildkite/cuda/test-ready.yml`) and **L3** (`.buildkite/cuda/test-merge.yml`) only. Bootstrap upload entry: `upload_pipeline.py --upload .buildkite/cuda/bootstrap-upload-steps.yml`.
+CUDA **L2** (`.buildkite/cuda/test-ready.yml`), **L3** (`.buildkite/cuda/test-merge.yml`), **L4** (`.buildkite/cuda/test-nightly.yml`), **L5** (`.buildkite/cuda/test-weekly.yml`), and NPU **L4** (`.buildkite/npu/test-npu-nightly.yml`). Bootstrap upload entry: `upload_pipeline.py --upload .buildkite/cuda/bootstrap-upload-steps.yml` (or `npu/bootstrap-upload-steps.yml`).
+
+**When filtering applies:** **PR label** uploads (`ready`, `merge-test`, `nightly-test`, `weekly-test`) and **post-merge `main` L3** (`test-merge.yml`). Scheduled **`main` + env** uploads that pass `--all` (`NIGHTLY=1`, `WEEKLY=1` / `NON_CRITICAL=1`) or `--e2e` (`WEEKLY=1` on L2/L3) keep every selected job and still strip the key. If **no** job-key prefix matches the diff, a change to the pipeline YAML being uploaded, or any path under the `source_filter_fallback` key in `ci_source_file_dependencies.yml`, keeps every job in that upload. `source_filter_fallback` is not a job key — do not attach it to a step. When any job-key prefix already matches, normal filtering wins (the fallback does not expand the selection).
 
 **Uploader-only keys** — removed before Buildkite sees the YAML; never used at runtime on agents:
 
 | Key | Purpose |
 | --- | ------- |
-| `source_file_dependencies` | List of path **prefixes**. If any changed file equals a prefix or starts with `prefix/`, keep the step (or group); otherwise omit it. |
+| `source_file_dependencies` | Named preset (or list of presets) from [`ci_source_file_dependencies.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/ci_source_file_dependencies.yml). The uploader expands only that key; it does **not** scan `commands`. If any changed file equals a listed prefix or starts with `prefix/`, keep the step (or group); otherwise omit it. Inline path-prefix lists are still accepted. |
 | `mirror_hardwares` | Expand to `agents` + `plugins` (+ optional `image`) from `ci_mirror_hardwares.yml`. |
 
 ### Policy
 
-- **Always uploaded** (no key): groups outside **E2E Test**—Simple Test, Diffusion unit tests, Engine/Model Executor, Distributed, Custom Pipeline, Entrypoints (L2), LoRA / Entrypoints (L3).
-- **Diff-gated**: every **E2E Test** leaf job. List the smallest prefix set per step—pytest file(s), model code under `vllm_omni/model_executor/models/` or `vllm_omni/diffusion/models/`, plus `stage_input_processors/` and `vllm_omni/deploy/*.yaml` when applicable. A **group** may define the key instead; the whole group drops if no prefix matches.
+- **Always uploaded** (no key): L2/L3 groups outside **E2E Test**—Simple Test, Diffusion unit tests, Engine/Model Executor, Distributed, Custom Pipeline, Entrypoints (L2), LoRA / Entrypoints (L3).
+- **Diff-gated**: every **E2E Test** leaf job in L2/L3, L4 nightly leaf jobs (CUDA and NPU), and L5 weekly jobs whose `if` includes the `weekly-test` label (Reliability / Perf). Jobs gated only by `WEEKLY=1` or `NON_CRITICAL=1` have no `source_file_dependencies`. Registry layout: YAML anchors are model business code only (`vllm_omni/model_executor/models/`, `vllm_omni/diffusion/models/`, plus `stage_input_processors/` and `vllm_omni/deploy/*.yaml` when applicable). Job keys alias the anchor and **also** list the scripts that job runs (pytest files, `run_cov_split.sh --offline` / `--online` targets, example README runners). Feature keys stay cross-cutting. A **group** may define the key instead; the whole group drops if no prefix matches.
 
 ### YAML examples
 
 ```yaml
       - label: "Diffusion · Qwen Image Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen_image.py
-          - vllm_omni/diffusion/models/qwen_image/
+        source_file_dependencies: diffusion_qwen_image_function
+        # diffusion_qwen_image_function: *qwen_image + tests/e2e/online_serving/test_qwen_image.py
         commands:
           - pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' ...
         mirror_hardwares: h100_1
 
       - label: "TTS · Qwen3-TTS CustomVoice Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
-          - vllm_omni/model_executor/models/qwen3_tts/
-          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
-          - vllm_omni/deploy/qwen3_tts.yaml
+        source_file_dependencies: tts_qwen3_tts_function
+        # tts_qwen3_tts_function: *qwen3_tts + tests/e2e/online_serving/test_qwen3_tts_customvoice.py
         commands:
           - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py ...
-        mirror_hardwares: l4_4
+        mirror_hardwares: l4_1
 ```
 
 ### Local dry-run
@@ -436,15 +448,16 @@ plugin, which provides the agent. `l4_*` jobs run on the EKS `l4-k8s` queue
 still needs `mount-buildkite-agent: true`.
 
 List both `run_cov_split.sh` and `pyproject.toml` in every opted-in job's
-`source_file_dependencies` — both change what the job measures, so without them a
+`source_file_dependencies` preset — both change what the job measures, so without them a
 change there is filtered out of normal PR builds and only surfaces in a later
 nightly. `tests/buildkite/test_upload_pipeline.py` covers the filter behavior with
 a synthetic job (it does not pin real merge labels).
-Editing only the surrounding CI YAML still does not schedule them, so a PR that
-touches just the wiring needs a full E2E run (or the commands run on a GPU host)
-to produce artifacts. When checking a new model's
-artifacts, compare `lines-covered` between the online and offline XML rather than
-just confirming both files exist.
+If no job-key prefix matches, editing the pipeline YAML being uploaded, or any
+path under the `source_filter_fallback` registry key, keeps every job in that
+upload (including coverage jobs) so the wiring can be validated before merge.
+When a job-key prefix already matches, bypass does not expand the selection. A
+change to a different pipeline YAML still does not schedule them. When checking a new model's artifacts, compare `lines-covered`
+between the online and offline XML rather than just confirming both files exist.
 
 ### Validation checklist
 

docs/contributing/ci/test_execution_guide.md

@@ -177,10 +177,10 @@ Failed jobs: 1/2
     cd tests
 
     # Stability: Qwen3-Omni
-    pytest -s -v dfx/stability/scripts/test_stability_qwen3_omni.py -m slow
+    pytest -s -v dfx/stability/scripts/run_stability_qwen3_omni.py -m slow
 
     # Stability: Wan2.2 (v1/videos diffusion benchmark loop)
-    pytest -s -v dfx/stability/scripts/test_stability_wan22.py -m slow
+    pytest -s -v dfx/stability/scripts/run_stability_wan22.py -m slow
 
     # Reliability: Qwen3-Omni (H100 × 2)
     pytest -s -v dfx/reliability/test_reliability_qwen3_omni.py -m slow