[MR V2] Migrate to Model Runner V2

Squashed migration rebased onto current main. Collapses the 46-commit
dev/migrate-MR-v2 history (10 merge commits + superseded pre-rebase work)
into a single commit, so the diff reflects only the genuine MR-V2 feature
(worker_v2/ runners + model_states, omni schedulers, stage engine pipeline,
pooling_output serialization bridge). Content already merged to main
separately (step_audio2, tts_adapters, npu/_310p, etc.) drops out.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.buildkite/pipeline-intel.yaml

@@ -10,7 +10,7 @@ steps:
           DOCKER_BUILDKIT: "1"
           # Buildkite will automatically replace this with the actual commit hash
           VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.22.0"
+          VLLM_VERSION: "v0.23.0"
         priority: 100
         timeout_in_minutes: 60
         soft_fail: false

.buildkite/pipeline.yml

@@ -1,14 +1,15 @@
-# Document 1: Buildkite loads only this block on first parse. The next step resolves docs-only skip-ci
-# from git diff, then uploads document 2. When docs-only skip applies, image-build still runs for the nightly
-# exception in upload_pipeline_with_skip_ci.sh (nightly-test label or main NIGHTLY=1); otherwise upload-nightly
-# can be skipped together with test-ready/test-merge. Weekly uploads also depend on image-build.
+# Document 1: Buildkite loads only this block on first parse. The next step resolves docs-only,
+# pytest skip-mark-only, or combined skip-ci from git diff, then uploads document 2. When skip-ci
+# applies, image-build still runs for the nightly exception in upload_pipeline.py (nightly-test
+# label or main NIGHTLY=1); otherwise upload-nightly can be skipped together with test-ready/test-merge.
+# Weekly uploads also depend on image-build.
 
-# Document 2: appended after `---`; same file, read by upload_pipeline_with_skip_ci.sh (not evaluated as a second pipeline by Buildkite). Child uploads include test-ready.yml, test-merge.yml, test-nightly.yml, and test-weekly.yml.
+# Document 2: appended after `---`; same file, read by upload_pipeline.py (not evaluated as a second pipeline by Buildkite). Child uploads include test-ready.yml, test-merge.yml, test-nightly.yml, and test-weekly.yml.
 steps:
-  - label: ":github: Resolve skip-ci & upload pipeline"
+  - label: ":github: Resolve skip-ci (docs / skip marks) & upload pipeline"
     key: upload-ci-pipeline
     commands:
-      - "bash .buildkite/scripts/upload_pipeline_with_skip_ci.sh"
+      - "python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/pipeline.yml"
     agents:
       queue: "cpu_queue_premerge"
 
@@ -31,7 +32,7 @@ steps:
     key: upload-ready-pipeline
     if: __UPLOAD_READY_IF__
     commands:
-      - buildkite-agent pipeline upload .buildkite/test-ready.yml
+      - python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/test-ready.yml
     agents:
       queue: "cpu_queue_premerge"
 
@@ -41,7 +42,7 @@ steps:
     key: upload-merge-pipeline
     if: __UPLOAD_MERGE_IF__
     commands:
-      - buildkite-agent pipeline upload .buildkite/test-merge.yml
+      - python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/test-merge.yml
     agents:
       queue: "cpu_queue_premerge"
 

.buildkite/scripts/hardware_ci/run-amd-test.sh

@@ -64,7 +64,8 @@ container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | hea
 docker pull "${image_name}"
 
 remove_docker_container() {
-   docker rm -f "${container_name}" || docker image rm -f "${image_name}" || true
+   docker ps -aq --filter "name=${container_name}" | xargs -r docker rm -f || true
+   docker image rm -f "${image_name}" || true
 }
 trap remove_docker_container EXIT
 
@@ -107,7 +108,6 @@ if [[ $commands == *"--shard-id="* ]]; then
         --network=host \
         --shm-size=16gb \
         --group-add "$render_gid" \
-        --rm \
         -e MIOPEN_DEBUG_CONV_DIRECT=0 \
         -e MIOPEN_DEBUG_CONV_GEMM=0 \
         -e VLLM_ROCM_USE_AITER=0 \
@@ -140,7 +140,6 @@ else
           --network=host \
           --shm-size=16gb \
           --group-add "$render_gid" \
-          --rm \
           -e MIOPEN_DEBUG_CONV_DIRECT=0 \
           -e MIOPEN_DEBUG_CONV_GEMM=0 \
           -e VLLM_ROCM_USE_AITER=0 \

.buildkite/scripts/upload_pipeline.py

@@ -0,0 +1,499 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""Render and optionally upload Buildkite pipeline YAML with diff-aware logic.
+
+Bootstrap mode (pipeline.yml with __IMAGE_BUILD_IF__ placeholders):
+  - Detect docs-only, pytest skip-mark-only, or combined docs+skip-mark PR/main changes and
+    substitute skip-ci ``if`` expressions.
+
+Test pipeline mode (e.g. test-merge.yml):
+  - Drop steps or groups whose ``source_file_dependencies`` do not match changed files.
+  - ``source_file_dependencies`` may be set on a leaf step or on a group step.
+  - Always strip ``source_file_dependencies`` from uploaded YAML (Buildkite does not
+    recognize this uploader-only key).
+
+Usage:
+  python3 upload_pipeline.py [--upload] <pipeline.yml>
+
+  # Bootstrap (replaces upload_pipeline_with_skip_ci.sh):
+  python3 upload_pipeline.py --upload .buildkite/pipeline.yml
+
+  # Test pipeline (replaces upload_test_pipeline_with_diff_skip.py):
+  python3 upload_pipeline.py --upload .buildkite/test-merge.yml
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+try:
+    import yaml
+except ImportError:
+    subprocess.check_call(
+        [sys.executable, "-m", "pip", "install", "-q", "pyyaml"],
+    )
+    import yaml
+
+LOG = "upload_pipeline"
+ROOT = Path(__file__).resolve().parent.parent.parent
+DOC_SEP = "\n---\n"
+BOOTSTRAP_MARKER = "__IMAGE_BUILD_IF__"
+
+
+def _log(message: str) -> None:
+    print(f"{LOG}: {message}", file=sys.stderr)
+
+
+def _git(*args: str) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        ["git", *args],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+
+
+def resolve_diff_range() -> str | None:
+    """Return a git diff range for PR or main builds, or None when unavailable."""
+    is_pr = os.environ.get("BUILDKITE_PULL_REQUEST", "false") != "false" and os.environ.get(
+        "BUILDKITE_PULL_REQUEST", ""
+    )
+    commit = os.environ.get("BUILDKITE_COMMIT", "")
+
+    if is_pr:
+        base_branch = os.environ.get("BUILDKITE_PULL_REQUEST_BASE_BRANCH", "main")
+        base_ref = f"origin/{base_branch}"
+        if _git("rev-parse", "--verify", base_ref).returncode != 0:
+            _log(f"origin/{base_branch} not found locally; trying fetch")
+            _git("fetch", "--depth=200", "origin", base_branch)
+        if _git("rev-parse", "--verify", base_ref).returncode != 0:
+            if _git("rev-parse", "--verify", base_branch).returncode == 0:
+                base_ref = base_branch
+            else:
+                _log(f"cannot resolve PR base {base_branch}; using safe defaults")
+                return None
+        return f"{base_ref}...{commit}"
+
+    if os.environ.get("BUILDKITE_BRANCH", "") == "main":
+        if _git("rev-parse", "--verify", f"{commit}^").returncode != 0:
+            _log("main commit has no parent; using safe defaults")
+            return None
+        return f"{commit}^..{commit}"
+
+    _log("not PR/main build; using safe defaults")
+    return None
+
+
+def resolve_changed_files() -> list[str] | None:
+    """Return changed file paths, or None when diff cannot be resolved."""
+    diff_range = resolve_diff_range()
+    if diff_range is None:
+        return None
+
+    result = _git("diff", "--name-only", diff_range)
+    if result.returncode != 0:
+        _log(f"git diff failed ({diff_range}); using safe defaults")
+        return None
+
+    files = [line for line in result.stdout.splitlines() if line.strip()]
+    _log(f"{len(files)} changed file(s)")
+    return files
+
+
+def _is_doc_file(file_path: str) -> bool:
+    if not file_path:
+        return False
+    if file_path.startswith("docs/"):
+        return True
+    if file_path.endswith(".md"):
+        return True
+    return file_path == "mkdocs.yml"
+
+
+def is_docs_only_change(changed_files: list[str]) -> bool:
+    has_any = False
+    for file_path in changed_files:
+        if not file_path:
+            continue
+        has_any = True
+        if not _is_doc_file(file_path):
+            return False
+    return has_any
+
+
+def _is_test_python_file(file_path: str) -> bool:
+    path = Path(file_path)
+    return path.suffix == ".py" and bool(path.parts) and path.parts[0] == "tests"
+
+
+_SKIP_MARK_RE = re.compile(r"pytest\.mark\.skip(?:if)?\b|pytest\.skip\s*\(")
+_PYTESTMARK_SKIP_RE = re.compile(r"pytest\.mark\.skip\b")
+_PYTEST_MARK_ONLY_RE = re.compile(r"pytest\.mark\.\w+")
+
+
+def _paren_balance(line: str) -> int:
+    return line.count("(") - line.count(")")
+
+
+def _is_skip_mark_related_content(line: str) -> bool:
+    stripped = line.strip()
+    if not stripped:
+        return True
+    if stripped.startswith("#"):
+        return True
+    return _SKIP_MARK_RE.search(stripped) is not None
+
+
+def _is_pytestmark_adjacent_content(line: str) -> bool:
+    """Allow pytestmark list refactors that only add skip/skipif alongside existing marks."""
+    stripped = line.strip().rstrip(",")
+    if not stripped:
+        return True
+    if stripped in {"[", "]"}:
+        return True
+    if stripped.startswith("#"):
+        return True
+    if stripped.startswith("pytestmark"):
+        return True
+    return _PYTEST_MARK_ONLY_RE.search(stripped) is not None
+
+
+def diff_only_contains_skip_mark_changes(diff_text: str) -> bool:
+    """True when diff only edits skip marks or reformats pytestmark to add them."""
+    pending_depth = 0
+    saw_change = False
+    has_skip_mark_edit = False
+    for raw_line in diff_text.splitlines():
+        if raw_line.startswith("@@"):
+            pending_depth = 0
+            continue
+        if not (raw_line.startswith("+") or raw_line.startswith("-")):
+            continue
+        if raw_line.startswith("+++") or raw_line.startswith("---"):
+            continue
+
+        saw_change = True
+        content = raw_line[1:]
+
+        if pending_depth > 0:
+            pending_depth += _paren_balance(content)
+            if pending_depth < 0:
+                pending_depth = 0
+            continue
+
+        if _is_skip_mark_related_content(content):
+            has_skip_mark_edit = True
+            pending_depth = max(0, _paren_balance(content))
+            continue
+
+        if _is_pytestmark_adjacent_content(content):
+            continue
+
+        return False
+
+    return saw_change and has_skip_mark_edit
+
+
+def _git_file_diff(file_path: str, diff_range: str) -> str | None:
+    result = _git("diff", diff_range, "--", file_path)
+    if result.returncode != 0:
+        _log(f"skip-mark-only: git diff failed for {file_path}")
+        return None
+    return result.stdout
+
+
+def _is_new_file_diff(diff_text: str) -> bool:
+    return "new file mode" in diff_text or "--- /dev/null" in diff_text
+
+
+def _file_has_module_level_pytest_skip(content: str) -> bool:
+    """True when ``pytestmark`` applies unconditional ``pytest.mark.skip`` to the module."""
+    match = re.search(r"^pytestmark\s*=\s*\[(.*?)^\s*\]", content, re.MULTILINE | re.DOTALL)
+    if match is not None:
+        return _PYTESTMARK_SKIP_RE.search(match.group(1)) is not None
+    match = re.search(r"^pytestmark\s*=\s*pytest\.mark\.skip\b", content, re.MULTILINE)
+    return match is not None
+
+
+def _file_content_at_commit(file_path: str, commit: str) -> str | None:
+    result = _git("show", f"{commit}:{file_path}")
+    if result.returncode != 0:
+        _log(f"skip-mark-only: cannot read {file_path} at {commit}")
+        return None
+    return result.stdout
+
+
+def _qualifies_as_skip_mark_test_change(file_path: str, diff_range: str) -> bool:
+    diff_text = _git_file_diff(file_path, diff_range)
+    if diff_text is None:
+        return False
+    if not diff_text.strip():
+        _log(f"skip-mark-only: empty diff for {file_path}")
+        return False
+
+    if diff_only_contains_skip_mark_changes(diff_text):
+        return True
+
+    if not _is_new_file_diff(diff_text):
+        _log(f"skip-mark-only: non-skip changes in {file_path}")
+        return False
+
+    commit = os.environ.get("BUILDKITE_COMMIT", "")
+    if not commit:
+        return False
+    content = _file_content_at_commit(file_path, commit)
+    if content is None:
+        return False
+    if _file_has_module_level_pytest_skip(content):
+        _log(f"skip-mark-only: new test file with module-level pytest.mark.skip: {file_path}")
+        return True
+
+    _log(f"skip-mark-only: new test file without module-level pytest.mark.skip: {file_path}")
+    return False
+
+
+def is_skip_mark_only_change(changed_files: list[str], *, diff_range: str) -> bool:
+    """True when every changed file is a tests/ module with only skip-mark edits."""
+    has_any = False
+    for file_path in changed_files:
+        if not file_path:
+            continue
+        has_any = True
+        if not _is_test_python_file(file_path):
+            return False
+        if not _qualifies_as_skip_mark_test_change(file_path, diff_range):
+            return False
+    return has_any
+
+
+def is_docs_or_skip_mark_only_change(changed_files: list[str], *, diff_range: str) -> bool:
+    """True when each changed file is a doc path or a qualifying skip-mark test change."""
+    has_any = False
+    for file_path in changed_files:
+        if not file_path:
+            continue
+        has_any = True
+        if _is_doc_file(file_path):
+            continue
+        if _is_test_python_file(file_path) and _qualifies_as_skip_mark_test_change(file_path, diff_range):
+            continue
+        _log(f"docs/skip-mark-only: rejecting {file_path}")
+        return False
+    return has_any
+
+
+def resolve_skip_ci(changed_files: list[str] | None, *, diff_range: str | None = None) -> bool:
+    if changed_files is None:
+        _log("skip-ci=0 (could not resolve changed files)")
+        return False
+
+    if diff_range is not None and is_docs_or_skip_mark_only_change(changed_files, diff_range=diff_range):
+        if is_docs_only_change(changed_files):
+            _log("docs-only change detected; skip-ci=1")
+        elif is_skip_mark_only_change(changed_files, diff_range=diff_range):
+            _log("pytest skip-mark-only change detected; skip-ci=1")
+        else:
+            _log("docs + pytest skip-mark-only change detected; skip-ci=1")
+        return True
+
+    if diff_range is None and is_docs_only_change(changed_files):
+        _log("docs-only change detected; skip-ci=1")
+        return True
+
+    _log("non-doc/non-skip-mark changes detected; skip-ci=0")
+    return False
+
+
+def render_bootstrap_pipeline(text: str, *, skip_ci: bool) -> str:
+    if DOC_SEP in text:
+        _, continuation = text.split(DOC_SEP, 1)
+    else:
+        continuation = text
+
+    nightly_only = (
+        '(build.pull_request.labels includes "nightly-test") || (build.branch == "main" && build.env("NIGHTLY") == "1")'
+    )
+    if skip_ci:
+        image_if = f"'{nightly_only}'"
+        ready_if = "'false'"
+        merge_if = "'false'"
+    else:
+        image_if = "'true'"
+        ready_if = '\'build.branch != "main" && build.pull_request.labels includes "ready"\''
+        merge_if = (
+            '\'(build.branch == "main" && build.env("NIGHTLY") != "1" '
+            '&& build.env("WEEKLY") != "1") || '
+            '(build.branch != "main" && build.pull_request.labels includes "merge-test")\''
+        )
+
+    return (
+        continuation.replace("__IMAGE_BUILD_IF__", image_if)
+        .replace("__UPLOAD_READY_IF__", ready_if)
+        .replace("__UPLOAD_MERGE_IF__", merge_if)
+    )
+
+
+def _matches_dependencies(changed_files: list[str], prefixes: list[str]) -> bool:
+    for path in changed_files:
+        for prefix in prefixes:
+            normalized = prefix.rstrip("/")
+            if path == normalized or path.startswith(f"{normalized}/"):
+                return True
+    return False
+
+
+def _strip_source_file_dependencies(step: dict[str, Any]) -> dict[str, Any]:
+    return {key: value for key, value in step.items() if key != "source_file_dependencies"}
+
+
+def _step_label(step: dict[str, Any]) -> str:
+    return str(step.get("group") or step.get("label") or "<step>")
+
+
+def _filter_steps(steps: list[Any], changed_files: list[str]) -> list[Any]:
+    filtered: list[Any] = []
+    for step in steps:
+        if not isinstance(step, dict):
+            filtered.append(step)
+            continue
+
+        deps = step.get("source_file_dependencies")
+        if deps is not None and not isinstance(deps, list):
+            raise ValueError(
+                f"source_file_dependencies must be a list in step {_step_label(step)!r}",
+            )
+        if deps is not None and not _matches_dependencies(changed_files, deps):
+            _log(
+                f"skip {_step_label(step)!r} (no changes under {deps})",
+            )
+            continue
+
+        nested = step.get("steps")
+        if nested is not None:
+            kept_nested = _filter_steps(nested, changed_files)
+            if not kept_nested:
+                _log(f"omit empty group {_step_label(step)!r}")
+                continue
+            new_step = _strip_source_file_dependencies(step)
+            new_step["steps"] = kept_nested
+            filtered.append(new_step)
+            continue
+
+        if deps is not None:
+            filtered.append(_strip_source_file_dependencies(step))
+        else:
+            filtered.append(step)
+
+    return filtered
+
+
+def _strip_uploader_metadata_from_steps(steps: list[Any]) -> list[Any]:
+    """Remove uploader-only keys while keeping all steps (no diff filtering)."""
+    stripped: list[Any] = []
+    for step in steps:
+        if not isinstance(step, dict):
+            stripped.append(step)
+            continue
+
+        deps = step.get("source_file_dependencies")
+        if deps is not None and not isinstance(deps, list):
+            raise ValueError(
+                f"source_file_dependencies must be a list in step {_step_label(step)!r}",
+            )
+
+        nested = step.get("steps")
+        new_step = _strip_source_file_dependencies(step)
+        if nested is not None:
+            new_step["steps"] = _strip_uploader_metadata_from_steps(nested)
+        stripped.append(new_step)
+
+    return stripped
+
+
+def render_test_pipeline(
+    doc: dict[str, Any],
+    changed_files: list[str] | None,
+) -> dict[str, Any]:
+    steps = doc.get("steps")
+    if not isinstance(steps, list):
+        return doc
+    if changed_files is not None:
+        steps = _filter_steps(steps, changed_files)
+    else:
+        steps = _strip_uploader_metadata_from_steps(steps)
+    return {**doc, "steps": steps}
+
+
+def resolve_pipeline_path(arg: str) -> Path:
+    path = Path(arg)
+    if path.is_absolute():
+        return path
+    return ROOT / path
+
+
+def render_pipeline(path: Path) -> str:
+    text = path.read_text(encoding="utf-8")
+    diff_range = resolve_diff_range()
+    changed_files = resolve_changed_files()
+
+    if BOOTSTRAP_MARKER in text:
+        rendered = render_bootstrap_pipeline(
+            text,
+            skip_ci=resolve_skip_ci(changed_files, diff_range=diff_range),
+        )
+        return rendered
+
+    doc = yaml.safe_load(text)
+    if not isinstance(doc, dict):
+        raise ValueError(f"invalid pipeline YAML: {path}")
+
+    doc = render_test_pipeline(doc, changed_files)
+
+    return yaml.safe_dump(doc, sort_keys=False)
+
+
+def upload_to_buildkite(content: str) -> None:
+    subprocess.run(
+        ["buildkite-agent", "pipeline", "upload"],
+        input=content,
+        text=True,
+        check=True,
+    )
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "pipeline",
+        nargs="?",
+        default=".buildkite/pipeline.yml",
+        help="Pipeline YAML path (default: .buildkite/pipeline.yml)",
+    )
+    parser.add_argument(
+        "--upload",
+        action="store_true",
+        help="Pipe rendered YAML to buildkite-agent pipeline upload",
+    )
+    args = parser.parse_args()
+
+    path = resolve_pipeline_path(args.pipeline)
+    if not path.is_file():
+        _log(f"missing pipeline file: {path}")
+        return 1
+
+    rendered = render_pipeline(path)
+    if args.upload:
+        upload_to_buildkite(rendered)
+    else:
+        sys.stdout.write(rendered)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.buildkite/scripts/upload_pipeline_with_skip_ci.sh

@@ -1,146 +0,0 @@
-#!/usr/bin/env bash
-# Evaluate docs-only skip-ci and upload continuation steps from the given pipeline YAML
-# (YAML document after the first `---`). Buildkite `if` is evaluated at upload time.
-#
-# Usage: upload_pipeline_with_skip_ci.sh [pipeline_yaml]
-#   pipeline_yaml: path relative to repo root or absolute (default: .buildkite/pipeline.yml)
-set -euo pipefail
-
-ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-PIPELINE_ARG="${1:-.buildkite/pipeline.yml}"
-if [[ "${PIPELINE_ARG}" = /* ]]; then
-  PIPELINE_YML="${PIPELINE_ARG}"
-else
-  PIPELINE_YML="${ROOT}/${PIPELINE_ARG}"
-fi
-
-# Prints a single digit to stdout: 1 = skip image CI, 0 = run. Logs go to stderr.
-is_docs_only_change() {
-  local file_path
-  local has_any=0
-
-  while IFS= read -r file_path; do
-    [[ -z "${file_path}" ]] && continue
-    has_any=1
-
-    if [[ "${file_path}" == docs/* ]]; then
-      continue
-    fi
-    if [[ "${file_path}" == *.md ]]; then
-      continue
-    fi
-    if [[ "${file_path}" == "mkdocs.yml" ]]; then
-      continue
-    fi
-    return 1
-  done
-
-  [[ "${has_any}" -eq 1 ]]
-}
-
-resolve_skip_ci() {
-  local is_pr_build=0
-  local files
-  local base_branch base_ref
-
-  if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" && -n "${BUILDKITE_PULL_REQUEST:-}" ]]; then
-    is_pr_build=1
-  fi
-
-  if [[ "${is_pr_build}" -eq 1 ]]; then
-    base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
-    if ! git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then
-      echo "resolve_skip_ci: origin/${base_branch} not found locally; trying fetch" >&2
-      git fetch --depth=200 origin "${base_branch}" >/dev/null 2>&1 || true
-    fi
-
-    base_ref=""
-    if git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then
-      base_ref="origin/${base_branch}"
-    elif git rev-parse --verify "${base_branch}" >/dev/null 2>&1; then
-      base_ref="${base_branch}"
-    else
-      echo "resolve_skip_ci: cannot resolve PR base ${base_branch}; skip-ci=0" >&2
-      echo -n 0
-      return 0
-    fi
-
-    if ! files="$(git diff --name-only "${base_ref}...${BUILDKITE_COMMIT}" 2>/dev/null)"; then
-      echo "resolve_skip_ci: failed to compute PR changed files; skip-ci=0" >&2
-      echo -n 0
-      return 0
-    fi
-  elif [[ "${BUILDKITE_BRANCH:-}" == "main" ]]; then
-    if ! git rev-parse --verify "${BUILDKITE_COMMIT}^" >/dev/null 2>&1; then
-      echo "resolve_skip_ci: commit has no parent on main; skip-ci=0" >&2
-      echo -n 0
-      return 0
-    fi
-    if ! files="$(git diff --name-only "${BUILDKITE_COMMIT}^..${BUILDKITE_COMMIT}" 2>/dev/null)"; then
-      echo "resolve_skip_ci: failed to compute main changed files; skip-ci=0" >&2
-      echo -n 0
-      return 0
-    fi
-  else
-    echo "resolve_skip_ci: not PR/main build; skip-ci=0" >&2
-    echo -n 0
-    return 0
-  fi
-
-  if is_docs_only_change <<< "${files}"; then
-    echo "resolve_skip_ci: docs-only change detected; skip-ci=1" >&2
-    echo -n 1
-    return 0
-  fi
-
-  echo "resolve_skip_ci: non-doc changes detected; skip-ci=0" >&2
-  echo -n 0
-}
-
-SKIP_CI="$(resolve_skip_ci)"
-
-if [[ ! -f "${PIPELINE_YML}" ]]; then
-  echo "upload_pipeline_with_skip_ci: missing ${PIPELINE_YML}" >&2
-  exit 1
-fi
-
-export ROOT SKIP_CI PIPELINE_YML
-python3 <<'PY' | buildkite-agent pipeline upload
-import os
-import pathlib
-
-path = pathlib.Path(os.environ["PIPELINE_YML"])
-text = path.read_text(encoding="utf-8")
-sep = "\n---\n"
-# Two supported layouts:
-#   - multi-doc: doc 1 = bootstrap (loaded by Buildkite), doc 2 (after `---`) = uploaded steps.
-#   - single-doc: whole file is the uploaded steps; caller (UI init step) is the bootstrap.
-if sep in text:
-    _, continuation = text.split(sep, 1)
-else:
-    continuation = text
-
-skip = os.environ.get("SKIP_CI") == "1"
-# When docs-only skip-ci: skip default CI image, but still build for L4 nightly (PR label nightly-test or
-# main NIGHTLY=1), otherwise upload-nightly (depends_on image-build) would be skipped too.
-nightly_only = (
-    '(build.pull_request.labels includes "nightly-test") '
-    '|| (build.branch == "main" && build.env("NIGHTLY") == "1")'
-)
-# Placeholder in pipeline.yml is `if: __IMAGE_BUILD_IF__` (valid YAML); replace value only.
-if skip:
-    rep = f"'{nightly_only}'"
-    ready_rep = "'false'"
-    merge_rep = "'false'"
-else:
-    rep = "'true'"
-    ready_rep = "'build.branch != \"main\" && build.pull_request.labels includes \"ready\"'"
-    merge_rep = "'(build.branch == \"main\" && build.env(\"NIGHTLY\") != \"1\" && build.env(\"WEEKLY\") != \"1\") || (build.branch != \"main\" && build.pull_request.labels includes \"merge-test\")'"
-rendered = (
-    continuation
-    .replace("__IMAGE_BUILD_IF__", rep)
-    .replace("__UPLOAD_READY_IF__", ready_rep)
-    .replace("__UPLOAD_MERGE_IF__", merge_rep)
-)
-print(rendered, end="")
-PY

.buildkite/test-amd-merge.yml

@@ -1,54 +1,111 @@
+env:
+  VLLM_WORKER_MULTIPROC_METHOD: spawn
+  HF_HUB_DOWNLOAD_TIMEOUT: 300
+  HF_HUB_ETAG_TIMEOUT: 60
+
 steps:
 
-- label: "Simple · Diffusion & Model Executor Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  timeout_in_minutes: 50
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_ROCM_USE_AITER=0
-    # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
-    # that is not supported on CPU
-    - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+- group: ":card_index_dividers: Simple Test"
+  steps:
+    - label: "Simple · Diffusion & Model Executor Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      timeout_in_minutes: 50
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
+        # that is not supported on CPU
+        - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
 
-- label: "Simple · Other Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  timeout_in_minutes: 50
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_ROCM_USE_AITER=0
-    # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
-    # that is not supported on CPU
-    - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+    - label: "Simple · Other Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      timeout_in_minutes: 50
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
+        # that is not supported on CPU
+        - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
 
-- label: "Diffusion Images API LoRA E2E"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - timeout 20m pytest -s -v tests/diffusion/lora/test_images_generations_lora.py
+- group: ":card_index_dividers: Diffusion Test"
+  steps:
+    - label: "Diffusion · LoRA Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - timeout 20m pytest -s -v tests/diffusion/lora/test_images_generations_lora.py
 
-- label: "Diffusion Model CPU offloading Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - |
-      timeout 20m bash -c '
-        set +e
-        pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
-        EXIT1=\$?
-        pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
-        EXIT2=\$?
-        exit \$((EXIT1 | EXIT2))
-      '
+    - label: "Diffusion Model CPU offloading Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - |
+          timeout 20m bash -c '
+            set +e
+            pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
+            EXIT1=\$?
+            pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
+            EXIT2=\$?
+            exit \$((EXIT1 | EXIT2))
+          '
+
+    - label: "Diffusion Sequence Parallelism Test (Need 4 GPUs)"
+      agent_pool: mi325_4
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py
+        - timeout 20m pytest -s -v tests/diffusion/distributed/test_ulysses_uaa_perf.py
+
+    # merge-only tests
+    - label: "Diffusion Tensor Parallelism Test"
+      agent_pool: mi325_2
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py
+
+    - label: "Diffusion Model Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - timeout 30m pytest -s -v tests/e2e/offline_inference/test_z_image.py
+
+    - label: "Diffusion GPU Worker Test"
+      agent_pool: mi325_2
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py
+
+    - label: "Diffusion · Wan22 Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - |
+          timeout 40m bash -c "
+            export VLLM_IMAGE_FETCH_TIMEOUT=60
+            pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+          "
 
 ## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274
 # - label: "Audio Generation Model Test"
@@ -62,42 +119,6 @@ steps:
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
 
-- label: "Diffusion Sequence Parallelism Test (Need 4 GPUs)"
-  agent_pool: mi325_4
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py
-    - timeout 20m pytest -s -v tests/diffusion/distributed/test_ulysses_uaa_perf.py
-
-# merge-only tests
-- label: "Diffusion Tensor Parallelism Test"
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py
-
-- label: "Diffusion Model Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - timeout 30m pytest -s -v tests/e2e/offline_inference/test_z_image.py
-
-- label: "Diffusion GPU Worker Test"
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py
-
 - label: "Engine Test"
   agent_pool: mi325_1
   depends_on: amd-build
@@ -111,50 +132,78 @@ steps:
 # This model seems to be important, so I am trying to run it here.
 # Since this is from test-nightly.yml,
 # I keep this non-blocking.
-- label: "Omni Model Test Qwen2-5-Omni"
-  agent_pool: mi325_4
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  commands:
-    - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-    - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
+- group: ":card_index_dividers: E2E Test"
+  steps:
+    - label: "Omni Model Test Qwen2-5-Omni"
+      agent_pool: mi325_4
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      commands:
+        - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+        - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
 
-- label: "Omni Model Test Qwen3-Omni"
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+    - label: "Omni Model Test Qwen3-Omni"
+      agent_pool: mi325_2
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
 
-- label: "Qwen3-TTS CustomVoice E2E Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - |
-      timeout 20m bash -c '
-        export VLLM_LOGGING_LEVEL=DEBUG
-        export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-        pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model"
-      '
+    - label: "Qwen3-TTS CustomVoice E2E Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - |
+          timeout 20m bash -c '
+            export VLLM_LOGGING_LEVEL=DEBUG
+            export VLLM_WORKER_MULTIPROC_METHOD=spawn
+            export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+            pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model"
+          '
 
-- label: "Qwen3-TTS Base E2E Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - |
-      timeout 30m bash -c '
-        export VLLM_LOGGING_LEVEL=DEBUG
-        export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-        pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model" --run-level "advanced_model"
-      '
+    - label: "Qwen3-TTS Base E2E Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - |
+          timeout 30m bash -c '
+            export VLLM_LOGGING_LEVEL=DEBUG
+            export VLLM_WORKER_MULTIPROC_METHOD=spawn
+            export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+            pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model" --run-level "advanced_model"
+          '
+
+    # It is the whisper ASR validation step that is slow, not the Voxtral TTS generation.
+    # whisper ASR runs on CPU.
+    - label: "Voxtral-TTS E2E Test (Online Serving)"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - |
+          timeout 40m bash -c '
+            export VLLM_WORKER_MULTIPROC_METHOD=spawn
+            pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
+          '
+
+    - label: "Voxtral-TTS E2E Test (Offline Inference)"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - |
+          timeout 30m bash -c '
+            export VLLM_WORKER_MULTIPROC_METHOD=spawn
+            pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
+          '
 
 # TODO: Bagel test on ROCm is very unstable. @tjtanaa
 # Need to debug before reneable numerical changes across large PRs
@@ -195,28 +244,3 @@ steps:
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
 #     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
-
-- label: "Diffusion · Wan22 Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - |
-      timeout 40m bash -c "
-        export VLLM_IMAGE_FETCH_TIMEOUT=60
-        pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
-      "
-
-- label: "Voxtral-TTS E2E Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - |
-      timeout 20m bash -c '
-        export VLLM_LOGGING_LEVEL=DEBUG
-        export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
-      '

.buildkite/test-amd-ready.yaml

@@ -1,42 +1,33 @@
+env:
+  VLLM_WORKER_MULTIPROC_METHOD: spawn
+  HF_HUB_DOWNLOAD_TIMEOUT: 300
+  HF_HUB_ETAG_TIMEOUT: 60
+
 steps:
 
-- label: "Simple · Diffusion & Model Executor Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_ROCM_USE_AITER=0
-    # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
-    # that is not supported on CPU
-    - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+- group: ":card_index_dividers: Simple Test"
+  steps:
+    - label: "Simple · Diffusion & Model Executor Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
+        # that is not supported on CPU
+        - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
 
-- label: "Simple · Other Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_ROCM_USE_AITER=0
-    # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
-    # that is not supported on CPU
-    - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-
-- label: "Voxtral TTS CUDA Unit Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - timeout 10m pytest -s -v tests/model_executor/models/voxtral_tts/test_cuda_graph_acoustic_transformer.py
-
-- label: "Diffusion · Batch Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - timeout 20m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
+    - label: "Simple · Other Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
+        # that is not supported on CPU
+        - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
 
 - label: "Custom Pipeline Test"
   agent_pool: mi325_1
@@ -46,23 +37,92 @@ steps:
   commands:
     - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model"
 
-- label: "Diffusion Model CPU offloading Test"
+
+- label: "Voxtral TTS CUDA Unit Test"
   agent_pool: mi325_1
   depends_on: amd-build
   mirror_hardwares: [amdproduction]
   grade: Blocking
   commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - |
-      timeout 20m bash -c '
-        set +e
-        pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
-        EXIT1=\$?
-        pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
-        EXIT2=\$?
-        exit \$((EXIT1 | EXIT2))
-      '
+    - timeout 10m pytest -s -v tests/model_executor/models/voxtral_tts/test_cuda_graph_acoustic_transformer.py
+
+- group: ":card_index_dividers: Diffusion Test"
+  steps:
+    - label: "Diffusion · Batch Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - timeout 20m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
+
+    - label: "Diffusion · Offloader Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - |
+          timeout 20m bash -c '
+            set +e
+            pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
+            EXIT1=\$?
+            pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
+            EXIT2=\$?
+            exit \$((EXIT1 | EXIT2))
+          '
+
+    - label: "Diffusion · Cache Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export GPU_ARCHS=gfx942
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - timeout 15m pytest -s -v -m "core_model and cache and diffusion and not distributed_cuda and L4"
+
+    - label: "Diffusion Sequence Parallelism Test"
+      agent_pool: mi325_2
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
+
+    - label: "Diffusion GPU Worker Test"
+      agent_pool: mi325_2
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py
+
+    - label: "Diffusion · Model Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
+
+    - label: "Diffusion · Wan22 Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - |
+          timeout 40m bash -c "
+            export VLLM_TEST_CLEAN_GPU_MEMORY=1
+            export VLLM_IMAGE_FETCH_TIMEOUT=60
+            pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
+          "
 
 ## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274
 # - label: "Audio Generation Model Test"
@@ -76,56 +136,6 @@ steps:
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
 
-- label: "Diffusion Cache Backend Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 15m pytest -s -v -m "core_model and cache and diffusion and not distributed_cuda and L4"
-
-- label: "Diffusion Sequence Parallelism Test"
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
-
-- label: "Diffusion GPU Worker Test"
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py
-
-- label: "Diffusion · Model Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
-
-- label: "Diffusion · Wan22 Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - |
-      timeout 40m bash -c "
-        export VLLM_TEST_CLEAN_GPU_MEMORY=1
-        export VLLM_IMAGE_FETCH_TIMEOUT=60
-        pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
-      "
-
 - label: "Engine Test"
   agent_pool: mi325_1
   depends_on: amd-build
@@ -149,43 +159,42 @@ steps:
 #     - timeout 10m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
 #     - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "core_model" --run-level "core_model"
 
-- label: "MiMo-Audio E2E Test with H100"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - |
-      timeout 30m bash -c '
-        export VLLM_LOGGING_LEVEL=DEBUG
-        export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        pytest -s -v tests/e2e/online_serving/test_mimo_audio.py -m "core_model" --run-level "core_model"
-      '
+- group: ":card_index_dividers: E2E Test"
+  steps:
+    - label: "MiMo-Audio E2E Test with H100"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - |
+          timeout 30m bash -c '
+            export VLLM_LOGGING_LEVEL=DEBUG
+            export VLLM_WORKER_MULTIPROC_METHOD=spawn
+            pytest -s -v tests/e2e/online_serving/test_mimo_audio.py -m "core_model" --run-level "core_model"
+          '
 
-- label: "Qwen3-TTS E2E Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-    - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"
+    - label: "Qwen3-TTS E2E Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+        - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"
 
-- label: "Voxtral-TTS E2E Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - |
-      timeout 20m bash -c '
-        export VLLM_LOGGING_LEVEL=DEBUG
-        export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
-        pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
-      '
+    - label: "CosyVoice3-TTS E2E Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - |
+          timeout 20m bash -c '
+            pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "full_model" --run-level "core_model"
+          '
 
 # TODO: Bagel test on ROCm is very unstable. @tjtanaa
 # Need to debug before reneable numerical changes across large PRs
@@ -225,14 +234,3 @@ steps:
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
 #     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
-
-- label: "CosyVoice3-TTS E2E Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - |
-      timeout 20m bash -c '
-        pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "full_model" --run-level "core_model"
-      '

.buildkite/test-merge.yml

@@ -149,7 +149,7 @@ steps:
         commands:
           - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model"
         agents:
-          queue: "gpu_4_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
+          queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
         plugins:
           - docker#v5.2.0:
               image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
@@ -164,7 +164,35 @@ steps:
   - group: ":card_index_dividers: E2E Test"
     depends_on: upload-merge-pipeline
     steps:
+      - label: "TTS · Step-Audio2 E2E Test"
+        timeout_in_minutes: 20
+        commands:
+          - |
+            timeout 20m bash -c "
+              export VLLM_LOGGING_LEVEL=DEBUG
+              export VLLM_WORKER_MULTIPROC_METHOD=spawn
+              pytest -s -v tests/e2e/offline_inference/test_step_audio2.py -m 'advanced_model' --run-level 'advanced_model'
+            "
+        agents:
+          queue: "gpu_4_queue"
+        plugins:
+          - docker#v5.2.0:
+              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              always-pull: true
+              propagate-environment: true
+              shm-size: "8gb"
+              environment:
+                - "HF_HOME=/fsx/hf_cache"
+              volumes:
+                - "/fsx/hf_cache:/fsx/hf_cache"
+
       - label: "TTS · Qwen3-TTS CustomVoice Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
+          - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
+          - vllm_omni/model_executor/models/qwen3_tts/
+          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+          - vllm_omni/deploy/qwen3_tts.yaml
         commands:
           - |
             timeout 20m bash -c "
@@ -173,7 +201,7 @@ steps:
               pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'
             "
         agents:
-          queue: "gpu_4_queue"
+          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
         plugins:
           - docker#v5.2.0:
               image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
@@ -186,6 +214,12 @@ steps:
                 - "/fsx/hf_cache:/fsx/hf_cache"
 
       - label: "TTS · Qwen3-TTS Base Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_tts_base.py
+          - tests/e2e/offline_inference/test_qwen3_tts_base.py
+          - vllm_omni/model_executor/models/qwen3_tts/
+          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+          - vllm_omni/deploy/qwen3_tts.yaml
         commands:
           - |
             timeout 20m bash -c "
@@ -194,7 +228,34 @@ steps:
               pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
             "
         agents:
-          queue: "gpu_4_queue"
+          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
+        plugins:
+          - docker#v5.2.0:
+              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              always-pull: true
+              propagate-environment: true
+              shm-size: "8gb"
+              environment:
+                - "HF_HOME=/fsx/hf_cache"
+              volumes:
+                - "/fsx/hf_cache:/fsx/hf_cache"
+
+      - label: "TTS · Ming-Omni-TTS Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_ming_tts.py
+          - tests/e2e/offline_inference/test_ming_tts.py
+          - vllm_omni/model_executor/models/ming_tts/
+          - vllm_omni/model_executor/stage_input_processors/ming_tts.py
+          - vllm_omni/deploy/ming_tts.yaml
+        timeout_in_minutes: 20
+        commands:
+          - |
+            timeout 20m bash -c "
+              export VLLM_LOGGING_LEVEL=DEBUG
+              pytest -s -v tests/e2e/online_serving/test_ming_tts.py tests/e2e/offline_inference/test_ming_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+            "
+        agents:
+          queue: "gpu_1_queue"
         plugins:
           - docker#v5.2.0:
               image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
@@ -207,6 +268,12 @@ steps:
                 - "/fsx/hf_cache:/fsx/hf_cache"
 
       - label: "TTS · Voxtral-TTS Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_voxtral_tts.py
+          - tests/e2e/online_serving/test_voxtral_tts.py
+          - vllm_omni/model_executor/models/voxtral_tts/
+          - vllm_omni/model_executor/stage_input_processors/voxtral_tts.py
+          - vllm_omni/deploy/voxtral_tts.yaml
         timeout_in_minutes: 20
         commands:
           - |
@@ -249,6 +316,12 @@ steps:
                       type: DirectoryOrCreate
 
       - label: "TTS · GLM-TTS Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_glm_tts.py
+          - tests/e2e/offline_inference/test_glm_tts.py
+          - vllm_omni/model_executor/models/glm_tts/
+          - vllm_omni/model_executor/stage_input_processors/glm_tts.py
+          - vllm_omni/deploy/glm_tts.yaml
         timeout_in_minutes: 20
         commands:
           - |
@@ -291,6 +364,12 @@ steps:
                       type: DirectoryOrCreate
 
       - label: "Omni · Qwen3-Omni Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_qwen3_omni.py
+          - tests/e2e/online_serving/test_qwen3_omni.py
+          - vllm_omni/model_executor/models/qwen3_omni/
+          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
+          - vllm_omni/deploy/qwen3_omni_moe.yaml
         timeout_in_minutes: 30
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
@@ -329,6 +408,11 @@ steps:
                       type: DirectoryOrCreate
 
       - label: "Omni · Mimo Audio Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_mimo_audio.py
+          - vllm_omni/model_executor/models/mimo_audio/
+          - vllm_omni/model_executor/stage_input_processors/mimo_audio.py
+          - vllm_omni/deploy/mimo_audio.yaml
         timeout_in_minutes: 30
         commands:
           - pytest -s -v tests/e2e/online_serving/test_mimo_audio.py -m 'advanced_model and cuda' --run-level 'advanced_model'
@@ -366,37 +450,19 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
-      - label: "Omni · VoxCPM2 Test"
-        timeout_in_minutes: 20
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_LOGGING_LEVEL=DEBUG
-              export VLLM_WORKER_MULTIPROC_METHOD=spawn
-              pytest -s -v tests/e2e/offline_inference/test_voxcpm2.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "TTS · VoxCPM2 Online Test"
+      - label: "TTS · VoxCPM2 Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_voxcpm2_tts.py
+          - tests/e2e/online_serving/test_voxcpm2_tts.py
+          - vllm_omni/model_executor/models/voxcpm2/
+          - vllm_omni/deploy/voxcpm2.yaml
         timeout_in_minutes: 30
         commands:
           - |
             timeout 30m bash -c "
               export VLLM_LOGGING_LEVEL=DEBUG
               export VLLM_WORKER_MULTIPROC_METHOD=spawn
-              pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+              pytest -s -v tests/e2e/offline_inference/test_voxcpm2_tts.py tests/e2e/online_serving/test_voxcpm2_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
             "
         agents:
           queue: "gpu_1_queue"
@@ -413,6 +479,12 @@ steps:
                 - "/fsx/hf_cache:/fsx/hf_cache"
 
       - label: "Omni · Higgs-Audio V2 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_higgs_audio_v2.py
+          - tests/e2e/offline_inference/test_higgs_audio_v2.py
+          - vllm_omni/model_executor/models/higgs_audio_v2/
+          - vllm_omni/model_executor/stage_input_processors/higgs_audio_v2.py
+          - vllm_omni/deploy/higgs_audio_v2.yaml
         timeout_in_minutes: 30
         commands:
           - |
@@ -438,6 +510,9 @@ steps:
                 - "/fsx/hf_cache:/fsx/hf_cache"
 
       - label: "Diffusion · Qwen Image Edit Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen_image_edit.py
+          - vllm_omni/diffusion/models/qwen_image/
         timeout_in_minutes: 20
         commands:
           - pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model'
@@ -476,6 +551,9 @@ steps:
                       type: DirectoryOrCreate
 
       - label: "Diffusion · Longcat Image Edit Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_longcat_image_edit.py
+          - vllm_omni/diffusion/models/longcat_image/
         timeout_in_minutes: 20
         commands:
           - pytest -s -v tests/e2e/online_serving/test_longcat_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model'
@@ -514,6 +592,9 @@ steps:
                       type: DirectoryOrCreate
 
       - label: "Diffusion · Longcat Image Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_longcat_image.py
+          - vllm_omni/diffusion/models/longcat_image/
         timeout_in_minutes: 20
         commands:
           - pytest -s -v tests/e2e/online_serving/test_longcat_image.py -m 'advanced_model and cuda' --run-level 'advanced_model'
@@ -552,6 +633,13 @@ steps:
                       type: DirectoryOrCreate
 
       - label: "Diffusion · Bagel Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_bagel.py
+          - tests/e2e/online_serving/test_bagel.py
+          - vllm_omni/model_executor/models/bagel/
+          - vllm_omni/diffusion/models/bagel/
+          - vllm_omni/model_executor/stage_input_processors/bagel.py
+          - vllm_omni/deploy/bagel.yaml
         timeout_in_minutes: 20
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_bagel.py tests/e2e/online_serving/test_bagel.py -m 'advanced_model and cuda' --run-level 'advanced_model'
@@ -591,6 +679,9 @@ steps:
 
 
       - label: "Diffusion · Qwen Image Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen_image.py
+          - vllm_omni/diffusion/models/qwen_image/
         commands:
           - |
             timeout 40m bash -c '
@@ -632,6 +723,9 @@ steps:
                       type: DirectoryOrCreate
 
       - label: "Diffusion · Qwen Image Layered Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen_image_layered.py
+          - vllm_omni/diffusion/models/qwen_image/
         commands:
           - |
             timeout 40m bash -c "
@@ -673,6 +767,10 @@ steps:
                       type: DirectoryOrCreate
 
       - label: "Diffusion · Wan22 Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_wan22_t2v.py
+          - tests/e2e/online_serving/test_wan22_t2v.py
+          - vllm_omni/diffusion/models/wan2_2/
         commands:
           - |
             timeout 40m bash -c "
@@ -714,6 +812,9 @@ steps:
                       type: DirectoryOrCreate
 
       - label: "Diffusion · HunyuanVideo 1.5 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_hunyuan_video_15.py
+          - vllm_omni/diffusion/models/hunyuan_video/
         timeout_in_minutes: 40
         commands:
           - |
@@ -755,6 +856,10 @@ steps:
                       type: DirectoryOrCreate
 
       - label: "Diffusion · Z Image Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_z_image.py
+          - tests/e2e/online_serving/test_z_image.py
+          - vllm_omni/diffusion/models/z_image/
         timeout_in_minutes: 30
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_z_image.py tests/e2e/online_serving/test_z_image.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
@@ -772,6 +877,10 @@ steps:
                 - "/fsx/hf_cache:/fsx/hf_cache"
 
       - label: "Diffusion · FLUX.2 Klein Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_flux2_klein.py
+          - tests/e2e/online_serving/test_flux2_klein.py
+          - vllm_omni/diffusion/models/flux2_klein/
         timeout_in_minutes: 30
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_flux2_klein.py tests/e2e/online_serving/test_flux2_klein.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
@@ -789,6 +898,9 @@ steps:
                 - "/fsx/hf_cache:/fsx/hf_cache"
 
       - label: "Diffusion · SD3 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_sd3.py
+          - vllm_omni/diffusion/models/sd3/
         timeout_in_minutes: 45
         commands:
           - |

.buildkite/test-nightly.yml

@@ -568,6 +568,55 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
+  - label: ":full_moon: TTS · Higgs-Audio-V3 · Perf Test"
+    key: nightly-tts-performance-higgs-audio-v3
+    depends_on: upload-nightly-pipeline
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
+    timeout_in_minutes: 180
+    commands:
+      - export BENCHMARK_DIR=tests/dfx/perf/results
+      - export VLLM_USE_DEEP_GEMM="0"
+      - export VLLM_MOE_USE_DEEP_GEMM="0"
+      - |
+        set +e
+        pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_higgs_audio_v3.json
+        EXIT=$$?
+        buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+        exit $$EXIT
+    agents:
+      queue: "mithril-h100-pool"
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 1
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /root/.cache/huggingface
+                env:
+                  - name: HF_HOME
+                    value: /root/.cache/huggingface
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            nodeSelector:
+              node.kubernetes.io/instance-type: gpu-h100-sxm
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: hf-cache
+                hostPath:
+                  path: /mnt/hf-cache
+                  type: DirectoryOrCreate
+
   # Diffusion X2I suite: x2i / x2a / x2t and related non-video paths; x2v is only in "Diffusion X2V Model Test" below.
   - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
     key: nightly-diffusion-x2iat-group
@@ -591,6 +640,7 @@ steps:
             tests/e2e/online_serving/test_longcat_image_edit_expansion.py
             tests/e2e/online_serving/test_flux_2_dev_expansion.py
             tests/e2e/online_serving/test_bagel_expansion.py
+            tests/e2e/offline_inference/test_sensenova_u1_expansion.py
             -m "full_model and diffusion and H100 and not distributed_cuda"
             --run-level "full_model"
         agents:
@@ -1462,6 +1512,7 @@ steps:
       - nightly-omni-performance-vllm-text
       - nightly-omni-performance-multi-replicas
       - nightly-tts-performance
+      - nightly-tts-performance-higgs-audio-v3
       - nightly-diffusion-x2iat-performance-qwen-image
       - nightly-diffusion-x2iat-performance-qwen-image-edit
       - nightly-diffusion-x2iat-performance-qwen-image-edit-2511
@@ -1475,6 +1526,7 @@ steps:
       - export DEFAULT_INPUT_DIR=tests/dfx/perf/results
       - export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results
       - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-higgs-audio-v3
       - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
       - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
       - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text

.buildkite/test-ready.yml

@@ -61,7 +61,7 @@ steps:
     steps:
       - label: "Diffusion · Offloader Test"
         commands:
-          - timeout 10m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
+          - timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
         agents:
           queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
         plugins:
@@ -215,10 +215,78 @@ steps:
                   path: /mnt/hf-cache
                   type: DirectoryOrCreate
 
+  - group: ":card_index_dividers: Entrypoints Test"
+    depends_on: upload-ready-pipeline
+    steps:
+      - label: "Entrypoints Test with H100"
+        commands:
+          - |
+            timeout 30m bash -c "
+              export VLLM_WORKER_MULTIPROC_METHOD=spawn
+              pytest -sv tests/entrypoints/ -m 'core_model and cuda and H100' --run-level 'core_model'
+            "
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 2
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: "Entrypoints Test with L4"
+        commands:
+          - |
+            timeout 30m bash -c "
+              export VLLM_WORKER_MULTIPROC_METHOD=spawn
+              pytest -sv tests/entrypoints/ -m 'core_model and cuda and L4' --run-level 'core_model'
+            "
+        agents:
+          queue: "gpu_1_queue"
+        plugins:
+          - docker#v5.2.0:
+              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              always-pull: true
+              propagate-environment: true
+              shm-size: "8gb"
+              environment:
+                - "HF_HOME=/fsx/hf_cache"
+                - "HF_TOKEN"
+              volumes:
+                - "/fsx/hf_cache:/fsx/hf_cache"
+
   - group: ":card_index_dividers: E2E Test"
     depends_on: upload-ready-pipeline
     steps:
       - label: "RLHF · VeRL-Omni E2E Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py
+          - tests/e2e/offline_inference/custom_pipeline/
+          - vllm_omni/diffusion/
+          - vllm_omni/engine/
+          - vllm_omni/entrypoints/
+          - vllm_omni/inputs/
+          - vllm_omni/outputs/
         commands:
           - |
             timeout 20m bash -c "
@@ -239,6 +307,11 @@ steps:
                 - "/fsx/hf_cache:/fsx/hf_cache"
 
       - label: "Omni · Qwen3-Omni Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_omni.py
+          - vllm_omni/model_executor/models/qwen3_omni/
+          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
+          - vllm_omni/deploy/qwen3_omni_moe.yaml
         commands:
           - |
             timeout 20m bash -c "
@@ -273,73 +346,18 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
-      - label: "Omni · Qwen3-Omni Realtime WebSocket"
-        commands:
-          - |
-            timeout 30m bash -c "
-              export VLLM_WORKER_MULTIPROC_METHOD=spawn
-              pytest -s -v tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py::TestQwen3OmniRealtimeWebSocket::test_streaming_audio_input_pcm_output_async_chunk_smoke -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Omni · VoxCPM2 Test"
-        timeout_in_minutes: 20
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_LOGGING_LEVEL=DEBUG
-              export VLLM_WORKER_MULTIPROC_METHOD=spawn
-              pytest -s -v tests/e2e/offline_inference/test_voxcpm2.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "TTS · VoxCPM2 Online Test"
+      - label: "TTS · VoxCPM2 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_voxcpm2_tts.py
+          - vllm_omni/model_executor/models/voxcpm2/
+          - vllm_omni/deploy/voxcpm2.yaml
         timeout_in_minutes: 30
         commands:
           - |
             timeout 30m bash -c "
               export VLLM_LOGGING_LEVEL=DEBUG
               export VLLM_WORKER_MULTIPROC_METHOD=spawn
-              pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'core_model and tts' --run-level 'core_model'
+              pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'core_model' --run-level 'core_model'
             "
         agents:
           queue: "gpu_1_queue"
@@ -356,6 +374,11 @@ steps:
                 - "/fsx/hf_cache:/fsx/hf_cache"
 
       - label: "TTS · Qwen3-TTS CustomVoice Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
+          - vllm_omni/model_executor/models/qwen3_tts/
+          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+          - vllm_omni/deploy/qwen3_tts.yaml
         commands:
           - |
             timeout 20m bash -c "
@@ -363,7 +386,7 @@ steps:
               pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'
             "
         agents:
-          queue: "gpu_4_queue"
+          queue: "gpu_1_queue"
         plugins:
           - docker#v5.2.0:
               image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
@@ -377,6 +400,11 @@ steps:
                 - "/fsx/hf_cache:/fsx/hf_cache"
 
       - label: "TTS · Qwen3-TTS Base Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_tts_base.py
+          - vllm_omni/model_executor/models/qwen3_tts/
+          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+          - vllm_omni/deploy/qwen3_tts.yaml
         commands:
           - |
             timeout 20m bash -c "
@@ -386,7 +414,35 @@ steps:
               pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m 'core_model' --run-level 'core_model'
             "
         agents:
-          queue: "gpu_4_queue"
+          queue: "gpu_1_queue"
+        plugins:
+          - docker#v5.2.0:
+              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              always-pull: true
+              propagate-environment: true
+              shm-size: "8gb"
+              environment:
+                - "HF_HOME=/fsx/hf_cache"
+                - "HF_TOKEN"
+              volumes:
+                - "/fsx/hf_cache:/fsx/hf_cache"
+
+      - label: "TTS · Higgs-Audio-V3 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_higgs_audio_v3.py
+          - vllm_omni/model_executor/models/higgs_audio_v3/
+          - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py
+          - vllm_omni/deploy/higgs_multimodal_qwen3.yaml
+        commands:
+          - |
+            timeout 20m bash -c "
+              export VLLM_WORKER_MULTIPROC_METHOD=spawn
+              export VLLM_USE_DEEP_GEMM=0
+              export VLLM_MOE_USE_DEEP_GEMM=0
+              pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'core_model' --run-level 'core_model'
+            "
+        agents:
+          queue: "gpu_1_queue"
         plugins:
           - docker#v5.2.0:
               image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
@@ -400,6 +456,9 @@ steps:
                 - "/fsx/hf_cache:/fsx/hf_cache"
 
       - label: "Diffusion · Qwen Image Edit Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen_image_edit.py
+          - vllm_omni/diffusion/models/qwen_image/
         commands:
           - |
             timeout 20m bash -c "
@@ -435,6 +494,12 @@ steps:
                       type: DirectoryOrCreate
 
       - label: "Diffusion · Bagel Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_bagel.py
+          - vllm_omni/model_executor/models/bagel/
+          - vllm_omni/diffusion/models/bagel/
+          - vllm_omni/model_executor/stage_input_processors/bagel.py
+          - vllm_omni/deploy/bagel.yaml
         commands:
           - |
             timeout 40m bash -c "
@@ -476,6 +541,9 @@ steps:
                       type: DirectoryOrCreate
 
       - label: "Diffusion · Wan22 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_wan22_t2v.py
+          - vllm_omni/diffusion/models/wan2_2/
         commands:
           - |
             timeout 40m bash -c "
@@ -518,6 +586,9 @@ steps:
 
 
       - label: "Diffusion · Qwen Image Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen_image.py
+          - vllm_omni/diffusion/models/qwen_image/
         commands:
           - |
             timeout 40m bash -c "
@@ -559,6 +630,9 @@ steps:
                       type: DirectoryOrCreate
 
       - label: "Diffusion · Qwen Image Layered Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen_image_layered.py
+          - vllm_omni/diffusion/models/qwen_image/
         commands:
           - |
             timeout 40m bash -c "

.buildkite/test-template-amd-omni.j2

@@ -6,6 +6,13 @@
 {% set docker_image_amd = "rocm/vllm-omni:$BUILDKITE_COMMIT" %}
 {% set default_working_dir = "/app/vllm-omni" %}
 
+{% if env %}
+env:
+{% for name in env %}
+  {{ name }}: "{{ env[name] }}"
+{% endfor %}
+{% endif %}
+steps:
   - group: "AMD Tests"
     depends_on: ~
     steps:
@@ -31,8 +38,15 @@
         agents:
           queue: amd-cpu
 
-    {% for step in steps %}
-    {% if step.mirror_hardwares and mirror_hw in step.mirror_hardwares %}
+{% for step_group in steps %}
+{% if step_group.group %}
+  - group: "{{ step_group.group }}"
+{% if step_group.depends_on %}
+    depends_on: {{ step_group.depends_on }}
+{% endif %}
+    steps:
+{% for step in step_group.steps %}
+{% if step.mirror_hardwares and mirror_hw in step.mirror_hardwares %}
       - label: "{{ step.agent_pool }}: {{ step.label }}"
         depends_on: amd-build
         agents:
@@ -46,8 +60,23 @@
         command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh
         env:
           DOCKER_BUILDKIT: "1"
+{% if step.env %}
+{% for name in step.env %}
+          {{ name }}: "{{ step.env[name] }}"
+{% endfor %}
+{% endif %}
           TEST_COMMAND: |-
             (command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }}
+{% if env %}
+{% for name in env %}
+            export {{ name }}="{{ env[name] }}"
+{% endfor %}
+{% endif %}
+{% if step.env %}
+{% for name in step.env %}
+            export {{ name }}="{{ step.env[name] }}"
+{% endfor %}
+{% endif %}
 {% if "mi250" in step.agent_pool %}
             python3 -m pip uninstall -y amd-aiter
 {% endif %}
@@ -61,5 +90,52 @@
         {% else %}
         soft_fail: true
         {% endif%}
+{% endif %}
+{% endfor %}
+{% elif step_group.mirror_hardwares and mirror_hw in step_group.mirror_hardwares %}
+{% set step = step_group %}
+  - label: "{{ step.agent_pool }}: {{ step.label }}"
+    depends_on: amd-build
+    agents:
+      {% if step.agent_pool %}
+      queue: amd_{{ step.agent_pool }}
+      {% else %}
+      queue: amd_mi325_1
+      {% endif %}
+{% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %}
+{% set indented_cmd = cmd_body | replace("\n", "\n        ") %}
+    command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh
+    env:
+      DOCKER_BUILDKIT: "1"
+{% if step.env %}
+{% for name in step.env %}
+      {{ name }}: "{{ step.env[name] }}"
+{% endfor %}
+{% endif %}
+      TEST_COMMAND: |-
+        (command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }}
+{% if env %}
+{% for name in env %}
+        export {{ name }}="{{ env[name] }}"
+{% endfor %}
+{% endif %}
+{% if step.env %}
+{% for name in step.env %}
+        export {{ name }}="{{ step.env[name] }}"
+{% endfor %}
+{% endif %}
+{% if "mi250" in step.agent_pool %}
+        python3 -m pip uninstall -y amd-aiter
+{% endif %}
+        {{ indented_cmd | safe }}
+    priority: 100
+    {% if step.timeout_in_minutes %}
+    timeout_in_minutes: {{ step.timeout_in_minutes }}
     {% endif %}
-    {% endfor %}
+    {% if step.grade and step.grade == "Blocking" %}
+    soft_fail: false
+    {% else %}
+    soft_fail: true
+    {% endif%}
+{% endif %}
+{% endfor %}

.claude/skills/add-diffusion-model/SKILL.md

@@ -288,9 +288,36 @@ Required updates:
 5. `examples/offline_inference/xxx/README.md` — offline example docs
 6. `examples/online_serve/xxx/README.md` — online serve docs
 
-### Step 8: Add E2E Tests (Recommended)
+### Step 8: Add E2E Tests
 
-Create `tests/e2e/online_serving/test_your_model_expansion.py`.
+**Follow the [vllm-omni-test skill](../vllm-omni-test/SKILL.md)** for markers, file naming, Buildkite wiring, and run commands. Also read [l4_functionality_tests.inc.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_examples/l4_functionality_tests.inc.md) and [CI_5levels.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/CI_5levels.md).
+
+Classify the model's **CI priority** first:
+
+| Priority | Required test levels | Files & markers |
+|----------|---------------------|-----------------|
+| **High** (listed in [#1832](https://github.com/vllm-project/vllm-omni/issues/1832) or on the diffusion hot path) | **L1** · **L2** online · **L3** online + offline · **L4** feature + performance | See table below |
+| **Medium** (*normal priority* in L4 docs) | **L3** online + offline · **L4** feature only | Fewer L4 parametrized rows |
+| **Low** | **L4** feature only | One or two `*_expansion.py` cases |
+
+**Per-level deliverables (diffusion / `pytest.mark.diffusion`):**
+
+| Level | Location | Marker | CI pipeline | Notes |
+|-------|----------|--------|-------------|-------|
+| **L1** | `tests/diffusion/models/{slug}/`, `tests/diffusion/cache/`, transformer unit tests | `core_model` + `cpu` | `test-ready.yml` | Weight remap, `_sp_plan`, cache enabler registration, shape contracts |
+| **L2** | `tests/e2e/online_serving/test_{slug}.py` (and offline if the category is offline-first) | **`core_model` + `advanced_model`** (both on baseline smoke) + `diffusion` + `@hardware_test` / `hardware_marks` | `test-ready.yml` | Default deploy smoke — minimal `num_inference_steps`, single prompt |
+| **L3** | `tests/e2e/online_serving/test_{slug}.py` **and** `tests/e2e/offline_inference/test_{slug}.py` when offline matters | Baseline smoke: **`core_model` + `advanced_model`**; heavier cases: `advanced_model` only (+ `diffusion`) | `test-merge.yml` **or** merged into nightly diffusion function job | Real weights, streaming/API paths, LoRA/offload smoke |
+| **L4** | `tests/e2e/online_serving/test_{slug}_expansion.py` (+ offline expansion if needed) | `full_model` + `diffusion` | `test-nightly.yml` (X2I / X2V / X2A function groups) | Feature combos per [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); perf → `tests/dfx/perf/tests/test_{model}_vllm_omni.json` |
+
+**L2 & L3 online — same file, dual marks on the baseline smoke:** The **first / simplest** case in `test_{slug}.py` (default deploy, minimal steps, single prompt) should carry **both** `@pytest.mark.core_model` and `@pytest.mark.advanced_model` on the **same** function so L2 (`test-ready.yml`) and L3 (`test-merge.yml`) share one smoke test. Heavier deploy variants or API paths in the same file use **`advanced_model` only**. When L3 moves to nightly, migrate those heavier cases into `test_{slug}_expansion.py` with `full_model` and remove the dedicated `test-merge.yml` job (see `test_longcat_image_expansion.py`, `test_qwen_image_expansion.py`).
+
+**L4 design (high priority):** Combine multiple supported features (Cache-DiT, TP, USP, CFG, HSDP, CPU offload, quantization) into **few parametrized** `OmniServerParams` rows so each feature appears in at least one case without exploding GPU jobs. Shard single-GPU vs multi-GPU cases across the nightly X2I/X2V function steps (1-GPU vs `distributed_cuda`).
+
+**L4 design (medium / low):** One or two parametrized rows covering the best quality/perf trade-off; skip perf JSON unless the model is high priority.
+
+**Reference implementations:** `tests/e2e/online_serving/test_qwen_image_edit_expansion.py`, `tests/e2e/online_serving/test_longcat_image_expansion.py`, `tests/e2e/online_serving/test_hunyuan_video_15_expansion.py`.
+
+**Keep model-specific code inside test modules — not `tests/helpers/{slug}.py`:** deploy constants, prompts, sampling dicts, and inline `request_config` / `form_data` belong in each `test_{slug}.py` and `test_{slug}_expansion.py`. Do not add per-model files under `tests/helpers/`; reuse only repo-wide harness (`mark`, `media`, `runtime`, `stage_config`, `assertions`). **L2+ online/offline e2e:** reuse or add `send_*_request` in `tests/helpers/runtime.py` — tests call the handler, not raw `omni.generate` / HTTP. See [vllm-omni-test skill](../vllm-omni-test/SKILL.md) § **Runtime send helpers**.
 
 ### Step 9: Add Cache-DiT Acceleration
 
@@ -568,6 +595,7 @@ See the [Profiling Single-Stage Diffusion](../../../docs/contributing/profiling.
 
 ## Reference Files
 
+- [vllm-omni-test skill](../vllm-omni-test/SKILL.md) — L1–L4 markers, naming, Buildkite wiring, run commands
 - [Transformer Adaptation](references/transformer-adaptation.md) — porting transformers from diffusers
 - [Custom Model Patterns](references/custom-model-patterns.md) — patterns for non-diffusers models
 - [Parallelism Patterns](references/parallelism-patterns.md) — TP, SP/USP, CFG parallel, HSDP implementation details

.claude/skills/add-tts-model/SKILL.md

@@ -305,13 +305,72 @@ def build_voice_clone_prompt(ref_audio_path: str, text: str, codec) -> list:
     ]
 ```
 
+### Test Case Writing (CI Levels)
+
+**Follow the [vllm-omni-test skill](../vllm-omni-test/SKILL.md)** for markers, file naming (`test_{slug}.py` / `test_{slug}_expansion.py`), Buildkite wiring, and copy-paste run commands. Also read [CI_5levels.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/CI_5levels.md) and [tests_style.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/tests_style.md).
+
+Classify the model's **CI priority** first (high / medium / low). High-priority TTS models are typically those on the integration hot path or listed in tracking issues such as [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); medium and low tiers cover the long tail. When unsure, ask the reviewer which tier applies.
+
+| Priority | Required test levels | Files & markers |
+|----------|---------------------|-----------------|
+| **High** | **L1** unit/logic · **L2** online smoke · **L3** online + offline integration · **L4** feature + performance | See table below |
+| **Medium** | **L3** online + offline · **L4** feature only | Skip dedicated L1/L2 unless fixing a logic bug |
+| **Low** | **L4** feature only | One or two `*_expansion.py` parametrized cases |
+
+**Per-level deliverables (TTS / `pytest.mark.tts`):**
+
+| Level | Location | Marker | CI pipeline | Notes |
+|-------|----------|--------|-------------|-------|
+| **L1** | `tests/model_executor/…`, `tests/entrypoints/openai_api/…`, stage-processor tests | `core_model` + `cpu` | `test-ready.yml` | Prompt assembly, async_chunk helpers, `serving_speech` validation — no GPU |
+| **L2** | `tests/e2e/online_serving/test_{slug}.py` | **`core_model` + `advanced_model`** (both on baseline smoke) + `tts` + `@hardware_test(...)` | `test-ready.yml` (`ready` label) | Default deploy smoke: single `/v1/audio/speech` or offline `OmniRunner` path |
+| **L3** | `tests/e2e/online_serving/test_{slug}.py` **and** `tests/e2e/offline_inference/test_{slug}.py` | Baseline smoke: **`core_model` + `advanced_model`**; heavier cases: `advanced_model` only (+ `tts`) | `test-merge.yml` **or** merged into nightly TTS function job | Streaming, voice clone, batch/queue, async_chunk |
+| **L4** | `tests/e2e/online_serving/test_{slug}_expansion.py`, optional offline expansion | `full_model` + `tts` | `test-nightly.yml` (`:full_moon: TTS · Function Test with L4`) | Feature matrix; perf → `tests/dfx/perf/tests/test_tts.json` |
+
+**L2 & L3 online — same file, dual marks on the baseline smoke:** The **first / simplest** case in `test_{slug}.py` (default deploy, single non-streaming `/v1/audio/speech` or equivalent offline path) should carry **both** `@pytest.mark.core_model` **and** `@pytest.mark.advanced_model` on the **same** function so it runs in L2 (`test-ready.yml`, `--run-level core_model`, basic validation) and L3 (`test-merge.yml`, `--run-level advanced_model`, deeper validation) without duplicating the test. In-tree examples: `test_voxcpm2_tts.py::test_text_to_audio_001`, `test_qwen3_tts_customvoice.py::test_text_to_audio_001`.
+
+Heavier scenarios in the same file use **`advanced_model` only** (streaming, extra languages, concurrency, async_chunk, batch). Example: `test_voice_clone_en_streaming_001` → `advanced_model` only. When migrating L3 to nightly, move those heavier cases into `test_{slug}_expansion.py` with `full_model` and drop the dedicated merge job (see `test_ming_tts_expansion.py`, `test_glm_tts_expansion.py`).
+
+```python
+@pytest.mark.core_model
+@pytest.mark.advanced_model
+@pytest.mark.tts
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
+@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
+def test_voice_clone_en_non_streaming_001(omni_server, openai_client) -> None:
+    openai_client.send_audio_speech_request({...})
+```
+
+**L4 consolidation:** Prefer parametrized `OmniServerParams` rows (`default`, `async_chunk`, feature flags) in one expansion module rather than many merge-only files ([#1832](https://github.com/vllm-project/vllm-omni/issues/1832)).
+
+**L4 performance (high-priority models):** Add latency / throughput / stress rows in `tests/dfx/perf/tests/test_tts.json`, or a dedicated `tests/dfx/perf/tests/test_{slug}.json` when the model must not join the shared nightly server matrix before integration lands (see VoxCPM2 / Coqui XTTS pattern). Register the model in `benchmarks/tts/model_configs.yaml` for local `bench_tts.py`. Wire a **separate** `test-nightly.yml` Perf Test step when the JSON is not merged into `test_tts.json` yet.
+
+**Keep model-specific code inside test modules — not `tests/helpers/{slug}.py`:**
+
+- Put `MODEL`, deploy path, vendored `REF_AUDIO_URL`, `get_prompt()`, and inline `request_config` dicts **in each** `test_{slug}.py`, `test_{slug}_expansion.py`, offline `test_{slug}.py`, and L1 `test_{slug}_*.py` as needed.
+- **Do not** add `tests/helpers/{slug}.py` (or `tests/helpers/{model_name}.py`) to deduplicate constants or request builders across those files. A little duplication is intentional; follow in-tree references such as `tests/e2e/online_serving/test_glm_tts.py` and `tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py`.
+- `tests/helpers/` is for **repo-wide** harness code only (`mark.py`, `media.py`, `runtime.py`, `stage_config.py`, `assertions.py`, `fixtures/`). Import those; do not extend the tree with per-model modules.
+
+**Runtime send helpers (`tests/helpers/runtime.py`) — online and offline e2e:**
+
+| Path | Fixture | Call |
+|------|---------|------|
+| Online `/v1/*` | `openai_client` | `openai_client.send_*_request(request_config)` |
+| Offline inference | `omni_runner_handler` | `omni_runner_handler.send_*_request(request_config)` |
+
+1. **Grep `runtime.py` first** — reuse `send_omni_request`, `send_diffusion_request`, `send_audio_speech_request` (online + offline Qwen-style TTS), `send_single_stage_tts_request` (Coqui XTTS / MOSS-TTS-Nano offline), etc.
+2. **No matching helper** → add `send_<feature>_request` (or `send_<route>_http_request` for negative/dfx) in **`runtime.py`** with general `assert_*` bundled inside, **then** call it from the test.
+3. **Test file** holds `request_config` dicts only — not `omni.generate`, not `_collect_audio()`, not raw HTTP/SDK.
+4. Wire `tests/helpers/runtime.py` into Buildkite `source_file_dependencies` when you add helpers.
+
+See [vllm-omni-test skill](../vllm-omni-test/SKILL.md) § **Runtime send helpers** for full tables and exceptions.
+
 ### Deliverables
 
 - Updated `serving_speech.py` with all 5 integration points (single commit)
 - Client scripts and server launcher under `examples/online_serving/text_to_speech/<model>/`
 - Gradio demo with streaming and voice cloning UI in the same dir
-- E2E online serving test (`tests/e2e/online_serving/test_<model>.py`)
-- Buildkite CI entry in `.buildkite/test-merge.yml`
+- E2E tests per **Test Case Writing (CI Levels)** above (priority tier determines L1–L4 scope)
+- Buildkite wired per level: `test-ready.yml` (L1/L2), `test-merge.yml` or nightly function job (L3), `test-nightly.yml` (L4) — see [vllm-omni-test skill](../vllm-omni-test/SKILL.md)
 - New section in `examples/online_serving/text_to_speech/README.md` (table row + per-model section). Do **not** create a top-level `examples/online_serving/<model>/` dir or a per-model `README.md` inside `text_to_speech/<model>/`.
 
 ### E2E test pitfalls to avoid
@@ -326,9 +385,9 @@ def build_voice_clone_prompt(ref_audio_path: str, text: str, codec) -> list:
 - **Use the harness readiness gate.** The fixture waits for HTTP 200 on
   `/health`; don't add `time.sleep` in tests. If warmup is incomplete, make
   `/health` return non-200 until you're actually ready.
-- **Mark with `@pytest.mark.core_model` + `hardware_test(res={"cuda": "H100"})`**
-  so the test lands in `test-ready.yml` (triggered by the `ready` label) rather
-  than only nightly.
+- **Mark tests per the CI Levels table** — baseline smoke: `core_model` + `advanced_model`; heavier cases: `advanced_model` only; L4 expansion: `full_model`
+- **No per-model helper modules** — do not create `tests/helpers/{slug}.py`; keep constants and `request_config` payloads in the test file
+- **Online and offline e2e go through `runtime.py`** — `openai_client.send_audio_speech_request` (online); `omni_runner_handler.send_audio_speech_request` (Qwen-style offline) or `send_single_stage_tts_request` (single-stage offline). Add a new `send_*_request` in `runtime.py` when none fits; do not embed `omni.generate` or HTTP in tests
 
 ## Phase 4: Async Chunk (Streaming)
 
@@ -461,8 +520,8 @@ Use this checklist when integrating a new TTS model:
 - [ ] Voice cloning works (if supported)
 - [ ] All response formats work (wav, mp3, flac, pcm)
 - [ ] Client scripts and server launcher created
-- [ ] E2E online serving test written (`tests/e2e/online_serving/test_<model>.py`)
-- [ ] Buildkite CI entry added to `.buildkite/test-merge.yml`
+- [ ] E2E tests added per model priority tier (see **Test Case Writing (CI Levels)**)
+- [ ] Buildkite entries match level: `test-ready.yml` / `test-merge.yml` or nightly TTS job / `test-nightly.yml`
 - [ ] Gradio demo working
 - [ ] Documentation added (offline + online docs, nav, supported models)
 
@@ -500,5 +559,6 @@ Project docs and adjacent skills:
 - [TTS audio skill](../vllm-omni-audio-tts/SKILL.md) — supported models and usage
 - [Fish Speech integration](../vllm-omni-audio-tts/references/fish-speech.md) — complete example of Phases 1–3
 - [Qwen3-TTS reference](../vllm-omni-audio-tts/references/qwen-tts.md) — complete example of all 5 phases
+- [vllm-omni-test skill](../vllm-omni-test/SKILL.md) — L1–L4 markers, naming, Buildkite wiring, run commands
 - [Adding a TTS model (developer guide)](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/model/adding_tts_model.md)
 - `plan/voxcpm2_native_ar_design.md` — VoxCPM2's vLLM-native AR + side-computation pattern (distinct from the generator-based single-stage described above)

.claude/skills/precheck-pr/SKILL.md

@@ -0,0 +1,97 @@
+---
+name: precheck-pr
+description: Self-check your branch before creating a PR — catch dead code, verify accuracy/perf claims, validate PR title format, and confirm merge readiness. Use when the user says "precheck", "self review", "pre-submit check", or "check my PR before I open it." Never posts to GitHub.
+---
+
+# PR Pre-Check
+
+Self-review your branch before creating a PR against `vllm-project/vllm-omni`. Two modes: **quick** catches showstoppers, **full** does a thorough maintainer-grade review. Never posts to GitHub; the report is for the contributor's terminal only.
+
+## Mode Selection
+
+| Mode | When | Time |
+|------|------|------|
+| **Quick** | About to push, final sanity check | ~3 min |
+| **Full** | Ready for review, want maintainer-level scan | ~10 min |
+
+Default to quick if unsure. Run full before marking a PR "ready for review."
+
+## Workflow
+
+### Step 1: Detect Base Branch
+
+```bash
+BASE_SHA=$(git merge-base HEAD origin/main 2>/dev/null \
+         || git merge-base HEAD main 2>/dev/null \
+         || echo origin/main)
+echo "diffing against ${BASE_SHA}"
+git diff --name-only ${BASE_SHA}...HEAD
+```
+
+### Step 2: Validate PR Title
+
+Check the most recent commit message (or branch name if no commit yet) against the project convention. Valid prefixes:
+
+| Prefix | Applies to |
+|--------|-----------|
+| `[Bugfix]` | Bug fixes |
+| `[CI/Build]` | Build or CI improvements |
+| `[Doc]` | Documentation changes |
+| `[Model]` | New/improved models (include model name) |
+| `[Frontend]` | Frontend changes (API server, OmniLLM class, etc.) |
+| `[Kernel]` | CUDA/kernel changes |
+| `[Core]` | Core logic changes (OmniProcessor, OmniARScheduler, etc.) |
+| `[Hardware][Vendor]` | Hardware-specific (e.g., `[Hardware][Ascend]`) |
+| `[Misc]` | Other changes (use sparingly) |
+
+✗ if: missing prefix, wrong case (`[bugfix]`), or WIP/Draft in title.
+⚠ if: `[Model]` prefix without the model identifier (e.g., `[Model] Add new model` — should be `[Model] Add <ModelName> ...`).
+
+### Step 3: Categorize the PR
+
+| Diff contains | PR type |
+|---------------|---------|
+| New files under `vllm_omni/model_executor/models/<name>/` | **New Model** |
+| Changes to `vllm_omni/diffusion/` | **Diffusion Model** |
+| `[Bugfix]` prefix or single-file fix | **Bug Fix** |
+| Perf/benchmark/throughput claims in commit msg or diff | **Performance** |
+| Everything else | **General** |
+
+If multiple rows apply (e.g., a diffusion model is also a new model), union the checklists.
+
+### Step 4: Run Checklist
+
+Ask: "Quick mode or full mode?" Then walk the checklist for the detected PR type from [references/checklists.md](references/checklists.md). Each item produces ✓, ✗, or ⚠.
+
+### Step 5: Print Report
+
+```
+Pre-check report for <branch>
+
+  Mode: quick | full
+  Type: <new-model | diffusion-model | bug-fix | perf | general>
+
+  Dimension          Result
+  ─────────────────  ──────
+  PR title format    ✓
+  PR desc integrity  ✓
+  Registry/config    ✓
+  Dead code          ⚠ 2 warnings
+  Accuracy           ✓
+  Benchmark          ✗ missing software versions
+
+  Verdict: 1 blocking | 2 warnings | recommend fixing ✗ before PR
+```
+
+**Severity:**
+
+| Mark | Meaning |
+|------|---------|
+| ✗ | Blocking — fix before opening PR |
+| ⚠ | Warning — consider fixing |
+| ✓ | Pass |
+| — | Skipped (not applicable) |
+
+### Stop Here
+
+Do not post comments, open PRs, or modify files. The report is for the contributor's terminal only.

.claude/skills/precheck-pr/references/checklists.md

@@ -0,0 +1,135 @@
+# Pre-Check Checklists
+
+Each item is a concrete check to run against the diff. ✗ = fix before PR. ⚠ = consider fixing.
+
+---
+
+## All PRs: Title Format (check first)
+
+PR title must follow the project convention documented in `docs/contributing/README.md`:
+
+| Prefix | Applies to |
+|--------|-----------|
+| `[Bugfix]` | Bug fixes |
+| `[CI/Build]` | Build or CI improvements |
+| `[Doc]` | Documentation changes |
+| `[Model]` | New/improved models (include model name) |
+| `[Frontend]` | Frontend changes (API server, OmniLLM class, etc.) |
+| `[Kernel]` | CUDA/kernel changes |
+| `[Core]` | Core logic changes (OmniProcessor, OmniARScheduler, etc.) |
+| `[Hardware][Vendor]` | Hardware-specific (e.g., `[Hardware][Ascend]`) |
+| `[Misc]` | Other changes (use sparingly) |
+
+**Examples:**
+```
+✓ [Model] Add MiniCPM-o 4.5 support
+✓ [Bugfix] Fix Ovis image text encoder dtype
+✓ [Core] Qwen3-Omni performance optimization
+✓ [Hardware][Ascend] Fix VoxCPM2 on Ascend
+✗ Fix Ovis image text encoder dtype              ← missing prefix
+✗ [bugfix] Fix Ovis image text encoder           ← wrong case
+✗ [WIP] Add model                                ← WIP in title
+```
+
+- [ ] **Title has valid prefix** — ✗ if missing prefix, wrong case, or WIP/Draft in title. For `[Model]` PRs, the title MUST also include the model identifier (e.g., `[Model] Add <ModelName> ...`). ⚠ if missing the model name.
+
+---
+
+## Bug Fix PRs
+
+### Quick
+
+- [ ] **PR body has 4 sections:** what broke (error message), repro steps, root cause, fix description
+- [ ] **Fix is minimal:** diff touches only files directly related to the bug. No unrelated refactors or style changes.
+- [ ] **Repro is runnable:** the PR body includes a copy-paste command that reproduces the bug
+- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
+
+### Full (adds)
+
+- [ ] **Regression test exists:** `git diff --name-only` includes at least one `tests/` file
+- [ ] **Regression test reproduces the original error:** the test would fail on main, pass on this branch
+- [ ] **Fix matches root cause exactly:** no "fix the symptom + something else" — if the root cause is dtype, the fix is dtype, not dtype + reformatting
+- [ ] **No silent failure risk:** no bare `except: pass`, no `try: ... except: return None`, no empty fallback added
+- [ ] **Upstream pattern match:** the fix follows the same pattern as existing code for similar cases (grep for analogous `from_pretrained` calls, etc.)
+- [ ] **Environment documented:** torch/transformers/vllm versions listed if the bug is version-dependent
+
+---
+
+## Performance PRs
+
+### Quick
+
+- [ ] **Before/after numbers in PR body:** at minimum, one metric (RTF, RPS, latency, VRAM) with main vs PR values
+- [ ] **Hardware specified:** GPU model, count, VRAM. "L4" or "H20" is sufficient. "tested on GPU" is not.
+- [ ] **Concurrency scaling:** if the optimization claims better throughput, numbers at 2+ concurrency levels
+- [ ] **No unexplained regressions:** if any metric goes down at any concurrency level, explain why
+- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
+
+### Full (adds)
+
+- [ ] **Software versions:** torch, CUDA, vllm, vllm-omni versions in PR body
+- [ ] **Warmup stated:** "1 warmup + 3 measured" or equivalent
+- [ ] **Stddev or range:** not just a single best run. "average of 2 rounds" is minimum.
+- [ ] **VRAM measured:** peak GPU memory before and after
+- [ ] **Benchmark script checked in:** under `tests/` or `examples/`, runnable by reviewer
+- [ ] **Exact command line in PR:** not pseudocode, not "run the benchmark"
+- [ ] **Pytest summary included:** `36 passed, 3 skipped, 17 warnings in 2897.66s` — proves the whole suite passes
+- [ ] **Config changes documented:** if `max_num_seqs`, `gpu_memory_utilization`, or deploy YAML changed, old and new values stated with reasoning
+
+---
+
+## New Model PRs
+
+### Quick
+
+- [ ] **PR body file list matches `git diff --name-only`:** every file path in the description exists in the diff
+- [ ] **Registry entries match claimed count:** count `_OMNI_MODELS` or `_DIFFUSION_MODELS` entries added vs claimed in body
+- [ ] **Pipeline config `custom_process_input_func` paths resolve:** each referenced module.function exists
+- [ ] **`__init__.py` exports match registry:** each registered class appears in `__init__.py` and `__all__`
+- [ ] **Shell script modes match README:** `run_curl.sh` supports every mode the README documents
+- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
+
+### Full (adds)
+
+- [ ] **Dead code scan (5 patterns):**
+  1. Dead `forward()` — defined but never called through the pipeline path
+  2. Dead factory/builder functions — instantiated nowhere
+  3. Dead wrappers — thin wrappers with no callers
+  4. Dead branch guards — `if version > X` where X is always satisfied
+  5. Unused parameters — `__init__` args never accessed
+- [ ] **Copy-paste detection:** no string constant defined in 3+ files, no cross-module validation duplication, no near-identical shape coercion fns
+- [ ] **Import hygiene:** no re-export-only imports, no module-level side effects without docs, no `import os` in function body when already at top. Function-body imports are acceptable when the import is only reachable from a closure or factory return value (i.e., `import` would otherwise execute at module load even though the user may never trigger that path). Either lift to top-of-file or annotate with `# noqa: PLC0415`.
+- [ ] **Accuracy:** at least one test compares output against reference; audio = valid WAV at correct sample rate; determinism verified if seed claimed. ⚠ if missing and not explicitly deferred to a follow-up PR or RFC section; — if explicitly deferred (the deferral target SHOULD be linked in the PR body).
+- [ ] **Performance:** hardware, software versions, warmup, and metrics (RTF, VRAM, RPS, latency) stated. ⚠ if missing and not explicitly deferred; — if explicitly deferred.
+- [ ] **Benchmark settings:** model config (TP, PP, max_model_len, enforce_eager, quant), runtime config (batch, input/output spec), environment versions
+- [ ] **Benchmark script checked in + exact command line + pytest summary**
+
+---
+
+## Diffusion Model PRs
+
+Diffusion models live under `vllm_omni/diffusion/`. In addition to the New Model checklist above, verify:
+
+- [ ] **Transformer adapter follows contract:** inherits from `nn.Module`, implements `load_weights()`, uses `vllm_omni.diffusion.attention.layer.Attention`
+- [ ] **Pipeline has both offline and online paths:** `forward()` for batch generation, streaming path for online serving
+- [ ] **Cache-DiT integration:** at least one acceleration (TeaCache, FBCache, etc.) is wired up. ✗ for new **top-level** pipelines. For **subclass** pipelines that inherit `__call__` from a parent where Cache-DiT is already absent, downgrade to ⚠ and recommend filing a follow-up issue against the parent pipeline family.
+- [ ] **Parallelism config:** TP/SP/USP/CFG-Parallel options exposed in pipeline config
+- [ ] **Docs table updated:** model added to `docs/models/supported_models.md` with correct metadata
+- [ ] **E2E test exists:** at least one test exercises the full generate path
+
+---
+
+## General PRs
+
+### Quick
+
+- [ ] **PR body matches diff:** description doesn't claim changes not in the diff
+- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
+- [ ] **CI gates passing:** DCO, pre-commit, build — check `gh pr view --json statusCheckRollup` if a PR already exists
+
+### Full (adds)
+
+- [ ] **No new dead code in changed files:** scan for unreachable paths, unused imports
+- [ ] **Test coverage:** if core code changed (`engine/`, `stages/`, `connectors/`), tests are added or existing tests cover the path
+- [ ] **Import hygiene:** no redundant imports added
+- [ ] **No unrelated changes:** diff scope matches PR description

.claude/skills/quantization/SKILL.md

@@ -0,0 +1,103 @@
+---
+name: quantization
+description: Work on vLLM-Omni quantization for diffusion, autoregressive, omni, or multi-stage models. Use when choosing or adding methods such as fp8, int8, gguf, mxfp8, mxfp4, mxfp4_dualscale, ModelOpt, AutoRound, INC, msModelSlim, awq, or gptq; debugging quantized loading; or validating memory, speed, and output quality.
+---
+
+# vLLM-Omni Quantization
+
+Use this skill for quantization work in `vllm-omni`. Start from the local
+code and docs, then pick the smallest validated path for the model, method,
+hardware, and quality target.
+
+## First Checks
+
+Before changing code or recommending a command, identify:
+
+- Model and task: AR text, omni/TTS, text-to-image, text-to-video, or multi-stage diffusion.
+- Hardware backend: CUDA generation, Ascend NPU, Intel XPU, or another platform.
+- Quantization mode: online from BF16/FP16 weights, pre-quantized checkpoint, runtime KV/FA quantization, or per-component routing.
+- Scope: transformer/DiT, thinker language model, text encoder, VAE, audio/vision encoder, talker, or a specific stage.
+- Validation target: memory reduction, latency/throughput, output quality, or all three.
+
+Do not assume a method is supported just because the CLI accepts the string.
+Check `docs/user_guide/quantization/` and the closest implementation first.
+
+## Quick Decision
+
+| Task | Start With |
+|------|------------|
+| Choose a method or command | `references/methods.md` and `references/modality-compat.md` |
+| Use `build_quant_config()` or per-component routing | `references/methods.md` |
+| Work on diffusion quantization | `references/diffusion.md` |
+| Add quantization to a new model | `references/adding-models.md` |
+| Convert or load ModelOpt FP8 checkpoints | `references/modelopt-fp8.md` |
+| Debug ModelOpt, GGUF, AutoRound, MXFP, or serialized Int8 loading | `references/diffusion.md` and method docs under `docs/user_guide/quantization/` |
+
+## Current Runtime Shape
+
+The unified entrypoint is:
+
+```python
+from vllm_omni.quantization import build_quant_config
+```
+
+It supports method strings, flat method dictionaries, per-component
+dictionaries, existing `QuantizationConfig` objects, and `None`. The factory
+delegates generic methods to upstream `vllm` and keeps vLLM-Omni overrides for
+diffusion or omni-specific routing:
+
+- `gguf`
+- `int8`
+- `mxfp8`
+- `mxfp4`
+- `mxfp4_dualscale`
+- `inc`, `auto-round`, `auto_round`
+- ModelOpt auto-detected configs: `modelopt`, `modelopt_fp4`, `modelopt_mixed`
+
+Use `ComponentQuantizationConfig` when only one stage or component should be
+quantized. Pre-quantized ModelOpt-style checkpoints should not spill into
+vision/audio encoders that have no corresponding scale tensors.
+
+## Ownership Boundary
+
+- Upstream `vllm` owns generic quantization configs, kernels, loader semantics,
+  hardware capability rules, and generic AR quantization methods.
+- `vllm-omni` owns unified config routing, diffusion-specific wrappers,
+  component scoping, GGUF or ModelOpt checkpoint adapters, model-specific
+  prefix mapping, docs, examples, and validation.
+
+If a new method needs missing generic kernels or loader behavior, fix upstream
+`vllm` first. In `vllm-omni`, add thin integration and model wiring.
+
+## Working Loop
+
+1. Read the local method doc under `docs/user_guide/quantization/`.
+2. Inspect the active factory and config class in `vllm_omni/quantization/`.
+3. Inspect the target model pipeline and transformer for stable prefixes and
+   whether every relevant vLLM linear layer receives `quant_config`.
+4. For pre-quantized checkpoints, inspect `transformer/config.json` and the
+   checkpoint tensor names before running full generation.
+5. Validate with the same prompt, seed, size, scheduler, steps, dtype, eager or
+   non-eager mode, and parallelism as the BF16 baseline.
+6. Report quality, latency, and memory separately. Do not call a method
+   supported until quality is checked.
+
+## Common Mistakes
+
+| Symptom | Likely Cause | Fix |
+|---------|--------------|-----|
+| `--quantization` has no visible effect | Wrong scope or unsupported model path | Check component routing and method docs |
+| Some layers stay BF16 unexpectedly | `quant_config` was not threaded into all vLLM linear layers | Audit transformer constructors and prefixes |
+| Quality collapses but loading succeeds | Too many sensitive layers were quantized | Add model-specific `ignored_layers` and compare to BF16 |
+| ModelOpt checkpoint loads but output is corrupted | Prefixes, packed-module mapping, scale routing, or BF16 fallback is wrong | Use `references/modelopt-fp8.md` |
+| GGUF shape or tensor mismatch | Missing architecture-specific adapter | Add explicit adapter mapping; avoid generic fallback |
+| Online and offline MXFP4 disagree | Wrong `mxfp4` vs `mxfp4_dualscale` mode | Check `docs/user_guide/quantization/mxfp4.md` |
+| Quantized path is slower than BF16 | Kernel path, compile mode, dtype, or shape mismatch | Compare eager-to-eager and non-eager-to-non-eager |
+
+## References
+
+- General methods and config routing: [references/methods.md](references/methods.md)
+- Modality and model scope: [references/modality-compat.md](references/modality-compat.md)
+- Diffusion implementation patterns: [references/diffusion.md](references/diffusion.md)
+- Adding model support: [references/adding-models.md](references/adding-models.md)
+- ModelOpt FP8 conversion and loading: [references/modelopt-fp8.md](references/modelopt-fp8.md)

.claude/skills/quantization/references/adding-models.md

@@ -0,0 +1,70 @@
+# Adding Quantization Support to a Model
+
+Use this checklist when extending quantization to a new architecture or stage.
+
+## Classify Ownership
+
+| Case | Owner |
+|------|-------|
+| New generic AR method, kernel, or loader semantics | upstream `vllm` first |
+| Existing upstream method needs diffusion wiring | `vllm-omni` model and config integration |
+| GGUF or ModelOpt tensor mapping for a diffusion architecture | `vllm-omni` adapter |
+| NPU MXFP or msModelSlim diffusion path | `vllm-omni` platform/method wrapper and docs |
+| Multi-stage omni checkpoint quantizes only the thinker LM | `vllm-omni` component scoping |
+
+Avoid building a private quantization stack in model code.
+
+## Model Wiring Checklist
+
+1. Find the closest supported model implementation.
+2. Confirm the model uses vLLM linear layers that accept `quant_config`.
+3. Add `quant_config` to the transformer constructor.
+4. Pass stable `prefix` values through every linear layer.
+5. Handle fused modules and packed mappings explicitly.
+6. Ensure `load_weights()` and any `WeightsMapper` remap `ignored_layers` or
+   `modules_to_not_convert`.
+7. Keep tokenizer, scheduler, VAE, text encoder, audio/vision encoders, and
+   talker BF16 unless method-specific docs validate them.
+8. Add or update a method doc under `docs/user_guide/quantization/`.
+
+## Checkpoint Adapter Checklist
+
+For pre-quantized checkpoints:
+
+- Inspect `transformer/config.json` first.
+- Check `quant_method`, `method`, `producer.name`, `quant_algo`, serialized
+  flags, `ignored_layers`, and any method-specific fields.
+- Verify tensor name mapping before generation.
+- Verify packed modules such as `to_qkv`, `add_kv_proj`, and `w13`.
+- Confirm scale tensor names and shapes.
+- Allow dequantization back to BF16 for intentionally skipped target layers.
+
+## Sensitive Layer Discovery
+
+Start conservative. Quantize the backbone first and keep output-side or routing
+layers BF16 until quality is proven.
+
+Frequently sensitive areas:
+
+- attention output projections such as `to_out`
+- FFN down projections such as `w2`
+- final projections such as `proj_out`, `final_layer`, and output heads
+- modulation, timestep, position, image, or text embedders
+- refiner blocks
+- MoE routers, experts selected by routing, and layernorm-adjacent modules
+- VAE, text/vision towers, audio encoders, and codec/talker stages
+
+Do not reuse one model family's ignore list blindly.
+
+## Tests and Evidence
+
+Minimum evidence for a PR:
+
+- loader/config test or smoke command
+- fixed-seed BF16 vs quantized output comparison
+- quality metrics and saved outputs for precision-changing work
+- memory and latency comparison under identical execution mode
+- failure behavior for unsupported models or missing checkpoint fields
+
+When the PR adds a method to a support table, include the exact model,
+hardware, command, quality threshold, and artifact path used for validation.

.claude/skills/quantization/references/diffusion.md

@@ -0,0 +1,113 @@
+# Diffusion Quantization Reference
+
+Use this file for DiT quantization, config routing, checkpoint adapters, and
+quality or loader debugging.
+
+## Supported Method Families
+
+The current local diffusion quantization families are:
+
+- CUDA/GPU online or checkpoint: `fp8`, `int8`, `gguf`, ModelOpt
+- Ascend NPU: `int8`, `mxfp8`, `mxfp4`, `mxfp4_dualscale`, msModelSlim paths
+- Intel XPU: AutoRound/INC and online MXFP8 paths where documented
+
+Always cross-check the method doc before claiming model support. The CLI
+accepting a method string is not enough.
+
+## Implementation Starting Points
+
+| Area | Files |
+|------|-------|
+| Unified factory | `vllm_omni/quantization/factory.py` |
+| Per-component routing | `vllm_omni/quantization/component_config.py` |
+| GGUF diffusion method | `vllm_omni/quantization/gguf_config.py` |
+| Int8 diffusion method | `vllm_omni/quantization/int8_config.py` |
+| MXFP8 diffusion method | `vllm_omni/quantization/mxfp8_config.py` |
+| MXFP4 and dual-scale method | `vllm_omni/quantization/mxfp4_config.py` |
+| AutoRound/INC omni extension | `vllm_omni/quantization/inc_config.py` |
+| ModelOpt checkpoint adapters | `vllm_omni/diffusion/model_loader/checkpoint_adapters/modelopt.py` |
+| FP8 scaled-mm runtime patches | `vllm_omni/patch.py` |
+| Similarity tool | `vllm_omni/quantization/tools/compare_diffusion_trajectory_similarity.py` |
+
+## Threading `quant_config`
+
+For a diffusion transformer:
+
+1. Accept `quant_config: QuantizationConfig | None` in the transformer.
+2. Pass `quant_config` and stable `prefix` values to every vLLM linear layer.
+3. Preserve packed-module mappings for fused QKV, FFN, or model-specific fused
+   projections.
+4. Call `apply_vllm_mapper()` on configs that need HF-to-vLLM name remapping.
+5. For component configs, keep prefixes specific enough to avoid quantizing VAE
+   or encoders by accident.
+
+Common skipped areas are `proj_out`, output projections, modulation layers,
+embedders, refiner blocks, MoE routers, and layernorm-adjacent paths.
+
+## Online vs Offline
+
+Online quantization:
+
+- starts from BF16/FP16 checkpoint weights
+- computes or stores quantized weights at load time
+- typically uses `--quantization <method>` or `quantization="<method>"`
+
+Offline or pre-quantized checkpoints:
+
+- include quantized weights and scales on disk
+- usually carry `quantization_config` in `transformer/config.json`
+- should auto-detect when the config is complete
+- may require model-specific adapter logic for tensor names and packed modules
+
+Do not mix online flags with incompatible offline checkpoint configs. If the
+active config and disk config differ, `resolve_quant_config_from_disk()` should
+raise or rebuild rather than silently loading corrupted weights.
+
+## GGUF Rules
+
+- Require `quantization_config.gguf_model`.
+- Use the normal base model for tokenizer, scheduler, text encoder, and VAE.
+- Add explicit architecture adapters for tensor-name mapping.
+- Use `named_parameters()` and `named_buffers()`, not `state_dict()`, to
+  discover loadable names.
+- Guard fused QKV/KV rewrites so tensors are not rewritten twice.
+- Flatten N-D activations around the GGUF matmul and restore shape.
+
+Unsupported architectures should fail clearly instead of falling back to a fake
+generic mapper.
+
+## MXFP Rules
+
+- `mxfp8` is W8A8 microscaling, currently centered on NPU and documented XPU
+  behavior.
+- `mxfp4` is online single-scale W4A4 on NPU.
+- `mxfp4_dualscale` is online/offline dual-scale W4A4 with BF16 fallback; the
+  offline path is the production path when calibrated `mul_scale` exists.
+- Offline dual-scale checkpoints must inject per-transformer `ignored_layers`
+  into config because cascade transformers can have different BF16 fallback
+  sets.
+
+Do not load an offline dual-scale checkpoint with the online single-scale path.
+
+## Common Failures
+
+| Symptom | Likely Cause | Fix |
+|---------|--------------|-----|
+| `Unknown quantization method` | Factory override or upstream registration is missing | Add the right override or upstream method |
+| Missing scale tensors | Wrong scope, wrong adapter, or BF16 layer not allowed to dequantize | Check config scope and adapter scale routing |
+| Output rank collapses from 3-D to 2-D | FP8 scaled-mm kernel ignored output shape | Check the patch in `vllm_omni/patch.py` |
+| Quality degrades silently | Sensitive layers quantized | Add `ignored_layers`; compare to BF16 |
+| Only one cascade transformer works | Disk config differs per transformer but active config is reused | Rebuild from each transformer's config |
+| Quantized path slower than BF16 | Kernel, dtype, compile, or benchmark mode mismatch | Compare eager-to-eager and non-eager-to-non-eager |
+
+## Verification
+
+For a new or changed diffusion quantization path, run:
+
+1. Config and loader smoke test.
+2. One fixed-seed generation against BF16.
+3. Numeric comparison with the similarity tool or equivalent metrics.
+4. Serving test if the path is user-facing online.
+5. Memory and latency check under the same execution mode as the baseline.
+
+Keep output artifacts when precision changes are part of the PR.

.claude/skills/quantization/references/methods.md

@@ -0,0 +1,82 @@
+# General Quantization Methods
+
+Use this file for method selection and unified `vllm-omni` config syntax. For
+diffusion-specific wiring, see `diffusion.md`. For ModelOpt checkpoint export
+and loading details, see `modelopt-fp8.md`.
+
+## Unified Entry Point
+
+`vllm_omni.quantization.build_quant_config()` accepts:
+
+- `None` or `"none"` to disable quantization
+- a method string such as `"fp8"`, `"int8"`, `"mxfp8"`, `"awq"`, or `"auto-round"`
+- a flat dict with `"method"` or `"quant_method"`
+- a per-component dict such as `{"transformer": {"method": "fp8"}, "vae": None}`
+- an already-built `QuantizationConfig`
+
+Per-component dicts are routed by `ComponentQuantizationConfig` using
+longest-prefix matching. Prefix stability matters: if a model remaps HF names
+to vLLM names, apply the mapper to `ignored_layers` or component prefixes.
+
+## Local Overrides
+
+The local factory delegates most methods to upstream `vllm`, but has vLLM-Omni
+overrides for methods that need diffusion, NPU, XPU, or component behavior:
+
+| Method | Local Config | Main Use |
+|--------|--------------|----------|
+| `gguf` | `DiffusionGGUFConfig` | Pre-quantized diffusion transformer weights |
+| `int8` | `DiffusionInt8Config` | Online or serialized W8A8 diffusion transformers |
+| `mxfp8` | `DiffusionMXFP8Config` | W8A8 MXFP8 diffusion on NPU and online XPU path |
+| `mxfp4` | `DiffusionMXFP4Config` | Online W4A4 MXFP4 diffusion on NPU |
+| `mxfp4_dualscale` | `DiffusionMXFP4DualScaleMixedConfig` | Online/offline W4A4 dual-scale plus BF16 fallback on NPU |
+| `inc`, `auto-round`, `auto_round` | `OmniINCConfig` | AutoRound/INC checkpoint loading, including MXFP8 checkpoint metadata |
+| ModelOpt config dicts | upstream ModelOpt configs through auto-detect | FP8, NVFP4, or mixed ModelOpt checkpoints |
+
+Check `SUPPORTED_QUANTIZATION_METHODS` in `vllm_omni/quantization/factory.py`
+before adding a new method name.
+
+## AR and Generic Quantization
+
+For AR-backed models, upstream `vllm` methods usually apply:
+
+- `awq` and `gptq`: 4-bit or 8-bit weight quantization with calibration,
+  commonly used for text and AR stages.
+- `fp8`: FP8 weight path or KV-cache FP8 depending on flags and hardware.
+- `inc` or `auto-round`: pre-quantized checkpoints with Intel Neural
+  Compressor / AutoRound metadata.
+
+Use these for the AR language model stage only when the checkpoint and model
+guide support it. Omni and TTS models often keep audio, vision, talker, and
+codec stages in BF16.
+
+## Diffusion and Offline Methods
+
+For diffusion transformers, prefer the method-specific docs:
+
+- `docs/user_guide/quantization/fp8.md`
+- `docs/user_guide/quantization/int8.md`
+- `docs/user_guide/quantization/gguf.md`
+- `docs/user_guide/quantization/modelopt.md`
+- `docs/user_guide/quantization/mxfp8.md`
+- `docs/user_guide/quantization/mxfp4.md`
+- `docs/user_guide/quantization/autoround.md`
+- `docs/user_guide/quantization/msmodelslim.md`
+
+Pre-quantized checkpoints should normally auto-detect from config. Runtime
+flags such as `--force-cutlass-fp8`, `--linear-backend cutlass`, and
+`--moe-backend cutlass` select kernels; they do not quantize a BF16 checkpoint.
+
+## Validation
+
+Use fixed-seed A/B comparisons and record:
+
+- model path and quantized checkpoint path
+- method config and `ignored_layers`
+- prompt, size, frame count, steps, scheduler, dtype, and seed
+- eager/non-eager mode, compile flags, and parallelism
+- output metrics such as PSNR, MAE, cosine similarity, SSIM, LPIPS, plus visual artifacts
+- latency and peak memory, reported separately from quality
+
+The helper `vllm_omni.quantization.tools.compare_diffusion_trajectory_similarity`
+is the default lightweight image/video comparison tool for diffusion paths.

.claude/skills/quantization/references/modality-compat.md

@@ -0,0 +1,47 @@
+# Quantization Compatibility by Modality
+
+Use this as a routing guide. The source of truth is the local method docs under
+`docs/user_guide/quantization/` and the current model implementation.
+
+## Summary
+
+- AR and omni language-model quantization mostly follows upstream `vllm`, with
+  vLLM-Omni scoping for multi-stage models.
+- Diffusion quantization is method-specific and model-specific. It usually
+  targets the diffusion transformer only.
+- Pre-quantized checkpoints should quantize only the component that has
+  quantized weights and scale tensors.
+
+## Model Type Matrix
+
+| Model Type | Typical Quantization Scope | Notes |
+|------------|----------------------------|-------|
+| AR text models | Language model | Upstream `vllm` methods usually apply |
+| Omni/TTS models | Thinker or AR language-model stage | Audio, vision, talker, and codec stages usually stay BF16 |
+| DiT image models | Diffusion transformer | Check FP8, Int8, GGUF, ModelOpt, AutoRound docs |
+| DiT video models | Diffusion transformer | HunyuanVideo FP8 and Wan2.2 MXFP paths are documented; validate other video paths |
+| Multi-stage diffusion | Selected stage only | Use per-component routing and per-stage validation |
+| Audio diffusion | Usually unsupported | Prefer dtype, offload, cache, or parallelism unless docs say otherwise |
+
+## Method Hints
+
+| Goal | Candidate |
+|------|-----------|
+| AR memory reduction | AWQ, GPTQ, AutoRound, ModelOpt where checkpoint supports it |
+| CUDA diffusion W8A8 | FP8 or Int8 |
+| Pre-quantized CUDA diffusion checkpoint | ModelOpt or GGUF |
+| Ascend NPU diffusion | Int8, MXFP8, MXFP4, MXFP4 DualScale, msModelSlim |
+| Intel XPU checkpoint | AutoRound/INC paths |
+| Conservative image quality | selective FP8/Int8 with `ignored_layers` |
+| Unsupported diffusion model | cache acceleration, CPU/layerwise offload, dtype, TP/SP/CFG parallelism |
+
+## Validation Rule
+
+Compatibility means more than successful loading. A method is compatible only
+after it has:
+
+- correct component scope
+- complete checkpoint/config metadata if offline
+- fixed-seed quality comparison against BF16
+- acceptable latency and memory behavior
+- docs or examples that state model, method, hardware, and caveats

.claude/skills/quantization/references/modelopt-fp8.md

@@ -0,0 +1,122 @@
+# ModelOpt FP8 Conversion and Loading
+
+Use this when exporting or loading ModelOpt FP8 checkpoints for diffusion
+transformers. This reference captures the practical conversion lessons from the
+2026-04-21 ModelOpt FP8 paste and aligns them with the current
+`vllm-omni` ModelOpt adapter.
+
+## Core Rule
+
+Treat ModelOpt FP8 as selective checkpoint quantization, not one-click
+whole-model FP8. Quantize the main transformer or DiT backbone, keep sensitive
+or peripheral modules BF16, serialize a complete ModelOpt checkpoint, and let
+vLLM-Omni auto-detect it from config.
+
+Do not pass `--quantization fp8` to force a ModelOpt checkpoint path. The
+checkpoint's `quantization_config` should select `modelopt`.
+
+## Conversion Strategy
+
+Prefer:
+
+- backbone attention Q/K/V and FFN expansion projections
+- conservative per-model ignore lists
+- MHA quantizers disabled until basic linear quantization is stable
+- full module-name prefixes in ignore/routing logic
+- BF16 fallback for output-side and routing-sensitive layers
+
+Avoid quantizing by default:
+
+- VAE
+- text, vision, or audio encoders
+- embedders and modulation layers
+- final heads and output projections
+- refiner modules
+- MoE routers and layernorm-sensitive blocks
+
+## Model-Specific Starting Points
+
+| Model Family | Quantize First | Keep BF16 First |
+|--------------|----------------|-----------------|
+| Z-Image | attention q/k/v, FFN w1/w3 | `to_out`, `w2`, `noise_refiner`, `context_refiner`, embedders, modulation, final/output |
+| Qwen-Image | transformer backbone linear layers | `proj_out`, `img_in`, `txt_in`, norms, modulation, position/time/embed layers |
+| FLUX2-dev / FLUX2-klein | backbone linear layers | `proj_out`, context/x embedders, `norm_out`, stream modulation, time embeddings |
+| HunyuanImage-3.0 | `model.model` backbone | VAE, vision tower, token/time/image embedders, final image head, LM head, routers, layernorm-sensitive blocks |
+
+These are starting points, not support claims. Validate with BF16 outputs.
+
+## Required Checkpoint Metadata
+
+`transformer/config.json` must include a complete `quantization_config`.
+At minimum it should identify ModelOpt and the algorithm:
+
+```json
+{
+  "quantization_config": {
+    "producer": {"name": "modelopt"},
+    "quant_method": "modelopt",
+    "quant_algo": "FP8"
+  }
+}
+```
+
+Current vLLM-Omni auto-detects:
+
+- `FP8` and `FP8_PER_CHANNEL_PER_TOKEN` -> `modelopt`
+- `NVFP4` -> `modelopt_fp4`
+- `MIXED_PRECISION` -> `modelopt_mixed`
+
+Packed modules and scales must map correctly:
+
+- `to_qkv` from `to_q`, `to_k`, `to_v`
+- `add_kv_proj` from `add_q_proj`, `add_k_proj`, `add_v_proj`
+- `w13` from `w1`, `w3`
+- `.input_scale`, `.weight_scale`, `.weight_scale_2`, `.weight_scale_inv`
+
+For selective FP8, target layers that remain BF16 must be allowed to
+dequantize from FP8 weights using the matching `weight_scale`, or the adapter
+must skip the quantized scale tensors for that BF16 target.
+
+## Runtime Signals
+
+Healthy loading should show ModelOpt auto-detection and a ModelOpt linear
+kernel path. Useful signals include:
+
+- auto-detected quantization from config
+- selected ModelOpt FP8 linear method
+- CUTLASS or FlashInfer FP8 scaled-mm backend when enabled and supported
+- non-eager path compiling the transformer when expected
+
+If online serving fails while offline inference works, inspect warmup and
+serving initialization before changing checkpoint conversion.
+
+## Validation Order
+
+1. Check config completeness.
+2. Run an export checker such as `examples/quantization/check_modelopt_fp8_export.py`.
+3. Confirm actual FP8 tensors and expected disk-size reduction.
+4. Run one offline generation with fixed prompt, seed, size, and steps.
+5. Run serving with `vllm serve --omni` and verify `/v1/images/generations`.
+6. Compare against BF16 using visual review plus metrics such as LPIPS, PSNR,
+   MAE, cosine similarity, and SSIM.
+7. Compare performance only under matched modes:
+   - BF16 eager vs FP8 eager
+   - BF16 non-eager vs FP8 non-eager
+
+FP8 is not automatically faster. Check whether execution reaches FP8 kernels,
+whether compile mode is comparable, and whether the workload shape fits the
+kernel path.
+
+## Triage
+
+| Symptom | First Check |
+|---------|-------------|
+| Startup failure | `config.json`, `model_index.json`, `quantization_config`, missing scales |
+| Loads but output is corrupted | Over-quantized sensitive layers or wrong full-name prefixes |
+| Offline works but online fails | Dummy warmup, serving init, compile path |
+| FP8 is not faster | Kernel selection, dtype, compile mode, workload shape |
+| BF16 fallback fails | Missing `weight_scale` for dequantization or wrong packed-module mapping |
+
+The success criterion is stable, correctly scoped checkpoint loading with
+quality preserved enough for the target use case, not maximizing the number of
+FP8 layers.

.claude/skills/readme.md

@@ -21,11 +21,24 @@ include:
 - `diffusion-perf-opt`: guides diffusion model performance optimization,
   including profiling traces, parallel strategies, stage timing analysis, and
   benchmark-driven tuning
+- `quantization`: guides quantization method selection, model integration,
+  checkpoint loading, and quality/performance validation for vLLM-Omni
 - `add-omni-model`: covers addition of new omni-modality model support
 - `add-tts-model`: covers integration of new TTS models and related serving
   workflows
 - `generate-release-note`: helps prepare release notes for repository changes
+- `precheck-pr`: self-check a branch before creating a PR — validates PR title
+  format, catches dead code, verifies accuracy/perf claims, and confirms merge
+  readiness
+- `vllm-omni-test`: guides generation and execution of CI-aligned tests (L1–L4),
+  pytest marker selection (`core_model` / `advanced_model` / `full_model`,
+  `omni` / `tts` / `diffusion`), Buildkite wiring (`test-ready.yml`,
+  `test-merge.yml`, `test-nightly.yml`, `test-weekly.yml`), and copy-paste
+  local plus CI-like `pytest` commands; see `references/test-routing.md` for
+  level-to-command mapping
 - `review-pr`: provides a structured workflow for reviewing pull requests
+- `vllm-omni-npu-model-runner-upgrade`: upgrades NPU model runners to align with the
+  latest vllm-ascend NPUModelRunner
 
 ## Maintenance Guidelines
 

.claude/skills/vllm-omni-test/SKILL.md

@@ -0,0 +1,890 @@
+---
+name: vllm-omni-test
+description: Generate and run tests for vllm-project/vllm-omni with CI-aligned levels and markers; wire new tests into Buildkite (test-ready.yml for L1/L2, test-merge.yml for L3, test-nightly.yml for L4). On completion, always provide copy-paste local and CI-like pytest commands plus prerequisites. Use when creating regression tests, adding L1-L4 coverage, selecting pytest markers, or validating fixes from issues/PRs.
+---
+
+# vLLM-Omni Test Generator & Runner
+
+## Purpose
+
+Use this skill to generate minimal, stable test cases and run them with the correct marker/level strategy for [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni).
+
+**Link convention:** Paths such as `.buildkite/` and `docs/contributing/` live at the **vllm-omni repo root**. Markdown links use repo-relative paths from this skill file (e.g. `../../../.buildkite/test-ready.yml`, `../../../docs/contributing/ci/CI_5levels.md`).
+
+Default priorities:
+
+1. Reproducible regression coverage for bug fixes
+2. Correct test level and marker selection
+3. Low flake, low dependency tests first
+4. CI-compatible run commands
+5. **Actionable run commands for the human**: whenever you add or change tests, always finish with **copy-paste-ready** `pytest` lines (local: single file and/or single test; CI-like: markers + `--run-level`), plus short **prerequisites** (GPU tier, HF cache, optional `model_prefix`). Do not assume the reader will infer commands from `test-routing.md` alone.
+
+## Inputs
+
+- Issue/PR link and summary
+- Changed files or suspected code path
+- Whether the user wants local quick validation or CI-equivalent validation
+- Hardware constraints (CPU only / CUDA / ROCm / NPU)
+
+## Workflow
+
+### Step 1: Classify Test Goal
+
+- **Bugfix regression**: start from a minimal failing scenario and add assertions that prevent recurrence. Before writing tests, output **`required` / `recommended` / `not_needed`**: **`required`** — stable logic/contract bug that should have been caught; **`recommended`** — environment-sensitive but a small regression still helps; **`not_needed`** — one-off external/config failure or existing tests already cover the path. Prefer the narrowest stable L1 (CPU) case; escalate to L2/L3 only when the bug needs real weights or serving.
+- **Feature coverage**: verify new behavior and one negative/boundary case.
+- **Perf/benchmark claim**: require benchmark-oriented tests and explicit metrics.
+
+### Step 2: Select Test Level
+
+- **L1**: unit/logic, deterministic, CPU-friendly, fastest feedback.
+- **L2**: basic e2e/integration and platform-dependent checks.
+- **L3/L4**: advanced model/integration/perf validation.
+
+Use [references/test-routing.md](references/test-routing.md) for level-to-marker and command mapping.
+
+### Step 3: Pick Markers
+
+Always attach markers deliberately:
+
+- **Level**: `core_model` (L1/L2) and/or `advanced_model` (L3) and/or `full_model` (L4 nightly)
+- **Model type** (required on model-centric e2e — pick exactly one):
+  - `omni` — end-to-end multimodal LLM pipelines (thinker/talker/stages; Qwen-Omni family)
+  - `tts` — speech synthesis / TTS-only models (`/v1/audio/speech`, voice clone, etc.)
+  - `diffusion` — generative diffusion models (image / audio / text / video from noise)
+- **Cross-cutting area** (when relevant): `parallel`, `cache`, `example`, `benchmark`
+- **Hardware**: `cpu`, `gpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `distributed_cuda`, …
+- Optional: `slow`, distributed markers when multi-card is required
+
+**Baseline smoke (L2 + L3):** The simplest e2e case per model — default deploy, minimal request — should usually carry **both** `@pytest.mark.core_model` and `@pytest.mark.advanced_model` on the **same** test function so `test-ready.yml` and `test-merge.yml` share one test. `send_*_request` picks validation depth from `--run-level`. References: `test_voxcpm2_tts.py::test_text_to_audio_001`, `test_qwen3_tts_customvoice.py::test_text_to_audio_001`. Heavier scenarios use **`advanced_model` only**; L4 expansion uses **`full_model`**.
+
+For hardware-aware tests, prefer `@hardware_test(...)` or `hardware_marks(...)` in `tests/helpers/mark.py`.
+
+**Diffusion nightly split** (`test-nightly.yml`): all diffusion tests use `pytest.mark.diffusion`, but CI groups them by **output modality**, not by separate pytest markers:
+
+| Nightly group | Scope | Typical models / paths |
+|---------------|--------|-------------------------|
+| **Diffusion X2I(&A&T) Model Test** | x2**i** (image), x2**a** (audio), x2**t** (text) and other **non-video** diffusion | Qwen-Image*, BAGEL, FLUX, SD3, Z-Image, LongCat, DreamZero, … — `test_*_expansion.py` under X2I shards; L4 sweep uses `-k "not test_wan and not test_bagel_expansion and not hunyuan"` for L4 |
+| **Diffusion X2V Model Test** | x2**v** (video) only | Wan2.2, HunyuanVideo 1.5, Wan VACE, … — e.g. `test_wan22_expansion.py`, `test_hunyuan_video_15_expansion.py` |
+
+Wire new **video** diffusion expansion tests into the **X2V** group; wire **image/audio/text** diffusion expansion tests into **X2I(&A&T)**. Do not place x2v modules in X2I shards (see comment in `test-nightly.yml` above the X2I group).
+
+### Naming: generated test module files (L2–L4, model-centric e2e)
+
+When adding **new** pytest modules whose primary scope is a **specific model** (typical under `tests/e2e/offline_inference/` or `tests/e2e/online_serving/`), use this filename pattern:
+
+| Level | Filename pattern | Example (model `Qwen/Qwen2.5-Omni-7B`) |
+|-------|------------------|----------------------------------------|
+| **L2**, **L3** | `test_{lowercase_model_slug}.py` | `test_qwen2_5_omni.py` |
+| **L4** | `test_{lowercase_model_slug}_expansion.py` | `test_qwen2_5_omni_expansion.py` |
+
+**Slug rules for `{lowercase_model_slug}`:**
+
+1. Start from the HuggingFace-style id (e.g. `Qwen/Qwen2.5-Omni-7B`), but **do not** put the org into the filename: use the **repo segment only** (`Qwen2.5-Omni-7B`), not `Qwen_...` / `qwen_qwen2_5_...`.
+2. **Lowercase**; replace `.`, `-`, and whitespace with a single `_` (e.g. `Qwen2.5-Omni-7B` → `qwen2_5_omni`). **Omit** trailing size tokens such as `7b` / `30b` in the basename when a single file covers that model line in the directory (matches `test_qwen2_5_omni.py` in-tree).
+3. If two checkpoints in the same folder need separate modules, add a **minimal** disambiguator (e.g. `_7b` vs `_3b`) only then.
+4. **L1** unit tests are **not** bound to this pattern; use `tests/<area>/test_<feature>.py` as today.
+
+Routing tables and commands: [references/test-routing.md](references/test-routing.md) § *Model-centric e2e filename convention*.
+
+Existing references: `tests/e2e/offline_inference/test_qwen2_5_omni.py` (L2-style omni), `tests/e2e/offline_inference/test_qwen3_5_9b.py` (L2-style omni, single-stage VL), `tests/e2e/online_serving/test_qwen3_omni_expansion.py` (L4-style omni), `tests/e2e/online_serving/test_qwen_image_edit_expansion.py` / `test_qwen_image_expansion.py` (L4-style diffusion).
+
+### Step 4: Generate Test Case Skeleton
+
+**1. Pick the functional scenario** (then choose directory, fixtures, and markers):
+
+| Scenario | Typical location | Fixtures / runner pattern | Baseline markers & level |
+|----------|------------------|---------------------------|---------------------------|
+| **Offline inference e2e** | `tests/e2e/offline_inference/` | **Module (default):** `omni_runner` + `omni_runner_handler`. **Function (isolation only):** `omni_runner_function` + `omni_runner_handler_function`. Diffusion/TTS may use `Omni(...).generate` directly | L2: `core_model` + **one of** `omni` / `tts` / `diffusion`; `@hardware_test(...)` when GPU/NPU is required |
+| **Online serving e2e** | `tests/e2e/online_serving/` | **Module (default):** `omni_server` + `openai_client`. **Function (isolation only):** `omni_server_function` + `openai_client_function`. Clients: `send_omni_request` (omni), `send_audio_speech_request` (tts), `send_diffusion_request` / `send_video_diffusion_request` / `send_images_generations_request` (diffusion) | Baseline smoke: **`core_model` + `advanced_model`**; heavier paths: `advanced_model` only; L4 expansion: `full_model` |
+| **Documentation / runnable examples** | `tests/examples/offline_inference/`, `tests/examples/online_serving/` | **Offline docs (preferred):** extract Python/Bash blocks from the doc README (e.g. `ReadmeSnippet.extract_readme_snippets`), `pytest.mark.parametrize` each snippet, run via `example_runner.run` with a stable `output_subfolder`. **Online docs:** copy client/request scripts into dedicated tests and keep them in sync with the doc page. | Usually **L4**: `advanced_model`, often `example` plus hardware marks matching the nightly docs-example job (see `.buildkite/test-nightly.yml`). Full conventions: [docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md) (introduced in [PR #1910](https://github.com/vllm-project/vllm-omni/pull/1910): naming, output directory layout, skip rules, avoid trimming `num_inference_steps` without a strong CI reason). |
+| **Performance / benchmark** | `tests/dfx/perf/tests/*.json` + `run_*_benchmark.py` | JSON or script-driven server + load config; assert explicit metrics / baselines | L4 Perf: `full_model` + `benchmark`; wire `test-nightly.yml` Perf steps |
+| **Invalid parameter / negative HTTP validation** | `tests/dfx/reliability/invalid_param_test/` | Live `omni_server` + low-level `send_*_http_request` with `err_code` / `err_message` | `pytest.mark.slow` + `omni` / `tts` / `diffusion` + `@hardware_marks` (`H100` or `L4`); CI in **`test-weekly.yml`** (not ready/merge/nightly) |
+
+**If the user’s test plan includes invalid parameter validation / invalid params / negative HTTP / 400 validation:** do **not** add those `test_*` functions to `tests/e2e/online_serving/test_*.py` or `*_expansion.py`. **Move or author them** under `tests/dfx/reliability/invalid_param_test/` in the **endpoint-matching script** (see **Invalid parameter validation** below). Success-path e2e and invalid-param dfx tests must stay in separate modules.
+
+**1b. Model type — `omni` vs `tts` vs `diffusion`**
+
+After choosing offline/online/docs/perf, classify the **product under test** and attach **exactly one** model-type marker. All three can live under the same `tests/e2e/` trees; conventions diverge:
+
+| Dimension | **Omni** (`pytest.mark.omni`) | **TTS** (`pytest.mark.tts`) | **Diffusion** (`pytest.mark.diffusion`) |
+|-----------|------------------------------|----------------------------|----------------------------------------|
+| **What it is** | Multimodal LLM pipeline (thinker/talker/stages; text + vision + audio I/O) | Speech synthesis / voice models | Generative diffusion (noise → image, audio, text, or video) |
+| **Examples** | Qwen2.5-Omni, Qwen3-Omni | Qwen3-TTS, VoxCPM2, Higgs-Audio, Step-Audio2 | Qwen-Image, BAGEL, Wan2.2, HunyuanVideo |
+| **Offline runner** | `omni_runner` + `omni_runner_handler`, `generate_multimodal` | `Omni(...).generate` or stage YAML + TTS params | `Omni(...).generate` + `OmniDiffusionSamplingParams` |
+| **Online client** | `openai_client.send_omni_request` (chat completions, modalities) | `openai_client.send_audio_speech_request` (`/v1/audio/speech`) | `send_diffusion_request` (chat/T2I), `send_video_diffusion_request` (`/v1/videos`, X2V), or `send_images_generations_http_request` / `send_images_edits_http_request` (DALL-E routes) |
+| **Typical assertions** | Stage outputs, text/audio keywords via `OmniRunnerHandler` / response handler | WAV bytes, stream chunks, speech endpoint contract | Image/video dimensions, `final_output_type`, `assert_diffusion_response` |
+| **Stage / deploy YAML** | Per-model omni stage configs (`ci/qwen3_omni_moe.yaml`, …) | `qwen3_tts.yaml`, `voxcpm2.yaml`, … | Often default serve; parallel/offload YAML for heavy DiT |
+| **Nightly group** (`test-nightly.yml`) | **Omni Model Test** — `-m "full_model and omni"` | **TTS Model Test** — `-m "full_model and tts"` | **Diffusion X2I(&A&T)** *or* **Diffusion X2V** (see Step 3 table; same `diffusion` marker, different YAML group / file shard) |
+| **L4 pressure** | Expansion per modality/model as needed | Expansion + accuracy/perf in TTS group | X2I: merge feature combos per [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); X2V: separate nightly group |
+
+Do **not** mix fixtures across types (e.g. do not use `omni_runner` layout for a pure diffusion or TTS model without mirroring an in-tree test in that family).
+
+**Diffusion only — X2I(&A&T) vs X2V (nightly routing, not extra markers):**
+
+- **X2I(&A&T)**: image / audio / text generation — Qwen-Image*, FLUX, SD3, Z-Image, BAGEL (expansion), LongCat, audio diffusion, etc.
+- **X2V**: video generation only — Wan2.2, HunyuanVideo 1.5, Wan VACE, LTX video similarity paths.
+
+When adding a new `test_*_expansion.py`, place it in the matching **nightly group** step (explicit file list in `test-nightly.yml`), not only by marker expression.
+
+**2. Use the narrowest deterministic skeleton for the scenario**
+
+*L1 unit / logic (CPU-first):*
+
+**Mocking rule (L1 only):** use **pytest** integration — **`mocker`** (`pytest-mock`) or **`monkeypatch`** (built-in). **Do not** import or call **`unittest.mock`** (`patch`, `MagicMock`, `@patch`, `with patch(...)`, etc.) in L1 tests; patches must auto-revert with the test lifecycle.
+
+```python
+import pytest
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+def test_<scenario_name>(mocker):
+    # Prefer mocker.patch / mocker.spy / mocker.Mock — not unittest.mock.patch
+    fake_fn = mocker.patch("vllm_omni.some.module.expensive_call", return_value=...)
+    # Act
+    # Assert
+    fake_fn.assert_called_once()
+
+
+def test_<env_or_attr>(monkeypatch):
+    # For simple env / attribute substitution without a Mock object
+    monkeypatch.setenv("SOME_FLAG", "1")
+    monkeypatch.setattr("vllm_omni.some.module.CONST", 42)
+    ...
+```
+
+**Avoid in L1:**
+
+```python
+# BAD — do not use in L1 unit tests
+from unittest.mock import patch, MagicMock
+
+@patch("vllm_omni.some.module.fn")
+def test_bad(mock_fn): ...
+
+def test_also_bad():
+    with patch("...") as m: ...
+```
+
+See **L1 unit test constraints (mocking)** below for the full do/don't list.
+
+*Offline multimodal e2e — **Omni** (representative):*
+
+```python
+@pytest.mark.core_model
+@pytest.mark.omni
+@hardware_test(...)
+@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
+def test_<scenario>(omni_runner, omni_runner_handler) -> None:
+    request_config = {"prompts": ..., "modalities": [...]}  # optional: images, videos, audios
+    omni_runner_handler.send_omni_request(request_config)
+```
+
+*Offline generative e2e — **Diffusion** (representative):*
+
+```python
+@pytest.mark.core_model
+@pytest.mark.diffusion
+@hardware_test(...)
+@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
+def test_text_to_image_001(omni_runner_handler) -> None:
+    omni_runner_handler.send_diffusion_request({"prompt": "...", "extra_body": {"num_inference_steps": 4, ...}})
+```
+
+*Offline TTS e2e — **Qwen3-TTS** (two-stage; representative):*
+
+```python
+@pytest.mark.advanced_model
+@pytest.mark.tts
+@hardware_test(...)
+@pytest.mark.parametrize("omni_runner", tts_server_params, indirect=True)
+def test_text_to_audio_001(omni_runner, omni_runner_handler) -> None:
+    omni_runner_handler.send_audio_speech_request({
+        "input": "...",
+        "task_type": "Base",
+        "ref_audio": REF_AUDIO_URL,
+        "ref_text": REF_TEXT,
+    })
+```
+
+*Offline TTS e2e — **single-stage** (Coqui XTTS, MOSS-TTS-Nano; representative):*
+
+```python
+@pytest.mark.advanced_model
+@pytest.mark.tts
+@hardware_test(...)
+@pytest.mark.parametrize("omni_runner", tts_server_params, indirect=True)
+def test_voice_clone_001(omni_runner_handler) -> None:
+    omni_runner_handler.send_single_stage_tts_request({
+        "input": "...",
+        "language": "en",
+        "prompt_audio_path": REF_AUDIO_PATH,
+        "response_format": "wav",
+        "run_level": "advanced_model",
+    })
+```
+
+*Online serving e2e — **Omni** (representative):*
+
+```python
+@pytest.mark.core_model
+@pytest.mark.omni
+@hardware_test(...)
+@pytest.mark.parametrize("omni_server", test_params, indirect=True)
+def test_text_to_text_001(omni_server, openai_client) -> None:
+    request_config = {"model": omni_server.model, "messages": ..., "modalities": ["text"]}
+    openai_client.send_omni_request(request_config)
+```
+
+*Online serving e2e — **TTS** (representative):*
+
+```python
+@pytest.mark.core_model
+@pytest.mark.advanced_model
+@pytest.mark.tts
+@hardware_test(...)
+@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
+def test_text_to_audio_001(omni_server, openai_client) -> None:
+    request_config = {"model": omni_server.model, "input": "...", "response_format": "wav", ...}
+    openai_client.send_audio_speech_request(request_config)
+```
+
+*Online serving e2e — **Diffusion** X2I (representative):*
+
+```python
+@pytest.mark.core_model
+@pytest.mark.diffusion
+@pytest.mark.parametrize("omni_server", _get_default_case(MODEL), indirect=True)
+def test_text_to_image_001(omni_server, openai_client) -> None:
+    openai_client.send_diffusion_request({...})  # chat completions + extra_body
+```
+
+*Online serving e2e — **Diffusion** X2V (representative):*
+
+```python
+@pytest.mark.core_model
+@pytest.mark.diffusion
+def test_text_to_video_001(omni_server, openai_client) -> None:
+    openai_client.send_video_diffusion_request({"model": ..., "form_data": {...}})  # /v1/videos
+```
+
+*Documentation example tests:* follow the **Preferred Test Strategy** in [l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md): dynamic extraction for offline READMEs; explicit copied client code for online pages until extraction is justified; use the documented **naming**, **output directory** (page folder + case id), and **skipping** rules (e.g. Gradio-only scripts).
+
+*Performance tests:* add or extend entries under `tests/perf/` (and JSON configs where the project uses them), with **explicit baselines** and the same marker/run-level pairing as the CI step that will execute them.
+
+**3. Cross-cutting rules**
+
+- Reuse existing fixtures for the chosen scenario; do not mix “online client” assumptions into offline `OmniRunner` tests without a clear reason.
+- Avoid external network dependency in assertions unless the scenario is explicitly “online serving” or doc examples that require a model hub (then align with CI secrets/cache).
+- Keep **one test function = one intent** (one modality combo, one endpoint contract, or one acceleration combo).
+- **E2E test function layout**: **one case → one `test_<scenario>` function** with a name that states what is validated (endpoint, size/`n`, server flag, or route). **Do not** merge multiple cases into a single test that branches on `request.node.callspec.id`, `param.id`, or `if case_id == ...`. Use `@pytest.mark.parametrize("omni_server", [...], indirect=True)` per function (usually **one** `OmniServerParams` per test). A loop inside one test is OK only when it serves **that function’s single intent** (e.g. three standard sizes in `test_*_sizes_256_512_1024`).
+- **Runtime fixture scope** (`tests/helpers/fixtures/runtime.py`): default **`omni_server` / `omni_runner`** (module) + matching client/handler; use **`omni_server_function` / `omni_runner_function`** only when each `test_*` must start a fresh instance (see below).
+- **L1 mocks**: never `unittest.mock`; use `mocker` or `monkeypatch` only (see below).
+- **API calls (L2+ e2e, online and offline)**: **reuse** `send_*_request` in `tests/helpers/runtime.py` when it exists; **otherwise add** the helper in `runtime.py` first, then call it from the test. General `assert_*` inside `send_*_request`; special `assert_*` only in the test. See **Runtime send helpers** below — do not call `omni.generate`, raw HTTP, or SDK clients from `test_*.py`.
+- **Response assertions**: reusable checks on API bodies / decoded media belong in **`tests/helpers/assertions.py`** — not as private `_assert_*` helpers inside `test_*.py` (see below).
+- **Model-specific payloads stay in test modules** — per-model `MODEL`, deploy path, `REF_AUDIO_URL`, `get_prompt()`, `_build_request()`, and inline `request_config` dicts live in `test_{slug}.py` / `test_{slug}_expansion.py` (and offline/L1 siblings). **Do not** create `tests/helpers/{slug}.py` to deduplicate them; a little copy across files is preferred (see `test_glm_tts.py`, `test_cosyvoice3_tts_expansion.py`). `tests/helpers/` is repo-wide harness only (`mark`, `media`, `runtime`, `stage_config`, `assertions`, `fixtures/`).
+
+#### L1 unit test constraints (mocking)
+
+L1 tests (`core_model and cpu`, under `tests/diffusion/`, `tests/engine/`, `tests/model_executor/`, etc.) must follow the repo’s **pytest-mock** convention (`pytest-mock>=3.10.0` in `pyproject.toml` `[project.optional-dependencies] dev`).
+
+| Do | Don't |
+|----|-------|
+| `def test_foo(mocker):` + `mocker.patch(...)`, `mocker.spy(...)`, `mocker.Mock()`, `mocker.MagicMock()`, `mocker.AsyncMock()` | `from unittest.mock import patch, MagicMock, Mock, AsyncMock` |
+| `def test_bar(monkeypatch):` + `monkeypatch.setattr` / `setenv` / `delenv` / `setitem` | `@patch(...)` decorator |
+| Let `mocker` auto-stop patches after the test | `with patch(...):` / `patch.object(...)` context managers |
+| Mirror neighboring L1 tests in the same directory | `unittest.mock.create_autospec` unless an existing file already documents an exception |
+
+**Rationale:** `mocker` ties patch lifecycle to pytest fixtures (no leaked patches across tests). `unittest.mock` decorators/context managers are easy to compose incorrectly with parametrized or async tests and are inconsistent with in-tree L1 style.
+
+**Minimal patterns:**
+
+```python
+def test_returns_cached_config(mocker):
+    loader = mocker.patch(
+        "vllm_omni.foo.load_yaml",
+        return_value={"stages": []},
+    )
+    result = get_deploy_config("ci/foo.yaml")
+    assert result["stages"] == []
+    loader.assert_called_once_with("ci/foo.yaml")
+
+
+def test_skips_when_env_unset(monkeypatch):
+    monkeypatch.delenv("VLLM_OMNI_FEATURE", raising=False)
+    assert should_enable_feature() is False
+```
+
+E2E levels (L2+) generally **avoid mocks**; if a rare L2 stub is unavoidable, still prefer `mocker` over `unittest.mock` for consistency.
+
+#### Runtime fixtures — scope (`tests/helpers/fixtures/runtime.py`)
+
+vLLM-Omni e2e tests start a real **OmniServer** (online) or **OmniRunner** (offline). Pick scope by **how often the process must be recreated**, not by test level alone.
+
+| Scope | Online fixtures | Offline fixtures | When to use |
+|-------|-----------------|------------------|-------------|
+| **Module** (default) | `omni_server` → `openai_client` | `omni_runner` → `omni_runner_handler` | **Default for L2/L3/L4 expansion** — amortize model/server init across `test_*` in the same module. Same `OmniServerParams` / runner config can be reused by multiple tests. |
+| **Function** | `omni_server_function` → `openai_client_function` | `omni_runner_function` → `omni_runner_handler_function` | **Only when required** — each `test_*` must get a **clean** server/runner (no shared engine/GPU state). Typical: `tests/dfx/reliability/`, sleep/wakeup, crash/restart, tests that mutate global server state. |
+
+**Rules:**
+
+1. **Default to module scope** (`omni_server` / `omni_runner`) unless the scenario **explicitly** needs a fresh instance per test function.
+2. **Indirect parametrize name must match the fixture name**: `@pytest.mark.parametrize("omni_server", ...)` with `omni_server` + `openai_client`; `@pytest.mark.parametrize("omni_server_function", ...)` with `omni_server_function` + `openai_client_function`. Do not mix module fixture with function client (or vice versa).
+3. Different `OmniServerParams` per `test_*` (e.g. default vs `--enable-cpu-offload`) is still OK with **module** `omni_server` — pytest parametrizes per test node; only switch to `_function` when isolation between tests matters, not merely because `server_args` differ.
+4. One `test_*` with **many** server configs (expansion matrix) → single function + `@pytest.mark.parametrize("omni_server", [...], indirect=True)` + module `omni_server` (see in-tree `test_qwen_image_expansion.py`).
+
+```python
+# Default — module-scoped server (L4 expansion)
+@pytest.mark.parametrize("omni_server", [pytest.param(OmniServerParams(model=MODEL), marks=H100)], indirect=True)
+def test_foo_images_generations_default_1024(omni_server, openai_client) -> None:
+    openai_client.send_images_generations_request({...})
+
+# Function-scoped — reliability / per-test clean state only
+@pytest.mark.parametrize(
+    "omni_server_function",
+    [pytest.param(OmniServerParams(model=MODEL), marks=H100)],
+    indirect=True,
+)
+def test_foo_sleep_wakeup_cycle(omni_server_function, openai_client_function) -> None:
+    openai_client_function.send_omni_sleep_http_request({...})
+    openai_client_function.send_omni_wakeup_http_request({...})
+```
+
+#### Runtime send helpers — online **and** offline (`tests/helpers/runtime.py`)
+
+**L2+ e2e (online serving and offline inference) must call APIs through `tests/helpers/runtime.py`.** Fixtures live in `tests/helpers/fixtures/runtime.py`; **send/assert implementation** lives in `tests/helpers/runtime.py` + `tests/helpers/assertions.py`.
+
+| Principle | Action |
+|-----------|--------|
+| **Reuse first** | Grep `runtime.py` for an existing **`send_*_request`** on `OpenAIClientHandler` (online) or `OmniRunnerHandler` (offline) that matches the endpoint / pipeline shape. |
+| **Extend when close** | If an existing `send_*_request` almost fits (e.g. missing one optional field), extend it in `runtime.py` — do not fork logic in the test file. |
+| **Add when missing** | No suitable helper → add **`send_<feature>_request`** (high-level: call + general `assert_*`) or **`send_<route>_<verb>_http_request`** (low-level HTTP for negative/dfx) in **`runtime.py` first**, then call it from tests. |
+| **Test module owns payload only** | In `test_*.py`: `MODEL`, deploy path, vendored media, `get_prompt()`, and inline **`request_config` dicts** only. **No** `omni.generate(...)`, raw `requests.post`, OpenAI SDK calls, or `_collect_audio()` / `_process_output()` in e2e tests. |
+
+**Online** (`openai_client` from `omni_server`): `OpenAIClientHandler.send_*_request`.
+
+**Offline** (`omni_runner_handler` from `omni_runner`): `OmniRunnerHandler.send_*_request`.
+
+| Do | Don't |
+|----|-------|
+| Online: `openai_client.send_omni_request`, `send_diffusion_request`, `send_audio_speech_request`, `send_video_diffusion_request`, `send_images_generations_request`, … | `requests.post(f"{base_url}/v1/...", json=…)` or `client.chat.completions.create(...)` inside a test |
+| Offline: `omni_runner_handler.send_omni_request`, `send_diffusion_request`, `send_audio_speech_request`, `send_single_stage_tts_request`, `send_single_stage_tts_batch_request`, … | `omni_runner.omni.generate(...)` + hand-rolled tensor/WAV extraction in `test_*.py` |
+| Add missing `send_*` to **`runtime.py`** first; bundle general `assert_*` inside the send helper | A one-off `def _post_*` / `def _collect_audio` at the bottom of a test module |
+| Mirror naming/style of neighboring `send_*` (docstring, `request_config` dict, optional `run_level`, `err_code` / `err_message` for negative cases) | Different parameter shapes per test file for the same endpoint |
+
+**Workflow when generating online/offline e2e tests:**
+
+1. Decide whether the needed check is **general** (every success call of this `send_*`) or **special** (one case / one parameter combo only). See **General vs special assert placement** below.
+2. **Search `runtime.py`** for a matching **`send_*_request`**; if missing, add it (with general `assert_*` inside) before writing the test body.
+3. In the test module: build `request_config` → call **`send_*_request` only** for the general contract; call **`assert_*` in the test file only** for special, case-specific checks (import from `assertions.py`).
+4. Reserve **low-level** `send_*_http_request` for negative/dfx tests (`err_code` / `err_message`) — not for ordinary L2+ success-path e2e.
+5. When wiring Buildkite `source_file_dependencies`, include **`tests/helpers/runtime.py`** and/or **`tests/helpers/assertions.py`** when new helpers were added.
+
+**Exceptions (document in the test docstring why):** models whose offline prompt path cannot go through existing handlers yet (e.g. `test_higgs_audio_v2.py`, `test_voxtral_tts.py` with custom tokenizer compose) may call `omni.generate` directly until a `send_*_request` is added to `runtime.py` — treat as **debt**, not the default for new TTS/diffusion/omni e2e.
+
+**General vs special assert placement:**
+
+| Kind | Definition | Where it lives | Test file calls |
+|------|------------|----------------|-----------------|
+| **General** | Default success contract for this endpoint/client method — e.g. HTTP 200, `data[]` shape, `n` count, `size` dimensions, decodable image/audio, bundled omni/diffusion fields | Implement in `assertions.py`, invoke **inside** `send_*_request` in `runtime.py` (after low-level send / SDK call, when `err_code` is not set) | **`send_*_request` only** — do **not** repeat the same `assert_*` |
+| **Special** | Extra check tied to **this test case only** — e.g. seed byte-identical replay, accuracy/CLIP threshold, perf ceiling, model-specific optional field | Add or reuse `assert_*` in `assertions.py` (never inline in test) | `send_*_request` **then** `assert_<special>(...)` **once** for that case |
+
+Changing `request_config` fields (`size`, `n`, `seed`, server flags) is **not** special validation — the general `assert_*` should read those from `request_config` inside `send_*`.
+
+**`send_*` ↔ `assert_*` pairing (online `OpenAIClientHandler`):**
+
+| High-level `send_*` (prefer in L2+ e2e) | Assert already invoked inside `runtime.py` | Low-level HTTP-only sibling |
+|----------------------------------------|--------------------------------------------|-----------------------------|
+| `send_omni_request` | `assert_omni_response` | `send_chat_completions_http_request` → `assert_http_error` only |
+| `send_diffusion_request` | `assert_diffusion_response` | — |
+| `send_audio_speech_request` | `assert_audio_speech_response` | `send_audio_speech_http_request` → `assert_http_error` only |
+| `send_video_diffusion_request` | `assert_diffusion_response` | `send_videos_*_http_request` → `assert_http_error` only |
+| `send_images_generations_request` *(add when needed)* | **`assert_images_generations_response`** *(general — inside send)* | `send_images_generations_http_request` → `assert_http_error` only |
+| `send_images_edits_request` *(add when needed)* | **`assert_images_edits_response`** *(general — inside send)* | `send_images_edits_http_request` → `assert_http_error` only |
+
+**Rule:** Tests call high-level `send_*_request` for the general contract. **Never** call the same bundled `assert_*` again. Call an extra `assert_*` in the test **only** for special, case-specific validation.
+
+**Common `OpenAIClientHandler` entry points** (non-exhaustive — grep `runtime.py` before adding):
+
+| Area | High-level (SDK + assert) | Low-level HTTP (`*_http_request`) |
+|------|---------------------------|-----------------------------------|
+| Omni chat | `send_omni_request` | `send_chat_completions_http_request` |
+| Diffusion T2I (chat route) | `send_diffusion_request` | — |
+| Diffusion X2V | `send_video_diffusion_request` | `send_videos_create_http_request`, `send_video_content_http_request`, … |
+| DALL-E T2I / edit | `send_images_generations_request`, `send_images_edits_request` | `send_images_generations_http_request`, `send_images_edits_http_request` |
+| TTS (online) | `send_audio_speech_request` | `send_audio_speech_http_request`, `send_audio_generate_http_request`, … |
+| Ops / meta | — | `send_health_http_request`, `send_models_http_request`, `send_omni_sleep_http_request`, … |
+
+**Common `OmniRunnerHandler` entry points** (offline — grep `runtime.py` before adding):
+
+| Area | High-level (`send_*_request` + assert) | Notes |
+|------|----------------------------------------|-------|
+| Omni multimodal | `send_omni_request` | `generate_multimodal` path |
+| Diffusion offline | `send_diffusion_request` | chat-route / `OmniTextPrompt` offline |
+| Qwen-style TTS | `send_audio_speech_request` | two-stage, `generate_multimodal` + `mm_processor_kwargs` |
+| Single-stage TTS (Coqui XTTS, MOSS-TTS-Nano, …) | `send_single_stage_tts_request`, `send_single_stage_tts_batch_request` | `prompt` + `additional_information` + `omni.generate` — **do not** duplicate in tests |
+| New model family | **Add** `send_<family>_request` here first | Then call from `tests/e2e/offline_inference/test_*.py` |
+
+L1 tests under `tests/entrypoints/` may use **FastAPI `TestClient`** or direct handler calls with mocks; they do **not** need `OpenAIClientHandler` / `OmniRunnerHandler`. **L2+ online and offline e2e** must use `runtime.py` `send_*` helpers.
+
+#### Invalid parameter validation (`tests/dfx/reliability/invalid_param_test/`)
+
+When the user asks for **invalid parameter validation**, **invalid request bodies**, **HTTP 4xx contract tests**, or any case that sends **malformed / out-of-range / mismatched** API payloads against a **live** server:
+
+1. **Do not** place these in `tests/e2e/online_serving/` or `*_expansion.py`. If already drafted there, **move** the `test_*` into the correct `invalid_param_test` script and delete the duplicate from e2e.
+2. **Pick the script by HTTP route** (extend an existing file; add a new `test_invalid_<area>.py` only when no in-tree script covers that route family):
+
+| API / area | Script |
+|------------|--------|
+| Omni chat completions, WebSocket video/realtime paths | `test_invalid_omni_chat.py` |
+| `POST /v1/audio/speech`, stream, batch, voices | `test_invalid_audio_speech.py` |
+| Audio diffusion endpoints | `test_invalid_audio_diffusion.py` |
+| `POST /v1/images/generations` | `test_invalid_image_generation.py` |
+| `POST /v1/images/edits` | `test_invalid_image_editing.py` |
+| `POST/GET/DELETE /v1/videos*` | `test_invalid_video_generation.py` |
+| Sleep / wakeup / server control | `test_invalid_server_control.py` |
+
+3. **Match in-tree style** in the chosen script:
+
+| Element | Convention |
+|---------|------------|
+| **Module markers** | `pytestmark = [pytest.mark.slow, pytest.mark.<omni or tts or diffusion>]` |
+| **Server fixture** | `_PARAMS` / `_QWEN3_TTS_SPEECH`-style list of `pytest.param(OmniServerParams(...), id="...", marks=hardware_marks(...))`; `@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)` |
+| **Hardware** | `hardware_marks(res={"cuda": "H100"})` for heavy diffusion/omni/video; `hardware_marks(res={"cuda": "L4"})` for smaller TTS models (must match weekly `-m "slow and L4"` step) |
+| **HTTP client** | **Low-level** `openai_client.send_*_http_request({..., "err_code": 400, "err_message": (...)} )` — **not** `send_*_request` (success path) |
+| **Case shape** | Prefer **one `test_*` per route family** + `@pytest.mark.parametrize("body_spec, err_message", [...])` with stable `id=` per case; or dedicated `test_<route>_malformed_json` when not parametrized |
+| **Body helpers** | `_minimal_<endpoint>_json(omni_server)` / `_minimal_*_form_data()`; merge overrides with `body.update(body_spec)` |
+| **Known gaps** | `pytest.mark.skip(reason="…#3649")` as `_SKIP_ISSUE_3649` when server validation is not yet strict (mirror neighboring cases) |
+| **Sections** | Route banner comments (`# ─── POST /v1/images/generations ───`) like existing files |
+| **Shared fixtures** | Reuse `tests/dfx/reliability/invalid_param_test/conftest.py` (`tiny_png_bytes`, env defaults) |
+
+4. **Adding a new model** to invalid-param coverage: append a `pytest.param(OmniServerParams(model="...", stage_config_path=..., server_args=...), ...)` entry to the script’s `_PARAMS` list — do **not** create `test_invalid_<model>.py` unless the route is new.
+
+5. **CI — `.buildkite/test-weekly.yml` only** (not `test-ready.yml` / `test-merge.yml` / `test-nightly.yml`):
+
+| Weekly step | Command | When your cases run |
+|-------------|---------|---------------------|
+| **Invalid parameters Test · H100** | `pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"` | Diffusion / omni / video invalid-param tests with `H100` hardware mark |
+| **Invalid parameters Test · L4** | `pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"` | TTS / lighter models marked `L4` |
+
+- **Trigger:** `build.env("WEEKLY") == "1"` or PR label `weekly-test`.
+- **Default:** extending an existing `invalid_param_test` script needs **no YAML edit** — the weekly steps already sweep the whole directory.
+- **Edit YAML only** when adding a **new hardware queue**, a **new top-level script** that must run in isolation, or a **model-specific weekly shard** (mirror neighboring reliability steps).
+- Weekly steps **do not** use `source_file_dependencies`.
+
+**Example — append to `test_invalid_image_generation.py`:**
+
+```python
+@pytest.mark.parametrize(
+    "body_spec, err_message",
+    [
+        pytest.param({"seed": -1}, ("seed", "greater_than_equal", "0"), id="seed_negative"),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
+def test_images_generations_invalid_requests(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    body_spec: dict[str, object],
+    err_message: str | tuple[str, ...],
+) -> None:
+    body = _minimal_images_gen_json(omni_server)
+    body.update(body_spec)
+    openai_client.send_images_generations_http_request(
+        {"json": body, "timeout": 300, "err_code": 400, "err_message": err_message}
+    )
+```
+
+See [references/test-routing.md](references/test-routing.md) **Invalid parameter / weekly CI**.
+
+#### Assertion helpers (`tests/helpers/assertions.py`)
+
+**Do not** add module-local helpers such as `_assert_images_generations_payload` or `_send_and_assert_*` in e2e test files. Response/media validation belongs in **`tests/helpers/assertions.py`**, grouped by **category**:
+
+| Category | Existing anchors | When to extend vs add |
+|----------|------------------|------------------------|
+| **Image** (chat diffusion + DALL-E JSON) | `assert_image_diffusion_response`, `assert_image_valid` | DALL-E `/v1/images/generations` JSON → add **`assert_images_generations_response`** beside image helpers (reuses `assert_image_valid`); do not fork decode logic into tests |
+| **Video** | `assert_video_diffusion_response`, `assert_video_valid` | Extend these for new video contracts |
+| **Audio** | `assert_audio_diffusion_response`, `assert_audio_speech_response`, `assert_audio_valid` | Extend for new TTS/audio endpoints |
+| **Omni multimodal** | `assert_omni_response` | Extend for new modality combos |
+| **HTTP errors** | `assert_http_error`, `assert_err_message_in_text` | Used inside low-level `send_*_http_request` and negative tests |
+
+| Do | Don't |
+|----|-------|
+| Put **general** `assert_*` inside `send_*_request` in `runtime.py`; tests call `send_*_request` only | Call `assert_diffusion_response` after `send_diffusion_request` (general assert already bundled) |
+| Put **special** `assert_*` in `assertions.py` and call it **in the test** after `send_*_request` when that case needs extra checks | Put general decode/count/size logic in the test because “this case uses `n=4`” (that belongs in general assert reading `request_config`) |
+| Extend the matching category function, or add `assert_<endpoint>_response` next to its category | `_send_and_assert_*` or PIL/base64 loops in `test_*.py` |
+| Use low-level `send_*_http_request` in dfx/negative tests (`err_code`) | Use low-level HTTP send + manual general assert in every L2+ expansion file |
+
+**Workflow when generating tests:**
+
+1. Implement or reuse **general** `assert_*` in `assertions.py` (by category).
+2. Wire it into **`send_*_request`** in `runtime.py` if not already bundled.
+3. In the test: `request_config` → `send_*_request`. Add a separate `assert_*` import/call **only** for special case validation.
+4. Update `source_file_dependencies` when `runtime.py` or `assertions.py` changes.
+
+**Example — DALL-E `/v1/images/generations`:**
+
+```python
+# tests/helpers/assertions.py — general contract (image category)
+def assert_images_generations_response(resp_body: dict, request_config: dict, *, run_level: str | None = None) -> None:
+    ...  # data[], n, size → assert_image_valid
+
+# tests/helpers/runtime.py — general assert INSIDE send
+def send_images_generations_request(self, request_config: dict[str, Any], ...) -> list[HttpResponse]:
+    responses = self.send_images_generations_http_request(request_config, ...)
+    if request_config.get("err_code") is None:
+        body = responses[0].json_body
+        assert isinstance(body, dict)
+        assert_images_generations_response(body, request_config, run_level=self.run_level)
+    return responses
+
+# tests/e2e/.../test_foo_expansion.py — one case per test_* (no case_id branching)
+@pytest.mark.parametrize("omni_server", [pytest.param(OmniServerParams(model=MODEL), marks=SINGLE_L4)], indirect=True)
+def test_foo_images_generations_default_1024(omni_server, openai_client) -> None:
+    openai_client.send_images_generations_request({"json": body, "timeout": 300})
+
+@pytest.mark.parametrize("omni_server", [pytest.param(OmniServerParams(model=MODEL), marks=SINGLE_L4)], indirect=True)
+def test_foo_images_generations_sizes_256_512_1024(omni_server, openai_client) -> None:
+    for size in ("256x256", "512x512", "1024x1024"):
+        openai_client.send_images_generations_request({"json": {**body, "size": size}, "timeout": 300})
+
+# chat diffusion — separate test; send_diffusion_request bundles assert_diffusion_response
+@pytest.mark.parametrize("omni_server", [pytest.param(OmniServerParams(model=MODEL), marks=SINGLE_L4)], indirect=True)
+def test_foo_chat_completions_t2i_fallback(omni_server, openai_client) -> None:
+    openai_client.send_diffusion_request({...})
+```
+
+L1 tests under `tests/entrypoints/` may keep **minimal** asserts next to the handler under test when validating a single branch; still prefer `assertions.py` when the same JSON/media check appears in more than one file or level (e.g. L1 protocol test + L4 e2e).
+
+#### Omni Test Writing Guidance (L1-L4 Layering)
+
+When the goal is a general Omni/multimodal test case, prioritize mapping the test to the correct purpose, directory, and resource assumptions aligned with the layers (see [CI_5levels.md](../../../docs/contributing/ci/CI_5levels.md)):
+
+- **L1**: Unit/logic validation on CPU (`core_model and cpu`). Cover input validation, branches, and exception paths (`tests/<component>/test_*.py`). **Mock with `mocker` / `monkeypatch` only — not `unittest.mock`.**
+- **L2**: Basic e2e (online/offline basic scenarios). Prefer dummy/lightweight models to validate the end-to-end request-to-output-structure/streaming chain (typically `tests/e2e/online_serving/` and `tests/e2e/offline_inference/`).
+- **L3/L4**: Important integration, performance, and accuracy validation. L4 emphasizes “full functional scenarios + performance/stress + runnable doc examples” (typically `*_expansion.py` plus related expansion cases).
+
+Also keep markers consistent with the run level: use `core_model` for L1/L2 and `advanced_model` for L3/L4, and pair with `--run-level` to select the intended CI strategy.
+
+#### Diffusion Test Writing Guidance (L4 Coverage Combinations)
+
+When the task involves diffusion models/features, organize L4 test cases following [`#1832`](https://github.com/vllm-project/vllm-omni/issues/1832): combine multiple diffusion features into as few test cases as possible to fit limited CI GPU resources.
+
+Implementation strategy:
+
+1. If “full L4 is too heavy”: first provide a **reduced local validation case in L1/L2** (so the key assertion and the contract fix point are covered deterministically).
+2. Then provide **CI/nightly-ready L3/L4 high-marked cases** (e.g. `advanced_model`) to broaden coverage under resource constraints.
+
+#### L4 nightly sub-pillars — Function vs Accuracy vs Perf
+
+**L4 is not only `test_*_expansion.py`.** In `test-nightly.yml`, each model-type group typically runs **separate jobs**:
+
+| Sub-pillar | Nightly step label pattern | What you add | Typical paths |
+|------------|----------------------------|--------------|---------------|
+| **Function** | `· Function Test with …` | E2e expansion / feature matrix | `tests/e2e/online_serving/test_<model>_expansion.py` (or offline) |
+| **Accuracy** | `· Accuracy Test` | Quality / similarity vs baseline | `tests/e2e/accuracy/test_<model>*.py` |
+| **Perf** | `· Perf Test · <Model>` | Throughput / latency / memory benchmarks | `tests/dfx/perf/tests/test_<model>_vllm_omni.json` + runner script |
+| **Doc** (optional) | `· Doc Test` | Runnable doc examples | `tests/examples/*/test_text_to_image.py`, … |
+
+**Default when the user asks for “L4 functional cases”:** deliver **Function** pillar only (`*_expansion.py` + Function Test shard in `test-nightly.yml`). **Do not** silently add Perf or Accuracy unless the user also asks for **performance** / **benchmark** / **accuracy** / **full L4** / **full L4 coverage**.
+
+**When the user explicitly asks for L4 perf (or full L4 including perf):**
+
+1. **Do not** put throughput/latency assertions inside `test_*_expansion.py` — perf uses the **dfx benchmark harness**, not `omni_server` e2e fixtures.
+2. Add or extend a **JSON config** under `tests/dfx/perf/tests/` (mirror in-tree names: `test_qwen_image_vllm_omni.json`, `test_bagel_vllm_omni.json`).
+3. Each JSON entry: `test_name`, `server_params` (`model`, `serve_args`), `benchmark_params[]` with explicit **`baseline`** (`throughput_qps`, `latency_mean`, `peak_memory_mb_mean`, …).
+4. Run locally via the matching script (model type → runner):
+
+| Model type | Runner script | Example config |
+|------------|---------------|----------------|
+| **Diffusion X2I/X2V** | `tests/dfx/perf/scripts/run_diffusion_benchmark.py` | `tests/dfx/perf/tests/test_qwen_image_vllm_omni.json` |
+| **TTS** | `tests/dfx/perf/scripts/run_benchmark.py` | `tests/dfx/perf/tests/test_tts.json` (shared) or `test_{slug}.json` (dedicated; use when the model must not join the shared nightly matrix before integration — e.g. `test_voxcpm2.json`, `test_coqui_tts.json`) |
+| **Omni** | (see existing Omni Perf steps — `run_benchmark.py` / dedicated omni configs in-tree) | `tests/dfx/perf/tests/test_omni*.json` |
+
+5. Wire **`test-nightly.yml` · Perf Test** step for that model (separate from Function Test): export `DIFFUSION_BENCHMARK_DIR` / `BENCHMARK_DIR`, run pytest on the script with `--test-config-file`, **upload artifacts** (`buildkite-agent artifact upload`), often **multi-GPU H100** for diffusion perf.
+6. **One benchmark scenario → one `test_name` block** in JSON (same spirit as one `test_*` per function case). Combine server `serve_args` + workload in one entry; use multiple `benchmark_params` rows for size/step sweeps under the same server config.
+
+**Perf local command (diffusion example):**
+
+```bash
+cd tests
+export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py \
+  --test-config-file dfx/perf/tests/test_<model>_vllm_omni.json
+```
+
+**Clarify in the test plan** which L4 pillars you deliver: `Function only` | `Function + Perf` | `Function + Accuracy + Perf`.
+
+See [references/test-routing.md](references/test-routing.md) **L4 nightly pillars** for Buildkite step patterns.
+
+### Step 5: Wire Buildkite (when CI must run the new test)
+
+If the test is not already collected by an existing pipeline command (for example, L1 tests marked `core_model and cpu` are already covered by the **Simple Unit Test** step in ready/merge), update the appropriate pipeline under `.buildkite/`:
+
+| Test level | Edit this file | Typical trigger / intent |
+|------------|----------------|---------------------------|
+| **L1** and **L2** | [`.buildkite/test-ready.yml`](../../../.buildkite/test-ready.yml) | PR **ready** label; L1 CPU + L2 GPU/basic e2e (steps labeled **Omni ·**, **TTS ·**, **Diffusion ·**) |
+| **L3** | [`.buildkite/test-merge.yml`](../../../.buildkite/test-merge.yml) | Post-merge; `advanced_model` integration per model type |
+| **L4** | [`.buildkite/test-nightly.yml`](../../../.buildkite/test-nightly.yml) | Nightly; grouped by model type (see below) |
+| **Invalid param / reliability (weekly)** | [`.buildkite/test-weekly.yml`](../../../.buildkite/test-weekly.yml) | Weekly; `tests/dfx/reliability/invalid_param_test/` — **not** L1–L4 e2e pipelines |
+
+**Level-specific Buildkite delivery (follow the user’s requested level):**
+
+| User asks for | Document / edit **only** | Do **not** add by default |
+|---------------|---------------------------|---------------------------|
+| **L4** (`*_expansion.py`, `full_model`) | `test-nightly.yml` — append file to the matching nightly shard (X2I / X2V / Omni / TTS) **or** note it is already collected by an existing `-m` / `-k` sweep | `test-merge.yml` (L3 / `advanced_model`) or `test-ready.yml` (L2) |
+| **L3** | `test-merge.yml` with `advanced_model` + `source_file_dependencies` | `test-nightly.yml` unless user also wants nightly |
+| **L2** | `test-ready.yml` with `core_model` + `source_file_dependencies` | merge / nightly pipelines |
+| **Invalid param** | `test-weekly.yml` — **Invalid parameters Test** group (`-m "slow and H100"` / `-m "slow and L4"`). Usually **no YAML edit** when appending cases to existing `invalid_param_test` scripts | `test-ready.yml`, `test-merge.yml`, `test-nightly.yml` |
+
+`test-nightly.yml` shards use **explicit pytest file paths** in `commands` (and PR labels like `diffusion-x2iat-test`); they generally **do not** use `source_file_dependencies`. Only suggest merge/ready YAML when the requested level is L3/L2 (or the user explicitly asks for multi-level CI).
+
+**`test-nightly.yml` top-level groups** (each uses `full_model` + type marker in pytest `-m`):
+
+| Group | Type marker | Notes |
+|-------|-------------|--------|
+| **Omni Model Test** | `omni` | Function / doc / accuracy / **perf** for Qwen-Omni family |
+| **TTS Model Test** | `tts` | `-m "full_model and L4 and tts"` (and H100 variants when enabled); **Perf Test** uses `run_benchmark.py` + `test_tts.json` |
+| **Diffusion X2I(&A&T) Model Test** | `diffusion` | Image/audio/text diffusion; explicit file shards for **Function**; **Perf Test · &lt;Model&gt;** uses `run_diffusion_benchmark.py` + JSON config |
+| **Diffusion X2V Model Test** | `diffusion` | Video-only (Wan, HunyuanVideo, …); PR label `diffusion-x2v-test`; separate **Perf** steps when in-tree |
+
+Guidelines when editing YAML:
+
+- **Match markers and `--run-level`** to the test: L1/L2 in `test-ready.yml` use `core_model` + `omni` / `tts` / `diffusion` as appropriate; L3 merge uses `advanced_model`; L4 nightly uses `full_model` with the same type marker.
+- **Diffusion L4**: add x2i/x2a/x2t expansion files to an **X2I(&A&T)** step; add x2v expansion files to **X2V** — never rely on a broad `tests/e2e/` sweep alone (shards exist to save GPU and avoid wrong queue).
+- **`source_file_dependencies` (required for new E2E jobs in `test-ready.yml` and `test-merge.yml`)**: every step under the **`:card_index_dividers: E2E Test`** group **must** declare `source_file_dependencies` so Buildkite only schedules the job when relevant paths change. Omitting it breaks path-based CI skipping for GPU e2e jobs.
+- **Reuse or extend an existing step** when the new test shares the same marker expression, queue, and timeout; otherwise add a new `steps:` entry with the correct `agents.queue`, `timeout_in_minutes`, and docker/kubernetes plugin blocks consistent with neighboring jobs.
+- **Job timeout — use `timeout_in_minutes` on the step, not `timeout` before pytest**: when adding or documenting a new E2E job in `test-ready.yml` / `test-merge.yml`, set `timeout_in_minutes` on the step and run `pytest` directly in `commands`. **Do not** wrap pytest in `timeout 40m bash -c "..."` / `timeout 20m bash -c "..."`; Buildkite already enforces the step deadline via `timeout_in_minutes`.
+- **Never omit `plugins` in skill examples or PR YAML snippets**: H100 E2E steps on `mithril-h100-pool` use the full `kubernetes` block (`resources`, `volumeMounts`, `env` with `HF_TOKEN` secret, `nodeSelector: gpu-h100-sxm`, `devshm` + `hf-cache` volumes). L4/docker steps on `gpu_1_queue` use the full `docker#v5.2.0` block (`shm-size`, `HF_HOME`, volumes). Do not write `# ... plugins unchanged` — paste the complete block from a sibling step.
+- **Platform forks** (e.g. AMD-ready / AMD-merge) live alongside these files; apply the same level → file mapping for those pipelines when the test is platform-specific.
+
+#### `source_file_dependencies` for E2E Test jobs (`test-ready.yml`, `test-merge.yml`)
+
+When adding or splitting a step inside the **E2E Test** group, **always** include `source_file_dependencies` on that step. List paths the job actually depends on — at minimum:
+
+1. **Test module(s)** — every `tests/e2e/...` file the `pytest` command runs (online and offline if both are in one step).
+2. **Model implementation** — `vllm_omni/model_executor/models/<family>/` and/or `vllm_omni/diffusion/models/<family>/` as applicable.
+3. **Stage / input plumbing** (omni & multi-stage TTS) — `vllm_omni/model_executor/stage_input_processors/<family>.py` when present.
+4. **Deploy / stage YAML** — `vllm_omni/deploy/<config>.yaml` or `vllm_omni/deploy/ci/<config>.yaml` referenced by the test’s `OmniServerParams` / `get_deploy_config_path(...)`.
+5. **Test helpers** (when changed in the same PR) — `tests/helpers/runtime.py` for new `send_*` client methods; `tests/helpers/assertions.py` for new shared `assert_*` helpers.
+
+Use **repo-relative paths** (no leading `./`). Prefer **directory** entries (`.../models/bagel/`) when the whole package matters; use **file** entries for single test modules and YAML configs. Mirror a neighboring step for the same model family (e.g. **Diffusion · Bagel Test** in `test-ready.yml`).
+
+**Example** (L2 ready — diffusion online smoke, H100 kubernetes):
+
+```yaml
+- label: "Diffusion · Bagel Test"
+  source_file_dependencies:
+    - tests/e2e/online_serving/test_bagel.py
+    - vllm_omni/model_executor/models/bagel/
+    - vllm_omni/diffusion/models/bagel/
+    - vllm_omni/model_executor/stage_input_processors/bagel.py
+    - vllm_omni/deploy/bagel.yaml
+  timeout_in_minutes: 40
+  commands:
+    - |
+      export VLLM_IMAGE_FETCH_TIMEOUT=60
+      pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model'
+  agents:
+    queue: "mithril-h100-pool"
+  plugins:
+    - kubernetes:
+        podSpec:
+          containers:
+            - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              resources:
+                limits:
+                  nvidia.com/gpu: 1
+              volumeMounts:
+                - name: devshm
+                  mountPath: /dev/shm
+                - name: hf-cache
+                  mountPath: /root/.cache/huggingface
+              env:
+                - name: HF_HOME
+                  value: /root/.cache/huggingface
+                - name: HF_TOKEN
+                  valueFrom:
+                    secretKeyRef:
+                      name: hf-token-secret
+                      key: token
+          nodeSelector:
+            node.kubernetes.io/instance-type: gpu-h100-sxm
+          volumes:
+            - name: devshm
+              emptyDir:
+                medium: Memory
+            - name: hf-cache
+              hostPath:
+                path: /mnt/hf-cache
+                type: DirectoryOrCreate
+```
+
+**Example** (L3 merge — TTS online + offline, L4 docker):
+
+```yaml
+- label: "TTS · Qwen3-TTS Base Test"
+  source_file_dependencies:
+    - tests/e2e/online_serving/test_qwen3_tts_base.py
+    - tests/e2e/offline_inference/test_qwen3_tts_base.py
+    - vllm_omni/model_executor/models/qwen3_tts/
+    - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+    - vllm_omni/deploy/qwen3_tts.yaml
+  timeout_in_minutes: 20
+  commands:
+    - |
+      export VLLM_LOGGING_LEVEL=DEBUG
+      export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
+      pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+  agents:
+    queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
+  plugins:
+    - docker#v5.2.0:
+        image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+        always-pull: true
+        propagate-environment: true
+        shm-size: "8gb"
+        environment:
+          - "HF_HOME=/fsx/hf_cache"
+        volumes:
+          - "/fsx/hf_cache:/fsx/hf_cache"
+```
+
+**Example** (L3 merge — diffusion online + offline, H100 kubernetes):
+
+```yaml
+- label: "Diffusion · Qwen Image Test"
+  source_file_dependencies:
+    - tests/e2e/online_serving/test_qwen_image.py
+    - tests/e2e/offline_inference/test_qwen_image.py
+    - vllm_omni/diffusion/models/qwen_image/
+  timeout_in_minutes: 40
+  commands:
+    - |
+      export VLLM_IMAGE_FETCH_TIMEOUT=60
+      pytest -s -v \
+        tests/e2e/online_serving/test_qwen_image.py \
+        tests/e2e/offline_inference/test_qwen_image.py \
+        -m 'advanced_model and cuda' --run-level 'advanced_model'
+  agents:
+    queue: "mithril-h100-pool"
+  plugins:
+    - kubernetes:
+        podSpec:
+          containers:
+            - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              resources:
+                limits:
+                  nvidia.com/gpu: 1
+              volumeMounts:
+                - name: devshm
+                  mountPath: /dev/shm
+                - name: hf-cache
+                  mountPath: /root/.cache/huggingface
+              env:
+                - name: HF_HOME
+                  value: /root/.cache/huggingface
+                - name: HF_TOKEN
+                  valueFrom:
+                    secretKeyRef:
+                      name: hf-token-secret
+                      key: token
+          nodeSelector:
+            node.kubernetes.io/instance-type: gpu-h100-sxm
+          volumes:
+            - name: devshm
+              emptyDir:
+                medium: Memory
+            - name: hf-cache
+              hostPath:
+                path: /mnt/hf-cache
+                type: DirectoryOrCreate
+```
+
+When extending an **existing** E2E step to run an additional test file, **append** that file (and any new model/deploy paths) to `source_file_dependencies` on the same step. When authoring a PR, mention the dependency list in the **Buildkite change** section of your test plan.
+
+#### L3 / L4: validating Buildkite from a feature branch (root `pipeline.yml`)
+
+Production behavior is driven by [`.buildkite/pipeline.yml`](../../../.buildkite/pipeline.yml): it builds the CI image, then uploads **L2** (`test-ready.yml` on non-`main`), **L3** (`test-merge.yml` when `build.branch == "main"`), and **L4** (`test-nightly.yml` when `build.env("NIGHTLY") == "1"`).
+
+When you add or debug **L3** or **L4** tests and need those child pipelines to run **off `main`** or **without** `NIGHTLY=1`, apply **temporary** edits (revert before merge):
+
+1. **Point the upload at the right file (optional shortcut)**  
+   In the **Upload Ready Pipeline** step, the command is `buildkite-agent pipeline upload .buildkite/test-ready.yml`. For a one-off run you may change that path to `.buildkite/test-merge.yml` (L3) or `.buildkite/test-nightly.yml` (L4), and adjust the step’s `if:` so it does not conflict with the other upload steps (avoid double-uploading unless intentional).
+
+2. **L3 — merge pipeline**  
+   The gate `if: build.branch == "main"` lives on the **Upload Merge Pipeline** step in `pipeline.yml` (not inside `test-merge.yml`). **Comment out that `if` line** so `test-merge.yml` is uploaded on your feature branch. Alternatively rely on step (1) instead of the dedicated merge upload step.
+
+3. **L4 — nightly pipeline**  
+   **Comment out** `if: build.env("NIGHTLY") == "1"` on the **Upload Nightly Pipeline** step in `pipeline.yml` so the nightly definition is uploaded without setting the env var.  
+   Child steps in [`test-nightly.yml`](../../../.buildkite/test-nightly.yml) often repeat `if: build.env("NIGHTLY") == "1"`; **comment out those lines on the steps you need to run**, otherwise they will still be skipped after upload.
+
+### Step 6: Run Tests
+
+Pick one command path:
+
+- **Quick local regression (preferred first)**: single file or `file.py::test_name`
+- **CI-like level run**: marker expression + `--run-level`
+
+Full templates live in [references/test-routing.md](references/test-routing.md). **After authoring tests, always emit concrete commands** (see **Output Format**).
+
+**Typical copy-paste examples** (run from repo `tests/` directory; requires matching vLLM/vllm-omni install and hardware):
+
+| Area | Example |
+|------|---------|
+| **Omni offline L2** | `pytest -s -v e2e/offline_inference/test_qwen2_5_omni.py -m "core_model and omni and not cpu" --run-level=core_model` |
+| **Omni online L2** | `pytest -s -v e2e/online_serving/test_qwen3_omni.py -m "core_model and omni" --run-level=core_model` |
+| **TTS online L2** | `pytest -s -v e2e/online_serving/test_qwen3_tts_base.py -m "core_model and tts" --run-level=core_model` |
+| **Diffusion X2I L2 online** | `pytest -s -v e2e/online_serving/test_qwen_image.py -m "core_model and diffusion" --run-level=core_model` |
+| **Diffusion X2V L2 online** | `pytest -s -v e2e/online_serving/test_wan22_t2v.py -m "core_model and diffusion" --run-level=core_model` |
+| **Diffusion X2I L4 nightly** | `pytest -s -v e2e/online_serving/test_qwen_image_expansion.py -m "full_model and diffusion and H100" --run-level=full_model` |
+| **Diffusion X2I L4 perf (nightly)** | `pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file dfx/perf/tests/test_qwen_image_vllm_omni.json` |
+| **Diffusion X2V L4 nightly** | `pytest -s -v e2e/online_serving/test_wan22_expansion.py -m "full_model and cuda" --run-level=full_model` |
+| **Invalid param (weekly H100)** | `pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and H100"` |
+| **Invalid param (weekly L4)** | `pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and L4"` |
+| **L1 CPU** | `pytest -s -v -m "core_model and cpu"` |
+
+**Prerequisites to mention when relevant**: GPU model (e.g. L4 vs H100), `HF_HOME` / token for hub weights, module-level `skipif` (NPU/XPU-only gaps), and whether CI already collects the path (e.g. `test_*_expansion.py` glob in `test-nightly.yml`).
+
+### Step 7: Validate Result Quality
+
+Before finishing:
+
+- Is the new assertion directly tied to the bug/feature contract?
+- Are API calls made via **`runtime.py` `send_*_request`** (not inline HTTP/SDK in the test file)?
+- Is the **general** `assert_*` bundled inside `send_*_request` (not duplicated in the test)?
+- Is each scenario a **separate `test_*` function** (no `case_id` / `if param.id` branching across cases)?
+- **Fixture scope**: default **`omni_server` / `omni_runner`** (module); **`_function`** variants only when each test must restart the instance?
+- Is the test deterministic (no fragile timing/network coupling)?
+- Is runtime appropriate for the selected level?
+- Are markers and `--run-level` consistent?
+- **Invalid-param cases** in `tests/dfx/reliability/invalid_param_test/` (not e2e), using `send_*_http_request` + `err_code`, with `pytest.mark.slow` and correct `H100` / `L4` hardware mark?
+
+## Output Format
+
+When completing a request, return:
+
+1. **Test plan** (level, markers, file target, **L4 pillar(s)** if applicable: Function / Accuracy / Perf / Doc; **or** **Invalid param** pillar with target `invalid_param_test/test_invalid_<area>.py`; module basename for e2e: `test_{slug}.py` / `test_{slug}_expansion.py`)
+2. **Generated/updated test file(s)** — model-specific constants and request payloads stay **inside** those test modules (**never** `tests/helpers/{slug}.py`). Extend **`tests/helpers/runtime.py`** / **`tests/helpers/assertions.py`** only when new **repo-wide** `send_*` or shared assert helpers are required (do not leave ad-hoc HTTP or `_assert_*` in test modules). **If L4 Perf was requested**, also list **`tests/dfx/perf/tests/*.json`**. **If invalid-param cases were requested**, list the **`invalid_param_test/` script** and parametrized `test_*` / new `id=` rows — **not** e2e paths.
+3. **Buildkite change** — **match the requested level and pillar**:
+   - **L4 Function**: `test-nightly.yml` **Function Test** shard (file list / note existing sweep). **No** `test-merge.yml` unless user also asked for L3.
+   - **L4 Perf**: `test-nightly.yml` **Perf Test · &lt;Model&gt;** step (new step or extend commands); include env exports + artifact upload pattern from a sibling perf job.
+   - **L4 Accuracy**: `test-nightly.yml` **Accuracy Test** step under the same model-type group.
+   - **L3**: `test-merge.yml` + full `source_file_dependencies` + `agents` + `plugins`.
+   - **L2**: `test-ready.yml` + same E2E block requirements.
+   - **Invalid param**: `test-weekly.yml` — note **Invalid parameters Test · H100/L4** group; usually **no YAML change** when only extending existing scripts.
+   If the user asked only for **L4 functional cases**, state explicitly that **Perf / Accuracy / Invalid param** were not included (offer to add if needed). If the plan mixes **success e2e** and **invalid param**, split deliverables across e2e vs `invalid_param_test/` and call out both CI files.
+4. **Run commands (required)** — always include, in fenced `bash` blocks:
+   - **Local — whole file**: `cd tests` then `pytest -s -v <path> …`
+   - **Local — single test** (optional but preferred when the change is one function): `pytest -s -v path::test_func …`
+   - **CI-like** (when not L1 CPU): the same **marker + `--run-level`** pairing the level uses (see [references/test-routing.md](references/test-routing.md) and **Step 6** table above)
+   - **Prerequisites** (one line): e.g. “Requires CUDA L4 + weights cached locally”; “Requires H100 + `HF_TOKEN`”; “Syntax-only validation when vllm is not installed locally”
+5. **Result summary** (pass/fail if you executed tests; if not executed, state that explicitly and what the user should run)
+
+## Additional Resources
+
+- Marker and command routing: [references/test-routing.md](references/test-routing.md)
+- CI pipelines (vllm-omni): [test-ready.yml](../../../.buildkite/test-ready.yml) (L1/L2), [test-merge.yml](../../../.buildkite/test-merge.yml) (L3), [test-nightly.yml](../../../.buildkite/test-nightly.yml) (L4), [test-weekly.yml](../../../.buildkite/test-weekly.yml) (invalid param / reliability)
+- L4 documentation example tests (naming, extraction vs copied scripts, output dirs, skips): [docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md) — see also [PR #1910](https://github.com/vllm-project/vllm-omni/pull/1910)

.claude/skills/vllm-omni-test/references/test-routing.md

@@ -0,0 +1,372 @@
+# Test Routing Reference
+
+Use this reference to map testing goals to levels, markers, and runnable commands.
+
+**Repo paths** (`.buildkite/`, `docs/contributing/…`): link with repo-relative paths from this file (e.g. `../../../../.buildkite/test-ready.yml`, `../../../../docs/contributing/ci/CI_5levels.md`).
+
+## Model-centric e2e filename convention (L2–L4)
+
+When generating a **new** test module tied to a **specific model** under `tests/e2e/offline_inference/` or `tests/e2e/online_serving/`:
+
+| Level | Pattern | Example (`Qwen/Qwen2.5-Omni-7B`) |
+|-------|---------|----------------------------------|
+| **L2**, **L3** | `test_{lowercase_model_slug}.py` | `test_qwen2_5_omni.py` |
+| **L4** | `test_{lowercase_model_slug}_expansion.py` | `test_qwen2_5_omni_expansion.py` |
+
+**Slug rules for `{lowercase_model_slug}`** (canonical: [SKILL.md](../SKILL.md) § *Naming: generated test module files*):
+
+1. Start from the HuggingFace-style id (e.g. `Qwen/Qwen2.5-Omni-7B`), but **do not** put the org into the filename: use the **repo segment only** (`Qwen2.5-Omni-7B`), not `Qwen_...` / `qwen_qwen2_5_...`.
+2. **Lowercase**; replace `.`, `-`, and whitespace with a single `_` (e.g. `Qwen2.5-Omni-7B` → `qwen2_5_omni`). **Omit** trailing size tokens such as `7b` / `30b` in the basename when a single file covers that model line in the directory (matches `test_qwen2_5_omni.py` in-tree).
+3. If two checkpoints in the same folder need separate modules, add a **minimal** disambiguator (e.g. `_7b` vs `_3b`) **only then**.
+4. **L1** unit tests are **not** bound to this pattern; use `tests/<area>/test_<feature>.py` as today.
+
+## Model type markers (`omni` / `tts` / `diffusion`)
+
+Every model-centric e2e test must declare **exactly one** type marker:
+
+| Marker | Model family | Typical APIs |
+|--------|--------------|--------------|
+| `pytest.mark.omni` | Multimodal LLM (Qwen-Omni, …) | `send_omni_request`, `omni_runner`, stage YAML |
+| `pytest.mark.tts` | TTS / speech synthesis | `send_audio_speech_request`, `/v1/audio/speech` |
+| `pytest.mark.diffusion` | Diffusion generative models | `send_diffusion_request`, `send_video_diffusion_request`, `send_images_generations_http_request`, `OmniDiffusionSamplingParams` |
+
+**Diffusion** uses one marker for all output modalities. Nightly CI splits diffusion by **group**, not by extra markers:
+
+| Nightly group (`test-nightly.yml`) | Diffusion scope | Examples |
+|-----------------------------------|-----------------|----------|
+| **Diffusion X2I(&A&T) Model Test** | x2**i** / x2**a** / x2**t** — image, audio, text (non-video) | Qwen-Image*, BAGEL, FLUX, SD3, Z-Image, LongCat, DreamZero |
+| **Diffusion X2V Model Test** | x2**v** — video only | Wan2.2, HunyuanVideo 1.5, Wan VACE |
+
+PR labels for selective nightly runs: `diffusion-x2iat-test`, `diffusion-x2v-test` (plus `nightly-test` / `NIGHTLY=1`).
+
+## Level and Marker Mapping
+
+| Goal | Suggested Level | Marker baseline | Typical location |
+|------|------------------|-----------------|------------------|
+| Unit logic, regression on pure code path | L1 | `core_model and cpu` | `tests/<component>/test_*.py` |
+
+### L1 unit tests — mocking (`mocker`, not `unittest.mock`)
+
+L1 modules run in **Simple Test** (`test-ready.yml` / `test-merge.yml`: `-m 'core_model and cpu'`). When isolation requires doubles:
+
+- **Use** the **`mocker`** fixture (`pytest-mock`): `mocker.patch`, `mocker.spy`, `mocker.Mock`, `mocker.MagicMock`, `mocker.AsyncMock`.
+- **Use** **`monkeypatch`** for env vars and simple `setattr` / `delattr` without mock objects.
+- **Do not use** `unittest.mock` — no `from unittest.mock import ...`, no `@patch`, no `with patch(...)`.
+
+```python
+# Good (L1)
+def test_handler(mocker):
+    mocker.patch("vllm_omni.pkg.fn", return_value=0)
+
+# Bad (L1)
+from unittest.mock import patch
+
+@patch("vllm_omni.pkg.fn")
+def test_handler(mock_fn): ...
+```
+
+E2E (L2+) should not rely on mocks unless documenting a rare exception; prefer real (or lightweight) execution paths.
+| Basic integration/e2e | L2 | `core_model` + **one of** `omni` / `tts` / `diffusion` | `tests/e2e/...` |
+| Advanced integration | L3 | `advanced_model` + type marker | `tests/e2e/...` |
+| Full function / nightly | L4 | `full_model` (nightly) + type marker | **Function:** `tests/e2e/*_expansion.py`; **Perf:** `tests/dfx/perf/tests/*.json`; **Accuracy:** `tests/e2e/accuracy/` |
+| Invalid HTTP / param validation | Weekly (dfx) | `pytest.mark.slow` + type marker + `H100` or `L4` | `tests/dfx/reliability/invalid_param_test/test_invalid_*.py` |
+
+## Marker Selection Rules
+
+1. **Level** (pick one):
+   - `core_model` — L1/L2 (`test-ready.yml`)
+   - `advanced_model` — L3 (`test-merge.yml`)
+   - `full_model` — L4 nightly (`test-nightly.yml`); some expansion tests still carry both `advanced_model` and `full_model` during migration
+2. **Model type** (pick one): `omni`, `tts`, or `diffusion`
+3. **Cross-cutting** when relevant: `parallel`, `cache`, `example`, `benchmark`
+4. **Hardware**: `cpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `distributed_cuda`, …
+5. Multi-card: `@hardware_test(...)` in `tests/helpers/mark.py`
+
+## Command Templates
+
+### Quick local checks
+
+```bash
+cd tests
+pytest -s -v test_xxxx.py
+```
+
+### L1
+
+```bash
+cd tests
+pytest -s -v -m "core_model and cpu"
+```
+
+### L2
+
+```bash
+cd tests
+pytest -s -v -m "core_model and not cpu" --run-level=core_model
+```
+
+### L3 (merge)
+
+```bash
+cd tests
+pytest -s -v -m "advanced_model" --run-level=advanced_model
+```
+
+### L4 (nightly)
+
+**Function** (e2e expansion — default when user asks for “L4 functional cases”):
+
+```bash
+cd tests
+pytest -s -v e2e/online_serving/test_qwen_image_expansion.py -m "full_model and diffusion and H100" --run-level=full_model
+```
+
+**Perf** (dfx benchmark harness — separate pillar; not `omni_server` fixtures):
+
+```bash
+cd tests
+export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py \
+  --test-config-file dfx/perf/tests/test_qwen_image_vllm_omni.json
+```
+
+Broad marker sweep (when no explicit file shard):
+
+```bash
+cd tests
+pytest -s -v -m "full_model" --run-level=full_model
+```
+
+## L4 nightly pillars (Function / Accuracy / Perf)
+
+Nightly **L4** for a model is often **multiple jobs** in `test-nightly.yml`, not a single pytest module:
+
+| Pillar | User says | Deliver | Nightly step |
+|--------|-----------|---------|--------------|
+| **Function** | “L4 functional cases” / “functional” / `*_expansion.py` | `tests/e2e/.../test_<model>_expansion.py` | `· Function Test with H100/L4` |
+| **Perf** | “performance” / “perf” / “benchmark” / “full L4” | `tests/dfx/perf/tests/test_<model>_vllm_omni.json` + runner | `· Perf Test · <Model>` |
+| **Accuracy** | “accuracy” / “similarity” | `tests/e2e/accuracy/test_<model>*.py` | `· Accuracy Test` |
+
+**Rule:** “L4 functional cases” → **Function only** by default. State that Perf/Accuracy are separate pillars; add them only when requested.
+
+**Diffusion perf JSON** (`tests/dfx/perf/tests/test_<slug>_vllm_omni.json`): array of objects with `test_name`, `server_params` (`model`, `serve_args`), `benchmark_params[]`, and per-workload **`baseline`** (`throughput_qps`, `latency_mean`, `peak_memory_mb_mean`). Copy structure from `test_qwen_image_vllm_omni.json` in-tree.
+
+**Diffusion perf nightly step** (under **Diffusion X2I(&A&T)**; copy full `kubernetes` plugins from `· Perf Test · Qwen-Image`):
+
+```yaml
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · <Model-Display-Name>"
+        key: nightly-diffusion-x2iat-performance-<slug>
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - export CACHE_DIT_VERSION=1.3.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_<slug>_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          # … full kubernetes block (often multi-GPU) — copy from Perf Test · Qwen-Image
+```
+
+| Model type | Perf runner | Config example |
+|------------|-------------|----------------|
+| Diffusion X2I/X2V | `dfx/perf/scripts/run_diffusion_benchmark.py` | `test_qwen_image_vllm_omni.json` |
+| TTS | `dfx/perf/scripts/run_benchmark.py` | `test_tts.json` (`BENCHMARK_DIR`) |
+| Omni | `run_benchmark.py` / omni configs in-tree | `test_omni*.json` |
+
+Do **not** put throughput/latency baselines inside `test_*_expansion.py` — that belongs in the dfx perf JSON + nightly Perf job.
+
+## Invalid parameter validation (weekly / dfx)
+
+When the user’s test plan includes **invalid parameter validation**, **invalid request bodies**, or **HTTP 4xx** validation against a live server:
+
+1. **Do not** author these in `tests/e2e/online_serving/test_*.py` or `*_expansion.py`. **Move** any drafted `test_*` into `tests/dfx/reliability/invalid_param_test/`.
+2. **Pick script by route** (extend in-tree file):
+
+| Route family | Script |
+|--------------|--------|
+| Omni chat / WS video·realtime | `test_invalid_omni_chat.py` |
+| `/v1/audio/speech` (+ stream / batch / voices) | `test_invalid_audio_speech.py` |
+| Audio diffusion | `test_invalid_audio_diffusion.py` |
+| `/v1/images/generations` | `test_invalid_image_generation.py` |
+| `/v1/images/edits` | `test_invalid_image_editing.py` |
+| `/v1/videos*` | `test_invalid_video_generation.py` |
+| Sleep / wakeup / server control | `test_invalid_server_control.py` |
+
+3. **Style:** `pytestmark = [pytest.mark.slow, pytest.mark.<type>]`; `_PARAMS` + `hardware_marks`; `send_*_http_request` with `err_code` + `err_message`; parametrized `body_spec` rows with `id=`; `_minimal_*_json()` helpers; `_SKIP_ISSUE_3649` when tracked in [#3649](https://github.com/vllm-project/vllm-omni/issues/3649).
+4. **CI:** [`.buildkite/test-weekly.yml`](../../../../.buildkite/test-weekly.yml) group **Reliability Test - Invalid parameters Test** — **not** ready/merge/nightly.
+
+```bash
+# Weekly H100 (diffusion / omni / video invalid-param)
+cd tests
+pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and H100"
+
+# Weekly L4 (e.g. Qwen3-TTS speech invalid-param)
+pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and L4"
+
+# Single script / case
+pytest -s -v dfx/reliability/invalid_param_test/test_invalid_image_generation.py::test_images_generations_invalid_requests -m "slow and H100"
+```
+
+**Trigger:** `WEEKLY=1` or PR label `weekly-test`. Appending cases to existing scripts usually needs **no YAML edit** (directory sweep). No `source_file_dependencies` on weekly steps.
+
+### Platform-targeted examples
+
+```bash
+cd tests
+pytest -s -v -m "core_model and distributed_cuda and L4" --run-level=core_model
+```
+
+### Concrete e2e paths (common in-tree)
+
+Paths are relative to `tests/` after `cd tests`.
+
+| Scenario | Example command |
+|----------|------------------|
+| **Omni** offline L2 | `pytest -s -v e2e/offline_inference/test_qwen2_5_omni.py -m "core_model and omni and not cpu" --run-level=core_model` |
+| **Omni** offline L2 — one test | `pytest -s -v e2e/offline_inference/test_qwen2_5_omni.py::test_text_to_text -m "core_model and omni and not cpu" --run-level=core_model` |
+| **Omni** online L2 | `pytest -s -v e2e/online_serving/test_qwen3_omni.py -m "core_model and omni" --run-level=core_model` |
+| **Omni** offline L2 (Qwen3.5-9B VL) | `pytest -s -v e2e/offline_inference/test_qwen3_5_9b.py -m "core_model and omni and not cpu" --run-level=core_model` |
+| **TTS** online L2 | `pytest -s -v e2e/online_serving/test_qwen3_tts_base.py -m "core_model and tts" --run-level=core_model` |
+| **TTS** online L2 — one test | `pytest -s -v e2e/online_serving/test_qwen3_tts_base.py::test_text_to_audio_001 -m "core_model and tts" --run-level=core_model` |
+| **Diffusion X2I** offline L2 | `pytest -s -v e2e/offline_inference/test_t2i_model.py -m "core_model and diffusion and not cpu" --run-level=core_model` |
+| **Diffusion X2I** L2 online smoke | `pytest -s -v e2e/online_serving/test_qwen_image.py e2e/online_serving/test_bagel.py -m "core_model and diffusion" --run-level=core_model` |
+| **Diffusion X2V** L2 online smoke | `pytest -s -v e2e/online_serving/test_wan22_t2v.py -m "core_model and diffusion" --run-level=core_model` |
+| **Diffusion X2I** L4 online expansion (H100) | `pytest -s -v e2e/online_serving/test_qwen_image_expansion.py -m "full_model and diffusion and H100" --run-level=full_model` |
+| **Diffusion X2I** L4 perf (nightly) | `pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file dfx/perf/tests/test_qwen_image_vllm_omni.json` |
+| **Diffusion X2I** L4 online expansion (Edit) | `pytest -s -v e2e/online_serving/test_qwen_image_edit_expansion.py -m "full_model and diffusion and H100" --run-level=full_model` |
+| **Diffusion X2V** L4 nightly | `pytest -s -v e2e/online_serving/test_wan22_expansion.py -m "full_model and cuda" --run-level=full_model` |
+| **Invalid param** weekly H100 | `pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and H100"` |
+| **Invalid param** weekly L4 | `pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and L4"` |
+
+### Agent / author completion checklist
+
+When adding or modifying tests, do not stop at “where the file lives” — also deliver:
+
+1. **Local**: `cd tests` + `pytest -s -v <path>` (and `path::test_func` when a single case is enough).
+2. **CI-like**: marker string + `--run-level` matching the pipeline (`core_model` / `advanced_model` / `full_model`).
+3. **L1 mocks**: `mocker` / `monkeypatch` only; no `unittest.mock`.
+4. **API client + assert placement**: **General** validation → implement in `assertions.py`, call **inside** `send_*_request` in `runtime.py`; tests call **`send_*_request` only**. **Special** case validation → `assert_*` in `assertions.py`, called **in the test** after `send_*_request`. Low-level `send_*_http_request` is for negative/dfx tests (`err_code`), not ordinary L2+ success e2e.
+5. **Shared assertions**: logic in **`tests/helpers/assertions.py`**; general checks inside `send_*_request`; special checks only in the test. No `_assert_*` in `test_*.py`.
+6. **One case → one `test_*`**: function name reflects what is validated; no `if case_id == ...` mega-test merging multiple scenarios.
+7. **Fixture scope**: default **`omni_server` + `openai_client`** / **`omni_runner` + `omni_runner_handler`** (module). Use **`omni_server_function` + `openai_client_function`** / **`omni_runner_function` + `omni_runner_handler_function`** only when each `test_*` must spawn a fresh instance. Parametrize name must match fixture (`omni_server` vs `omni_server_function`).
+8. **Type marker**: `omni`, `tts`, or `diffusion` on every model e2e module.
+9. **Diffusion L4 Function**: wire `*_expansion.py` into **X2I(&A&T)** or **X2V** **Function Test** in **`test-nightly.yml` only** — do not add `test-merge.yml` unless the user also requested L3.
+10. **Diffusion L4 Perf** (only when requested): add `tests/dfx/perf/tests/test_<slug>_vllm_omni.json` + **Perf Test · &lt;Model&gt;** step (artifact upload); not part of “L4 functional cases” by default.
+11. **E2E Buildkite (L2/L3 only)**: `test-ready.yml` / `test-merge.yml` steps need **`source_file_dependencies`** + full **`agents` + `plugins`**. **L4 nightly** uses explicit file lists or perf scripts in `test-nightly.yml` (no merge job).
+12. **Invalid param**: cases in **`tests/dfx/reliability/invalid_param_test/`** (route-matching script), `send_*_http_request` + `err_code`, `pytest.mark.slow` + `H100`/`L4`; CI = **`test-weekly.yml`** only — do not put in e2e or nightly.
+13. **Prerequisites**: GPU tier, HF cache/token, and any module `skipif` / platform-only YAML.
+
+## Buildkite pipeline mapping
+
+| Level | Repo file | Model-type grouping |
+|-------|-----------|---------------------|
+| L1, L2 | [`.buildkite/test-ready.yml`](../../../../.buildkite/test-ready.yml) | Steps prefixed **Omni ·**, **TTS ·**, **Diffusion ·** under **E2E Test** — **`source_file_dependencies` required** |
+| L3 | [`.buildkite/test-merge.yml`](../../../../.buildkite/test-merge.yml) | Per-model E2E steps; `-m "advanced_model and …"` — **`source_file_dependencies` required** |
+| L4 | [`.buildkite/test-nightly.yml`](../../../../.buildkite/test-nightly.yml) | **Omni / TTS / Diffusion X2I(&A&T) / X2V** — each group may have **Function**, **Accuracy**, **Perf**, **Doc** steps |
+| Invalid param (weekly) | [`.buildkite/test-weekly.yml`](../../../../.buildkite/test-weekly.yml) | **Invalid parameters Test · H100** / **· L4** — sweeps `tests/dfx/reliability/invalid_param_test/` |
+
+### `source_file_dependencies` (E2E Test only — ready & merge)
+
+Required on each step inside the **E2E Test** group in `test-ready.yml` and `test-merge.yml`. Typical entries:
+
+| Category | Path pattern |
+|----------|----------------|
+| Test module | `tests/e2e/online_serving/test_<slug>.py`, `tests/e2e/offline_inference/test_<slug>.py` |
+| API client helpers | `tests/helpers/runtime.py` (when the e2e job uses newly added `send_*` methods) |
+| Shared assert helpers | `tests/helpers/assertions.py` (when the e2e job uses newly added `assert_*` helpers) |
+| AR / omni model | `vllm_omni/model_executor/models/<family>/` |
+| Diffusion model | `vllm_omni/diffusion/models/<family>/` |
+| Stage processor | `vllm_omni/model_executor/stage_input_processors/<family>.py` |
+| Deploy config | `vllm_omni/deploy/<name>.yaml` or `vllm_omni/deploy/ci/<name>.yaml` |
+
+Copy the dependency block from the closest in-tree E2E step for the same model family. Update the list whenever the pytest command or server YAML changes.
+
+### E2E `agents` + `plugins` blocks (required — do not truncate)
+
+When generating or documenting E2E steps, **always include the complete `plugins` section**. Set **`timeout_in_minutes`** on the step for the job deadline; run **`pytest` directly** in `commands` (no `timeout 40m bash -c` wrapper around pytest).
+
+Common patterns in `test-ready.yml` / `test-merge.yml`:
+
+| Queue | Plugin | Required fields |
+|-------|--------|-----------------|
+| `mithril-h100-pool` | `kubernetes` | `resources.limits.nvidia.com/gpu`, `volumeMounts` (`devshm`, `hf-cache`), `env` (`HF_HOME`, `HF_TOKEN` via `secretKeyRef`), `nodeSelector: gpu-h100-sxm`, `volumes` (`devshm` emptyDir, `hf-cache` hostPath `/mnt/hf-cache`) |
+| `gpu_1_queue` / `gpu_4_queue` | `docker#v5.2.0` | `image`, `always-pull`, `propagate-environment`, `shm-size: "8gb"`, `environment: HF_HOME`, `volumes: /fsx/hf_cache` (add `HF_TOKEN` when the step needs hub access) |
+
+**H100 kubernetes template** (copy from **Diffusion · Bagel Test** / **Diffusion · Qwen Image Test** in-tree):
+
+```yaml
+  agents:
+    queue: "mithril-h100-pool"
+  plugins:
+    - kubernetes:
+        podSpec:
+          containers:
+            - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              resources:
+                limits:
+                  nvidia.com/gpu: 1
+              volumeMounts:
+                - name: devshm
+                  mountPath: /dev/shm
+                - name: hf-cache
+                  mountPath: /root/.cache/huggingface
+              env:
+                - name: HF_HOME
+                  value: /root/.cache/huggingface
+                - name: HF_TOKEN
+                  valueFrom:
+                    secretKeyRef:
+                      name: hf-token-secret
+                      key: token
+          nodeSelector:
+            node.kubernetes.io/instance-type: gpu-h100-sxm
+          volumes:
+            - name: devshm
+              emptyDir:
+                medium: Memory
+            - name: hf-cache
+              hostPath:
+                path: /mnt/hf-cache
+                type: DirectoryOrCreate
+```
+
+**L4 docker template** (copy from **TTS · Qwen3-TTS Base Test** in `test-merge.yml`):
+
+```yaml
+  agents:
+    queue: "gpu_1_queue"
+  plugins:
+    - docker#v5.2.0:
+        image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+        always-pull: true
+        propagate-environment: true
+        shm-size: "8gb"
+        environment:
+          - "HF_HOME=/fsx/hf_cache"
+        volumes:
+          - "/fsx/hf_cache:/fsx/hf_cache"
+```
+
+When extending an existing step (e.g. add `tests/e2e/offline_inference/test_qwen_image.py` to **Diffusion · Qwen Image Test**), update `source_file_dependencies` and `commands` only; **keep the existing `agents` + `plugins` block unchanged** unless the hardware tier changes.
+
+The root [`.buildkite/pipeline.yml`](../../../../.buildkite/pipeline.yml) decides **which** child file is uploaded. To run **L3** on a feature branch, comment out `if: build.branch == "main"` on the merge upload step. To run **L4** without `NIGHTLY=1`, comment out the nightly upload step’s `if: build.env("NIGHTLY") == "1"` and the same `if` on relevant steps inside `test-nightly.yml`. Revert such edits before merging.
+
+Platform-specific pipelines (e.g. AMD) follow the same level → file pairing under `.buildkite/`.
+
+## Diffusion RFC (#1832) Alignment Tips
+
+For **X2I(&A&T)** coverage planning:
+
+- Prioritize high-value feature combinations with minimal case count.
+- Split into lightweight L1/L2 validation plus L4 `*_expansion.py` under the **X2I** nightly shards.
+- **X2V** models (Wan, HunyuanVideo) stay in the **X2V** group — do not merge into X2I sweeps.
+
+If hardware is insufficient, provide an executable reduced case plus a deferred full CI/nightly plan.

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,21 +1,15 @@
 <!-- markdownlint-disable -->
-PLEASE FILL IN THE PR DESCRIPTION HERE ENSURING ALL CHECKLIST ITEMS (AT THE BOTTOM) HAVE BEEN CONSIDERED.
+PLEASE FILL IN THE PR DESCRIPTION HERE.
 
 ## Purpose
 
 ## Test Plan
 
+**vLLM Version:**
+
+**vLLM-Omni Commit:**
+
 ## Test Result
 
----
-<details>
-<summary> Essential Elements of an Effective PR Description Checklist </summary>
-
-- [ ] The purpose of the PR, such as "Fix some issue (link existing issues this PR will resolve)".
-- [ ] The test plan. Please provide the test scripts & test commands. Please state the reasons if your codes don't require additional test scripts. For test file guidelines, please check the [test style doc](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/tests_style/)
-- [ ] The test results. Please paste the results comparison before and after, or the e2e results.
-- [ ] (Optional) The necessary documentation update, such as updating `supported_models.md` and `examples` for a new model. **Please run `mkdocs serve` to sync the documentation editions to `./docs`.**
-- [ ] (Optional) Release notes update. If your change is user-facing, please update the release notes draft.
-</details>
-
-**BEFORE SUBMITTING, PLEASE READ <https://github.com/vllm-project/vllm-omni/blob/main/CONTRIBUTING.md>** (anything written below this line will be removed by GitHub Actions)
+**BEFORE SUBMITTING:** read [CONTRIBUTING.md](https://github.com/vllm-project/vllm-omni/blob/main/CONTRIBUTING.md) and run the [precheck-pr skill](https://github.com/vllm-project/vllm-omni/blob/main/.claude/skills/precheck-pr/SKILL.md) with the code agent for a self-check against project conventions.
+(anything written below this line will be removed by GitHub Actions)

CONTRIBUTING.md

@@ -1,3 +1,5 @@
 # Contributing to vLLM-Omni
 
 You may find information about contributing to vLLM-Omni on [Contributing](https://vllm-omni.readthedocs.io/en/latest/contributing/)
+
+Before submitting a PR, run the [precheck-pr skill](https://github.com/vllm-project/vllm-omni/blob/main/.claude/skills/precheck-pr/SKILL.md) with the code agent for a self-check against project conventions.

README.md

@@ -16,6 +16,7 @@ Easy, fast, and cheap omni-modality model serving for everyone
 ---
 
 *Latest News* 🔥
+- [2026/06] We released [0.22.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.22.0) - an **omnimodal world-model** release aligned with vLLM 0.22, featuring [Nvidia Cosmos3](recipes/cosmos3/Cosmos3-Nano.md)/DreamZero world model support, expanded quantization coverage across Blackwell/NPU/XPU, TTS production improvements, new models including MiniCPM-o 4.5, MOSS-TTS, and Lance, plus RL integration with [VeRL-Omni](https://github.com/verl-project/verl-omni).
 - [2026/05] We released [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0) - refreshes the serving/runtime stack for large-scale omni workloads, and improves diffusion model performance, quantization, and hardware readiness across CUDA, ROCm, MUSA, NPU, and XPU backends.
 - [2026/03] We released [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0) - strengthens the core runtime through a large entrypoint refactor and scheduler/runtime cleanups, expands unified quantization and diffusion execution, broadens multimodal model coverage, and improves production readiness across audio, omni, image, video, RL, and multi-platform deployments.
 - [2026/03] Check out our first public [project deepdive](https://youtu.be/sgwNfsNnR9I) at the vLLM Hong Kong Meetup!
@@ -56,8 +57,9 @@ vLLM-Omni is flexible and easy to use with:
 
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
-- Omni-modality models (e.g. Qwen-Omni)
-- Multi-modality generation models (e.g. Qwen-Image)
+- **Omni-modality models** (e.g. Qwen3-Omni, Cosmos, HunyuanImage, BAGEL)
+- **TTS models** (e.g. Qwen3-TTS, VoxCPM2, Ming-Omni-TTS, CosyVoice3)
+- **Diffusion models** — image, video, and audio generation (e.g. Qwen-Image, Wan2.2, FLUX)
 
 ## Getting Started
 
@@ -66,6 +68,7 @@ Visit our [documentation](https://vllm-omni.readthedocs.io/en/latest/) to learn
 - [Installation](https://vllm-omni.readthedocs.io/en/latest/getting_started/installation/)
 - [Quickstart](https://vllm-omni.readthedocs.io/en/latest/getting_started/quickstart/)
 - [List of Supported Models](https://vllm-omni.readthedocs.io/en/latest/models/supported_models/)
+- [Deployment Recipes](https://recipes.vllm.ai) for vLLM-Omni model serving
 
 ## Contributing
 

benchmarks/diffusion/README.md

@@ -32,6 +32,8 @@ python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 - By default, image tasks talk to `http://<host>:<port>/v1/chat/completions`; video tasks talk to `/v1/videos`.
 - If you run the server on another host or port, pass `--base-url` accordingly.
+- To compare diffusion multi-replica serving, restart the server with different
+  `runtime.num_replicas` values and keep the benchmark parameters unchanged.
 
 ## 2. Supported Datasets
 
@@ -93,6 +95,35 @@ You can point to your own trace using `--dataset-path`.
 
 ## 3. Benchmark Parameters
 
+### Multi-replica sweep
+
+For replica scaling, use the same model, prompt count, image/video size, and
+step count for every run. Change only the serving stage config:
+
+```yaml
+runtime:
+  devices: "0,1,2,3"
+  num_replicas: 4
+```
+
+Then run a fixed benchmark and write one JSON file per replica count:
+
+```bash
+python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
+  --base-url http://localhost:8099 \
+  --model Qwen/Qwen-Image \
+  --task t2i \
+  --dataset random \
+  --num-prompts 32 \
+  --max-concurrency 8 \
+  --request-rate inf \
+  --num-inference-steps 10 \
+  --width 512 --height 512 \
+  --output-file diffusion_replica_4.json
+```
+
+Repeat for `num_replicas=1`, `2`, `3`, and `4`, using matching `runtime.devices`.
+
 ### Basic flags
 
 - `--base-url`: Server address; `--endpoint` selects the path appended to this base URL.

benchmarks/diffusion/diffusion_benchmark_serving.py

@@ -79,6 +79,7 @@ Usage:
 import argparse
 import ast
 import asyncio
+import base64
 import glob
 import json
 import logging
@@ -106,6 +107,9 @@ from tqdm.asyncio import tqdm
 
 logger = logging.getLogger(__name__)
 
+_STAGE_METRICS_ENDPOINTS = {"/v1/chat/completions"}
+_RETURN_STAGE_METRICS_FIELD = "return_stage_metrics"
+
 
 class BaseDataset(ABC):
     def __init__(self, args, api_url: str, model: str):
@@ -1013,6 +1017,73 @@ def calculate_metrics(
     return metrics
 
 
+def _save_generated_outputs(
+    outputs: list[RequestFuncOutput],
+    requests_list: list[RequestFuncInput],
+    save_dir: str,
+) -> None:
+    """Decode and save base64 images/videos from successful responses."""
+    os.makedirs(save_dir, exist_ok=True)
+    saved = 0
+    failed = 0
+
+    for idx, (req, out) in enumerate(zip(requests_list, outputs)):
+        if not out.success or not out.response_body:
+            continue
+
+        media_urls: list[str] = []
+
+        # Chat-completions style: choices[*].message.content[*].image_url.url
+        choices = out.response_body.get("choices", [])
+        if isinstance(choices, list):
+            for choice in choices:
+                content = (choice or {}).get("message", {}).get("content")
+                if not isinstance(content, list):
+                    continue
+                for item in content:
+                    if not isinstance(item, dict) or item.get("type") != "image_url":
+                        continue
+                    url = (item.get("image_url") or {}).get("url", "")
+                    if isinstance(url, str) and url.startswith("data:"):
+                        media_urls.append(url)
+
+        # Images endpoint style: data[*].b64_json
+        data_items = out.response_body.get("data", [])
+        if isinstance(data_items, list):
+            for data_item in data_items:
+                if not isinstance(data_item, dict):
+                    continue
+                b64_json = data_item.get("b64_json", "")
+                if isinstance(b64_json, str) and b64_json:
+                    media_urls.append(f"data:image/png;base64,{b64_json}")
+
+        for img_idx, url in enumerate(media_urls):
+            if "," not in url:
+                continue
+
+            try:
+                header, b64_data = url.split(",", 1)
+                ext = "png"
+                if "image/jpeg" in header:
+                    ext = "jpg"
+                elif "image/webp" in header:
+                    ext = "webp"
+                elif "video/mp4" in header:
+                    ext = "mp4"
+
+                img_bytes = base64.b64decode(b64_data)
+                fname = f"req_{idx:04d}_{img_idx}.{ext}"
+                fpath = os.path.join(save_dir, fname)
+                with open(fpath, "wb") as f:
+                    f.write(img_bytes)
+                saved += 1
+            except Exception as e:
+                failed += 1
+                logger.warning(f"Failed to save image for request {idx}: {e}", exc_info=True)
+
+    logger.info(f"Saved {saved} generated image(s) to {save_dir}. Failed to save {failed} image(s).")
+
+
 def wait_for_service(base_url: str, timeout: int = 120) -> None:
     print(f"Waiting for service at {base_url}...")
     start_time = time.time()
@@ -1095,6 +1166,10 @@ async def benchmark(args):
     requests_list = dataset.get_requests()
     print(f"Prepared {len(requests_list)} requests from {args.dataset} dataset.")
 
+    if args.return_stage_metrics and args.endpoint in _STAGE_METRICS_ENDPOINTS:
+        for req in requests_list:
+            req.extra_body.setdefault(_RETURN_STAGE_METRICS_FIELD, True)
+
     if args.endpoint == "/v1/images/edits":
         for req in requests_list:
             req.default_bot_task = args.bot_task
@@ -1202,6 +1277,9 @@ async def benchmark(args):
 
     print("\n" + "=" * 60)
 
+    if args.save_dir:
+        _save_generated_outputs(outputs, requests_list, args.save_dir)
+
     if args.output_file:
         with open(args.output_file, "w") as f:
             json.dump(metrics, f, indent=2)
@@ -1278,11 +1356,18 @@ if __name__ == "__main__":
         default=1,
         help="Number of warmup requests to run before measurement.",
     )
+    # NOTE Changed default from 1 to 2 because some models (e.g., Bagel) run
+    # `num_timesteps - 1` denoising iterations. A default of 1 results in 0 steps,
+    # which causes errors.
+    # TODO If this slightly longer warmup causes regression issues for other
+    # diffusion pipelines in the future, consider implementing model-specific
+    # overrides instead of a global default.
     parser.add_argument(
         "--warmup-num-inference-steps",
         type=int,
-        default=1,
-        help="num_inference_steps used for warmup requests.",
+        default=2,
+        help="Number of inference steps used for warmup requests. "
+        "Default is 2 to ensure at least one denoising step is executed.",
     )
     parser.add_argument(
         "--warmup-concurrency",
@@ -1323,6 +1408,13 @@ if __name__ == "__main__":
         default=3.0,
         help="SLO target multiplier: slo_ms = estimated_exec_time_ms * slo_scale (default: 3).",
     )
+    parser.add_argument(
+        "--save-dir",
+        type=str,
+        default=None,
+        help="Directory to save generated images/outputs for visual inspection. "
+        "If not set, generated outputs are discarded after metric collection.",
+    )
     parser.add_argument("--disable-tqdm", action="store_true", help="Disable progress bar.")
     parser.add_argument(
         "--enable-negative-prompt",
@@ -1358,6 +1450,11 @@ if __name__ == "__main__":
         default="think",
         help=("bot_task form field for --endpoint /v1/images/edits (think, recaption, think_recaption, vanilla)."),
     )
+    parser.add_argument(
+        "--return-stage-metrics",
+        action="store_true",
+        help="Request stage duration metrics from endpoints that support return_stage_metrics.",
+    )
 
     args = parser.parse_args()
     asyncio.run(benchmark(args))

benchmarks/tts/model_configs.yaml

@@ -33,6 +33,15 @@ models:
       voice_clone:
         task_type: Base
 
+  bosonai/higgs-audio-v3-tts-4b:
+    supported_tasks: [default_voice, voice_clone]
+    backend: openai-audio-speech
+    endpoint: /v1/audio/speech
+    trust_remote_code: true
+    task_extra_body:
+      default_voice: {}
+      voice_clone: {}
+
   openbmb/VoxCPM2:
     supported_tasks: [voice_clone, default_voice]
     trust_remote_code: true

docker/Dockerfile.ci

@@ -2,7 +2,7 @@
 # The pinned VLLM_PRECOMPILED_WHEEL_COMMIT and VLLM_BASE_TAG below must
 # be updated on every rebase to match the new upstream vllm commit/tag.
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai
-ARG VLLM_BASE_TAG=v0.22.0
+ARG VLLM_BASE_TAG=v0.23.0
 FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
 ARG APP_DIR=/workspace/vllm-omni
 WORKDIR ${APP_DIR}
@@ -16,6 +16,7 @@ RUN apt-get update && \
 
 RUN uv pip install --system ".[dev]"
 
+
 RUN ln -sf /usr/bin/python3 /usr/bin/python
 
 ENTRYPOINT []

docker/Dockerfile.cuda

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai:v0.22.0
+ARG BASE_IMAGE=vllm/vllm-openai:v0.23.0
 FROM ${BASE_IMAGE}
 
 ARG COMMON_WORKDIR=/app

docker/Dockerfile.rocm

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.22.0
+ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.23.0
 FROM ${BASE_IMAGE} AS base
 
 # Declare a variable to know if we want to use the nightly build or the stable build.

docker/Dockerfile.xpu

@@ -88,7 +88,7 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
 ENV UV_INDEX_STRATEGY="unsafe-best-match"
 ENV UV_LINK_MODE="copy"
 
-ARG VLLM_VERSION=v0.22.0
+ARG VLLM_VERSION=v0.23.0
 RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
 WORKDIR /workspace/vllm
 
@@ -142,15 +142,12 @@ WORKDIR /workspace/vllm-omni
 COPY . .
 
 ENV VLLM_OMNI_TARGET_DEVICE=xpu
-RUN uv pip install --no-cache-dir ".[dev]" --no-build-isolation
-
-# FIX triton
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip uninstall triton triton-xpu && \
-    uv pip install triton-xpu==3.7.0
-
-# remove torch bundled oneccl to avoid conflicts
 RUN --mount=type=cache,target=/root/.cache/uv \
+    echo "triton @ https://example.com/blocked" > /tmp/vllm_omni_constraints.txt && \
+    PIP_CONSTRAINT=/tmp/vllm_omni_constraints.txt \
+    uv pip install --no-build-isolation \
+    --extra-index-url https://mirrors.ustc.edu.cn/pypi/simple/ \
+    ".[dev]" && \
     uv pip uninstall oneccl oneccl-devel
 
 FROM vllm-omni AS vllm-omni-openai

docs/.nav.yml

@@ -87,6 +87,8 @@ nav:
       - AutoRound: user_guide/quantization/autoround.md
       - msModelSlim: user_guide/quantization/msmodelslim.md
     - ComfyUI: features/comfyui.md
+  - Fault Tolerance:
+    - Failure Modes: user_guide/fault_injection_reliability_matrix.md
 - Developer Guide:
   - General:
     - contributing/README.md

docs/api/README.md

@@ -92,7 +92,6 @@ Core scheduling and caching components.
 Configuration classes.
 
 - [vllm_omni.config.model.OmniModelConfig][]
-- [vllm_omni.config.stage_config.ModelPipeline][]
 - [vllm_omni.config.stage_config.StageConfig][]
 - [vllm_omni.config.stage_config.StageConfigFactory][]
 - [vllm_omni.config.stage_config.StageType][]

docs/contributing/README.md

@@ -116,9 +116,40 @@ Only specific types of PRs will be reviewed. The PR title is prefixed appropriat
 !!! note
     If the PR spans more than one category, please include all relevant prefixes.
 
+### Pre-Check Before Submitting
+
+Before submitting a PR, run the [precheck-pr skill](https://github.com/vllm-project/vllm-omni/blob/main/.claude/skills/precheck-pr/SKILL.md) with the code agent for a self-review against project conventions:
+
+The skill offers two modes:
+- **Quick (~3 min):** catches showstoppers — PR title format, missing benchmark claims, rebase status
+- **Full (~10 min):** thorough maintainer-grade review — dead code scan, copy-paste detection, import hygiene
+
+The precheck covers five PR types: Bug Fix, Performance, New Model, Diffusion Model, and General. Each type has a tailored checklist that validates evidence quality (repro steps, A/B benchmarks, registry entries, etc.). See the [precheck-pr skill](https://github.com/vllm-project/vllm-omni/blob/main/.claude/skills/precheck-pr/SKILL.md) for the full checklist.
+
 ### Local Test
 Please run the L1 and L2 test cases locally first and attach the results before contacting us to add the "ready" label. Please refer to the [test instructions](./ci/test_guide.md) for running the test cases.
 
+### Automatic skip-ci (docs and pytest skip marks)
+
+On pull requests and `main` pushes, the bootstrap step in [`.buildkite/pipeline.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/pipeline.yml) runs [`.buildkite/scripts/upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/scripts/upload_pipeline.py) against the git diff. When every changed file qualifies, **L2 (`ready`) and L3 (`merge-test`) pipelines are not uploaded**, so the default GPU CI jobs are skipped.
+
+| Change per file | Examples |
+| --- | --- |
+| Documentation | `docs/**`, any `*.md`, `mkdocs.yml` |
+| Pytest skip marks only (under `tests/`) | Add/remove/edit `@pytest.mark.skip`, `@pytest.mark.skipif`, or `pytest.skip(...)`; reformat `pytestmark` only to add a skip/skipif alongside existing `pytest.mark.*` entries |
+| New skipped test module | New `tests/**/*.py` whose `pytestmark` includes unconditional `pytest.mark.skip` |
+
+These PR shapes all trigger skip-ci:
+
+- Documentation only
+- Qualifying skip-mark edits in `tests/**/*.py` only
+- **A mix of documentation and qualifying skip-mark test edits**
+
+Skip-ci does **not** apply when the diff also touches product code (for example `vllm_omni/`), or when test files change assertions, imports, fixtures, or other non-skip logic. If the diff cannot be resolved (non-PR branches outside `main`), CI runs as usual.
+
+!!! note
+    Skipping L2/L3 does **not** disable the Docker image build step. Nightly (L4) upload can still run when the PR has a `nightly-test` label or on scheduled `main` builds with `NIGHTLY=1`. See [CI levels](./ci/CI_5levels.md) for how bootstrap skip-ci relates to diff-gated E2E jobs.
+
 ### Code Quality
 
 The PR needs to meet the following code quality standards:

docs/contributing/ci/CI_5levels.md

@@ -11,6 +11,10 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
     <tr>
       <th>Level</th>
       <th>Scope & Focus</th>
+      <th>Model Coverage Strategy</th>
+      <th>Feature Coverage Strategy</th>
+      <th>Interface Coverage Strategy</th>
+      <th>Tags</th>
       <th>Time Cost</th>
       <th>Test Dir</th>
       <th>Doc</th>
@@ -24,6 +28,10 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
       <td>Contribution Guideline & PR checklist</td>
       <td>/</td>
       <td>/</td>
+      <td>/</td>
+      <td>/</td>
+      <td>/</td>
+      <td>/</td>
       <td>.github/PULL_REQUEST_TEMPLATE.md <a href="../tests_style/"> Test Style (PR Checklist)</a></td>
       <td>/</td>
       <td>/</td>
@@ -32,6 +40,10 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
       <td>CI Failure Description</td>
       <td>/</td>
       <td>/</td>
+      <td>/</td>
+      <td>/</td>
+      <td>/</td>
+      <td>/</td>
       <td><a href="../failures/"> CI Failures</a></td>
       <td>/</td>
       <td>/</td>
@@ -39,6 +51,10 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
     <tr>
       <td><strong>L1</strong><br>(Unit & Logic)</td>
       <td>Unit tests for components like entrypoints, models</td>
+      <td>/</td>
+      <td>/</td>
+      <td>/</td>
+      <td><code>core_model and cpu</code></td>
       <td rowspan="2">&lt;15min</td>
       <td>/tests/{component_name}/test_xxx</td>
       <td>
@@ -50,10 +66,18 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
     </tr>
     <tr>
       <td><strong>L2</strong><br>(E2E across models & GPU-required UT)</td>
-      <td>Online & Offline (basic deployment scenarios):<br>dummy, normal inference function (output format, stream), some instance startup UT</td>
+      <td>Online (basic deployment scenarios):<br>dummy, normal inference function (output format, stream), some instance startup UT</td>
+      <td>High-priority models + online basic scenarios + request success validation</td>
+      <td>High-priority features (using random lightweight models)</td>
+      <td>High-priority interfaces (using random lightweight models)</td>
+      <td><code>core_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
       <td>
+        <strong>Model tests:</strong><br>
         /tests/e2e/online_serving/test_{model_name}.py<br>
-        /tests/e2e/offline_inference/test_{model_name}.py
+        <strong>Feature tests:</strong><br>
+        /tests/{component_name}/test_xxx<br>
+        <strong>Interface tests:</strong><br>
+        /tests/entrypoints/test_xxx
       </td>
       <td>
         <a href="#chapter-1-l1-l2-level-testing-unit-testing-and-basic-end-to-end-verification">Chapter 1</a><br>
@@ -65,10 +89,19 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
     <tr>
       <td><strong>L3</strong><br>(Important Perf & Integration & Accuracy)</td>
       <td>Online & Offline (multiple deployment scenarios):<br>real model, normal inference function, normal accuracy</td>
+      <td>High/medium-priority models with real weights + online/offline key scenarios + basic accuracy validation</td>
+      <td>Medium-priority features (using random lightweight models)</td>
+      <td>Medium-priority interfaces (using random lightweight models)</td>
+      <td><code>advanced_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
       <td>&lt;30min</td>
       <td>
+        <strong>Model tests:</strong><br>
         /tests/e2e/online_serving/test_{model_name}.py<br>
-        /tests/e2e/offline_inference/test_{model_name}.py
+        /tests/e2e/offline_inference/test_{model_name}.py<br>
+        <strong>Feature tests:</strong><br>
+        /tests/{component_name}/test_xxx<br>
+        <strong>Interface tests:</strong><br>
+        /tests/entrypoints/test_xxx
       </td>
       <td>
         <a href="#chapter-2-l3-level-testing-core-integration-performance-and-accuracy-verification">Chapter 2</a><br>
@@ -79,18 +112,27 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
     </tr>
     <tr>
       <td><strong>L4</strong><br>(Perf & Integration & Accuracy)</td>
-      <td>Online & Offline: full functional scenarios + performance test + doc test</td>
+      <td>Online: full functional scenarios + performance test + doc test + accuracy test</td>
+      <td>High-priority models: function, performance, accuracy, and doc testing<br>Medium/low-priority models: function and doc testing</td>
+      <td>Low-priority features (using random lightweight models)</td>
+      <td>Low-priority interfaces (using random lightweight models)</td>
+      <td><code>full_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
       <td>&lt;3 hour</td>
       <td>
-        <strong>Full Function:</strong><br>
+        <strong>Model tests:</strong><br>
         /tests/e2e/online_serving/test_{model_name}_expansion.py<br>
-        /tests/e2e/offline_inference/test_{model_name}_expansion.py<br>
+        <strong>Feature tests:</strong><br>
+        /tests/{component_name}/test_xxx<br>
+        <strong>Interface tests:</strong><br>
+        /tests/entrypoints/test_xxx<br>
         <strong>Performance:</strong><br>
         /tests/dfx/perf/tests/test_qwen_omni.json (Omni), test_tts.json (TTS),<br>
         and /tests/dfx/perf/tests/test_{diffusion_model}_vllm_omni.json (Diffusion)<br>
         <strong>Doc Test:</strong><br>
         tests/example/online_serving/test_{model_name}.py<br>
-        tests/example/offline_inference/test_{model_name}.py
+        tests/example/offline_inference/test_{model_name}.py<br>
+        <strong>Accuracy Test:</strong><br>
+        /tests/e2e/accuracy/test_{model_name}.py
       </td>
       <td>
         <a href="#chapter-3-l4-level-testing-full-functionality-performance-and-documentation-testing">Chapter 3</a><br>
@@ -101,7 +143,11 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
     </tr>
     <tr>
       <td><strong>L5</strong><br>(Stability & Reliability)</td>
-      <td>Online & Offline: long-term stability test + reliability test</td>
+      <td>Online: long-term stability test + reliability test</td>
+      <td>Long-term stability and reliability testing for high-priority models</td>
+      <td>/</td>
+      <td>Invalid-parameter validation for high-priority interfaces</td>
+      <td><code>slow and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
       <td> Depends on reality </td>
       <td>
         <strong>Stability:</strong><br>
@@ -268,6 +314,120 @@ Before entering specific testing levels, the project establishes two common spec
 1.  ***PR Checklist ([Tests Style](../ci/tests_style.md))***: This template defines the self-check items that must be completed before submitting a code review (Pull Request). It ensures that each code change meets basic requirements such as code style, dependency updates, and documentation synchronization before entering the automated testing pipeline, serving as the first manual line of defense for quality assurance.
 2.  ***CI Failure Explanation ([CI Failures](../ci/failures.md))***: This document archives and explains common failure patterns in the Continuous Integration (CI) pipeline, error log interpretation, and preliminary troubleshooting steps. It helps developers and testers quickly diagnose the causes of automated test failures, improving problem-solving efficiency.
 
+## Notes
+
+### Diff-aware Buildkite uploads (`source_file_dependencies`)
+
+L2 (`.buildkite/test-ready.yml`) and L3 (`.buildkite/test-merge.yml`) pipelines can **skip unrelated GPU jobs at upload time** based on the PR diff. This is implemented by `.buildkite/scripts/upload_pipeline.py`, which filters steps before calling `buildkite-agent pipeline upload`.
+
+#### What `source_file_dependencies` is
+
+- A **uploader-only** YAML key on a Buildkite step or group. **Buildkite does not understand it**; `upload_pipeline.py` always **removes** it from the YAML that is uploaded.
+- A list of path **prefixes** (directories or individual files). If **any** changed file in the diff equals a prefix or starts with `prefix/`, the step is kept; otherwise the step (or entire group) is **omitted** from the uploaded pipeline.
+
+#### When filtering runs
+
+| Build context | Changed files used for matching |
+| --- | --- |
+| Pull request | `git diff --name-only origin/<base>...<BUILDKITE_COMMIT>` |
+| `main` branch push | `git diff --name-only <commit>^..<commit>` |
+| Other (e.g. local dry-run, non-PR branch) | Diff cannot be resolved → **no filtering**; all steps are uploaded and `source_file_dependencies` is still stripped |
+
+Docs-only PRs are handled earlier in bootstrap (`pipeline.yml`) via skip-ci logic; `source_file_dependencies` applies only to the **uploaded** L2/L3 test pipelines.
+
+#### Where it is configured
+
+| Level | Pipeline file | Upload entry |
+| --- | --- | --- |
+| L2 | `.buildkite/test-ready.yml` | `upload_pipeline.py --upload .buildkite/test-ready.yml` (from `pipeline.yml`) |
+| L3 | `.buildkite/test-merge.yml` | `upload_pipeline.py --upload .buildkite/test-merge.yml` (from `pipeline.yml`) |
+
+Steps **without** `source_file_dependencies` are always uploaded (subject to the usual label conditions: `ready` for L2, `merge-test` for L3).
+
+#### Current skip policy (L2 / L3)
+
+To balance CI cost and coverage:
+
+- **Always run** (no `source_file_dependencies`): baseline groups outside E2E Test—e.g. Simple Test, Diffusion unit tests, Engine/Model Executor, Distributed, Custom Pipeline, Entrypoints (L2), LoRA / Entrypoints (L3).
+- **Diff-gated** (`source_file_dependencies` set): **every leaf job under the E2E Test group** in `test-ready.yml` and `test-merge.yml`, regardless of queue (`mithril-h100-pool`, `gpu_1_queue`, or `gpu_4_queue`). Each step lists the smallest set of prefixes that should trigger it—typically:
+  - pytest file(s) exercised by the job (online and/or offline);
+  - model code under `vllm_omni/model_executor/models/` or `vllm_omni/diffusion/models/`;
+  - related `vllm_omni/model_executor/stage_input_processors/` and `vllm_omni/deploy/*.yaml` when applicable.
+
+Adding a new E2E step: add `source_file_dependencies` on the leaf job with those prefixes. Prefer **per-step** deps rather than a broad group-level list unless every child shares the same paths.
+
+#### YAML examples
+
+H100 E2E (kubernetes / `mithril-h100-pool`):
+
+```yaml
+      - label: "Diffusion · Qwen Image Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen_image.py
+          - vllm_omni/diffusion/models/qwen_image/
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' ...
+        agents:
+          queue: "mithril-h100-pool"
+```
+
+Docker E2E (`gpu_1_queue` / `gpu_4_queue`)—same key, same upload-time filtering:
+
+```yaml
+      - label: "TTS · Qwen3-TTS CustomVoice Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
+          - vllm_omni/model_executor/models/qwen3_tts/
+          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+          - vllm_omni/deploy/qwen3_tts.yaml
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py ...
+        agents:
+          queue: "gpu_4_queue"
+```
+
+A **group** may also define `source_file_dependencies`; nested steps inherit filtering as a unit—the whole group is dropped if no prefix matches.
+
+#### Local dry-run
+
+```bash
+# Render filtered YAML to stdout (no upload)
+python3 .buildkite/scripts/upload_pipeline.py .buildkite/test-ready.yml
+
+# Confirm uploader-only keys are stripped
+python3 .buildkite/scripts/upload_pipeline.py .buildkite/test-merge.yml | grep source_file_dependencies
+# (no output expected)
+```
+
+On a PR build, Buildkite logs from `upload_pipeline.py` include lines such as `skip '…' (no changes under …)` for omitted steps.
+
+#### Related
+
+- Implementation: `.buildkite/scripts/upload_pipeline.py`
+- L2/L3 diff skip does **not** replace label-based triggers (`ready`, `merge-test`); it only reduces which steps appear **after** the pipeline is already scheduled.
+
+### Test helper environment variables
+
+Some shared helpers under `tests/helpers/` honor optional environment variables for local debugging. These are **not** set in CI by default.
+
+| Variable | Accepted values | Description |
+| -------- | --------------- | ----------- |
+| `VLLM_OMNI_KEEP_REQUEST_MEDIA` | `1`, `true`, `yes` (case-insensitive) | When enabled, temporary WAV files created by `tests.helpers.media.convert_audio_bytes_to_text` are **not** deleted when the pytest process exits. By default, each call writes a unique file under the system temp directory via `tempfile.mkstemp` and registers `atexit` cleanup. Use this when debugging audio output validation (Whisper transcription, keyword checks, text–audio similarity). The saved path is logged as `audio data is saved: <path>`. |
+
+Example (Linux / macOS):
+
+```bash
+export VLLM_OMNI_KEEP_REQUEST_MEDIA=1
+pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -k test_mix_to_text_audio
+```
+
+Example (Windows PowerShell):
+
+```powershell
+$env:VLLM_OMNI_KEEP_REQUEST_MEDIA = "1"
+pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -k test_mix_to_text_audio
+```
+
 ## Chapter 1: L1 & L2 Level Testing - Unit Testing and Basic End-to-End Verification
 
 ### 1.1 Testing Purpose
@@ -302,6 +462,7 @@ A clear directory structure is key to managing test cases efficiently.
 ### 1.4 Execution Method and Example
 
 -   ***Trigger Timing***: **`PR with ready label`**. That is, when a developer adds a "ready for review" or similar label to a PR on platforms like GitHub, L1 and L2 tests are automatically triggered.
+-   ***Diff-aware step skipping***: On L2, **E2E Test** jobs may be omitted at pipeline upload when the PR diff does not touch their [`source_file_dependencies`](#diff-aware-buildkite-uploads-source_file_dependencies) prefixes (H100 and docker queues alike); non-E2E groups still always upload. See [Diff-aware Buildkite uploads](#diff-aware-buildkite-uploads-source_file_dependencies).
 -   ***Execution Environment***: L1 uses ***CPU*** environment; L2 requires ***GPU*** environment.
 -   ***Script Example***:
 
@@ -404,7 +565,8 @@ L3 level testing executes after code is merged into the main branch. Its core pu
 
 ### 2.4 Execution Method and Example
 
--   ***Trigger Timing***: **`PR Merged`**. Automatically triggered after code review is approved and merged into the main branch.
+-   ***Trigger Timing***: **`PR Merged`**. Automatically triggered after code review is approved and merged into the main branch (typically via `merge-test` label on the PR before merge).
+-   ***Diff-aware step skipping***: Same [`source_file_dependencies`](#diff-aware-buildkite-uploads-source_file_dependencies) mechanism as L2—**all E2E Test** leaf steps are diff-gated; other L3 groups always upload. See [Diff-aware Buildkite uploads](#diff-aware-buildkite-uploads-source_file_dependencies).
 -   ***Execution Environment***: ***GPU*** servers.
 -   ***Script Example***:
 
@@ -520,6 +682,8 @@ L3 level testing executes after code is merged into the main branch. Its core pu
 
     **Implicit Validation**: The `send_omni_request` and `send_diffusion_request` methods internally includes validation logic dynamically selected based on the --run-level parameter: core_model performs basic validation, while advanced_model and full_model perform deep validation.
 
+    **Audio output debugging**: Deep validation may transcribe returned audio via `convert_audio_bytes_to_text` (Whisper). If an audio keyword or text–audio similarity assertion fails, set `VLLM_OMNI_KEEP_REQUEST_MEDIA=1` before running pytest to keep the intermediate WAV files for inspection (see [Test helper environment variables](#test-helper-environment-variables)).
+
 -   ***Run Command (L3 merge)***: `pytest -s -v /tests/e2e/online_serving/test_{model_name}.py -m advanced_model --run-level=advanced_model`
 
 -   ***Run Command (L4 nightly expansion)***: `pytest -s -v /tests/e2e/online_serving/test_{model_name}_expansion.py -m full_model --run-level=full_model`

docs/contributing/ci/failures.md

@@ -69,6 +69,8 @@ If the failure is already tracked, leave a comment on your PR noting that the fa
 
 If the failure appears to be new, investigate whether your changes caused it.
 
+For resilience behavior under injected process signals/OOM pressure (validation scenarios, not root-cause equivalence), see [Failure Mode Injection Scenarios and Expected Behavior Matrix](../../user_guide/fault_injection_reliability_matrix.md).
+
 ### Reading Buildkite Logs
 
 1. Click **Details** next to the Buildkite check on your PR.

docs/design/feature/disaggregated_inference.md

@@ -8,6 +8,7 @@ Backend-specific setup lives in separate docs:
 - [SharedMemoryConnector](omni_connectors/shared_memory_connector.md)
 - [MooncakeStoreConnector](omni_connectors/mooncake_store_connector.md)
 - [MooncakeTransferEngineConnector](omni_connectors/mooncake_transfer_engine_connector.md)
+- [MoriTransferEngineConnector](omni_connectors/mori_transfer_engine_connector.md)
 - [YuanrongConnector](omni_connectors/yuanrong_connector.md)
 - [YuanrongTransferEngineConnector](omni_connectors/yuanrong_transfer_engine_connector.md)
 
@@ -23,6 +24,7 @@ Current connectors operate in D2H2D (device to host to device) mode.
 | Single node | SharedMemoryConnector | Auto-configured if no connector is specified. |
 | Multi node (Mooncake Store) | MooncakeStoreConnector | TCP-based, requires Mooncake Master + metadata server. |
 | Multi node (Mooncake RDMA) | MooncakeTransferEngineConnector | RDMA/TCP direct transfer with managed memory pool. Fastest. |
+| Multi node (Mori RDMA) | MoriTransferEngineConnector | RDMA direct transfer via Mori IOEngine. |
 | Multi node (Yuanrong) | YuanrongConnector | Requires Yuanrong Datasystem + etcd. |
 | Ascend NPU P2P (Yuanrong TE) | YuanrongTransferEngineConnector | Uses Yuanrong TransferEngine directly. Configure NPU device IPv4 and `memory_pool_device: "npu"`. |
 

docs/design/feature/omni_connectors/mori_transfer_engine_connector.md

@@ -0,0 +1,76 @@
+# MoriTransferEngineConnector
+
+## When to Use
+
+Currently supports intra-node deployment with Mori.
+
+As noted in #1742, inter-node support will be added back in a future
+refactor.
+
+## Mechanism
+
+Uses Mori's `IOEngine` / `MemoryDesc` API for zero-copy RDMA transfers.
+
+- Data Plane: RDMA (InfiniBand/RoCE) with managed memory pool.
+- Control Plane: ZMQ for pull-request handshake and async completion.
+
+## Installation
+
+See the [Mori repository](https://github.com/ROCm/mori) for installation instructions.
+
+## Configuration
+
+Mori is configured through the new deploy-config schema (see
+[`docs/configuration/stage_configs.md`](../../../configuration/stage_configs.md)).
+Define the connector at the top level of the deploy YAML and reference it
+by name from each stage's `input_connectors` / `output_connectors`:
+
+```yaml
+connectors:
+  mori_connector:
+    name: MoriTransferEngineConnector
+    extra:
+      host: "auto"
+      zmq_port: 50051
+      device_name: ""
+      memory_pool_size: 536870912
+      memory_pool_device: "cuda"
+
+stages:
+  - stage_id: 0
+    output_connectors:
+      to_stage_1: mori_connector
+
+  - stage_id: 1
+    input_connectors:
+      from_stage_0: mori_connector
+```
+
+A ready-to-run intra-node example for Qwen3-Omni-MoE on AMD MI300X lives
+at `vllm_omni/deploy/qwen3_omni_moe_mori_intranode.yaml` and can be loaded
+with:
+
+```bash
+vllm-omni serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --log-stats \
+    --deploy-config vllm_omni/deploy/qwen3_omni_moe_mori_intranode.yaml
+```
+
+The yaml wires `MoriTransferEngineConnector` (with `backend_type: xgmi`) to
+the chunk_transfer_adapter path (`async_chunk: true`) so stage-to-stage
+hidden-state and codec-frame streams ship GPU-to-GPU over AMD Infinity
+Fabric instead of SHM.  Qwen2.5-Omni + Mori is not yet functional on
+the chunk path: it needs `thinker2talker_async_chunk` /
+`talker2code2wav_async_chunk` input processors that do not exist yet
+(the orchestrator-level path the upstream PR originally targeted was
+lost during the entrypoints → engine refactor and removed in #1742).
+
+Parameters:
+
+- host: local RDMA IP (`"auto"` for auto-detect).
+- zmq_port: ZMQ base port for control-plane communication.
+- device_name: RDMA device (e.g., `"mlx5_0"`), empty for auto-detect.
+- memory_pool_size: RDMA memory pool size in bytes.
+- memory_pool_device: `"cpu"` (pinned) or `"cuda"` (GPUDirect / XGMI RDMA).
+
+For more details, refer to the
+[Mori repository](https://github.com/ROCm/mori).

docs/design/module/dit_module.md

@@ -305,8 +305,8 @@ class GPUWorker:
         )
 
         # Load model
-        model_loader = DiffusersPipelineLoader(load_config)
-        self.pipeline = model_loader.load_model(od_config, load_device=f"cuda:{rank}")
+        model_loader = DiffusersPipelineLoader(load_config, self.od_config)
+        self.pipeline = model_loader.load_model(load_device=f"cuda:{rank}")
 
         # Setup cache backend
         from vllm_omni.diffusion.cache.selector import get_cache_backend

docs/getting_started/installation/gpu.md

@@ -34,7 +34,7 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
 
 ### Pre-built wheels
 
-Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, and 0.22.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
+Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, and 0.23.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
 
 === "NVIDIA CUDA"
 
@@ -94,6 +94,10 @@ Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc
 
 ### Build your own docker image
 
+=== "NVIDIA CUDA"
+
+    --8<-- "docs/getting_started/installation/gpu/cuda.inc.md:build-docker"
+
 === "AMD ROCm"
 
     --8<-- "docs/getting_started/installation/gpu/rocm.inc.md:build-docker"

docs/getting_started/installation/gpu/cuda.inc.md

@@ -20,7 +20,7 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new**
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm==0.22.0 --torch-backend=auto
+uv pip install vllm==0.23.0 --torch-backend=auto
 ```
 
 #### Installation of vLLM-Omni
@@ -39,13 +39,13 @@ uv pip install 'vllm-omni[demo]'
 # --8<-- [start:build-wheel-from-source]
 
 #### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.22.0 release version of the library
+If you do not need to modify source code of vLLM, you can directly install the stable 0.23.0 release version of the library
 
 ```bash
-uv pip install vllm==0.22.0 --torch-backend=auto
+uv pip install vllm==0.23.0 --torch-backend=auto
 ```
 
-The 0.22.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The 0.23.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
 
 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -66,12 +66,12 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.22.0
+git checkout v0.23.0
 ```
 Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
 ```bash
-#For CUDA 13.0 (the default for v0.22.0; the wheel filename has no `+cu130` suffix)
-export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.22.0/vllm-0.22.0-cp38-abi3-manylinux_2_35_x86_64.whl
+#For CUDA 13.0 (the default for v0.23.0; the wheel filename has no `+cu130` suffix)
+export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.23.0/vllm-0.23.0-cp38-abi3-manylinux_2_28_x86_64.whl
 ```
 Install vllm with command below (If you have no existing PyTorch).
 ```bash
@@ -102,7 +102,7 @@ docker run --runtime nvidia --gpus 2 \
     --env "HF_TOKEN=$HF_TOKEN" \
     -p 8091:8091 \
     --ipc=host \
-    vllm/vllm-omni:v0.22.0 \
+    vllm/vllm-omni:v0.23.0 \
     vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```
 
@@ -110,3 +110,53 @@ docker run --runtime nvidia --gpus 2 \
     The CUDA image does not define a default entrypoint, so include `vllm serve ... --omni` after the image name.
 
 # --8<-- [end:pre-built-images]
+
+# --8<-- [start:build-docker]
+
+#### Build docker image
+
+```bash
+DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.cuda -t vllm-omni-cuda .
+```
+
+If you want to specify the base vLLM version:
+
+```bash
+DOCKER_BUILDKIT=1 docker build \
+  -f docker/Dockerfile.cuda \
+  --build-arg BASE_IMAGE=vllm/vllm-openai:v0.22.1 \
+  -t vllm-omni-cuda .
+```
+
+#### Launch the docker image
+
+##### Launch with OpenAI API Server
+
+!!! note
+    The model `Qwen/Qwen3-Omni-30B-A3B-Instruct` requires significant GPU memory. The example below has been verified on 2 x H100's.
+
+```bash
+docker run --runtime nvidia --gpus 2 \
+  -v ${HF_HOME:-$HOME/.cache/huggingface}:/root/.cache/huggingface \
+  --env "HF_TOKEN=$HF_TOKEN" \
+  -p 8091:8091 \
+  --ipc=host \
+  vllm-omni-cuda \
+  vllm serve --omni --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 8091
+```
+
+By default, this mounts `$HOME/.cache/huggingface` as the model cache directory. To use a custom location, set the `HF_HOME` environment variable before running the command (e.g., `export HF_HOME=/data/models`).
+
+##### Launch with interactive session for development
+
+```bash
+docker run --runtime nvidia --gpus all -it --rm \
+  -v ${HF_HOME:-$HOME/.cache/huggingface}:/root/.cache/huggingface \
+  --env "HF_TOKEN=$HF_TOKEN" \
+  -p 8091:8091 \
+  --ipc=host \
+  --entrypoint bash \
+  vllm-omni-cuda
+```
+
+# --8<-- [end:build-docker]

docs/getting_started/installation/gpu/rocm.inc.md

@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm==0.22.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.22.0/rocm722
+uv pip install vllm==0.23.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.23.0/rocm722
 ```
 
 #### Installation of vLLM-Omni
@@ -34,13 +34,13 @@ uv pip install onnxruntime-rocm
 # --8<-- [start:build-wheel-from-source]
 
 #### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.22.0 release version of the library
+If you do not need to modify source code of vLLM, you can directly install the stable 0.23.0 release version of the library
 
 ```bash
-uv pip install vllm==0.22.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.22.0/rocm722
+uv pip install vllm==0.23.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.23.0/rocm722
 ```
 
-The pre-built 0.22.0 vLLM wheel targets ROCm 7.2.2. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The pre-built 0.23.0 vLLM wheel targets ROCm 7.2.2. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
 
 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -58,7 +58,7 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.22.0
+git checkout v0.23.0
 python3 -m pip install -r requirements/rocm.txt
 python3 setup.py develop
 ```
@@ -130,7 +130,7 @@ docker run --rm \
   -v ~/.cache/huggingface:/root/.cache/huggingface \
   --env "HF_TOKEN=$HF_TOKEN" \
   -p 8091:8091 \
-  vllm/vllm-omni-rocm:v0.22.0 \
+  vllm/vllm-omni-rocm:v0.23.0 \
   --model Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```
 
@@ -149,7 +149,7 @@ docker run --rm -it \
   -v ~/.cache/huggingface:/root/.cache/huggingface \
   --env "HF_TOKEN=$HF_TOKEN" \
   --entrypoint bash \
-  vllm/vllm-omni-rocm:v0.22.0
+  vllm/vllm-omni-rocm:v0.23.0
 ```
 
 # --8<-- [end:pre-built-images]

docs/getting_started/quickstart.md

@@ -19,10 +19,10 @@ uv venv --python 3.12 --seed
 source .venv/bin/activate
 
 # On CUDA
-uv pip install vllm==0.22.0 --torch-backend=auto
+uv pip install vllm==0.23.0 --torch-backend=auto
 
 # On ROCm
-uv pip install vllm==0.22.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.22.0/rocm721
+uv pip install vllm==0.23.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.23.0/rocm722
 
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
@@ -35,7 +35,7 @@ For additional installation methods — please see the [installation guide](inst
 !!! note
     It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.
 
-    If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.22.0` and vLLM Omni `0.22.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
+    If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.23.0` and vLLM Omni `0.23.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
 
 ## Offline Inference
 

docs/models/supported_models.md

@@ -6,6 +6,8 @@ vLLM-Omni supports unified multimodal comprehension and generation models across
 
 If vLLM-Omni natively supports a model, its implementation can be found in <gh-file:vllm_omni/model_executor/models> and <gh-file:vllm_omni/diffusion/models>.
 
+For deployment recipes (TTS and multimodal), see [recipes.vllm.ai](https://recipes.vllm.ai) and the in-repo [`recipes/`](https://github.com/vllm-project/vllm-omni/tree/main/recipes) directory.
+
 ## List of Supported Models
 
 <style>
@@ -22,6 +24,7 @@ th {
 | `MingFlashOmniForConditionalGeneration` + `MingImagePipeline` | Ming-flash-omni-2.0 (omni-speech + imagegen<sup>1</sup>) | `Jonathan1909/Ming-flash-omni-2.0` | ✅︎ |   |   |   |
 | `BagelForConditionalGeneration` | BAGEL (DiT-only) | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ | | ✅︎ |
 | `InternVLAA1Pipeline` | InternVLA-A1 | `InternRobotics/InternVLA-A1-3B` | ✅︎ | ✅︎ | | |
+| `Gr00tN1d7Pipeline` | GR00T N1.7 | `nvidia/GR00T-N1.7-3B` | ✅︎ | | | |
 | `HunyuanImage3ForCausalMM` | HunyuanImage3.0 (DiT-only) | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `QwenImagePipeline` | Qwen-Image-2512 | `Qwen/Qwen-Image-2512` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
@@ -33,7 +36,7 @@ th {
 | `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V, T2V with sound, action policy | `nvidia/Cosmos3-Nano` | ✅︎ | | | |
+| `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V, V2V, T2V with sound, action policy | `nvidia/Cosmos3-Nano` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `WanSpeechToVideoPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `LTX2Pipeline` | LTX-2-T2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
@@ -42,14 +45,17 @@ th {
 | `LTX2ImageToVideoTwoStagesPipeline` | LTX-2-I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
 | `LTX23Pipeline` | LTX-2.3-T2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
 | `LTX23ImageToVideoPipeline` | LTX-2.3-I2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
-| `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | | | |
+| `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | |
 | `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | |
 | `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ |
 | `LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
+| `StableDiffusionXLPipeline` | Stable-Diffusion-XL | `stabilityai/stable-diffusion-xl-base-1.0` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ |
 | `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ |
+| `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | |
+| `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | |
 | `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | |
 | `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | |
@@ -60,8 +66,14 @@ th {
 | `AudioXPipeline` | AudioX | `zhangj1an/AudioX` | ✅︎ | ✅︎ | | |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
+| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
+| `MingTTSForConditionalGeneration` | Ming-omni-tts dense 0.5B | `inclusionAI/Ming-omni-tts-0.5B` | ✅︎ | | | |
 | `GLMTTSForConditionalGeneration` | GLM-TTS | `zai-org/GLM-TTS` | ✅︎ | | | |
+| `MossTTSNanoForCausalLM` | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✅︎ | | | |
+| `MossTTSDelayModel` | MOSS-TTS, MOSS-TTSD, MOSS-SoundEffect, MOSS-VoiceGenerator | `OpenMOSS-Team/MOSS-VoiceGenerator` | ✅︎ | | | |
+| `MossTTSRealtime` | MOSS-TTS-Realtime | `OpenMOSS-Team/MOSS-TTS-Realtime` | ✅︎ | | | |
+| `HiggsAudioV2ForConditionalGeneration` | Higgs-Audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` | ✅︎ | | | |
+| `HiggsMultimodalQwen3ForConditionalGeneration` | Higgs-Audio v3 (TTS) | `bosonai/higgs-audio-v3-tts-4b` | ✅︎ | | | |
 | `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ |
 | `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | |
 | `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | |
@@ -69,11 +81,13 @@ th {
 | `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | |
 | `DreamIDOmniPipeline` | DreamID-Omni | `XuGuo699/DreamID-Omni` | ✅︎ | ✅︎ | | |
 | `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only) | `SenseNova/SenseNova-U1-8B-MoT` | ✅︎ | | | |
+| `LancePipeline` | Lance | `bytedance-research/Lance` | ✅︎ | | | |
 | `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | |
 | `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | |
 | `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | |
 | `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | |
 |`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | |
+| `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | |
 | `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 |`HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | |
 

docs/serving/speech_api.md

@@ -9,6 +9,13 @@ vLLM-Omni provides an OpenAI-compatible API for text-to-speech (TTS) generation.
 
 See the [Supported Models](#supported-models) section below for the full list, including OmniVoice, VoxCPM2, and MOSS-TTS-Nano.
 
+!!! tip "Deployment recipes"
+    TTS deployment recipes are published at
+    [recipes.vllm.ai](https://recipes.vllm.ai) (e.g.
+    [Qwen3-TTS](https://recipes.vllm.ai/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice),
+    [Higgs-Audio v3](https://recipes.vllm.ai/bosonai/higgs-audio-v3-tts-4b)).
+    The in-repo runbooks live under [`recipes/`](https://github.com/vllm-project/vllm-omni/tree/main/recipes).
+
 Each server instance runs a single model (specified at startup via `vllm serve <model> --omni`).
 
 ## Quick Start
@@ -114,9 +121,10 @@ Content-Type: application/json
 | `instructions` | string | "" | Voice style/emotion instructions |
 | `max_new_tokens` | integer | 2048 | Maximum tokens to generate |
 | `initial_codec_chunk_frames` | integer | null | Per-request initial chunk size override for TTFA tuning. When null, IC is computed dynamically based on server load. |
+| `non_streaming_mode` | bool | null | Qwen3-TTS prompt construction mode override. Does not affect HTTP response streaming or async-chunk pipelining. When null, Qwen3-TTS uses model defaults: Base=false, CustomVoice/VoiceDesign=true. |
 | `stream` | bool | false | Stream raw PCM chunks as they are decoded (requires `response_format="pcm"`) |
 
-**Supported languages:** Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian
+**Supported languages:** Only applicable to Qwen3-TTS. Derived from the model configuration (`talker_config.codec_language_id` in the checkpoint's `config.json`), plus `Auto`, which is always accepted. Official Qwen3-TTS checkpoints support: Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian.
 
 #### Voice Clone Parameters (Base task)
 
@@ -332,7 +340,8 @@ curl -X POST http://localhost:8091/v1/audio/speech \
         "input": "Hello, this is a cloned voice",
         "task_type": "Base",
         "ref_audio": "https://example.com/reference.wav",
-        "ref_text": "Original transcript of the reference audio"
+        "ref_text": "Original transcript of the reference audio",
+        "non_streaming_mode": true
     }' --output cloned.wav
 ```
 
@@ -454,6 +463,7 @@ Content-Type: application/json
 | `ref_audio` | string | null | Default reference audio (Base task) |
 | `ref_text` | string | null | Default reference transcript (Base task) |
 | `max_new_tokens` | integer | null | Default max tokens |
+| `non_streaming_mode` | bool | null | Default Qwen3-TTS prompt construction mode override. Does not affect HTTP response streaming or async-chunk pipelining. When null, Qwen3-TTS uses model defaults: Base=false, CustomVoice/VoiceDesign=true. |
 
 Each item in the `items` array requires only `input` (the text). All other fields are optional and override the batch-level defaults when set:
 
@@ -469,6 +479,7 @@ Each item in the `items` array requires only `input` (the text). All other field
 | `ref_audio` | string | Override reference audio |
 | `ref_text` | string | Override reference transcript |
 | `max_new_tokens` | integer | Override max tokens |
+| `non_streaming_mode` | bool | Override Qwen3-TTS prompt construction mode. Does not affect HTTP response streaming or async-chunk pipelining. When null, inherits the batch-level value (then the model default). |
 
 ### Response Format
 

docs/serving/videos_api.md

@@ -67,8 +67,10 @@ curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4
 
 | Parameter | Type | Default | Description |
 |-----------|------|---------|-------------|
-| `input_reference` | file | null | Uploaded reference image for image-to-video requests |
-| `image_reference` | string | null | JSON-encoded reference image payload; do not combine with `input_reference` |
+| `input_reference` | file | null | Uploaded reference image or video for image-to-video/video-to-video requests |
+| `image_reference` | string | null | JSON-encoded reference image payload; do not combine with `input_reference` or `video_reference` |
+| `video_reference` | string | null | JSON-encoded reference video payload; do not combine with `input_reference` or `image_reference` |
+| `audio_reference` | string | null | JSON-encoded audio reference for speech-to-video: `{"audio_url": "..."}` — supports HTTP(s) URLs or base64 data URLs |
 | `width` | integer | model default | Output video width |
 | `height` | integer | model default | Output video height |
 | `num_frames` | integer | model default | Number of generated frames |
@@ -80,6 +82,8 @@ curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4
 | `flow_shift` | number | null | Scheduler flow-shift value |
 | `true_cfg_scale` | number | null | True CFG scale when supported by the model |
 | `seed` | integer | null | Random seed for reproducibility |
+| `generate_sound` | boolean | false | Request model-generated audio for video models that support sound generation |
+| `sound_duration` | number | null | Duration in seconds for generated audio; defaults to generated video duration |
 | `negative_prompt` | string | null | Text describing what to avoid in the generated video |
 | `enable_frame_interpolation` | boolean | null | Enable post-generation frame interpolation |
 | `frame_interpolation_exp` | integer | null | Interpolation exponent; `1=2x`, `2=4x`, and so on |
@@ -123,6 +127,60 @@ curl -s http://localhost:8091/v1/videos \
   -F "fps=16"
 ```
 
+### Video-to-Video
+
+For models that support video conditioning, upload the reference video with
+`input_reference`:
+
+```bash
+curl -s http://localhost:8091/v1/videos \
+  -F "prompt=continue this motion with consistent subjects and lighting" \
+  -F "input_reference=@input.mp4;type=video/mp4" \
+  -F "width=1280" \
+  -F "height=720" \
+  -F "num_frames=80" \
+  -F "fps=16"
+```
+
+You can also pass a JSON-safe video URL or `data:video/...;base64,...` payload
+through `video_reference`. Do not send `video_reference` together with
+`input_reference` or `image_reference`.
+
+```bash
+curl -s http://localhost:8091/v1/videos \
+  -F "prompt=continue this motion with consistent subjects and lighting" \
+  -F 'video_reference={"video_url":"https://example.com/input.mp4"}' \
+  -F "width=1280" \
+  -F "height=720" \
+  -F "num_frames=80" \
+  -F "fps=16"
+```
+
+JSON references currently support `image_url`/`video_url`; `file_id` references
+are not implemented yet. Models may expose additional V2V controls through
+`extra_params`. For example, Cosmos3 supports
+`condition_frame_indexes_vision` and `condition_video_keep` to select which
+decoded reference frames are used as clean conditioning.
+
+### Speech-to-Video
+
+For models that support audio-driven generation (e.g., Wan2.2-S2V), pass both
+an image reference and an audio reference. The `audio_reference` field accepts a
+JSON string with `audio_url` pointing to an HTTP(s) URL or base64 data URL.
+
+```bash
+curl -s http://localhost:8091/v1/videos \
+  -F "prompt=A person singing" \
+  -F 'image_reference={"image_url": "https://example.com/face.png"}' \
+  -F 'audio_reference={"audio_url": "https://example.com/speech.mp3"}' \
+  -F "width=832" \
+  -F "height=480" \
+  -F "num_inference_steps=40" \
+  -F "guidance_scale=4.5" \
+  -F "fps=16"
+```
+
+
 ### Synchronous Generation
 
 ```bash
@@ -137,16 +195,24 @@ curl -X POST http://localhost:8091/v1/videos/sync \
 
 ## Storage
 
-Set `VLLM_OMNI_STORAGE_PATH` to control where asynchronous video outputs are
+Set `VLLM_OMNI_SERVER_STORAGE__PATH` to control where asynchronous video outputs are
 stored:
 
 ```bash
-export VLLM_OMNI_STORAGE_PATH=/var/tmp/vllm-omni-videos
+export VLLM_OMNI_SERVER_STORAGE__PATH=/var/tmp/vllm-omni-videos
 ```
 
+> `VLLM_OMNI_STORAGE_PATH` is deprecated and will be removed in a future release;
+> use `VLLM_OMNI_SERVER_STORAGE__PATH` instead.
+
 ## Model-Specific Examples
 
-For complete text-to-video and image-to-video walkthroughs, see:
+For complete text-to-video, image-to-video, and model-specific video-to-video
+walkthroughs, see:
 
 - [Text-to-Video](../user_guide/examples/online_serving/text_to_video.md)
 - [Image-to-Video](../user_guide/examples/online_serving/image_to_video.md)
+- [Speech-to-Video](../user_guide/examples/online_serving/speech_to_video.md)
+  for Wan2.2-S2V audio-driven lip-sync generation
+- [Cosmos3 recipes](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-Nano.md)
+  for model-specific video-to-video examples and conditioning controls

docs/user_guide/diffusion/cpu_offload_diffusion.md

@@ -193,6 +193,7 @@ Factory function `get_offload_backend()` selects the appropriate backend based o
 | NextStep11Pipeline | `stepfun-ai/NextStep-1.1` | `NextStepModel` | - | ✓ | `"layers"` |
 | OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | - | ✓ | `"transformer"` |
 | QwenImagePipeline | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | ✓ | ✓ | `"transformer_blocks"` |
+| StableDiffusionXLPipeline | `stabilityai/stable-diffusion-xl-base-1.0` | `SDXLUNet2DConditionModel` | ✓ | ✓ | `"down_blocks"`, `"up_blocks"` |
 | StableDiffusion3Pipeline | `stabilityai/stable-diffusion-3.5-medium` | `SD3Transformer2DModel` | - | ✓ | `"transformer_blocks"` |
 | Wan22I2VPipeline | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | `"blocks"` |
 | Wan22Pipeline | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | `"blocks"` |

docs/user_guide/diffusion/mot_config.md

@@ -0,0 +1,50 @@
+This directory contains auto-tuned Triton kernel configurations for the
+MoT (Mixture-of-Tokens) GEMM and RMSNorm operators used by BAGEL and other
+MoT-architecture diffusion models.
+
+File naming convention:
+    device_name=<GPU_NAME>,dtype=<DTYPE>.json
+
+For example:
+    device_name=NVIDIA_A100-SXM4-80GB,dtype=w16a16.json
+
+Each JSON file maps (K, N) matrix shapes to a dictionary of batch sizes (M)
+and their optimal Triton tile configurations:
+
+    {
+        "3584_4608": {              // K=3584, N=4608 (QKV projection)
+            "1024": {
+                "BLOCK_SIZE_M": 64,
+                "BLOCK_SIZE_N": 128,
+                "BLOCK_SIZE_K": 64,
+                "GROUP_SIZE_M": 8,
+                "num_warps": 4,
+                "num_stages": 3
+            },
+            ...
+        },
+        "3584_3584": { ... }        // K=3584, N=3584 (output projection)
+    }
+
+Config loading order (3-tier, see ops/mot_gemm.py):
+    1. $VLLM_TUNED_CONFIG_FOLDER/<filename>  (env override)
+    2. This directory: vllm_omni/diffusion/layers/mot/configs/<filename>
+    3. Conservative default config (compiles everywhere, sub-optimal perf)
+
+If no config file matches the current device, a warning is printed with
+instructions to run the auto-tuning benchmark.
+
+To generate configs for your hardware:
+
+    python benchmarks/kernels/mot_linear_benchmarks.py \
+        --model ByteDance-Seed/BAGEL-7B-MoT \
+        --tp-size 1 --dtype w16a16 --tune \
+        --save-dir vllm_omni/diffusion/layers/mot/configs/
+
+For multi-GPU tuning (uses Ray for parallel search):
+
+    python benchmarks/kernels/mot_linear_benchmarks.py \
+        --model ByteDance-Seed/BAGEL-7B-MoT \
+        --tp-size 2 --tune
+
+See benchmarks/kernels/mot_linear_benchmarks.py for full options.

docs/user_guide/diffusion/multi_replica_serving.md

@@ -0,0 +1,80 @@
+# Diffusion Multi-Replica Serving
+
+Diffusion stages can be replicated in online serving by setting `runtime.num_replicas` on a `stage_type: diffusion` stage. Each replica owns an independent diffusion engine process and receives requests through round-robin routing with request affinity.
+
+## When to Use
+
+Use multi-replica serving when one diffusion process does not saturate the available GPUs, or when you want higher request throughput for many independent image/video generation requests. It improves concurrency throughput; it does not make a single request faster.
+
+## Configuration
+
+A replica consumes the devices required by that diffusion stage. For single-device diffusion pipelines, set `runtime.devices` to one device per replica:
+
+```yaml
+stages:
+  - stage_id: 0
+    stage_type: diffusion
+    runtime:
+      devices: "0,1,2,3"
+      num_replicas: 4
+    engine_args:
+      model: Qwen/Qwen-Image
+      parallel_config:
+        world_size: 1
+```
+
+For multi-device diffusion pipelines, `parallel_config.world_size` is treated as the number of devices per replica. For example, four 2-GPU replicas need eight devices:
+
+```yaml
+stages:
+  - stage_id: 0
+    stage_type: diffusion
+    runtime:
+      devices: "0,1,2,3,4,5,6,7"
+      num_replicas: 4
+    engine_args:
+      model: Wan-AI/Wan2.2-T2V-A14B-Diffusers
+      parallel_config:
+        world_size: 2
+```
+
+The runtime splits the device list into contiguous chunks per replica. With the second example, replica 0 uses `0,1`, replica 1 uses `2,3`, replica 2 uses `4,5`, and replica 3 uses `6,7`.
+
+## Serving
+
+Start the server with the stage config:
+
+```bash
+vllm serve Qwen/Qwen-Image \
+  --omni \
+  --port 8099 \
+  --stage-configs-path /path/to/diffusion_multi_replica.yaml
+```
+
+Then send normal OpenAI-compatible diffusion chat requests. The orchestrator routes new requests across replicas and keeps updates/abort calls on the same replica for each request.
+
+## Benchmarking
+
+Benchmark each replica count by changing `runtime.num_replicas` and `runtime.devices`, restarting the server, then running:
+
+```bash
+python benchmarks/diffusion/diffusion_benchmark_serving.py \
+  --base-url http://localhost:8099 \
+  --model Qwen/Qwen-Image \
+  --task t2i \
+  --dataset random \
+  --num-prompts 8 \
+  --request-rate inf \
+  --num-inference-steps 2 \
+  --width 256 --height 256 \
+  --output-file diffusion_replica_4.json
+```
+
+For a quick scaling sweep, test `num_replicas` values `1`, `2`, `3`, and `4` with the same prompt count and generation parameters. Compare throughput and end-to-end latency from the JSON outputs.
+
+## Notes and Limitations
+
+- Multi-replica serving is currently supported for diffusion stages. Single-stage mode still rejects non-diffusion replicas.
+- `runtime.devices` must provide exactly `num_replicas * devices_per_replica` devices.
+- Request-level latency may stay similar or slightly increase because each request still runs on one replica.
+- Throughput improves only when enough concurrent requests are issued and each replica has enough GPU memory.

docs/user_guide/diffusion_features.md

@@ -130,6 +130,7 @@ The following tables show which models support each feature:
 | **Qwen-Image-Edit-2509** |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |        ✅ (decode)         |          ✅           |       ❌        |        ❌         |
 | **Qwen-Image-Layered**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **SenseNova-U1**         |     ❌     |     ✅      |           ❌           |       ❌        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **Stable-Diffusion-XL**  |     ❌     |     ❌      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Stable-Diffusion3.5**  |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Z-Image**              |     ✅     |     ✅      |           ✅           |       ❓        |   ✅ (TP=2 only)   |          ❌          |   ✅    |             ❌             |      ✅ (decode)      |       ✅        |        ❌         |
 | **ERNIE-Image**          |     ❌     |     ✅      |           ✅           |       ❓        |         ✅         |          ❌          |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
@@ -148,7 +149,7 @@ The following tables show which models support each feature:
 | **Wan2.2-S2V**               |     ❌     |     ✅      |           ❌           |       ❌        |         ✅         |         ❌         |   ❌    |             ✅             |  ✅ (encode/decode)   |       ❌        |        ❌         |
 | **Wan2.1-VACE**              |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **LTX-2**                    |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
-| **LTX-2.3**                  |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
+| **LTX-2.3**                  |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ❌    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Helios**                   |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **HunyuanVideo-1.5 T2V I2V** |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
 | **DreamID-Omni**             |     ❌     |     ❌      |           ❌           |       ✅        |         ❌         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |

docs/user_guide/examples/online_serving/glm_image.md

@@ -19,7 +19,7 @@ both stages on a single GPU, override per stage:
 
 ```bash
 vllm serve zai-org/GLM-Image --omni --port 8091 \
-    --stage-0-devices 0 --stage-1-devices 0
+    --stage-overrides '{"0": {"devices": "0"}, "1": {"devices": "0"}}'
 ```
 
 ## API Calls

docs/user_guide/examples/online_serving/image_to_video.md

@@ -84,16 +84,26 @@ curl -X POST http://localhost:8091/v1/videos/sync \
 Generated video files are stored on local disk by the async video API.
 Local file storage behavior can be controlled via the following environment variables:
 
-- `VLLM_OMNI_STORAGE_PATH`: directory used for generated files (default: `/tmp/storage`)
-- `VLLM_OMNI_STORAGE_MAX_CONCURRENCY`: max concurrent save/delete operations (default: `4`)
+- `VLLM_OMNI_SERVER_STORAGE__PATH`: directory used for generated files
+  (default: `/tmp/storage`)
+- `VLLM_OMNI_SERVER_STORAGE__FILE_CONCURRENCY`: max concurrent save/delete/open
+  operations (default: `4`)
+- `VLLM_OMNI_SERVER_STORAGE__FILE_TTL`: optional TTL for generated files in
+  seconds
+- `VLLM_OMNI_SERVER_STORAGE__TTL_SWEEP_INTERVAL`: optional sweep interval in
+  seconds for enforcing file TTL; defaults to `300` when TTL is set
 
 Example:
 
 ```bash
-export VLLM_OMNI_STORAGE_PATH=/var/tmp/vllm-omni-videos
-export VLLM_OMNI_STORAGE_MAX_CONCURRENCY=8
+export VLLM_OMNI_SERVER_STORAGE__PATH=/var/tmp/vllm-omni-videos
+export VLLM_OMNI_SERVER_STORAGE__FILE_CONCURRENCY=8
+export VLLM_OMNI_SERVER_STORAGE__FILE_TTL=86400
+export VLLM_OMNI_SERVER_STORAGE__TTL_SWEEP_INTERVAL=300
 ```
 
+See also: [Configuration Options](../../../configuration/README.md)
+
 ## API Calls
 
 ### Method 1: Using curl

docs/user_guide/examples/online_serving/speech_to_video.md

@@ -0,0 +1,128 @@
+# Speech-To-Video
+
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/speech_to_video>.
+
+This example demonstrates how to deploy the Wan2.2 speech-to-video (S2V) model for online video generation using vLLM-Omni.
+
+## Supported Models
+
+| Model | Model ID |
+|-------|----------|
+| Wan2.2 S2V (14B) | `Wan-AI/Wan2.2-S2V-14B` |
+
+## Start Server
+
+### Basic Start
+
+```bash
+VLLM_WORKER_MULTIPROC_METHOD=spawn \
+vllm serve Wan-AI/Wan2.2-S2V-14B --omni \
+  --model-class-name WanS2VPipeline \
+  --tensor-parallel-size 2 \
+  --flow-shift 3.0 \
+  --vae-use-slicing --vae-use-tiling \
+  --cache-backend cache_dit \
+  --port 8091
+```
+
+### Start with Script
+
+```bash
+bash run_server.sh
+```
+
+The script allows overriding:
+- `MODEL` (default: `Wan-AI/Wan2.2-S2V-14B`)
+- `PORT` (default: `8091`)
+- `FLOW_SHIFT` (default: `3.0`)
+- `TP` (default: `2`)
+- `CACHE_BACKEND` (default: `cache_dit`)
+
+### Start with 4 GPUs
+
+```bash
+TP=4 bash run_server.sh
+```
+
+## API Endpoints
+
+The server provides OpenAI-compatible video generation endpoints:
+
+- `POST /v1/videos`: create a video generation job (async)
+- `POST /v1/videos/sync`: generate a video and return raw bytes (sync, for benchmarks)
+- `GET /v1/videos/{video_id}`: retrieve the current job status and metadata
+- `GET /v1/videos`: list stored video jobs
+- `GET /v1/videos/{video_id}/content`: download the generated video file
+- `DELETE /v1/videos/{video_id}`: delete the job and any stored output
+
+## Sync API (Recommended for Testing)
+
+`POST /v1/videos/sync` blocks until generation completes and returns the raw
+video bytes (`video/mp4`) directly in the response body.
+
+```bash
+bash run_curl_speech_to_video.sh
+```
+
+## Async Job API
+
+`POST /v1/videos` creates an asynchronous job and returns immediately.
+
+```bash
+no_proxy=127.0.0.1 \
+create_response=$(curl -sS -X POST http://127.0.0.1:8091/v1/videos \
+  -H "Accept: application/json" \
+  -F "prompt=A person singing" \
+  -F 'image_reference={"image_url": "https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/examples/Five%20Hundred%20Miles.png"}' \
+  -F 'audio_reference={"audio_url": "https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/examples/Five%20Hundred%20Miles.MP3"}' \
+  -F "width=832" -F "height=480" \
+  -F "num_inference_steps=40" \
+  -F "guidance_scale=4.5" \
+  -F "fps=16")
+
+video_id=$(echo "$create_response" | jq -r '.id')
+
+# Poll until complete
+while true; do
+  status=$(curl -s "http://127.0.0.1:8091/v1/videos/${video_id}" | jq -r '.status')
+  if [ "$status" = "completed" ]; then
+    break
+  fi
+  if [ "$status" = "failed" ]; then
+    echo "Video generation failed"
+    exit 1
+  fi
+  sleep 2
+done
+
+# Download
+curl -L "http://127.0.0.1:8091/v1/videos/${video_id}/content" -o s2v_output.mp4
+```
+
+## Request Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `prompt` | str | - | Text description of the desired video |
+| `image_reference` | JSON str | - | Image reference: `{"image_url": "..."}` — supports HTTP(s) URLs or base64 data URLs |
+| `audio_reference` | JSON str | - | Audio reference: `{"audio_url": "..."}` — supports HTTP(s) URLs or base64 data URLs |
+| `width` | int | None | Video width in pixels |
+| `height` | int | None | Video height in pixels |
+| `fps` | int | None | Output video frame rate |
+| `num_inference_steps` | int | None | Number of denoising steps (40 recommended) |
+| `guidance_scale` | float | None | CFG guidance scale |
+| `seed` | int | None | Random seed for reproducibility |
+
+## Notes
+
+- S2V requires both a reference image (`image_reference`) and an audio reference
+  (`audio_reference`). The generated video will show a person matching the reference
+  image with lip movements synchronized to the audio.
+- `audio_reference` accepts a JSON string: `{"audio_url": "..."}` where the URL can be:
+  - An HTTP/HTTPS URL (e.g., `https://example.com/audio.mp3`)
+  - A base64 data URL (e.g., `data:audio/mp3;base64,...`)
+- `--model-class-name WanS2VPipeline` is required on the server to select the
+  S2V pipeline (distinct from the T2V/I2V pipelines).
+- `--cache-backend cache_dit` enables DiT caching for ~2x speedup on cached steps.
+- Audio is muxed into the output MP4 automatically.
+- Always pass `fps=16` to ensure correct video/audio alignment.

docs/user_guide/examples/online_serving/text_to_video.md

@@ -90,16 +90,26 @@ curl -X POST http://localhost:8091/v1/videos/sync \
 Generated video files are stored on local disk by the async video API.
 Local file storage behavior can be controlled via the following environment variables:
 
-- `VLLM_OMNI_STORAGE_PATH`: directory used for generated files (default: `/tmp/storage`)
-- `VLLM_OMNI_STORAGE_MAX_CONCURRENCY`: max concurrent save/delete operations (default: `4`)
+- `VLLM_OMNI_SERVER_STORAGE__PATH`: directory used for generated files
+  (default: `/tmp/storage`)
+- `VLLM_OMNI_SERVER_STORAGE__FILE_CONCURRENCY`: max concurrent save/delete/open
+  operations (default: `4`)
+- `VLLM_OMNI_SERVER_STORAGE__FILE_TTL`: optional TTL for generated files in
+  seconds
+- `VLLM_OMNI_SERVER_STORAGE__TTL_SWEEP_INTERVAL`: optional sweep interval in
+  seconds for enforcing file TTL; defaults to `300` when TTL is set
 
 Example:
 
 ```bash
-export VLLM_OMNI_STORAGE_PATH=/var/tmp/vllm-omni-videos
-export VLLM_OMNI_STORAGE_MAX_CONCURRENCY=8
+export VLLM_OMNI_SERVER_STORAGE__PATH=/var/tmp/vllm-omni-videos
+export VLLM_OMNI_SERVER_STORAGE__FILE_CONCURRENCY=8
+export VLLM_OMNI_SERVER_STORAGE__FILE_TTL=86400
+export VLLM_OMNI_SERVER_STORAGE__TTL_SWEEP_INTERVAL=300
 ```
 
+See also: [Configuration Options](../../../configuration/README.md)
+
 ## API Calls
 
 ### Method 1: Using curl

docs/user_guide/fault_injection_reliability_matrix.md

@@ -0,0 +1,117 @@
+# vLLM-Omni Failure Mode Injection Scenarios and Expected Behavior Matrix
+
+## Background and Goals
+
+The online serving pipeline of vLLM-Omni can be abstracted as: `Serve` (service entry process) -> `Engine` (model orchestration unit) -> `Worker` (execution process) -> `GPU` (compute resource).
+
+- `Serve` provides external APIs, accepts requests, and manages lifecycle.
+- `Engine` handles request scheduling and inference orchestration; the number and organization of engines can vary by model type.
+- `Worker` performs concrete model computation, usually bound to underlying GPU resources.
+- `GPU` is the resource layer that ultimately carries inference compute and memory usage.
+
+<div markdown style="text-align: center">
+
+```mermaid
+%%{init: {'flowchart': {'useMaxWidth': false}}}%%
+flowchart TD
+    R(request)--> A(serve)
+    A --> B(engine)
+    A --> C(engine)
+    A --> D(engine)
+    B --> E(worker)
+    C --> F(worker)
+    D --> I(worker)
+    E --> G(GPU)
+    F --> H(GPU)
+    I --> L(GPU)
+```
+
+</div>
+
+The diagram above shows an abstract relationship. Omni models and Diffusion models share the same hierarchy; differences mainly come from engine decomposition, worker count, and scheduling strategy.
+
+In real customer environments, the system will inevitably encounter accidental operations or environmental disturbances, such as unintended process termination, container/node jitter, or short-term resource contention. From the system perspective, many of these events map to two fault categories: "process receives abnormal signals" or "GPU memory is squeezed." For example, pressing `Ctrl+C` essentially sends a `SIGINT` signal to the `serve` process; running `kill -15 <pid>` maps to graceful termination via `SIGTERM`; running `kill -9 <pid>` maps to forced termination via `SIGKILL`; when `docker kill` is executed or a Pod is deleted, the container runtime usually sends `SIGKILL`.
+
+This document currently covers two core failure modes, and the system is expected to show predictable and observable behavior under each mode:
+
+- **Failure Mode 1: Process receives abnormal signals (`SIGINT` / `SIGTERM` / `SIGKILL`)**  
+  Focus on three aspects: whether processes exit as expected and complete cleanup, whether requests fail fast or connections are interrupted, and whether GPU resources are eventually fully released with no residue.
+- **Failure Mode 2: OOM (occupy all free GPU memory via an extra process)**  
+  Focus on two aspects: whether service health degrades to the expected state (for example, `503`), and whether requests fail within an acceptable time without hanging.
+
+More failure modes (for example, network jitter and network interruption) will be added in future iterations to improve end-to-end reliability validation coverage.
+
+> Note: independent fault injection on the `engine` component is not covered in the current version and will be added gradually in future versions.
+
+## Fault Injection Scenario Matrix
+
+| Scenario | Fault Type | System Behavior | Current Status |
+|------|----------------------------|----------|----------|
+| No load | Send `SIGKILL` to Worker | Worker process is killed immediately; main service detects child-process loss and turns unavailable; API enters stable 5xx |  |
+| No load | Send `SIGTERM` to Worker | Worker exits after receiving termination signal; main service is marked unavailable; API enters stable 5xx |  |
+| No load | Send `SIGKILL` to serve main process | serve main process exits instantly; request connections are interrupted; related child processes are cleaned up with no residue; GPU memory is released quickly | [#3725](https://github.com/vllm-project/vllm-omni/issues/3725) <br>[#43060](https://github.com/vllm-project/vllm/issues/43060) |
+| No load | Send `SIGTERM` to serve main process | serve enters graceful shutdown and stops serving; then exits and completes cleanup; GPU memory is released |  |
+| No load | Send `SIGINT` to serve main process (equivalent to `Ctrl+C`) | Triggers serve shutdown path; service stops responding and becomes unavailable; related child processes exit and resources are released |  |
+| No load | Send `SIGKILL` to all related processes | All related processes terminate immediately; service becomes unavailable at once; no residual processes remain; GPU memory is released quickly |  |
+| No load | Send `SIGTERM` to all related processes | All processes enter exit flow and complete shutdown; service becomes unavailable; resource release is completed |  |
+| Under load | Send `SIGKILL` to Worker | In-flight requests are hard interrupted (5xx/connection drop); main service becomes unavailable | [#3683](https://github.com/vllm-project/vllm-omni/issues/3683) |
+| Under load | Send `SIGTERM` to Worker | In-flight requests are canceled or fail fast; main service becomes unavailable |  |
+| Under load | Send `SIGKILL` to serve main process | serve is hard-killed and current connections are interrupted; in-flight requests fail; after cleanup there are no residual processes and GPU memory is released | [#3683](https://github.com/vllm-project/vllm-omni/issues/3683) |
+| Under load | Send `SIGTERM` to serve main process | serve stops accepting new requests and executes shutdown flow; in-flight requests fail; no residue remains and GPU memory is released | [#3683](https://github.com/vllm-project/vllm-omni/issues/3683) |
+| Under load | Send `SIGINT` to serve main process (equivalent to `Ctrl+C`) | `Ctrl+C`-style serve shutdown; in-flight requests fail (5xx/connection interruption); service unavailable; after exit there is no residue and GPU memory is released | [#3683](https://github.com/vllm-project/vllm-omni/issues/3683) |
+| Under load | Send `SIGKILL` to all related processes | All processes terminate instantly; all in-flight requests fail; service becomes unavailable immediately; GPU memory is released quickly |  |
+| Under load | Send `SIGTERM` to all related processes | All processes exit gracefully; in-flight requests fail; service unavailable; GPU memory is released | [#3683](https://github.com/vllm-project/vllm-omni/issues/3683) |
+| OOM | Occupy all free GPU memory via an extra process | After OOM injection process starts, GPU memory is continuously saturated; service enters unavailable/degraded state and health check drops to 503; different request types (chat/speech, etc.) fail fast within a fixed time and return 500 (no hanging) | [#4285](https://github.com/vllm-project/vllm-omni/issues/4285) |
+
+## Source of Conclusions
+
+The behaviors and conclusions above are summarized from current fault injection validation results on `Qwen3-Omni` and `Wan2.2`.
+
+### Example: `SIGTERM` fault injection log (Qwen3-Omni)
+
+The following full log is from a real run where a `SIGTERM` signal was sent to the serve root process.
+
+```text
+Launching OmniServer with: /workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --stage-configs-path vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
+Server ready on 127.0.0.1:60675 (OmniServer startup took 206.023s)
+OmniServer started successfully
+[reliability][process-kill] current_server_proc pid=556855 name=python3 cmdline=/workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --stage-configs-path vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
+[reliability][process-kill] current_server_proc pid=557118 name=python3 cmdline=/workspace/.venv/bin/python3 -c from multiprocessing.resource_tracker import main;main(57)
+[reliability][process-kill] current_server_proc pid=557119 name=VLLM::StageEngineCoreProc_noid_replica0_DP0 cmdline=VLLM::StageEngineCoreProc_noid_replica0_DP0
+[reliability][process-kill] current_server_proc pid=557122 name=VLLM::StageEngineCoreProc_noid_replica0_DP0 cmdline=VLLM::StageEngineCoreProc_noid_replica0_DP0
+[reliability][process-kill] current_server_proc pid=558201 name=VLLM::StageEngineCoreProc_noid_replica0_DP0 cmdline=VLLM::StageEngineCoreProc_noid_replica0_DP0
+
+[reliability][process-kill] root-kill pid=556855 name=python3 signal=SIGTERM cmdline=/workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --stage-configs-path vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
+[0;36m(APIServer pid=556855)[0;0m INFO 05-19 12:24:51 [omni_base.py:463] [AsyncOmni] Shutting down
+[0;36m(APIServer pid=556855)[0;0m INFO 05-19 12:24:51 [async_omni_engine.py:2397] [AsyncOmniEngine] Shutting down Orchestrator
+[0;36m(APIServer pid=556855)[0;0m INFO 05-19 12:24:51 [orchestrator.py:351] [Orchestrator] Received shutdown signal
+[0;36m(APIServer pid=556855)[0;0m INFO 05-19 12:24:51 [orchestrator.py:1359] [Orchestrator] Shutting down all 3 client(s)
+
+[0;36m(StageEngineCoreProc pid=557119)[0;0m [32mINFO[0m [90m05-19 12:24:51[0m [90m[core.py:1242][0m Shutdown initiated (timeout=0)
+[0;36m(StageEngineCoreProc pid=557119)[0;0m [32mINFO[0m [90m05-19 12:24:51[0m [90m[core.py:1265][0m Shutdown complete
+[0;36m(APIServer pid=556855)[0;0m INFO:     Shutting down
+[0;36m(APIServer pid=556855)[0;0m INFO:     Waiting for application shutdown.
+[0;36m(APIServer pid=556855)[0;0m INFO:     Application shutdown complete.
+[0;36m(APIServer pid=556855)[0;0m INFO:     Finished server process [556855]
+[0;36m(APIServer pid=556855)[0;0m INFO 05-19 12:24:51 [omni_base.py:463] [AsyncOmni] Shutting down
+[rank0]:[W519 12:24:53.427894833 ProcessGroupNCCL.cpp:1575] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+[0;36m(APIServer pid=556855)[0;0m INFO 05-19 12:24:56 [stage_pool.py:731] [StagePool] Stage 0 replica 0 shut down
+[0;36m(StageEngineCoreProc pid=557122)[0;0m [32mINFO[0m [90m05-19 12:24:56[0m [90m[core.py:1242][0m Shutdown initiated (timeout=0)
+[0;36m(StageEngineCoreProc pid=557122)[0;0m [32mINFO[0m [90m05-19 12:24:56[0m [90m[core.py:1265][0m Shutdown complete
+[rank0]:[W519 12:24:57.569425065 ProcessGroupNCCL.cpp:1575] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+[0;36m(APIServer pid=556855)[0;0m INFO 05-19 12:25:00 [stage_pool.py:731] [StagePool] Stage 1 replica 0 shut down
+[0;36m(StageEngineCoreProc pid=558201)[0;0m [32mINFO[0m [90m05-19 12:25:00[0m [90m[core.py:1242][0m Shutdown initiated (timeout=0)
+[0;36m(StageEngineCoreProc pid=558201)[0;0m [32mINFO[0m [90m05-19 12:25:00[0m [90m[core.py:1265][0m Shutdown complete
+[0;36m(APIServer pid=556855)[0;0m WARNING 05-19 12:25:01 [async_omni_engine.py:2406] [AsyncOmniEngine] Orchestrator thread did not exit in time
+[rank0]:[W519 12:25:01.867242016 ProcessGroupNCCL.cpp:1575] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+```
+
+### How this maps to the matrix
+
+- This log corresponds to the `Send SIGTERM to serve main process` scenario in the matrix.
+- The `root-kill ... signal=SIGTERM` line confirms the fault injection action is applied as expected.
+- The sequence from `Shutting down Orchestrator` to stage-level `Shutdown complete` and `Stage ... shut down` matches the expected graceful shutdown path and child-process cleanup behavior described in the matrix.
+
+## Related Documentation
+
+- For practical CI debugging procedures and triage guidance, see [CI Failures](../contributing/ci/failures.md).

docs/user_guide/quantization/fp8.md

@@ -28,6 +28,50 @@ in deep DiT blocks.
 Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
 guide. FP8 on Ampere may use a weight-only path where available.
 
+### Faster FP8 GEMM on Blackwell (quack)
+
+On Blackwell (SM 100+), vLLM runs FP8 linears through the FlashInfer kernel, which
+applies the bias as a separate kernel after the GEMM. On the small GEMMs in video
+DiTs this bias add is a significant overhead. Installing the optional `quack` kernel
+lets vLLM-Omni fuse `alpha * (A @ B) + bias` into a single CuteDSL GEMM, recovering
+that overhead (e.g. HunyuanVideo-1.5 FP8 goes from slower-than-BF16 to faster).
+
+```bash
+# CUDA 12.9
+pip install vllm-omni[quack]
+
+# CUDA 13.x
+pip install 'quack-kernels[cu13]' --extra-index-url https://download.pytorch.org/whl/cu130
+```
+
+It is enabled automatically once installed (no flag needed) and is **Blackwell-only**:
+on Hopper/Ada the CUTLASS FP8 kernel already fuses bias, so quack is not used there.
+Set `VLLM_OMNI_USE_QUACK_FP8=0` to force the FlashInfer path. If `quack-kernels` is
+not installed, FP8 still works — it just keeps the unfused FlashInfer path.
+
+#### Compile cache and warmup
+
+quack JIT-compiles its kernel once per distinct GEMM shape (tens of seconds, longer
+the first time across all autotuned configs). The compiled `.o` files are cached on
+disk and reused on later runs, so this is a one-time cost — **not** per request.
+
+vLLM-Omni points that cache at `~/.cache/vllm_omni/quack` (override with
+`QUACK_CACHE_DIR`) instead of quack's default under `/tmp`, so it survives restarts.
+In containers, set `QUACK_CACHE_DIR` to a mounted/persistent path — or bake it into
+the image — so the first cold start does not recompile. The engine's startup dummy
+run already exercises the kernels, so with a warm cache the first real request is fast.
+
+To pre-warm specific shapes (e.g. at image build time):
+
+```python
+from vllm_omni.quantization.quack_fp8 import warmup_quack_fp8
+# (M, K, N) per linear; M = number of tokens for your resolution/frame count
+warmup_quack_fp8([(14040, 2048, 6144), (14040, 2048, 2048)])
+```
+
+> The PyPI package is `quack-kernels` (imported as `quack`); plain `pip install
+> quack` is an unrelated statistics library. Requires CUDA 12.9+ and Python 3.12.
+
 ## Model Type Support
 
 ### Diffusion Model (Qwen-Image, Wan2.2)
@@ -41,6 +85,7 @@ guide. FP8 on Ampere may use a weight-only path where available.
 | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B` | Yes | Yes | All layers | None | |
 | HunyuanImage-3.0 | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | Yes | Yes | All layers; use the Hunyuan stage config for multi-stage runs | None | |
 | HunyuanVideo-1.5 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `720p_t2v`, `480p_i2v` | Yes | Yes | All layers | None | |
+| Cosmos3 | `nvidia/Cosmos3-Nano` | Yes | Not validated | All layers | None | |
 
 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
 

docs/user_guide/quantization/modelopt.md

@@ -89,11 +89,54 @@ and
 | Model | Scope | Status |
 |-------|-------|--------|
 | Qwen3-Omni | Thinker language-model stage | ModelOpt FP8 checkpoint path |
+| Qwen3-Omni | Thinker language-model stage (W4A4 NVFP4) | Validated; see [Qwen3-Omni NVFP4 W4A4](#qwen3-omni-nvfp4-w4a4-thinker) below |
 | Qwen3-TTS | TTS language-model stage | Not validated |
 
 Audio encoder, vision encoder, talker, and code2wav stages stay in BF16 unless
 a model-specific guide documents otherwise.
 
+#### Qwen3-Omni NVFP4 W4A4 (thinker)
+
+vLLM-Omni serves ModelOpt NVFP4 W4A4 quantizations of the
+Qwen3-Omni-30B-A3B-Instruct thinker language model. The thinker text body
+(attention + MoE experts) is quantized to NVFP4 with FP8 per-tensor input
+scales; the audio encoder, vision encoder, talker, and code2wav stay in BF16.
+
+| Variant | HF checkpoint | Hardware |
+|---------|---------------|----------|
+| W4A4 NVFP4 (full thinker) | `YihongJin/Qwen3-Omni-30B-A3B-Instruct-NVFP4-W4A4-full-thinker-awqclip` | sm_100+ (Blackwell, FlashInfer FP4 GEMM) |
+
+Calibration uses ModelOpt `mtq.NVFP4_DEFAULT_CFG` with the `awq_clip` algorithm
+on 1024 ultrachat samples chat-templated through the Qwen3-Omni tokenizer.
+Excluded modules: `*audio_tower*`, `*visual*`, `*talker*`, `*code2wav*`,
+`*lm_head*`, `*mlp.gate*`. See `scripts/nvfp4/calibrate.py` for the reference
+recipe.
+
+!!! note "ModelOpt 0.44 NaN regression workaround"
+    ModelOpt 0.44's float32 -> FP8 E4M3 cast of per-block weight scales
+    occasionally emits literal NaN bytes (E4M3 encoding 0x7F / 0xFF) for
+    blocks whose pre-cast scale rounds above the FP8 max of 448 after the
+    global-scale division. A single NaN byte in any `weight_scale`
+    propagates through the FP4 GEMM and collapses the served model output
+    to `!!!!`. vLLM-Omni's `vllm_omni.patch` installs a defensive override
+    of `ModelOptNvFp4LinearMethod.process_weights_after_loading` that
+    clamps these bytes to the FP8 E4M3 max at load time. The override
+    self-extinguishes once vllm-omni's vllm pin moves to a release
+    containing the upstream fix. Set `VLLM_OMNI_SKIP_NVFP4_NAN_CLAMP=1`
+    to disable the override for diagnostics.
+
+Serving:
+
+```bash
+vllm serve YihongJin/Qwen3-Omni-30B-A3B-Instruct-NVFP4-W4A4-full-thinker-awqclip \
+    --omni --port 8000
+```
+
+> Do **not** pass `--enforce-eager` for production / benchmarks. CUDA graphs
+> amortize launch overhead and unlock the FP4 throughput wins; with
+> `--enforce-eager` set, W4A4 TPOT degrades ~10x relative to the CUDA-graph
+> configuration.
+
 ### Multi-Stage Diffusion Model
 
 ModelOpt checkpoints must be routed to the stage whose checkpoint contains the

examples/offline_inference/bagel/README.md

@@ -1,391 +0,0 @@
-# BAGEL-7B-MoT
-
-## Setup
-
-Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
-
-## Architecture
-
-BAGEL-7B-MoT is a Mixture-of-Transformers (MoT) model supporting both image generation and understanding. It offers two deployment topologies:
-
-| Topology | Stages | Description |
-| :------- | :----- | :---------- |
-| **Two-stage** (default) | Stage 0 (Thinker, AR) + Stage 1 (DiT, Diffusion) | Thinker handles text/understanding via vLLM AR engine; DiT handles image generation. KV cache is transferred between stages. |
-| **Single-stage** | Stage 0 (DiT, Diffusion) only | The DiT stage contains a full LLM, ViT, VAE, and tokenizer internally. All modalities are handled within a single diffusion process. |
-
-Both topologies support all four modalities: `text2img`, `img2img`, `img2text`, `text2text`.
-
-## Quick Start
-
-```bash
-cd examples/offline_inference/bagel
-
-# Default two-stage mode (auto-detected)
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2img \
-                  --prompts "A cute cat"
-
-# Single-stage mode
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2img \
-                  --prompts "A cute cat" \
-                  --deploy-config vllm_omni/deploy/bagel_single_stage.yaml
-```
-
-> **Note**: These examples work with the default configuration on an **NVIDIA A100 (80GB)**. For dual-GPU setups, modify the deploy YAML to distribute stages across devices.
-
-## Modality Control
-
-Control the mode using the `--modality` argument:
-
-| Modality | Input | Output | Description |
-| :------- | :---- | :----- | :---------- |
-| `text2img` | Text | Image | Generate images from text prompts |
-| `img2img` | Image + Text | Image | Transform images using text guidance |
-| `img2text` | Image + Text | Text | Generate text descriptions from images |
-| `text2text` | Text | Text | Pure text generation (language model mode) |
-
-### Text to Image (text2img)
-
-```bash
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2img \
-                  --prompts "A cute cat" \
-                  --steps 50
-```
-
-### Image to Image (img2img)
-
-```bash
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality img2img \
-                  --image-path /path/to/image.jpg \
-                  --prompts "Let the woman wear a blue dress"
-```
-
-### Image to Text (img2text)
-
-```bash
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality img2text \
-                  --image-path /path/to/image.jpg \
-                  --prompts "Describe this image in detail"
-```
-
-### Text to Text (text2text)
-
-```bash
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2text \
-                  --prompts "What is the capital of France?"
-
-# Load prompts from a text file (one prompt per line):
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2text \
-                  --txt-prompts /path/to/prompts.txt
-```
-
-## Think Mode
-
-Think mode enables the model to generate `<think>...</think>` planning/reasoning tokens before producing the final output. This improves generation quality for complex prompts.
-
-- **Two-stage**: The Thinker (AR) stage decodes think tokens, then transfers the augmented KV cache to the DiT stage for image generation.
-- **Single-stage**: The DiT's internal LLM generates think tokens in-place before proceeding to denoise.
-
-```bash
-# Think + text2img: plan before generating
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2img \
-                  --prompts "A futuristic city with flying cars" \
-                  --think \
-                  --max-think-tokens 1000
-
-# Think + img2img: reason about the edit
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality img2img \
-                  --image-path /path/to/image.jpg \
-                  --prompts "Make it look like a watercolor painting" \
-                  --think
-
-# Think + img2text: reason before describing
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality img2text \
-                  --image-path /path/to/image.jpg \
-                  --prompts "What is happening in this image?" \
-                  --think
-
-# Think + text2text: chain-of-thought reasoning
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2text \
-                  --prompts "Solve: 23 * 47" \
-                  --think
-```
-
-Think mode parameters:
-
-| Argument | Default | Description |
-| :------- | :------ | :---------- |
-| `--think` | `False` | Enable thinking mode |
-| `--max-think-tokens` | `1000` | Maximum tokens for think generation |
-| `--do-sample` | `False` | Enable sampling (vs. greedy) for text generation |
-| `--text-temperature` | `0.3` | Temperature for text generation sampling |
-
-## Classifier-Free Guidance (CFG)
-
-CFG controls the trade-off between prompt fidelity and diversity. These parameters apply to image generation modalities (`text2img`, `img2img`).
-
-```bash
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2img \
-                  --prompts "A photorealistic portrait" \
-                  --cfg-text-scale 6.0 \
-                  --cfg-img-scale 2.0 \
-                  --negative-prompt "blurry, low quality, distorted" \
-                  --cfg-interval 0.4 1.0 \
-                  --cfg-renorm-type global \
-                  --cfg-renorm-min 0.0
-```
-
-| Argument | Default | Description |
-| :------- | :------ | :---------- |
-| `--cfg-text-scale` | `4.0` | Text CFG scale (higher = more prompt-adherent) |
-| `--cfg-img-scale` | `1.5` | Image CFG scale (for img2img) |
-| `--negative-prompt` | `None` | Negative prompt for CFG conditioning |
-| `--cfg-interval` | pipeline default | CFG active interval `[start, end]` as fractions of total timesteps |
-| `--cfg-renorm-type` | `None` | Renormalization type: `global`, `text_channel`, `channel` |
-| `--cfg-renorm-min` | `None` | Minimum renormalization value |
-| `--cfg-parallel-size` | `1` | CFG parallel size: `1` = batched (single GPU), `2` = 2-branch parallel, `3` = full 3-GPU parallel |
-
-## Deployment Topologies
-
-### Two-Stage (Default)
-
-The default topology auto-detected from the model. No extra flags needed.
-
-```bash
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2img \
-                  --prompts "A cute cat"
-```
-
-The pipeline is defined in [`bagel.yaml`](../../../vllm_omni/deploy/bagel.yaml). Stage 0 (Thinker) and Stage 1 (DiT) share GPU 0 by default. For dual-GPU setups, customize the deploy YAML and set `devices: "1"` for stage 1.
-
-### Single-Stage
-
-Pass the single-stage deploy config via `--deploy-config`:
-
-```bash
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2img \
-                  --prompts "A cute cat" \
-                  --deploy-config vllm_omni/deploy/bagel_single_stage.yaml
-```
-
-See [`bagel_single_stage.yaml`](../../../vllm_omni/deploy/bagel_single_stage.yaml) for configuration details. The `pipeline: bagel_single_stage` field selects the single-stage topology from the pipeline registry.
-
-### Tensor Parallelism (TP)
-
-For larger models or multi-GPU environments, customize the deploy YAML (see [`bagel.yaml`](../../../vllm_omni/deploy/bagel.yaml)) and set per-stage `tensor_parallel_size` and `devices`:
-
-```yaml
-# Example: TP=2 on GPUs 0,1 for the Thinker stage
-stages:
-  - stage_id: 0
-    tensor_parallel_size: 2
-    devices: "0,1"
-```
-
-### VAE Patch Parallelism
-
-[VAE Patch Parallelism](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/parallelism/vae_patch_parallel.html) splits Bagel VAE **decode/encode** tiles across multiple GPUs on the **DiT stage**, reducing **per-GPU peak memory during VAE decode**. Use it when high-resolution `text2img` or `img2img` hits VAE OOM or large decode spikes.
-
-**Bagel-specific notes:**
-
-- Implemented in `BagelPipeline` via `DistributedAutoEncoder` (DiT stage only).
-- **Single-stage** is the simplest path: one DiT process with TP + VAE patch parallel.
-- **Two-stage**: enable on **stage 1 (DiT)** only; stage 0 (Thinker) keeps encoder-only `VAEEncoder` and does not use VAE patch parallel.
-- You need a DiT `world_size` ≥ `vae_patch_parallel_size` (typically `tensor_parallel_size=2` on that stage). VAE PP reuses the DiT process group; it is not a standalone second-GPU VAE worker.
-
-**Single-stage via deploy YAML** (recommended for `end2end.py`):
-
-```yaml
-pipeline: bagel_single_stage
-async_chunk: false
-
-stages:
-  - stage_id: 0
-    max_num_batched_tokens: 32768
-    max_num_seqs: 1
-    enforce_eager: true
-    trust_remote_code: true
-    enable_prefix_caching: false
-    devices: "0,1"
-    vae_use_tiling: true
-    parallel_config:
-      tensor_parallel_size: 2
-      vae_patch_parallel_size: 2
-    default_sampling_params:
-      seed: 52
-```
-
-```bash
-cd examples/offline_inference/bagel
-
-CUDA_VISIBLE_DEVICES=0,1 python end2end.py \
-    --model /path/to/BAGEL-7B-MoT \
-    --deploy-config /path/to/bagel_single_stage_vae_pp.yaml \
-    --modality text2img \
-    --prompts "A cute cat" \
-    --steps 10 \
-    --output ./out_vae_pp
-```
-
-**Single-stage via `Omni` kwargs** (same flags as online serving):
-
-```python
-from vllm_omni.entrypoints.omni import Omni
-
-omni = Omni(
-    model="ByteDance-Seed/BAGEL-7B-MoT",
-    deploy_config="vllm_omni/deploy/bagel_single_stage.yaml",
-    tensor_parallel_size=2,
-    vae_patch_parallel_size=2,
-    vae_use_tiling=True,
-)
-# Then call omni.generate(...) as in end2end.py
-```
-
-**Two-stage (VAE PP on DiT only):**
-
-```yaml
-stages:
-  - stage_id: 0
-    devices: "0"
-    # AR Thinker — no vae_patch_parallel here
-
-  - stage_id: 1
-    devices: "0,1"
-    vae_use_tiling: true
-    parallel_config:
-      tensor_parallel_size: 2
-      vae_patch_parallel_size: 2
-```
-
-```bash
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-    --deploy-config /path/to/bagel_vae_pp.yaml \
-    --modality text2img \
-    --prompts "A cute cat"
-```
-
-**Startup log checks:**
-
-```text
-INFO ... vae_patch_parallel_size=2 requires vae_use_tiling; automatically enabling it.
-```
-
-| Setting | Role |
-| :------ | :--- |
-| `parallel_config.tensor_parallel_size` | DiT world size / TP (must be ≥ `vae_patch_parallel_size`) |
-| `parallel_config.vae_patch_parallel_size` | Number of ranks for distributed VAE tiles (`1` = off) |
-| `vae_use_tiling` | Enable spatial tiling (auto-enabled when `vae_patch_parallel_size > 1`) |
-
-#### Hybrid Sharded Data Parallel (HSDP)
-
-For larger Bagel deployments on multiple GPUs, you can enable HSDP (Hybrid Sharded Data Parallel) by modifying the stage configuration (for example, [`bagel.yaml`](../../../vllm_omni/deploy/bagel.yaml)). HSDP shards transformer weights across GPUs to reduce per-GPU memory usage.
-
-1. **Enable HSDP**: Set `use_hsdp: true`.
-2. **Set shard size**: Set `hsdp_shard_size` to the number of GPUs used for sharding (for example, `4`).
-3. **Set replicate size**: Usually keep `hsdp_replicate_size: 1` unless you want replicated HSDP groups.
-4. **Set devices**: Specify the comma-separated GPU IDs used by the diffusion stage (for example, `"0,1,2,3"`).
-
-Example configuration for HSDP across 4 GPUs:
-```yaml
-  - stage_id: 1
-    devices: "0,1,2,3"
-    parallel_config:
-      use_hsdp: true
-      hsdp_shard_size: 4
-      hsdp_replicate_size: 1
-```
-
-Then pass the custom deploy YAML:
-
-```bash
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2img \
-                  --prompts "A cute cat" \
-                  --deploy-config /path/to/custom_bagel.yaml
-```
-
-### FP8 Quantization
-
-```bash
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2img \
-                  --prompts "A cute cat" \
-                  --quantization fp8
-```
-
-## Command Line Reference
-
-### Core Arguments
-
-| Argument | Type | Default | Description |
-| :------- | :--- | :------ | :---------- |
-| `--model` | string | `ByteDance-Seed/BAGEL-7B-MoT` | Model path or HuggingFace name |
-| `--modality` | choice | `text2img` | `text2img`, `img2img`, `img2text`, `text2text` |
-| `--prompts` | list | `None` | Input text prompts |
-| `--txt-prompts` | string | `None` | Path to text file with one prompt per line |
-| `--image-path` | string | `None` | Input image path (required for `img2img`/`img2text`) |
-| `--output` | string | `.` | Output directory for saved images |
-| `--steps` | int | `50` | Number of diffusion inference steps |
-| `--seed` | int | `None` | Random seed for reproducibility |
-
-### Think Mode Arguments
-
-| Argument | Type | Default | Description |
-| :------- | :--- | :------ | :---------- |
-| `--think` | flag | `False` | Enable `<think>...</think>` planning/reasoning |
-| `--max-think-tokens` | int | `1000` | Maximum tokens for think generation |
-| `--do-sample` | flag | `False` | Use sampling instead of greedy decoding |
-| `--text-temperature` | float | `0.3` | Sampling temperature for text generation |
-
-### CFG Arguments
-
-| Argument | Type | Default | Description |
-| :------- | :--- | :------ | :---------- |
-| `--cfg-text-scale` | float | `4.0` | Text CFG guidance scale |
-| `--cfg-img-scale` | float | `1.5` | Image CFG guidance scale |
-| `--negative-prompt` | string | `None` | Negative prompt for CFG |
-| `--cfg-parallel-size` | int | `1` | CFG parallel GPU count (1, 2, or 3) |
-| `--cfg-interval` | float[2] | pipeline default | CFG active window `[start, end]` |
-| `--cfg-renorm-type` | string | `None` | `global`, `text_channel`, or `channel` |
-| `--cfg-renorm-min` | float | `None` | Minimum renormalization value |
-
-### Engine Arguments
-
-| Argument | Type | Default | Description |
-| :------- | :--- | :------ | :---------- |
-| `--deploy-config` | string | `None` | Path to deploy YAML (auto-detected if omitted) |
-| `--worker-backend` | choice | `process` | `process` or `ray` |
-| `--ray-address` | string | `None` | Ray cluster address |
-| `--quantization` | string | `None` | Quantization method (e.g. `fp8`) |
-| `--log-stats` | flag | `False` | Enable statistics logging |
-| `--init-timeout` | int | `300` | Initialization timeout (seconds) |
-| `--batch-timeout` | int | `5` | Batch timeout (seconds) |
-| `--enable-diffusion-pipeline-profiler` | flag | `False` | Profile diffusion stage durations |
-
-## FAQ
-
-- If you encounter OOM errors, try decreasing `max_model_len` or `gpu_memory_utilization` in the deploy YAML.
-
-**Two-stage VRAM usage:**
-
-| Stage | VRAM |
-| :---- | :--- |
-| Stage 0 (Thinker) | **15.04 GiB + KV Cache** |
-| Stage 1 (DiT) | **26.50 GiB** |
-| Total | **~42 GiB + KV Cache** |
-
-**Single-stage VRAM usage:** The DiT loads the full model (~42 GiB) in one process.

examples/offline_inference/bagel/end2end.py

@@ -1,278 +0,0 @@
-import argparse
-import os
-
-from vllm_omni.inputs.data import OmniPromptType
-from vllm_omni.model_executor.stage_input_processors.bagel import (
-    GEN_THINK_SYSTEM_PROMPT,
-    VLM_THINK_SYSTEM_PROMPT,
-)
-
-
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--model",
-        default="ByteDance-Seed/BAGEL-7B-MoT",
-        help="Path to merged model directory.",
-    )
-    parser.add_argument("--prompts", nargs="+", default=None, help="Input text prompts.")
-    parser.add_argument(
-        "--txt-prompts",
-        type=str,
-        default=None,
-        help="Path to a .txt file with one prompt per line (preferred).",
-    )
-    parser.add_argument("--prompt-type", default="text", choices=["text"])
-
-    parser.add_argument(
-        "--modality",
-        default="text2img",
-        choices=["text2img", "img2img", "img2text", "text2text"],
-        help="Modality mode to control stage execution.",
-    )
-
-    parser.add_argument(
-        "--image-path",
-        type=str,
-        default=None,
-        help="Path to input image for img2img.",
-    )
-
-    parser.add_argument(
-        "--output",
-        type=str,
-        default=".",
-        help="Output directory to save images.",
-    )
-
-    # OmniLLM init args
-    parser.add_argument("--log-stats", action="store_true", default=False)
-    parser.add_argument("--init-sleep-seconds", type=int, default=20)
-    parser.add_argument("--batch-timeout", type=int, default=5)
-    parser.add_argument("--init-timeout", type=int, default=300)
-    parser.add_argument("--shm-threshold-bytes", type=int, default=65536)
-    parser.add_argument("--worker-backend", type=str, default="process", choices=["process", "ray"])
-    parser.add_argument("--ray-address", type=str, default=None)
-    parser.add_argument(
-        "--deploy-config",
-        type=str,
-        default=None,
-        help="Path to deploy YAML. If unset, auto-loads vllm_omni/deploy/bagel.yaml based on the HF model_type.",
-    )
-    parser.add_argument("--steps", type=int, default=50, help="Number of inference steps.")
-
-    parser.add_argument("--cfg-text-scale", type=float, default=4.0, help="Text CFG scale (default: 4.0)")
-    parser.add_argument("--cfg-img-scale", type=float, default=1.5, help="Image CFG scale (default: 1.5)")
-    parser.add_argument(
-        "--negative-prompt", type=str, default=None, help="Negative prompt for CFG (default: empty prompt)"
-    )
-    parser.add_argument(
-        "--cfg-parallel-size",
-        type=int,
-        default=1,
-        choices=[1, 2, 3],
-        help="CFG parallel size: 1=batched (single GPU), 2=parallel with 2 branches (text CFG only), 3=parallel (3 GPUs).",
-    )
-    parser.add_argument("--seed", type=int, default=None, help="Random seed for generation.")
-    parser.add_argument(
-        "--cfg-interval",
-        type=float,
-        nargs=2,
-        default=None,
-        help="CFG interval [start, end] (default: pipeline default)",
-    )
-    parser.add_argument(
-        "--cfg-renorm-type", type=str, default=None, help="CFG renorm type: global, text_channel, channel"
-    )
-    parser.add_argument("--cfg-renorm-min", type=float, default=None, help="CFG renorm min")
-    parser.add_argument(
-        "--enable-diffusion-pipeline-profiler",
-        action="store_true",
-        help="Enable diffusion pipeline profiler to display stage durations.",
-    )
-    parser.add_argument(
-        "--quantization",
-        type=str,
-        default=None,
-        help="Quantization method (e.g. 'fp8').",
-    )
-    parser.add_argument(
-        "--think",
-        action="store_true",
-        default=False,
-        help="Enable thinking mode: AR stage decodes <think>...</think> planning tokens before image generation.",
-    )
-    parser.add_argument(
-        "--max-think-tokens",
-        type=int,
-        default=1000,
-        help="Maximum number of tokens for thinking text generation (default: 1000).",
-    )
-    parser.add_argument(
-        "--do-sample",
-        action="store_true",
-        default=False,
-        help="Enable sampling for text generation (default: greedy).",
-    )
-    parser.add_argument(
-        "--text-temperature",
-        type=float,
-        default=0.3,
-        help="Temperature for text generation sampling (default: 0.3).",
-    )
-
-    args = parser.parse_args()
-    return args
-
-
-def main():
-    args = parse_args()
-    os.makedirs(args.output, exist_ok=True)
-    model_name = args.model
-    prompts: list[OmniPromptType] = []
-    try:
-        if getattr(args, "txt_prompts", None) and args.prompt_type == "text":
-            with open(args.txt_prompts, encoding="utf-8") as f:
-                lines = [ln.strip() for ln in f.readlines()]
-            prompts = [ln for ln in lines if ln != ""]
-            print(f"[Info] Loaded {len(prompts)} prompts from {args.txt_prompts}")
-        else:
-            prompts = args.prompts
-    except Exception as e:
-        print(f"[Error] Failed to load prompts: {e}")
-        raise
-
-    if not prompts:
-        prompts = ["A cute cat"]
-        print(f"[Info] No prompts provided, using default: {prompts}")
-
-    from PIL import Image
-
-    from vllm_omni.entrypoints.omni import Omni
-
-    omni_kwargs = vars(args).copy()
-    omni_kwargs.pop("model", None)
-    deploy_config = args.deploy_config
-    if args.think and deploy_config is None:
-        deploy_config = "vllm_omni/deploy/bagel_think.yaml"
-        print(f"[Info] Think mode enabled, using deploy config: {deploy_config}")
-    if deploy_config:
-        omni_kwargs["deploy_config"] = deploy_config
-
-    if args.quantization:
-        omni_kwargs["quantization_config"] = args.quantization
-
-    omni = Omni(model=model_name, **omni_kwargs)
-
-    formatted_prompts = []
-    for p in prompts:
-        if args.modality == "img2img":
-            if not args.image_path or not os.path.exists(args.image_path):
-                raise ValueError(f"img2img requires --image-path pointing to an existing file, got: {args.image_path}")
-            loaded_image = Image.open(args.image_path).convert("RGB")
-            think_prefix = f"<|im_start|>{GEN_THINK_SYSTEM_PROMPT}<|im_end|>" if args.think else ""
-            final_prompt_text = f"{think_prefix}<|fim_middle|><|im_start|>{p}<|im_end|>"
-            prompt_dict = {
-                "prompt": final_prompt_text,
-                "multi_modal_data": {"img2img": loaded_image},
-                "modalities": ["img2img"],
-            }
-            if args.negative_prompt is not None:
-                prompt_dict["negative_prompt"] = args.negative_prompt
-            formatted_prompts.append(prompt_dict)
-        elif args.modality == "img2text":
-            if args.image_path:
-                loaded_image = Image.open(args.image_path).convert("RGB")
-                think_prefix = f"<|im_start|>system\n{VLM_THINK_SYSTEM_PROMPT}<|im_end|>\n" if args.think else ""
-                final_prompt_text = (
-                    f"{think_prefix}<|im_start|>user\n<|image_pad|>\n{p}<|im_end|>\n<|im_start|>assistant\n"
-                )
-                prompt_dict = {
-                    "prompt": final_prompt_text,
-                    "multi_modal_data": {"image": loaded_image},
-                    "modalities": ["text"],
-                }
-                formatted_prompts.append(prompt_dict)
-        elif args.modality == "text2text":
-            think_prefix = f"<|im_start|>{VLM_THINK_SYSTEM_PROMPT}<|im_end|>" if args.think else ""
-            final_prompt_text = f"{think_prefix}<|im_start|>{p}<|im_end|><|im_start|>"
-            prompt_dict = {"prompt": final_prompt_text, "modalities": ["text"]}
-            formatted_prompts.append(prompt_dict)
-        else:
-            think_prefix = f"<|im_start|>{GEN_THINK_SYSTEM_PROMPT}<|im_end|>" if args.think else ""
-            final_prompt_text = f"{think_prefix}<|im_start|>{p}<|im_end|>"
-            prompt_dict = {"prompt": final_prompt_text, "modalities": ["image"]}
-            if args.negative_prompt is not None:
-                prompt_dict["negative_prompt"] = args.negative_prompt
-            formatted_prompts.append(prompt_dict)
-
-    params_list = omni.default_sampling_params_list
-    # Bagel exposes 1 sampling param set for single-stage (DiT-only) and
-    # 2 for two-stage (Thinker + DiT).  This heuristic may need updating
-    # if future pipelines break that 1:1 mapping.
-    is_single_stage = len(params_list) == 1
-
-    diffusion_params_idx = 0 if is_single_stage else (1 if len(params_list) > 1 else 0)
-    diffusion_params = params_list[diffusion_params_idx]
-
-    if args.modality in ("text2img", "img2img"):
-        diffusion_params.num_inference_steps = args.steps  # type: ignore
-        diffusion_params.cfg_parallel_size = args.cfg_parallel_size  # type: ignore
-        if args.seed is not None:
-            diffusion_params.seed = args.seed  # type: ignore
-
-    extra = getattr(diffusion_params, "extra_args", {}) or {}
-    extra["cfg_text_scale"] = args.cfg_text_scale
-    extra["cfg_img_scale"] = args.cfg_img_scale
-    if args.cfg_interval is not None:
-        extra["cfg_interval"] = tuple(args.cfg_interval)
-    if args.cfg_renorm_type is not None:
-        extra["cfg_renorm_type"] = args.cfg_renorm_type
-    if args.cfg_renorm_min is not None:
-        extra["cfg_renorm_min"] = args.cfg_renorm_min
-    if args.negative_prompt is not None:
-        extra["negative_prompt"] = args.negative_prompt
-
-    needs_text_gen = is_single_stage and (args.think or args.modality in ("text2text", "img2text"))
-    if needs_text_gen:
-        if args.think:
-            extra["think"] = True
-        extra["max_think_tokens"] = args.max_think_tokens
-        extra["do_sample"] = args.do_sample
-        extra["text_temperature"] = args.text_temperature
-    diffusion_params.extra_args = extra  # type: ignore
-
-    omni_outputs = list(omni.generate(prompts=formatted_prompts, sampling_params_list=params_list))
-
-    img_idx = 0
-    for req_output in omni_outputs:
-        # 2-stage think mode: text output from thinker stage
-        ro = getattr(req_output, "request_output", None)
-        if ro and getattr(ro, "outputs", None):
-            txt = "".join(getattr(o, "text", "") or "" for o in ro.outputs)
-            if txt:
-                if args.think:
-                    print(f"[Think]\n{txt}")
-                else:
-                    print(f"[Output] Text:\n{txt}")
-
-        # Single-stage DiT: text from custom_output
-        custom = getattr(req_output, "_custom_output", {}) or {}
-        if custom.get("think_text"):
-            print(f"[Think]\n{custom['think_text']}")
-        if custom.get("text_output"):
-            print(f"[Output] Text:\n{custom['text_output']}")
-
-        images = getattr(req_output, "images", None)
-        if not images:
-            continue
-
-        for j, img in enumerate(images):
-            save_path = os.path.join(args.output, f"output_{img_idx}_{j}.png")
-            img.save(save_path)
-            print(f"[Output] Saved image to {save_path}")
-        img_idx += 1
-
-
-if __name__ == "__main__":
-    main()

examples/offline_inference/cosmos3/inputs/v2v.json

@@ -0,0 +1,14 @@
+{
+    "prompt": "A robotic arm, primarily white with black joints and cables, is shown in a clean, modern indoor setting with a white tabletop. The arm, equipped with a gripper holding a small, light green pitcher, is positioned above a clear glass containing a reddish-brown liquid and a spoon. The robotic arm is in the process of pouring a transparent liquid into the glass. To the left of the pitcher, there is an opened jar with a similar reddish-brown substance visible through its transparent body. In the background, a vase with white flowers and a brown couch are partially visible, adding to the contemporary ambiance. The lighting is bright, casting soft shadows on the table. The robotic arm's movements are smooth and controlled, demonstrating precision in its task. As the video progresses, the robotic arm completes the pour, leaving the glass half-filled with the reddish-brown liquid. The jar remains untouched throughout the sequence, and the spoon inside the glass remains stationary. The other robotic arm on the right side also stays stationary throughout the video. The final frame captures the robotic arm with the pitcher finishing the pour, with the glass now filled to a higher level, while the pitcher is slightly tilted but still held securely by the gripper.",
+    "negative_prompt": "",
+    "vision_path": "https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_pouring.mp4",
+    "height": 720,
+    "width": 1280,
+    "num_frames": 189,
+    "num_inference_steps": 35,
+    "guidance_scale": 6.0,
+    "flow_shift": 10.0,
+    "fps": 24,
+    "condition_frame_indexes_vision": [0, 1],
+    "condition_video_keep": "first"
+}

examples/offline_inference/dreamzero/client_schedule.py

@@ -0,0 +1,129 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""DreamZero AR client frame schedule aligned with upstream ``test_client_AR.py``."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import numpy as np
+
+# Frame schedule constants (matching upstream debug_inference.py / test_client_AR.py).
+ACTION_HORIZON = 24
+DEFAULT_ACTION_DIM = 8
+RELATIVE_OFFSETS = [-23, -16, -8, 0]
+DEFAULT_NUM_AR_CHUNKS = 15
+
+CAMERA_FILES = {
+    "observation/exterior_image_0_left": "exterior_image_1_left.mp4",
+    "observation/exterior_image_1_left": "exterior_image_2_left.mp4",
+    "observation/wrist_image_left": "wrist_image_left.mp4",
+}
+
+
+def build_frame_schedule(
+    total_frames: int,
+    num_chunks: int,
+    *,
+    logger: logging.Logger | None = None,
+) -> list[list[int]]:
+    """Build the frame index schedule for multi-frame chunks.
+
+    Args:
+        total_frames: Number of frames available in each camera video.
+        num_chunks: Number of 4-frame chunks to schedule after the initial frame.
+
+    Returns:
+        A list of frame-index lists. Each inner list has four indices. The
+        returned list may be shorter than ``num_chunks`` when the videos run
+        out of frames (upstream stops early instead of erroring).
+    """
+    chunks: list[list[int]] = []
+    current_frame = 23
+    for _ in range(num_chunks):
+        indices = [max(current_frame + offset, 0) for offset in RELATIVE_OFFSETS]
+        if indices[-1] >= total_frames:
+            if logger is not None:
+                logger.info(
+                    "Frame %s >= %s, stopping at %s chunks",
+                    indices[-1],
+                    total_frames,
+                    len(chunks),
+                )
+            break
+        chunks.append(indices)
+        current_frame += ACTION_HORIZON
+    return chunks
+
+
+def make_obs_from_video(
+    camera_frames: dict[str, np.ndarray],
+    frame_indices: list[int],
+    *,
+    prompt: str,
+    session_id: str,
+) -> dict[str, Any]:
+    """Build an observation dict from real video frames.
+
+    For one frame each image key is ``(H, W, 3)``. For four frames each key is
+    ``(4, H, W, 3)``.
+    """
+    obs: dict[str, Any] = {}
+    for camera_key, all_frames in camera_frames.items():
+        selected = all_frames[frame_indices]
+        obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected
+
+    obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
+    obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
+    obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
+    obs["prompt"] = prompt
+    obs["session_id"] = session_id
+    return obs
+
+
+def build_ar_observations(
+    camera_frames: dict[str, np.ndarray],
+    *,
+    prompt: str,
+    session_id: str,
+    num_chunks: int = DEFAULT_NUM_AR_CHUNKS,
+    repeat_chunk_observations: bool = False,
+    logger: logging.Logger | None = None,
+) -> list[dict[str, Any]]:
+    """Build the AR observation sequence used by upstream ``test_client_AR.py``.
+
+    Step 0 sends frame ``[0]``. Each subsequent step sends one 4-frame chunk.
+
+    ``num_chunks`` counts only the 4-frame chunks after the initial frame, not
+    the total number of inferences.
+    """
+    if num_chunks < 0:
+        raise ValueError("num_chunks must be non-negative")
+
+    total_frames = min(frames.shape[0] for frames in camera_frames.values())
+    observations = [
+        make_obs_from_video(
+            camera_frames,
+            [0],
+            prompt=prompt,
+            session_id=session_id,
+        )
+    ]
+
+    chunk_schedule = build_frame_schedule(total_frames, num_chunks, logger=logger)
+    if repeat_chunk_observations and chunk_schedule and len(chunk_schedule) < num_chunks:
+        while len(chunk_schedule) < num_chunks:
+            chunk_schedule.append(chunk_schedule[-1])
+
+    for frame_indices in chunk_schedule:
+        observations.append(
+            make_obs_from_video(
+                camera_frames,
+                frame_indices,
+                prompt=prompt,
+                session_id=session_id,
+            )
+        )
+    return observations

examples/offline_inference/dreamzero/export_prediction_video.py

@@ -11,6 +11,11 @@ from pathlib import Path
 import cv2
 import numpy as np
 import torch
+from client_schedule import (
+    CAMERA_FILES,
+    DEFAULT_NUM_AR_CHUNKS,
+    build_ar_observations,
+)
 from PIL import Image
 
 from vllm_omni import Omni
@@ -20,13 +25,6 @@ from vllm_omni.outputs import OmniRequestOutput
 WORKER_EXTENSION = "vllm_omni.diffusion.models.dreamzero.video_export_worker.DreamZeroVideoExportWorkerExtension"
 ASSET_REPO_ID = "YangshenDeng/vllm-omni-dreamzero-assets"
 DEFAULT_SESSION_PREFIX = "dreamzero-export"
-RELATIVE_OFFSETS = [-23, -16, -8, 0]
-ACTION_HORIZON = 24
-CAMERA_FILES = {
-    "observation/exterior_image_0_left": "exterior_image_1_left.mp4",
-    "observation/exterior_image_1_left": "exterior_image_2_left.mp4",
-    "observation/wrist_image_left": "wrist_image_left.mp4",
-}
 
 
 def _parse_args() -> argparse.Namespace:
@@ -49,6 +47,17 @@ def _parse_args() -> argparse.Namespace:
         default="Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan",
     )
     parser.add_argument("--session-id", default=None)
+    parser.add_argument(
+        "--num-chunks",
+        type=int,
+        default=DEFAULT_NUM_AR_CHUNKS,
+        help="Number of 4-frame chunks to send after the initial frame (default: 15).",
+    )
+    parser.add_argument(
+        "--repeat-chunk-observations",
+        action="store_true",
+        help="Repeat the last valid chunk observation when bundled assets run out of frames.",
+    )
     parser.add_argument("--save-input-video", action="store_true")
     parser.add_argument("--save-gif", action="store_true")
     parser.add_argument("--save-actions", action="store_true")
@@ -84,57 +93,25 @@ def _load_camera_frames(video_dir: Path) -> dict[str, np.ndarray]:
     return camera_frames
 
 
-def _build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
-    chunks: list[list[int]] = []
-    current_frame = 23
-    for _ in range(num_chunks):
-        indices = [max(current_frame + offset, 0) for offset in RELATIVE_OFFSETS]
-        if indices[-1] >= total_frames:
-            break
-        chunks.append(indices)
-        current_frame += ACTION_HORIZON
-    return chunks
-
-
-def _make_obs_from_video(
-    camera_frames: dict[str, np.ndarray],
-    frame_indices: list[int],
-    *,
+def _build_observations(
+    video_dir: Path,
     prompt: str,
     session_id: str,
-) -> dict:
-    obs: dict = {}
-    for camera_key, all_frames in camera_frames.items():
-        selected = all_frames[frame_indices]
-        obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected
-
-    obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
-    obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
-    obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
-    obs["prompt"] = prompt
-    obs["session_id"] = session_id
-    return obs
-
-
-def _build_observations(video_dir: Path, prompt: str, session_id: str) -> tuple[dict[str, np.ndarray], list[dict]]:
+    *,
+    num_chunks: int,
+    repeat_chunk_observations: bool,
+) -> tuple[dict[str, np.ndarray], list[dict]]:
     camera_frames = _load_camera_frames(video_dir)
-    total_frames = min(frames.shape[0] for frames in camera_frames.values())
-    chunks = _build_frame_schedule(total_frames, 1)
-    observations = [
-        _make_obs_from_video(camera_frames, [0], prompt=prompt, session_id=session_id),
-    ]
-    if chunks:
-        observations.append(
-            _make_obs_from_video(
-                camera_frames,
-                chunks[0],
-                prompt=prompt,
-                session_id=session_id,
-            )
-        )
+    observations = build_ar_observations(
+        camera_frames,
+        prompt=prompt,
+        session_id=session_id,
+        num_chunks=num_chunks,
+        repeat_chunk_observations=repeat_chunk_observations,
+    )
     if len(observations) < 2:
-        raise RuntimeError("Need at least two DreamZero example observations to export a prediction video.")
-    return camera_frames, observations[:2]
+        raise RuntimeError("Need at least two DreamZero observations to export a prediction video.")
+    return camera_frames, observations
 
 
 def _extract_latents(output: OmniRequestOutput) -> torch.Tensor:
@@ -229,6 +206,26 @@ def _run_generation(
     return omni, outputs
 
 
+def _decode_accumulated_with_worker(omni: Omni, session_id: str) -> np.ndarray:
+    """Decode server-side accumulated latents when the pipeline supports it."""
+    stage_client = omni.engine.stage_clients[0]
+    engine = getattr(stage_client, "_engine", None)
+    if engine is None:
+        raise RuntimeError("DreamZero export requires inline diffusion stage access.")
+
+    decoded = engine.executor.collective_rpc(
+        "decode_accumulated_video_latents_to_uint8",
+        args=(session_id,),
+        unique_reply_rank=0,
+        exec_all_ranks=True,
+    )
+    if isinstance(decoded, torch.Tensor):
+        decoded = decoded.numpy()
+    if not isinstance(decoded, np.ndarray):
+        raise TypeError(f"Unexpected decoded output type: {type(decoded)!r}")
+    return decoded
+
+
 def _decode_with_worker(omni: Omni, full_latents: torch.Tensor) -> np.ndarray:
     stage_client = omni.engine.stage_clients[0]
     engine = getattr(stage_client, "_engine", None)
@@ -248,6 +245,15 @@ def _decode_with_worker(omni: Omni, full_latents: torch.Tensor) -> np.ndarray:
     return decoded
 
 
+def _decode_video(omni: Omni, session_id: str, outputs: list[OmniRequestOutput]) -> np.ndarray:
+    """Decode full AR rollout by concatenating per-step latents from all outputs."""
+    latent_steps = [_extract_latents(output) for output in outputs]
+    latent_t = [step.shape[2] for step in latent_steps]
+    print(f"Decoding {len(latent_steps)} latent steps, latent_t={latent_t}, total_t={sum(latent_t)}")
+    full_latents = torch.cat(latent_steps, dim=2)
+    return _decode_with_worker(omni, full_latents)
+
+
 def main() -> None:
     args = _parse_args()
     session_id = args.session_id or f"{DEFAULT_SESSION_PREFIX}-{uuid.uuid4()}"
@@ -256,6 +262,12 @@ def main() -> None:
         video_dir=args.video_dir,
         prompt=args.prompt,
         session_id=session_id,
+        num_chunks=args.num_chunks,
+        repeat_chunk_observations=args.repeat_chunk_observations,
+    )
+    print(
+        f"Running {len(observations)} inferences "
+        f"(1 initial + {len(observations) - 1} chunks; requested {args.num_chunks} post-initial chunks)"
     )
 
     args.output_dir.mkdir(parents=True, exist_ok=True)
@@ -273,17 +285,19 @@ def main() -> None:
             deploy_config_path=args.deploy_config,
             observations=observations,
         )
-        latent_steps = [_extract_latents(output) for output in outputs]
-        full_latents = torch.cat(latent_steps, dim=2)
-        frames = _decode_with_worker(omni, full_latents)
+        frames = _decode_video(omni, session_id, outputs)
     finally:
         if omni is not None:
             omni.close()
 
+    expected_frames = 8 * len(observations) + 1
     mp4_path = args.output_dir / f"{args.output_stem}.mp4"
 
     _write_mp4(mp4_path, frames, fps=args.fps)
-    print(f"SAVED_MP4={mp4_path}")
+    print(
+        f"SAVED_MP4={mp4_path} frames={len(frames)} "
+        f"expected≈{expected_frames} duration≈{len(frames) / max(args.fps, 1):.1f}s"
+    )
 
     if args.save_gif:
         gif_path = args.output_dir / f"{args.output_stem}.gif"
@@ -294,8 +308,10 @@ def main() -> None:
         npz_path = args.output_dir / f"{args.output_stem}_actions.npz"
         np.savez(
             npz_path,
-            step0=np.asarray(outputs[0].multimodal_output.get("actions")),
-            step1=np.asarray(outputs[1].multimodal_output.get("actions")),
+            **{
+                f"step{index}": np.asarray(output.multimodal_output.get("actions"))
+                for index, output in enumerate(outputs)
+            },
         )
         print(f"SAVED_ACTIONS={npz_path}")
 

examples/offline_inference/image_to_image/image_edit.py

@@ -97,9 +97,16 @@ import torch
 from PIL import Image
 
 from vllm_omni.diffusion.data import DiffusionParallelConfig
+from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
 from vllm_omni.entrypoints.omni import Omni
+from vllm_omni.entrypoints.openai.stage_params import clone_sampling_params
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-from vllm_omni.outputs import OmniRequestOutput
+from vllm_omni.model_extras import (
+    build_image_to_image_prompt,
+    get_extra_body_params,
+    get_model_class_name,
+    should_init_extra_args_for_non_diffusion_stages,
+)
 from vllm_omni.platforms import current_omni_platform
 
 
@@ -143,6 +150,20 @@ def parse_args() -> argparse.Namespace:
         default=None,
         required=False,
     )
+    parser.add_argument(
+        "--width",
+        type=int,
+        default=None,
+        metavar="W",
+        help="Output image width in pixels. Default: None (pipeline's default).",
+    )
+    parser.add_argument(
+        "--height",
+        type=int,
+        default=None,
+        metavar="H",
+        help="Output image height in pixels. Default: None (pipeline's default).",
+    )
     parser.add_argument(
         "--seed",
         type=int,
@@ -173,6 +194,12 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--guidance-scale-2", type=float, default=None, help="image guidance scale for image-to-image generation."
     )
+    parser.add_argument(
+        "--extra-args",
+        type=parse_profiler_config,
+        default=None,
+        help="JSON object copied to OmniDiffusionSamplingParams.extra_args, e.g. '{\"cfg_text_scale\": 4.0}'.",
+    )
     parser.add_argument(
         "--output",
         type=str,
@@ -236,8 +263,8 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--resolution",
         type=int,
-        default=640,
-        help="Bucket in (640, 1024) to determine the condition and output resolution",
+        default=None,
+        help="Bucket in (640, 1024) to determine the condition and output resolution. If width and height are not provided, this will be set to default 640.",
     )
 
     parser.add_argument(
@@ -384,6 +411,15 @@ def parse_args() -> argparse.Namespace:
 def main():
     args = parse_args()
 
+    if args.resolution and (args.width or args.height):
+        raise ValueError("--resolution and --width/--height cannot be specified together")
+    if args.width is not None and args.width <= 0:
+        raise ValueError("--width must be a positive integer")
+    if args.height is not None and args.height <= 0:
+        raise ValueError("--height must be a positive integer")
+    if not args.width and not args.height and not args.resolution:
+        args.resolution = 640
+
     # Validate input images exist and load them
     input_images = []
     for image_path in args.image:
@@ -445,6 +481,8 @@ def main():
         enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
         profiler_config=args.profiler_config,
     )
+    model_class_name = get_model_class_name(omni)
+    declared_extra_body_params = get_extra_body_params(model_class_name)
     print("Pipeline loaded")
 
     profiler_enabled = args.profiler_config is not None
@@ -455,6 +493,8 @@ def main():
     print(f"  Model: {args.model}")
     print(f"  Inference steps: {args.num_inference_steps}")
     print(f"  Cache backend: {args.cache_backend if args.cache_backend else 'None (no acceleration)'}")
+    if args.height is not None or args.width is not None:
+        print(f"  Output size: {args.width or 'auto'}x{args.height or 'auto'}")
     if isinstance(input_image, list):
         print(f"  Number of input images: {len(input_image)}")
         for idx, img in enumerate(input_image):
@@ -472,24 +512,65 @@ def main():
         print("[Profiler] Starting profiling...")
         omni.start_profile()
 
-    # Generate edited image
-    outputs = omni.generate(
-        {
-            "prompt": args.prompt,
-            "negative_prompt": args.negative_prompt,
-            "multi_modal_data": {"image": input_image},
-        },
-        OmniDiffusionSamplingParams(
-            generator=generator,
-            true_cfg_scale=args.cfg_scale,
-            guidance_scale=args.guidance_scale,
-            guidance_scale_2=args.guidance_scale_2,
-            num_inference_steps=args.num_inference_steps,
-            num_outputs_per_prompt=args.num_outputs_per_prompt,
-            layers=args.layers,
-            resolution=args.resolution,
-        ),
+    prompt_dict = build_image_to_image_prompt(
+        model_class_name=model_class_name,
+        prompt=args.prompt,
+        negative_prompt=args.negative_prompt,
+        input_image=input_image,
+        height=args.height,
+        width=args.width,
     )
+
+    extra_args_from_cli = dict(args.extra_args or {})
+    if args.negative_prompt is not None:
+        extra_args_from_cli.setdefault("negative_prompt", args.negative_prompt)
+
+    diffusion_params = OmniDiffusionSamplingParams(
+        generator=generator,
+        true_cfg_scale=args.cfg_scale,
+        guidance_scale=args.guidance_scale,
+        guidance_scale_2=args.guidance_scale_2,
+        num_inference_steps=args.num_inference_steps,
+        num_outputs_per_prompt=args.num_outputs_per_prompt,
+        layers=args.layers,
+        resolution=args.resolution,
+        height=args.height,
+        width=args.width,
+    )
+    if declared_extra_body_params:
+        apply_declared_extra_args(
+            diffusion_params,
+            declared_extra_body_params,
+            extra_args_from_cli,
+        )
+    else:
+        diffusion_params.extra_args.update({k: v for k, v in extra_args_from_cli.items() if v is not None})
+
+    # Build per-stage sampling params for multi-stage models
+    init_non_diffusion = should_init_extra_args_for_non_diffusion_stages(
+        model_class_name,
+    )
+    defaults = list(omni.default_sampling_params_list or [])
+    sampling_params_list = [clone_sampling_params(p) for p in defaults]
+    if not sampling_params_list:
+        sampling_params_list = [diffusion_params]
+
+    diffusion_replaced = False
+    for idx, params in enumerate(sampling_params_list):
+        if isinstance(params, OmniDiffusionSamplingParams):
+            merged_extra = dict(getattr(params, "extra_args", {}) or {})
+            merged_extra.update(diffusion_params.extra_args)
+            diffusion_params.extra_args = merged_extra
+            sampling_params_list[idx] = diffusion_params
+            diffusion_replaced = True
+        elif init_non_diffusion and hasattr(params, "extra_args"):
+            if params.extra_args is None:
+                params.extra_args = {}
+
+    if not diffusion_replaced and len(sampling_params_list) == 1:
+        sampling_params_list = [diffusion_params]
+
+    outputs = omni.generate(prompt_dict, sampling_params_list=sampling_params_list)
     generation_end = time.perf_counter()
     generation_time = generation_end - generation_start
 
@@ -516,17 +597,16 @@ def main():
     if not outputs:
         raise ValueError("No output generated from omni.generate()")
 
-    # Extract images from OmniRequestOutput
-    # omni.generate() returns list[OmniRequestOutput], extract images from request_output.images
-    first_output = outputs[0]
-    if not hasattr(first_output, "request_output") or not first_output.request_output:
-        raise ValueError("No request_output found in OmniRequestOutput")
+    images = None
+    for output in outputs:
+        images = getattr(output, "images", None)
+        if images:
+            break
+        req_out = getattr(output, "request_output", None)
+        images = getattr(req_out, "images", None) if req_out is not None else None
+        if images:
+            break
 
-    req_out = first_output.request_output
-    if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"):
-        raise ValueError("Invalid request_output structure or missing 'images' key")
-
-    images = req_out.images
     if not images:
         raise ValueError("No images found in request_output")
 

examples/offline_inference/qwen3_omni/end2end_async_chunk.py

@@ -257,8 +257,11 @@ async def run_single_request(
                         new_chunks = audio_data[audio_list_consumed:]
                         audio_list_consumed = len(audio_data)
                     elif isinstance(audio_data, torch.Tensor):
-                        new_chunks = [audio_data]
-                        audio_last_tensor = audio_data
+                        if audio_data is not audio_last_tensor:
+                            new_chunks = [audio_data]
+                            audio_last_tensor = audio_data
+                        else:
+                            new_chunks = []
                     else:
                         new_chunks = []
 

examples/offline_inference/qwen3_omni/run_tests.sh

@@ -0,0 +1,140 @@
+#!/usr/bin/env bash
+# =============================================================================
+# Qwen3-Omni end2end functional test script
+# Uses default built-in assets, iterates over query-type x output-modalities combinations
+# =============================================================================
+
+set -euo pipefail
+
+export VLLM_OMNI_USE_V2_RUNNER=1
+
+MODEL_PATH="/models/Qwen3-Omni-30B-A3B-Instruct"
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+END2END="$SCRIPT_DIR/end2end.py"
+LOG_DIR="$SCRIPT_DIR/test_logs"
+SUMMARY_FILE="$LOG_DIR/summary.log"
+
+rm -rf "$LOG_DIR"
+mkdir -p "$LOG_DIR"
+> "$SUMMARY_FILE"
+
+PASS=0
+FAIL=0
+TOTAL=0
+
+run_test() {
+    local name="$1"
+    shift
+    local log_file="$LOG_DIR/${name}.log"
+    local output_dir="$LOG_DIR/${name}_output"
+
+    TOTAL=$((TOTAL + 1))
+    echo "========================================"
+    echo "[${TOTAL}] Running: ${name}"
+    echo "  Command: python $END2END $*"
+    echo "========================================"
+
+    local start_time
+    start_time=$(date +%s)
+
+    if python "$END2END" "$@" --output-dir "$output_dir" > "$log_file" 2>&1; then
+        local end_time
+        end_time=$(date +%s)
+        local elapsed=$((end_time - start_time))
+        echo "  => PASS (${elapsed}s)"
+        echo "[PASS] ${name} (${elapsed}s)" >> "$SUMMARY_FILE"
+        PASS=$((PASS + 1))
+    else
+        local exit_code=$?
+        local end_time
+        end_time=$(date +%s)
+        local elapsed=$((end_time - start_time))
+        echo "  => FAIL (exit code: ${exit_code}, ${elapsed}s)"
+        echo "[FAIL] ${name} (exit code: ${exit_code}, ${elapsed}s)" >> "$SUMMARY_FILE"
+        # Append last 30 lines of log to summary
+        echo "  --- last 30 lines of log ---" >> "$SUMMARY_FILE"
+        tail -n 30 "$log_file" >> "$SUMMARY_FILE"
+        echo "  --- end ---" >> "$SUMMARY_FILE"
+        FAIL=$((FAIL + 1))
+    fi
+    echo ""
+}
+
+# =============================================================================
+# Test cases
+# =============================================================================
+
+# --- 1. Text-only input ---
+run_test "text_to_text" \
+    --model "$MODEL_PATH" --query-type text --modalities text
+
+run_test "text_to_text_audio" \
+    --model "$MODEL_PATH" --query-type text --modalities text,audio
+
+run_test "text_to_audio" \
+    --model "$MODEL_PATH" --query-type text --modalities audio
+
+# --- 2. Image input ---
+run_test "image_to_text" \
+    --model "$MODEL_PATH" --query-type use_image --modalities text
+
+run_test "image_to_text_audio" \
+    --model "$MODEL_PATH" --query-type use_image --modalities text,audio
+
+# --- 3. Audio input ---
+run_test "audio_to_text" \
+    --model "$MODEL_PATH" --query-type use_audio --modalities text
+
+run_test "audio_to_text_audio" \
+    --model "$MODEL_PATH" --query-type use_audio --modalities text,audio
+
+# --- 4. Video input ---
+run_test "video_to_text" \
+    --model "$MODEL_PATH" --query-type use_video --modalities text
+
+run_test "video_to_text_audio" \
+    --model "$MODEL_PATH" --query-type use_video --modalities text,audio
+
+# --- 5. Multi-audio input ---
+run_test "multi_audios_to_text" \
+    --model "$MODEL_PATH" --query-type use_multi_audios --modalities text
+
+run_test "multi_audios_to_text_audio" \
+    --model "$MODEL_PATH" --query-type use_multi_audios --modalities text,audio
+
+# --- 6. Mixed modality input (audio+image+video) ---
+run_test "mixed_to_text" \
+    --model "$MODEL_PATH" --query-type use_mixed_modalities --modalities text
+
+run_test "mixed_to_text_audio" \
+    --model "$MODEL_PATH" --query-type use_mixed_modalities --modalities text,audio
+
+# --- 7. Audio from video ---
+run_test "audio_in_video_to_text" \
+    --model "$MODEL_PATH" --query-type use_audio_in_video --modalities text
+
+run_test "audio_in_video_to_text_audio" \
+    --model "$MODEL_PATH" --query-type use_audio_in_video --modalities text,audio
+
+# --- 8. py-generator mode ---
+run_test "text_to_text_generator" \
+    --model "$MODEL_PATH" --query-type text --modalities text --py-generator
+
+run_test "audio_to_text_audio_generator" \
+    --model "$MODEL_PATH" --query-type use_audio --modalities text,audio --py-generator
+
+# --- 9. Multi-prompt batch inference ---
+run_test "text_to_text_batch3" \
+    --model "$MODEL_PATH" --query-type text --modalities text --num-prompts 3
+
+# =============================================================================
+# Summary
+# =============================================================================
+echo "========================================" | tee -a "$SUMMARY_FILE"
+echo "Tests done: total ${TOTAL}, passed ${PASS}, failed ${FAIL}" | tee -a "$SUMMARY_FILE"
+echo "========================================" | tee -a "$SUMMARY_FILE"
+echo ""
+echo "Detailed logs: $LOG_DIR"
+echo "Summary report: $SUMMARY_FILE"
+
+exit $FAIL

examples/offline_inference/qwen3_omni/run_tests_async_chunk.sh

@@ -0,0 +1,143 @@
+#!/usr/bin/env bash
+# =============================================================================
+# Qwen3-Omni end2end_async_chunk functional test script
+# Uses default built-in assets, iterates over AsyncOmni query-type x output-modalities combinations
+# =============================================================================
+
+set -euo pipefail
+
+export VLLM_OMNI_USE_V2_RUNNER=1
+
+MODEL_PATH="/models/Qwen3-Omni-30B-A3B-Instruct"
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+END2END="$SCRIPT_DIR/end2end_async_chunk.py"
+STAGE_CONFIGS="$SCRIPT_DIR/../../../vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml"
+LOG_DIR="$SCRIPT_DIR/test_logs_async_chunk"
+SUMMARY_FILE="$LOG_DIR/summary.log"
+
+rm -rf "$LOG_DIR"
+mkdir -p "$LOG_DIR"
+> "$SUMMARY_FILE"
+
+PASS=0
+FAIL=0
+TOTAL=0
+
+run_test() {
+    local name="$1"
+    shift
+    local log_file="$LOG_DIR/${name}.log"
+    local output_dir="$LOG_DIR/${name}_output"
+
+    TOTAL=$((TOTAL + 1))
+    echo "========================================"
+    echo "[${TOTAL}] Running: ${name}"
+    echo "  Command: python $END2END $*"
+    echo "========================================"
+
+    local start_time
+    start_time=$(date +%s)
+
+    if python "$END2END" "$@" --output-dir "$output_dir" > "$log_file" 2>&1; then
+        local end_time
+        end_time=$(date +%s)
+        local elapsed=$((end_time - start_time))
+        echo "  => PASS (${elapsed}s)"
+        echo "[PASS] ${name} (${elapsed}s)" >> "$SUMMARY_FILE"
+        PASS=$((PASS + 1))
+    else
+        local exit_code=$?
+        local end_time
+        end_time=$(date +%s)
+        local elapsed=$((end_time - start_time))
+        echo "  => FAIL (exit code: ${exit_code}, ${elapsed}s)"
+        echo "[FAIL] ${name} (exit code: ${exit_code}, ${elapsed}s)" >> "$SUMMARY_FILE"
+        # Append last 30 lines of log to summary
+        echo "  --- last 30 lines of log ---" >> "$SUMMARY_FILE"
+        tail -n 30 "$log_file" >> "$SUMMARY_FILE"
+        echo "  --- end ---" >> "$SUMMARY_FILE"
+        FAIL=$((FAIL + 1))
+    fi
+    echo ""
+}
+
+# =============================================================================
+# Test cases
+# =============================================================================
+
+# --- 1. Text-only input ---
+run_test "text_to_text" \
+    --model "$MODEL_PATH" --stage-configs-path "$STAGE_CONFIGS" \
+    --query-type text --modalities text
+
+run_test "text_to_text_audio" \
+    --model "$MODEL_PATH" --stage-configs-path "$STAGE_CONFIGS" \
+    --query-type text --modalities text,audio
+
+run_test "text_to_audio" \
+    --model "$MODEL_PATH" --stage-configs-path "$STAGE_CONFIGS" \
+    --query-type text --modalities audio
+
+# --- 2. Image input ---
+run_test "image_to_text" \
+    --model "$MODEL_PATH" --stage-configs-path "$STAGE_CONFIGS" \
+    --query-type use_image --modalities text
+
+run_test "image_to_text_audio" \
+    --model "$MODEL_PATH" --stage-configs-path "$STAGE_CONFIGS" \
+    --query-type use_image --modalities text,audio
+
+# --- 3. Audio input ---
+run_test "audio_to_text" \
+    --model "$MODEL_PATH" --stage-configs-path "$STAGE_CONFIGS" \
+    --query-type use_audio --modalities text
+
+run_test "audio_to_text_audio" \
+    --model "$MODEL_PATH" --stage-configs-path "$STAGE_CONFIGS" \
+    --query-type use_audio --modalities text,audio
+
+# --- 4. Video input ---
+run_test "video_to_text" \
+    --model "$MODEL_PATH" --stage-configs-path "$STAGE_CONFIGS" \
+    --query-type use_video --modalities text
+
+run_test "video_to_text_audio" \
+    --model "$MODEL_PATH" --stage-configs-path "$STAGE_CONFIGS" \
+    --query-type use_video --modalities text,audio
+
+# --- 5. Multi-prompt batch inference ---
+run_test "text_to_text_batch3" \
+    --model "$MODEL_PATH" --stage-configs-path "$STAGE_CONFIGS" \
+    --query-type text --modalities text --num-prompts 3
+
+# --- 6. Concurrent requests (max-in-flight) ---
+run_test "text_to_text_audio_concurrent2" \
+    --model "$MODEL_PATH" --stage-configs-path "$STAGE_CONFIGS" \
+    --query-type text --modalities text,audio --num-prompts 2 --max-in-flight 2
+
+# --- 7. stream-audio-to-disk mode ---
+run_test "audio_to_text_audio_stream" \
+    --model "$MODEL_PATH" --stage-configs-path "$STAGE_CONFIGS" \
+    --query-type use_audio --modalities text,audio --stream-audio-to-disk
+
+# --- 8. request-timeout test (long enough timeout, should PASS) ---
+run_test "text_to_text_with_timeout" \
+    --model "$MODEL_PATH" --stage-configs-path "$STAGE_CONFIGS" \
+    --query-type text --modalities text --request-timeout-s 600
+
+# --- 9. batch-timeout test (long enough timeout, should PASS) ---
+run_test "text_to_text_audio_batch_timeout" \
+    --model "$MODEL_PATH" --stage-configs-path "$STAGE_CONFIGS" \
+    --query-type text --modalities text,audio --batch-timeout-s 600
+
+# =============================================================================
+# Summary
+# =============================================================================
+echo "========================================" | tee -a "$SUMMARY_FILE"
+echo "Tests done: total ${TOTAL}, passed ${PASS}, failed ${FAIL}" | tee -a "$SUMMARY_FILE"
+echo "========================================" | tee -a "$SUMMARY_FILE"
+echo ""
+echo "Detailed logs: $LOG_DIR"
+echo "Summary report: $SUMMARY_FILE"
+
+exit $FAIL

examples/offline_inference/sensenova_u1/README.md

@@ -145,7 +145,7 @@ python end2end.py \
 ## Reducing GPU Memory Usage
 
 For hardware with limited VRAM, enable **module-wise CPU offload** with
-`--enable-cpu-offload`. The pipeline implements `SupportsModuleOffload`, so the
+`--enable-cpu-offload`. The pipeline implements `SupportsComponentDiscovery`, so the
 vision encoder (`vision_model`) and the Qwen3 LLM (`language_model`) are
 swapped between CPU and GPU on demand:
 

examples/offline_inference/step_audio2/README.md

@@ -0,0 +1,375 @@
+# Step-Audio2 Offline Inference Examples
+
+This directory contains examples for running offline inference with Step-Audio2 using vLLM-Omni.
+
+## Model Overview
+
+Step-Audio2 is a two-stage audio model:
+
+- **Stage 0 (Thinker)**: Audio understanding → Text + Audio tokens
+  - Input: Audio (16kHz)
+  - Output: Text transcription + Audio tokens for synthesis
+
+- **Stage 1 (Token2Wav)**: Audio synthesis
+  - Input: Audio tokens + Speaker prompt wav
+  - Output: Synthesized audio waveform (24kHz)
+
+## Hardware Requirements
+
+| Mode | GPU Configuration | VRAM Required |
+|------|-------------------|---------------|
+| ASR (S2T) | 1x GPU | ~20-25GB |
+| TTS/S2ST (single GPU) | 1x GPU | ~40-50GB |
+| TTS/S2ST (multi GPU) | 2x GPU | GPU0: ~28GB, GPU1: ~22GB |
+
+**Tested on:**
+- 1x NVIDIA H100 80GB (single-card S2ST)
+- 2x NVIDIA A10 40GB (multi-card S2ST)
+
+**Notes:**
+- Single GPU mode requires high VRAM due to both stages sharing memory
+- Multi GPU mode separates Stage 0 (Thinker) and Stage 1 (Token2Wav) across GPUs
+- VRAM usage can be adjusted via `gpu_memory_utilization` in stage config
+
+## Performance Benchmark
+
+### vLLM-Omni vs Official Step-Audio2
+
+Single request latency comparison between vLLM-Omni and official Step-Audio2 implementation.
+
+| Task | Tokens | vllm-omni | Step-Audio2 | Speedup |
+|------|--------|-----------|-------------|---------|
+| S2ST | ~85 | 5.45s | 7.36s | **1.35x** |
+| S2ST | ~160 | 6.67s | 13.92s | **2.09x** |
+| S2ST | ~315 | 9.42s | 31.50s | **3.34x** |
+| TTS | ~1024 | 16.65s | ~87s | **~5.2x** |
+
+**Key observations:**
+- Speedup increases with sequence length due to vLLM's efficient KV cache management
+- TTS (pure generation) shows the largest speedup (~5x)
+- S2ST benefits from optimized multi-stage pipeline
+
+**Benchmark environment:**
+- GPU: NVIDIA H100 80GB (single card)
+- Model: Step-Audio2-mini
+- Warmup: 1 run, Measured: 3 runs (averaged)
+
+### Async Chunk Streaming Performance
+
+Comparison between sequential (non-async) and async chunk modes via `/v1/audio/speech` TTS endpoint.
+
+| Mode | Mean TTFP | Mean E2E | Mean RTF | Audio Throughput |
+|------|-----------|----------|----------|-----------------|
+| Sequential | 4316ms | 4316ms | 0.938 | 1.07x realtime |
+| **Async Chunk** | **1437ms** | 4362ms | 0.949 | 1.06x realtime |
+| **Improvement** | **-67% (3x faster)** | ~same | ~same | ~same |
+
+**Key observations:**
+- Async chunk reduces **time-to-first-audio (TTFP) by 67%** (4.3s → 1.4s)
+- E2E latency remains comparable — async chunk overlaps Thinker decode with Token2Wav synthesis
+- RTF < 1 in both modes (real-time capable)
+- Sequential mode: TTFP ≈ E2E (must wait for all audio tokens before synthesis starts)
+- Async chunk mode: Token2Wav starts after first 28 tokens (chunk_size=25 + lookahead=3)
+
+**Benchmark environment:**
+- GPU: 4x NVIDIA RTX 3090 24GB (TP=2 for Thinker, 1 GPU for Token2Wav)
+- Model: Step-Audio2-mini
+- Endpoint: `/v1/audio/speech` (10 prompts, concurrency=1)
+- Measured via `bench_tts_serve.py`
+
+## Installation
+
+Make sure you have installed vLLM-Omni and all required dependencies:
+
+```bash
+# Install vLLM-Omni
+pip install vllm-omni
+
+# Install Step-Audio2 (REQUIRED for Token2Wav stage)
+pip install step-audio2
+```
+
+## Model Setup
+
+### Option 1: Auto-download from HuggingFace (Recommended)
+
+The script will **automatically download** the model on first run:
+
+```bash
+# Just run without specifying --model, it will auto-download stepfun-ai/Step-Audio2-mini
+python end2end.py --query-type audio_to_text
+
+# Or explicitly specify the HuggingFace model
+python end2end.py --query-type audio_to_text --model stepfun-ai/Step-Audio2-mini
+```
+
+Models will be cached in `~/.cache/huggingface/hub/` for future use.
+
+**Available models**:
+- `stepfun-ai/Step-Audio2-mini` (smaller, faster)
+- `stepfun-ai/Step-Audio2-7B` (larger, better quality)
+
+### Option 2: Manual Download (for offline use)
+
+Download and use locally:
+
+```bash
+# Download from HuggingFace
+huggingface-cli download stepfun-ai/Step-Audio2-mini --local-dir ./models/Step-Audio2-mini
+
+# Then use the local path
+python end2end.py --query-type audio_to_text --model ./models/Step-Audio2-mini
+```
+
+Ensure the model directory contains:
+```
+Step-Audio2-mini/
+├── config.json
+├── model.safetensors (or pytorch_model.bin)
+├── tokenizer.json
+├── tokenizer_config.json
+└── token2wav/                           # Token2Wav models (REQUIRED)
+    ├── speech_tokenizer_v2_25hz.onnx   # Audio tokenizer
+    ├── campplus.onnx                    # Speaker encoder
+    ├── flow.yaml                        # Flow model config
+    ├── flow.pt                          # Flow model weights
+    └── hift.pt                          # HiFT vocoder weights
+```
+
+## Usage Examples
+
+### 1. Audio to Text (ASR - Speech Recognition)
+
+Transcribe audio to text:
+
+```bash
+# Quick start - Using default model and test audio
+python end2end.py --query-type audio_to_text
+
+# Using your own audio file (model will auto-download)
+python end2end.py --query-type audio_to_text \
+    --audio-path /path/to/input.wav
+
+# With specific model
+python end2end.py --query-type audio_to_text \
+    --audio-path /path/to/input.wav \
+    --model stepfun-ai/Step-Audio2-7B
+
+# With custom question
+python end2end.py --query-type audio_to_text \
+    --audio-path input.wav \
+    --question "What is the speaker saying?"
+```
+
+**Output**: Text transcription saved to `output_step_audio2/00000_text.txt`
+
+### 2. Text to Audio (TTS - Speech Synthesis)
+
+Convert text to speech:
+
+```bash
+# Basic TTS (model auto-downloads)
+python end2end.py --query-type text_to_audio \
+    --text "Hello, this is a test of Step Audio 2 synthesis."
+
+# With specific model
+python end2end.py --query-type text_to_audio \
+    --text "Hello, this is a test." \
+    --model stepfun-ai/Step-Audio2-7B
+```
+
+**Note**: Speaker voice is controlled by the `STEP_AUDIO2_DEFAULT_PROMPT_WAV` environment variable or the default prompt wav bundled with the model.
+
+**Output**:
+- Text: `output_step_audio2/00000_text.txt`
+- Audio: `output_step_audio2/00000_output.wav` (24kHz)
+
+### 3. Audio to Audio (Voice Conversion)
+
+Process input audio and generate output audio:
+
+```bash
+# Basic voice conversion (model auto-downloads)
+python end2end.py --query-type audio_to_audio \
+    --audio-path /path/to/source_audio.wav
+
+# With specific model
+python end2end.py --query-type audio_to_audio \
+    --audio-path source.wav \
+    --model stepfun-ai/Step-Audio2-7B
+```
+
+This mode:
+1. Understands the content in `--audio-path` (source)
+2. Generates audio output with the default voice
+
+**Note**: To use a custom speaker voice, set the `STEP_AUDIO2_DEFAULT_PROMPT_WAV` environment variable.
+
+### Advanced Options
+
+```bash
+# Use custom stage configuration
+python end2end.py --query-type audio_to_text \
+    --stage-configs-path /path/to/custom_config.yaml
+
+# Multiple prompts (for batch testing)
+python end2end.py --query-type audio_to_text \
+    --audio-path input.wav \
+    --num-prompts 5
+
+# Custom output directory
+python end2end.py --query-type text_to_audio \
+    --text "Test synthesis" \
+    --output-dir ./my_outputs
+
+# Enable detailed logging
+python end2end.py --query-type audio_to_text \
+    --audio-path input.wav \
+    --enable-stats
+
+# Adjust generation parameters
+python end2end.py --query-type audio_to_text \
+    --audio-path input.wav \
+    --max-tokens 2048
+
+# Use custom speaker voice via environment variable
+STEP_AUDIO2_DEFAULT_PROMPT_WAV=/path/to/speaker.wav python end2end.py \
+    --query-type text_to_audio \
+    --text "Hello world"
+
+# Use Ray backend for distributed processing
+python end2end.py --query-type text_to_audio \
+    --text "Hello world" \
+    --worker-backend ray \
+    --ray-address "auto"
+```
+
+## Configuration
+
+### Stage Configuration
+
+The default configuration (`step_audio_2.yaml`) uses:
+
+- **Stage 0 (Thinker)**: GPU 0, 80% memory
+- **Stage 1 (Token2Wav)**: GPU 1, 30% memory
+
+For **single GPU** setup, edit the config to use `devices: "0"` for both stages.
+
+### Sampling Parameters
+
+- **Thinker (Stage 0)**:
+  - Temperature: 0.7 (balanced creativity)
+  - Top-p: 0.9
+  - Max tokens: 1024 (configurable)
+
+- **Token2Wav (Stage 1)**:
+  - Temperature: 0.0 (deterministic)
+  - Operates in generation mode (not sampling)
+
+## Common Issues
+
+### 1. ImportError: No module named 's3tokenizer'
+
+**Solution**: Install Step-Audio2 package:
+```bash
+pip install step-audio2
+```
+
+### 2. FileNotFoundError: prompt_wav file not found
+
+**Solution**: Set the `STEP_AUDIO2_DEFAULT_PROMPT_WAV` environment variable to a valid audio file:
+```bash
+export STEP_AUDIO2_DEFAULT_PROMPT_WAV=/path/to/speaker.wav
+python end2end.py --query-type text_to_audio --text "Hello"
+```
+Or ensure the default prompt wav (`default_female.wav`) exists in your model directory.
+
+### 3. FileNotFoundError: token2wav models not found
+
+**Solution**: Ensure your model directory has the complete `token2wav/` subdirectory with all ONNX and PyTorch models.
+
+### 4. CUDA Out of Memory
+
+**Solutions**:
+- Use single GPU mode (set both stages to `devices: "0"`)
+- Reduce `gpu_memory_utilization` in config
+- Reduce `max_num_batched_tokens`
+- Process fewer prompts at once
+
+### 5. Model not found in registry
+
+**Solution**: Ensure you're using vLLM-Omni's entry point with `--omni` flag or install vllm-omni properly:
+```bash
+pip install vllm-omni
+```
+
+## Output Files
+
+The script generates files in the output directory (default: `output_step_audio2/`):
+
+```
+output_step_audio2/
+├── 00000_text.txt        # Text output from Thinker stage
+├── 00000_output.wav      # Audio output from Token2Wav stage (24kHz)
+├── 00001_text.txt        # (if multiple prompts)
+└── 00001_output.wav
+```
+
+## Performance Tips
+
+1. **First run is slow**: Stage initialization takes 20-60 seconds
+2. **Single GPU**: Set both stages to `devices: "0"` in config
+3. **Multiple prompts**: Use `--num-prompts N` for batch testing
+4. **Ray backend**: For multi-node or advanced scheduling
+5. **Logging**: Use `--enable-stats` to debug performance issues
+
+## Speaker Voice Configuration
+
+The Token2Wav stage requires a speaker prompt wav for voice conditioning. It is automatically resolved in this order:
+
+1. `STEP_AUDIO2_DEFAULT_PROMPT_WAV` environment variable (if set)
+2. `{model_dir}/assets/default_female.wav`
+3. `{model_dir}/default_female.wav`
+
+If none are found, set the environment variable explicitly:
+```bash
+export STEP_AUDIO2_DEFAULT_PROMPT_WAV=/path/to/speaker.wav
+```
+
+**Guidelines for custom speaker prompt:**
+
+- **Duration**: 3-10 seconds recommended
+- **Quality**: Clean audio, minimal background noise
+- **Format**: WAV, MP3, FLAC (will be resampled internally)
+- **Content**: Clear speech, representative of target voice
+
+## Example Workflow
+
+Complete example from audio to final output:
+
+```bash
+# 1. ASR: Transcribe audio
+python end2end.py --query-type audio_to_text \
+    --audio-path interview.wav \
+    --model ./models/Step-Audio2-7B \
+    --output-dir ./outputs
+
+# 2. Check the transcription
+cat ./outputs/00000_text.txt
+
+# 3. TTS: Synthesize new speech (with custom voice)
+STEP_AUDIO2_DEFAULT_PROMPT_WAV=./speaker_samples/female_voice.wav \
+python end2end.py --query-type text_to_audio \
+    --text "The quick brown fox jumps over the lazy dog" \
+    --model ./models/Step-Audio2-7B \
+    --output-dir ./outputs
+
+# 4. Listen to the result
+# Audio saved to: ./outputs/00000_output.wav
+```
+
+## References
+
+- [Step-Audio2 Paper](https://arxiv.org/abs/2507.16632)
+- [vLLM-Omni Documentation](https://vllm-omni.readthedocs.io)
+- [Model on HuggingFace](https://huggingface.co/stepfun-ai/Step-Audio-2-mini)

examples/offline_inference/step_audio2/end2end.py

@@ -0,0 +1,527 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+This example shows how to use vLLM-Omni for running offline inference
+with Step-Audio2 model for audio understanding and generation.
+
+Step-Audio2 is a two-stage model:
+  - Stage 0 (Thinker): Audio understanding → Text + Audio tokens
+  - Stage 1 (Token2Wav): Audio tokens → Waveform (24kHz)
+
+Usage examples:
+  # Audio to text (ASR)
+  python end2end.py --query-type audio_to_text --audio-path input.wav --model-path "$MODEL_PATH"
+
+  # Text to audio (TTS with default voice)
+  python end2end.py --query-type text_to_audio --model-path "$MODEL_PATH"
+
+  # Audio to audio (Voice conversion with default voice)
+  python end2end.py --query-type audio_to_audio --audio-path input.wav --model-path "$MODEL_PATH"
+"""
+
+import os
+import sys
+from pathlib import Path
+from typing import NamedTuple
+
+import librosa  # noqa: TID251
+import numpy as np
+import soundfile as sf
+import torch
+from vllm.assets.audio import AudioAsset
+from vllm.sampling_params import SamplingParams
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+from vllm_omni.entrypoints.omni import Omni
+
+SEED = 42
+
+
+class QueryResult(NamedTuple):
+    inputs: dict
+    limit_mm_per_prompt: dict[str, int]
+
+
+def get_audio_to_text_query(
+    audio_path: str | None = None,
+    question: str | None = None,
+    sampling_rate: int = 16000,
+) -> QueryResult:
+    """
+    Audio Speech Recognition (ASR) - Convert audio to text.
+
+    Args:
+        audio_path: Path to audio file
+        question: Question about the audio (optional)
+        sampling_rate: Target sampling rate (16kHz for Step-Audio2)
+
+    Returns:
+        QueryResult with prompt and audio data
+    """
+    if question is None:
+        question = "请记录下你所听到的语音内容。"
+
+    prompt = (
+        f"<|im_start|>system\n{question}<|im_end|>\n<|im_start|>user\n<audio_patch><|im_end|>\n<|im_start|>assistant\n"
+    )
+
+    # Load audio
+    if audio_path:
+        if not os.path.exists(audio_path):
+            raise FileNotFoundError(f"Audio file not found: {audio_path}")
+        audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
+        audio_data = (audio_signal.astype(np.float32), sr)
+    else:
+        # Use default test audio
+        audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
+        # Resample to 16kHz if needed
+        if audio_data[1] != sampling_rate:
+            audio_signal = librosa.resample(audio_data[0], orig_sr=audio_data[1], target_sr=sampling_rate)
+            audio_data = (audio_signal.astype(np.float32), sampling_rate)
+
+    return QueryResult(
+        inputs={
+            "prompt": prompt,
+            "multi_modal_data": {
+                "audio": audio_data,
+            },
+        },
+        limit_mm_per_prompt={"audio": 1},
+    )
+
+
+def get_text_to_audio_query(
+    text: str | None = None,
+    sampling_rate: int = 16000,
+) -> QueryResult:
+    """
+    Text-to-Speech (TTS) - Convert text to audio.
+
+    Args:
+        text: Text to synthesize
+        sampling_rate: Target sampling rate (16kHz for Step-Audio2)
+
+    Returns:
+        QueryResult with text prompt and speaker wav path
+
+    Note:
+        For audio generation, the prompt must end with "<tts_start>"
+        to signal the model to generate audio tokens instead of just text.
+    """
+    if text is None:
+        text = "Hello, this is a test of Step Audio 2 text to speech synthesis."
+
+    prompt = (
+        "<|im_start|>system\n"
+        "请逐字朗读用户提供的文本，不要改写或补充。<|im_end|>\n"
+        "<|im_start|>user\n"
+        f"{text}<|im_end|>\n"
+        "<|im_start|>assistant\n"
+        "<tts_start>"
+    )
+
+    inputs = {
+        "prompt": prompt,
+    }
+
+    return QueryResult(
+        inputs=inputs,
+        limit_mm_per_prompt={},
+    )
+
+
+def get_audio_to_audio_query(
+    audio_path: str | None = None,
+    question: str | None = None,
+    sampling_rate: int = 16000,
+) -> QueryResult:
+    """
+    Audio-to-Audio - Voice conversion/cloning.
+
+    Args:
+        audio_path: Path to source audio file
+        question: Question/instruction about the audio
+        sampling_rate: Target sampling rate (16kHz for Step-Audio2)
+
+    Returns:
+        QueryResult with audio input
+
+    Note:
+        This mode processes input audio and generates output audio.
+        Speaker voice is controlled by STEP_AUDIO2_DEFAULT_PROMPT_WAV env var.
+    """
+    if question is None:
+        question = "请仔细聆听这段语音，然后复述其内容。"
+
+    prompt = (
+        "<|im_start|>system\n"
+        f"{question}<|im_end|>\n"
+        "<|im_start|>user\n"
+        "<audio_patch><|im_end|>\n"
+        "<|im_start|>assistant\n"
+        "<tts_start>"
+    )
+
+    # Load source audio
+    if audio_path:
+        if not os.path.exists(audio_path):
+            raise FileNotFoundError(f"Audio file not found: {audio_path}")
+        audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
+        audio_data = (audio_signal.astype(np.float32), sr)
+
+    else:
+        audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
+        if audio_data[1] != sampling_rate:
+            audio_signal = librosa.resample(audio_data[0], orig_sr=audio_data[1], target_sr=sampling_rate)
+            audio_data = (audio_signal.astype(np.float32), sampling_rate)
+
+    return QueryResult(
+        inputs={
+            "prompt": prompt,
+            "multi_modal_data": {
+                "audio": audio_data,
+            },
+        },
+        limit_mm_per_prompt={"audio": 1},
+    )
+
+
+query_map = {
+    "audio_to_text": get_audio_to_text_query,
+    "text_to_audio": get_text_to_audio_query,
+    "audio_to_audio": get_audio_to_audio_query,
+}
+
+
+def main(args):
+    # Use local model path or HuggingFace model name
+    if not args.model:
+        sys.exit(1)
+
+    model_name = args.model
+
+    # Check if it's a local path
+    if os.path.isdir(model_name):
+        # Verify essential files exist
+        config_path = os.path.join(model_name, "config.json")
+        if not os.path.exists(config_path):
+            sys.exit(1)
+
+    # Get query configuration
+    query_func = query_map[args.query_type]
+
+    # Build query based on type
+    if args.query_type == "audio_to_text":
+        query_result = query_func(
+            audio_path=args.audio_path,
+            question=args.question,
+            sampling_rate=args.sampling_rate,
+        )
+    elif args.query_type == "text_to_audio":
+        query_result = query_func(
+            text=args.text,
+            sampling_rate=args.sampling_rate,
+        )
+    elif args.query_type == "audio_to_audio":
+        query_result = query_func(
+            audio_path=args.audio_path,
+            question=args.question,
+            sampling_rate=args.sampling_rate,
+        )
+    else:
+        raise ValueError(f"Unknown query type: {args.query_type}")
+
+    # Initialize vLLM-Omni with Step-Audio2
+    # Resolve stage config path
+    if args.stage_configs_path:
+        stage_config_path = args.stage_configs_path
+    else:
+        configs_dir = Path(__file__).parent.parent.parent.parent / "vllm_omni/model_executor/stage_configs"
+        if args.query_type == "audio_to_text":
+            # ASR only needs Thinker (Stage 0), no Token2Wav
+            stage_config_path = str(configs_dir / "step_audio_2_asr.yaml")
+            if not os.path.exists(stage_config_path):
+                stage_config_path = str(configs_dir / "step_audio_2.yaml")
+        else:
+            # TTS/S2ST need both stages
+            stage_config_path = str(configs_dir / "step_audio_2.yaml")
+
+    omni_llm = Omni(
+        model=model_name,
+        log_stats=args.enable_stats,
+        log_file=("step_audio2_pipeline.log" if args.enable_stats else None),
+        init_sleep_seconds=args.init_sleep_seconds,
+        batch_timeout=args.batch_timeout,
+        init_timeout=args.init_timeout,
+        shm_threshold_bytes=args.shm_threshold_bytes,
+        stage_configs_path=stage_config_path,
+        worker_backend=args.worker_backend,
+        ray_address=args.ray_address,
+    )
+
+    # Configure sampling parameters for each stage
+    # Stage 0 (Thinker): Generate text and audio tokens
+    # Note: For audio generation tasks, we need to allow the model to generate audio tokens
+    # Audio tokens are in range [151696, 158257], so we should not stop too early
+    # Qwen EOS token is typically 151645 (<|im_end|>), but we want to allow audio generation
+    # So we don't set stop_token_ids for audio tasks, but we do set it for text-only tasks
+    stop_token_ids = None
+    if args.query_type == "audio_to_text":
+        # For ASR, stop at EOS token
+        stop_token_ids = [151645]  # Qwen EOS token <|im_end|>
+
+    # Adjust sampling parameters based on query type
+    # For audio generation tasks, we need higher repetition_penalty to avoid text loops
+    # and allow the model to transition to audio token generation
+    repetition_penalty = 1.1 if args.query_type in ["text_to_audio", "audio_to_audio"] else 1.05
+
+    thinker_sampling_params = SamplingParams(
+        temperature=0.7,
+        top_p=0.9,
+        top_k=-1,
+        max_tokens=args.max_tokens,
+        seed=SEED,
+        detokenize=True,
+        repetition_penalty=repetition_penalty,
+        stop_token_ids=stop_token_ids,
+    )
+
+    # Stage 1 (Token2Wav): Convert audio tokens to waveform
+    # Note: This stage doesn't actually sample, it's deterministic generation
+    token2wav_sampling_params = SamplingParams(
+        temperature=0.0,
+        top_p=1.0,
+        top_k=-1,
+        max_tokens=1,
+        seed=SEED,
+        detokenize=False,
+    )
+
+    if args.query_type == "audio_to_text":
+        # ASR: single stage, only Thinker
+        sampling_params_list = [thinker_sampling_params]
+    else:
+        # TTS/S2ST: two stages
+        sampling_params_list = [thinker_sampling_params, token2wav_sampling_params]
+
+    # Prepare prompts
+    if args.num_prompts > 1:
+        prompts = [query_result.inputs for _ in range(args.num_prompts)]
+    else:
+        prompts = [query_result.inputs]
+
+    # Run inference
+    omni_outputs = omni_llm.generate(prompts, sampling_params_list)
+
+    # Create output directory
+    output_dir = args.output_dir
+    os.makedirs(output_dir, exist_ok=True)
+
+    def _get_request_index(req_id: str, fallback: int) -> int:
+        """Parse request index from vLLM request_id.
+
+        vLLM may return IDs like "0_<uuid>". Use the leading integer if present,
+        otherwise fall back to the loop index.
+        """
+        try:
+            return int(req_id)
+        except (TypeError, ValueError):
+            if isinstance(req_id, str):
+                head = req_id.split("_", 1)[0]
+                if head.isdigit():
+                    return int(head)
+        return fallback
+
+    # Process outputs from each stage
+    for stage_idx, stage_outputs in enumerate(omni_outputs):
+        # request_output may be a single object or a list
+        raw_output = stage_outputs.request_output
+        outputs_list = raw_output if isinstance(raw_output, list) else [raw_output]
+
+        if stage_outputs.final_output_type == "text":
+            # Stage 0 (Thinker) text output
+            for i, output in enumerate(outputs_list):
+                request_id = _get_request_index(output.request_id, i)
+                text_output = output.outputs[0].text
+
+                # Save to file
+                out_txt = os.path.join(output_dir, f"{request_id:05d}_text.txt")
+                with open(out_txt, "w", encoding="utf-8") as f:
+                    f.write(f"Prompt:\n{prompts[request_id]['prompt']}\n\n")
+                    f.write(f"Output:\n{text_output}\n")
+                print(f"Text output saved to: {out_txt}")
+                print(f"Text: {text_output[:200]}")
+
+        elif stage_outputs.final_output_type == "audio":
+            # Stage 1 (Token2Wav) audio output
+            for i, output in enumerate(outputs_list):
+                request_id = _get_request_index(output.request_id, i)
+
+                # Get audio tensor (24kHz)
+                mm_out = {}
+                if hasattr(output, "multimodal_output") and output.multimodal_output:
+                    mm_out = output.multimodal_output
+                elif output.outputs and hasattr(output.outputs[0], "multimodal_output"):
+                    mm_out = output.outputs[0].multimodal_output or {}
+                audio_tensor = None
+                for key in ("audio", "wav", "waveform", "audio_pcm", "pcm", "model_outputs"):
+                    if key in mm_out and mm_out[key] is not None:
+                        audio_tensor = mm_out[key]
+                        break
+
+                if audio_tensor is not None:
+                    # Save audio file
+                    output_wav = os.path.join(output_dir, f"{request_id:05d}_output.wav")
+                    if isinstance(audio_tensor, list):
+                        chunks = []
+                        for chunk in audio_tensor:
+                            if chunk is None:
+                                continue
+                            if isinstance(chunk, torch.Tensor):
+                                chunk_np = chunk.detach().cpu().numpy()
+                            else:
+                                chunk_np = np.asarray(chunk)
+                            if chunk_np.size > 0:
+                                chunks.append(chunk_np)
+                        if not chunks:
+                            continue
+                        audio_numpy = np.concatenate(chunks, axis=-1)
+                    elif isinstance(audio_tensor, torch.Tensor):
+                        audio_numpy = audio_tensor.detach().cpu().numpy()
+                    else:
+                        audio_numpy = np.asarray(audio_tensor)
+                    sf.write(output_wav, audio_numpy, samplerate=24000)
+
+
+def parse_args():
+    parser = FlexibleArgumentParser(description="Demo on using vLLM-Omni for offline inference with Step-Audio2")
+
+    # Model and config
+    parser.add_argument(
+        "--model",
+        "-m",
+        type=str,
+        required=True,
+        help=(
+            "Model path (required). Can be:\n"
+            "  - Local path: /path/to/step-audio-2\n"
+            "  - HuggingFace ID: stepfun-ai/Step-Audio2-mini"
+        ),
+    )
+    parser.add_argument(
+        "--stage-configs-path",
+        type=str,
+        default=None,
+        help="Path to stage config YAML file (default: use built-in step_audio_2.yaml)",
+    )
+
+    # Query configuration
+    parser.add_argument(
+        "--query-type",
+        "-q",
+        type=str,
+        default="audio_to_text",
+        choices=query_map.keys(),
+        help="Query type: audio_to_text (ASR), text_to_audio (TTS), audio_to_audio (voice conversion)",
+    )
+    parser.add_argument(
+        "--audio-path",
+        "-a",
+        type=str,
+        default=None,
+        help="Path to input audio file (for audio_to_text and audio_to_audio)",
+    )
+    parser.add_argument(
+        "--text",
+        "-t",
+        type=str,
+        default=None,
+        help="Text to synthesize (for text_to_audio mode)",
+    )
+    parser.add_argument(
+        "--question",
+        type=str,
+        default=None,
+        help="Question/instruction about the audio",
+    )
+    parser.add_argument(
+        "--sampling-rate",
+        type=int,
+        default=16000,
+        help="Audio sampling rate for input (default: 16000 Hz)",
+    )
+
+    # Generation parameters
+    parser.add_argument(
+        "--max-tokens",
+        type=int,
+        default=1024,
+        help="Maximum tokens to generate in Thinker stage (default: 1024)",
+    )
+    parser.add_argument(
+        "--num-prompts",
+        type=int,
+        default=1,
+        help="Number of times to repeat the prompt (for testing)",
+    )
+
+    # System configuration
+    parser.add_argument(
+        "--enable-stats",
+        action="store_true",
+        default=False,
+        help="Enable detailed statistics logging",
+    )
+    parser.add_argument(
+        "--init-sleep-seconds",
+        type=int,
+        default=20,
+        help="Sleep seconds after starting each stage (default: 20)",
+    )
+    parser.add_argument(
+        "--batch-timeout",
+        type=int,
+        default=5,
+        help="Batch timeout in seconds (default: 5)",
+    )
+    parser.add_argument(
+        "--init-timeout",
+        type=int,
+        default=300,
+        help="Initialization timeout in seconds (default: 300)",
+    )
+    parser.add_argument(
+        "--shm-threshold-bytes",
+        type=int,
+        default=65536,
+        help="Shared memory threshold in bytes (default: 65536)",
+    )
+    parser.add_argument(
+        "--worker-backend",
+        type=str,
+        default="multi_process",
+        choices=["multi_process", "ray"],
+        help="Worker backend (default: multi_process)",
+    )
+    parser.add_argument(
+        "--ray-address",
+        type=str,
+        default=None,
+        help="Ray cluster address (for ray backend)",
+    )
+
+    # Output
+    parser.add_argument(
+        "--output-dir",
+        "-o",
+        type=str,
+        default="output_step_audio2",
+        help="Output directory for generated files (default: output_step_audio2)",
+    )
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    main(args)

examples/offline_inference/text_to_image/text_to_image.py

@@ -10,10 +10,18 @@ from typing import Any
 import torch
 
 from vllm_omni.diffusion.data import logger
+from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
 from vllm_omni.entrypoints.omni import Omni
+from vllm_omni.entrypoints.openai.stage_params import clone_sampling_params
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 from vllm_omni.lora.request import LoRARequest
 from vllm_omni.lora.utils import stable_lora_int_id
+from vllm_omni.model_extras import (
+    build_text_to_image_prompt,
+    get_extra_body_params,
+    get_model_class_name,
+    should_init_extra_args_for_non_diffusion_stages,
+)
 from vllm_omni.platforms import current_omni_platform
 
 
@@ -259,6 +267,19 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="[NextStep-1.1 only] Apply layer normalization to sampled tokens.",
     )
+    parser.add_argument(
+        "--extra-body",
+        type=parse_profiler_config,
+        default=None,
+        help=(
+            "Model-specific generation params as a JSON object, e.g. "
+            '\'{"timestep_shift": 3.0, "cfg_text_scale": 4.0, "cfg_interval": [0.4, 1.0]}\'. '
+            "Each key is filtered against the model's declared extra_body_params "
+            "(see vllm_omni/model_extras), so unknown keys for the chosen model are "
+            "silently dropped. Values here take precedence over the equivalent "
+            "model-specific flags above."
+        ),
+    )
     parser.add_argument(
         "--enable-diffusion-pipeline-profiler",
         action="store_true",
@@ -405,6 +426,8 @@ def main():
         # NextStep-1.1 requires explicit pipeline class
         omni_kwargs["model_class_name"] = "NextStep11Pipeline"
     omni = Omni(**omni_kwargs)
+    model_class_name = get_model_class_name(omni)
+    declared_extra_body_params = get_extra_body_params(model_class_name)
 
     if profiler_enabled:
         print("[Profiler] Starting profiling...")
@@ -445,34 +468,79 @@ def main():
         )
 
     generation_start = time.perf_counter()
-    extra_args = {
+
+    prompt_dict = build_text_to_image_prompt(
+        model_class_name=model_class_name,
+        prompt=args.prompt,
+        negative_prompt=args.negative_prompt,
+        height=args.height,
+        width=args.width,
+    )
+
+    diffusion_params = OmniDiffusionSamplingParams(
+        height=args.height,
+        width=args.width,
+        seed=args.seed,
+        generator=generator,
+        true_cfg_scale=args.cfg_scale,
+        guidance_scale=args.guidance_scale,
+        guidance_scale_2=args.guidance_scale_2,
+        num_inference_steps=args.num_inference_steps,
+        num_outputs_per_prompt=args.num_images_per_prompt,
+    )
+
+    # Base layer: backward-compatible model-specific flags. New model params should
+    # instead be declared in vllm_omni/model_extras and passed via --extra-body, so
+    # this dict does not need to grow per-model.
+    user_extra = {
+        "cfg_scale": args.cfg_scale,
+        "cfg_text_scale": args.cfg_scale,
+        "negative_prompt": args.negative_prompt,
+        "timestep_shift": args.timesteps_shift,
         "timesteps_shift": args.timesteps_shift,
         "cfg_schedule": args.cfg_schedule,
         "use_norm": args.use_norm,
         "use_system_prompt": args.use_system_prompt,
         "system_prompt": args.system_prompt,
     }
-    if lora_request:
-        extra_args["lora_request"] = lora_request
-        extra_args["lora_scale"] = args.lora_scale
+    # Override layer: generic JSON passthrough wins over the flags above. Keys are
+    # still filtered against the model's declared extra_body_params downstream.
+    if args.extra_body:
+        user_extra.update(args.extra_body)
+    if declared_extra_body_params:
+        apply_declared_extra_args(diffusion_params, declared_extra_body_params, user_extra)
+    else:
+        diffusion_params.extra_args.update({k: v for k, v in user_extra.items() if v is not None})
 
-    outputs = omni.generate(
-        {
-            "prompt": args.prompt,
-            "negative_prompt": args.negative_prompt,
-        },
-        OmniDiffusionSamplingParams(
-            height=args.height,
-            width=args.width,
-            generator=generator,
-            true_cfg_scale=args.cfg_scale,
-            guidance_scale=args.guidance_scale,
-            guidance_scale_2=args.guidance_scale_2,
-            num_inference_steps=args.num_inference_steps,
-            num_outputs_per_prompt=args.num_images_per_prompt,
-            extra_args=extra_args,
-        ),
+    if lora_request:
+        diffusion_params.extra_args["lora_request"] = lora_request
+        diffusion_params.extra_args["lora_scale"] = args.lora_scale
+
+    # Build per-stage sampling params for multi-stage models (e.g. BAGEL),
+    # or wrap single diffusion params for single-stage models.
+    init_non_diffusion = should_init_extra_args_for_non_diffusion_stages(
+        model_class_name,
     )
+    defaults = list(omni.default_sampling_params_list or [])
+    sampling_params_list = [clone_sampling_params(p) for p in defaults]
+    if not sampling_params_list:
+        sampling_params_list = [diffusion_params]
+
+    diffusion_replaced = False
+    for idx, params in enumerate(sampling_params_list):
+        if isinstance(params, OmniDiffusionSamplingParams):
+            sampling_params_list[idx] = diffusion_params
+            diffusion_replaced = True
+        elif init_non_diffusion and hasattr(params, "extra_args"):
+            if params.extra_args is None:
+                params.extra_args = {}
+            if args.seed is not None and hasattr(params, "seed"):
+                params.seed = args.seed
+
+    if not diffusion_replaced and len(sampling_params_list) == 1:
+        sampling_params_list = [diffusion_params]
+
+    outputs = omni.generate(prompt_dict, sampling_params_list=sampling_params_list)
 
     generation_end = time.perf_counter()
     generation_time = generation_end - generation_start
@@ -502,15 +570,16 @@ def main():
         raise ValueError("No output generated from omni.generate()")
     logger.info(f"Outputs: {outputs}")
 
-    first_output = outputs[0]
-    if not hasattr(first_output, "request_output") or not first_output.request_output:
-        raise ValueError("No request_output found in OmniRequestOutput")
+    images = None
+    for output in outputs:
+        images = getattr(output, "images", None)
+        if images:
+            break
+        req_out = getattr(output, "request_output", None)
+        images = getattr(req_out, "images", None) if req_out is not None else None
+        if images:
+            break
 
-    req_out = first_output.request_output
-    if not hasattr(req_out, "images"):
-        raise ValueError("Invalid request_output structure or missing 'images'.")
-
-    images = req_out.images
     if not images:
         raise ValueError("No images found in request_output")
 

examples/offline_inference/text_to_speech/README.md

@@ -17,6 +17,7 @@ list of supported architectures across all modalities, see
 | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ | — | 24 kHz |
 | Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ | — | 44.1 kHz |
 | GLM-TTS | `zai-org/GLM-TTS` | 2 (AR + DiT) | ✓ (required) | ✓ | — | 24 kHz |
+| Ming-omni-tts | `inclusionAI/Ming-omni-tts-0.5B` | 2 (AR + audio VAE) | ✓ | ✓ | style / IP / dialect / TTA / podcast | 44.1 kHz |
 | Ming-flash-omni-TTS | `Jonathan1909/Ming-flash-omni-2.0` | single (talker only) | — (caption-controlled) | — | style / IP / basic captions | 44.1 kHz |
 | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | single (AR + codec) | ✓ (required) | ✓ | voice_clone, continuation | 48 kHz |
 | OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design, language hint | 24 kHz |
@@ -159,6 +160,46 @@ Streaming requires `async_chunk: true` in the stage config.
 
 ---
 
+## Ming-omni-tts
+
+Dense 0.5B two-stage TTS pipeline (`AR + flow` + audio VAE) at 44.1 kHz. The example covers style, IP voice, music-only generation, text-to-audio events, emotion, dialect, zero-shot cloning, podcast, speech+BGM, and speech+environment-sound cases.
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
+    --case style \
+    --deploy-config vllm_omni/deploy/ming_tts.yaml \
+    --enforce-eager
+```
+
+### Voice cloning
+```bash
+python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
+    --case zero_shot \
+    --ref-audio /path/to/reference.wav \
+    --ref-text "在此奉劝大家别乱打美白针。" \
+    --deploy-config vllm_omni/deploy/ming_tts.yaml \
+    --enforce-eager
+```
+
+### Streaming
+```bash
+python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
+    --case basic \
+    --ref-audio /path/to/reference.wav \
+    --streaming \
+    --deploy-config vllm_omni/deploy/ming_tts.yaml \
+    --enforce-eager
+```
+
+### Notes
+- `style`, `ip`, `bgm`, and `tta` do not require reference audio.
+- Reference-audio cases use `--ref-audio`; `zero_shot` also requires `--ref-text`.
+- `podcast` uses multiple references via `--ref-audio-paths`.
+- Full case details live in [`ming_tts/README.md`](ming_tts/README.md).
+
+---
+
 ## Ming-flash-omni-TTS
 
 Standalone talker-only deployment of Ming-flash-omni-2.0 at 44.1 kHz. Voice is controlled through caption fields (`风格` / `IP` / `语速`/`基频`/`音量`) rather than reference audio.

examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py

@@ -0,0 +1,204 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Offline higgs-audio v3 inference example.
+
+Runs Stage 0 (Qwen3 talker) + Stage 1 (HiggsAudio codec) end-to-end
+through the vLLM-Omni engine without going through the HTTP server, and
+saves a 24 kHz mono WAV per prompt.
+
+Example (plain TTS):
+
+    python examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py \\
+        --texts "Hello world." \\
+        --output-dir results/higgs_v3_wavs
+
+Example (voice clone):
+
+    python examples/offline_inference/text_to_speech/higgs_audio_v3/end2end.py \\
+        --texts "Hello world." \\
+        --ref-audio path/to/reference.wav \\
+        --ref-text "Transcript of the reference clip." \\
+        --output-dir results/higgs_v3_wavs
+"""
+
+from __future__ import annotations
+
+import os
+
+os.environ.setdefault("VLLM_USE_DEEP_GEMM", "0")
+os.environ.setdefault("VLLM_MOE_USE_DEEP_GEMM", "0")
+
+import re
+import time
+from pathlib import Path
+
+import numpy as np
+import soundfile as sf
+import torch
+
+from vllm_omni import Omni
+
+SAMPLE_RATE = 24_000
+DEFAULT_TEXTS = (
+    "Hello world.",
+    "The quick brown fox jumps over the lazy dog.",
+    "Today is a beautiful day for a walk in the park.",
+)
+
+
+def parse_args():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Offline higgs-audio v3 inference")
+    parser.add_argument(
+        "--model",
+        type=str,
+        default="bosonai/higgs-audio-v3-tts-4b",
+        help="Stage-0 talker model id or path.",
+    )
+    parser.add_argument(
+        "--texts",
+        type=str,
+        nargs="+",
+        default=list(DEFAULT_TEXTS),
+        help="One or more plain-text prompts to synthesize.",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default="results/higgs_v3_wavs",
+        help="Directory to write per-prompt WAV files.",
+    )
+    parser.add_argument(
+        "--deploy-config",
+        type=str,
+        default=None,
+        help="Override the deploy config path.",
+    )
+    parser.add_argument(
+        "--max-new-tokens",
+        type=int,
+        default=2048,
+        help="Cap on Stage-0 codec frames per request.",
+    )
+    parser.add_argument(
+        "--ref-audio",
+        type=str,
+        default=None,
+        help="Reference audio for voice clone (WAV/FLAC/MP3 path).",
+    )
+    parser.add_argument(
+        "--ref-text",
+        type=str,
+        default=None,
+        help="Transcript of the reference audio. Optional but improves fidelity.",
+    )
+    return parser.parse_args()
+
+
+def _slugify(text: str) -> str:
+    slug = re.sub(r"\s+", "_", text.strip().lower())
+    slug = re.sub(r"[^a-z0-9_]+", "", slug)
+    return slug[:48] or "prompt"
+
+
+def _extract_pcm(multimodal_output: dict) -> torch.Tensor:
+    audio = multimodal_output.get("model_outputs")
+    if audio is None:
+        audio = multimodal_output.get("audio")
+    if audio is None:
+        raise ValueError(f"no audio key in multimodal_output: {list(multimodal_output.keys())}")
+    if isinstance(audio, list):
+        valid = [torch.as_tensor(a).float().cpu().reshape(-1) for a in audio if a is not None]
+        if not valid:
+            raise ValueError("audio list is empty")
+        return torch.cat(valid, dim=0) if len(valid) > 1 else valid[0]
+    return torch.as_tensor(audio).float().cpu().reshape(-1)
+
+
+def _pcm_to_int16(pcm: torch.Tensor) -> np.ndarray:
+    arr = pcm.numpy()
+    if arr.dtype.kind == "f":
+        arr = np.clip(arr, -1.0, 1.0)
+        arr = (arr * 32767.0).astype(np.int16)
+    else:
+        arr = arr.astype(np.int16)
+    return arr
+
+
+def main():
+    args = parse_args()
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    engine = Omni(model=args.model, deploy_config=args.deploy_config, trust_remote_code=True)
+
+    from transformers import AutoTokenizer
+
+    from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
+        HiggsAudioV3TokenizerAdapter,
+        apply_delay_pattern,
+        encode_reference_audio,
+    )
+
+    tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
+    adapter = HiggsAudioV3TokenizerAdapter(tokenizer)
+
+    # Load and encode reference audio once (if voice cloning)
+    ref_codes_delayed: torch.Tensor | None = None
+    if args.ref_audio is not None:
+        ref_wav, ref_sr = sf.read(args.ref_audio, always_2d=False)
+        if ref_wav.ndim == 2:
+            ref_wav = ref_wav.mean(axis=1)
+        ref_codes_raw = encode_reference_audio(ref_wav, int(ref_sr))
+        ref_codes_delayed = apply_delay_pattern(ref_codes_raw)
+        print(f"Reference   : {args.ref_audio}")
+        print(f"Ref codes   : {ref_codes_raw.shape[0]} frames -> {ref_codes_delayed.shape[0]} delayed")
+        if args.ref_text:
+            print(f"Ref text    : {args.ref_text}")
+
+    print(f"Model       : {args.model}")
+    print(f"Prompts     : {len(args.texts)}")
+    print(f"Output dir  : {output_dir}")
+    print(f"Voice clone : {'yes' if ref_codes_delayed is not None else 'no'}")
+
+    total_elapsed = 0.0
+    total_dur = 0.0
+    for text in args.texts:
+        if ref_codes_delayed is not None:
+            prompt_ids = adapter.build_prompt(
+                text,
+                num_ref_tokens=int(ref_codes_delayed.shape[0]),
+                reference_text=args.ref_text,
+            )
+            prompt = {
+                "prompt_token_ids": prompt_ids,
+                "additional_information": {
+                    "audio_input_ids": ref_codes_delayed.to(torch.long),
+                    "audio_input_ids_mask": torch.ones(ref_codes_delayed.shape[0], dtype=torch.bool),
+                },
+            }
+        else:
+            prompt_ids = adapter.build_prompt(text)
+            prompt = {"prompt_token_ids": prompt_ids}
+
+        t_start = time.perf_counter()
+        outputs = engine.generate([prompt])
+        elapsed = time.perf_counter() - t_start
+        total_elapsed += elapsed
+
+        mm = outputs[0].outputs[0].multimodal_output
+        pcm = _extract_pcm(mm)
+        slug = _slugify(text)
+        out_path = output_dir / f"{slug}.wav"
+        sf.write(str(out_path), _pcm_to_int16(pcm), SAMPLE_RATE, format="WAV", subtype="PCM_16")
+        dur = pcm.numel() / SAMPLE_RATE
+        total_dur += dur
+        print(f"  {slug:<50} dur={dur:5.2f}s  -> {out_path}")
+
+    rtf = total_elapsed / total_dur if total_dur > 0 else float("inf")
+    print(f"Total infer : {total_elapsed:.2f}s  total audio: {total_dur:.2f}s  RTF: {rtf:.3f}")
+
+
+if __name__ == "__main__":
+    main()

examples/offline_inference/text_to_speech/ming_tts/README.md

@@ -0,0 +1,248 @@
+# Ming-omni-tts Offline Inference
+
+`end2end.py` runs Ming dense 0.5B end to end with vLLM-Omni. It uses the in-repo Ming prompt assembly helper directly, so the example request shape matches the real integration instead of a simplified wrapper.
+
+## Files
+
+| File | Purpose |
+|---|---|
+| `end2end.py` | Driver: CLI, case loading, prompt construction, orchestration (~150 lines) |
+| `cases.yaml` | All 11 built-in case definitions (prompt, text, instruction, ref-audio flags, flow controls) |
+| `runner.py` | Engine management and audio output (streaming + blocking paths) |
+
+## Model Overview
+
+Ming dense 0.5B is exposed here as a two-stage offline pipeline:
+
+- **Stage 0**: Qwen2-based AR generation with Ming prompt formatting and inline flow controls
+- **Stage 1**: audio VAE decode to mono 44.1 kHz waveform
+
+`config_ming_tts.py` adapts the checkpoint's HuggingFace config fields
+(LLM, DiT, aggregator, AudioVAE, token ids). `vllm_omni/deploy/ming_tts.yaml`
+selects the vLLM-Omni pipeline and stage runtime topology, including
+connectors, async chunking, memory limits, and sampling defaults.
+
+The example supports both:
+
+- **Blocking eager** via `vllm_omni/deploy/ming_tts.yaml`
+- **Async chunk eager** via `vllm_omni/deploy/ming_tts.yaml` (default `async_chunk: true`)
+
+## Setup
+
+Install vLLM-Omni with the platform requirements for your accelerator:
+
+```bash
+uv pip install -e .
+```
+
+The Ming offline example does not require a separate upstream Ming package.
+Reference-audio cases use the repo dependencies for audio loading,
+resampling, and CampPlus speaker extraction, including `soundfile`,
+`torchaudio`, and `onnxruntime-rocm` in the documented ROCm environment.
+
+The tested ROCm environment is summarized in the
+[Ming recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/inclusionAI/Ming-omni-tts-0.5B.md).
+
+## Supported Cases
+
+These cases cover the upstream dense 0.5B cookbook surface that maps cleanly onto the current vLLM-Omni example:
+
+- `style`: zero-speaker style-conditioned speech
+- `ip`: zero-speaker IP voice generation
+- `bgm`: music-only generation
+- `tta`: text-to-audio event generation with FlowLoss controls
+- `emotion`: reference-audio speech with emotion control
+- `basic`: reference-audio speech with speed / pitch / volume control
+- `dialect`: reference-audio speech with dialect control
+- `zero_shot`: reference-audio cloning with explicit transcript
+- `podcast`: multi-reference dialogue generation with automatic speaker embedding extraction
+- `speech_bgm`: speech with background music conditioning
+- `speech_sound`: speech with environmental sound conditioning
+
+## Quick Start
+
+Run the zero-speaker style example:
+
+```bash
+python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
+    --case style \
+    --deploy-config vllm_omni/deploy/ming_tts.yaml \
+    --enforce-eager
+```
+
+Run zero-shot cloning with a transcript:
+
+```bash
+python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
+    --case zero_shot \
+    --ref-audio /path/to/10002287-00000094.wav \
+    --ref-text "在此奉劝大家别乱打美白针。" \
+    --deploy-config vllm_omni/deploy/ming_tts.yaml \
+    --enforce-eager
+```
+
+Run emotion-controlled speech:
+
+```bash
+python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
+    --case emotion \
+    --ref-audio /path/to/emotion_prompt.wav \
+    --deploy-config vllm_omni/deploy/ming_tts.yaml \
+    --enforce-eager
+```
+
+Run podcast generation with two reference clips:
+
+```bash
+python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
+    --case podcast \
+    --ref-audio-paths /path/to/CTS-CN-F2F-2019-11-11-423-012-A.wav /path/to/CTS-CN-F2F-2019-11-11-423-012-B.wav \
+    --deploy-config vllm_omni/deploy/ming_tts.yaml \
+    --enforce-eager
+```
+
+The script automatically extracts one 192-d speaker embedding per reference WAV using the Ming model's `campplus.onnx`.
+
+If you already have precomputed multi-speaker embeddings, you can override extraction with:
+
+```bash
+--speaker-embedding /path/to/podcast_speaker_embeddings.json
+```
+
+where the JSON is a list of speaker embeddings, one 192-d vector per speaker.
+
+Run text-to-audio event generation:
+
+```bash
+python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
+    --case tta \
+    --deploy-config vllm_omni/deploy/ming_tts.yaml \
+    --enforce-eager
+```
+
+Use async_chunk streaming:
+
+```bash
+python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
+    --case basic \
+    --ref-audio /path/to/10002287-00000095.wav \
+    --streaming \
+    --deploy-config vllm_omni/deploy/ming_tts.yaml \
+    --enforce-eager
+```
+
+`--streaming` uses `AsyncOmni` and the async_chunk stage config. It currently
+supports one prompt per process invocation; use blocking mode for
+`--num-prompts > 1`.
+
+Collect runtime stats and a manifest:
+
+```bash
+python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
+    --case style \
+    --deploy-config vllm_omni/deploy/ming_tts.yaml \
+    --enforce-eager \
+    --enable-stats \
+    --stats-log-file output_audio/ming_style_pipeline.log \
+    --metadata-json output_audio/ming_style_manifest.json
+```
+
+## Reference Fixtures
+
+The upstream Ming cookbook uses these public audio fixtures from `inclusionAI/Ming-omni-tts/data/wavs`:
+
+- `10002287-00000094.wav` for zero-shot cloning
+- `10002287-00000095.wav` for `basic`
+- `emotion_prompt.wav` for `emotion`
+- `yue_prompt.wav` for `dialect`
+- `00000309-00000300.wav` for `speech_bgm` and `speech_sound`
+- `CTS-CN-F2F-2019-11-11-423-012-A.wav` and `CTS-CN-F2F-2019-11-11-423-012-B.wav` for `podcast`
+
+## Validation Matrix
+
+The repo-facing example is intended to cover the same dense TTS workflows used
+by the local Ming validation script:
+
+| Case | Blocking `deploy/ming_tts.yaml` | Async chunk `deploy/ming_tts.yaml` | Extra inputs |
+|---|---:|---:|---|
+| `style` | Yes | Optional smoke test | none |
+| `ip` | Yes | Optional smoke test | none |
+| `bgm` | Yes | Optional smoke test | none |
+| `tta` | Yes | Optional smoke test | none |
+| `emotion` | Yes | Yes | `--ref-audio emotion_prompt.wav` |
+| `basic` | Yes | Yes | `--ref-audio 10002287-00000095.wav` |
+| `dialect` | Yes | Yes | `--ref-audio yue_prompt.wav` |
+| `zero_shot` | Yes | Yes | `--ref-audio 10002287-00000094.wav --ref-text ...` |
+| `podcast` | Yes | Yes | two `--ref-audio-paths` |
+| `speech_bgm` | Yes | Yes | `--ref-audio 00000309-00000300.wav` |
+| `speech_sound` | Yes | Yes | `--ref-audio 00000309-00000300.wav` |
+
+## Validated Outputs
+
+The following measurements are retained from an earlier L4 CUDA validation;
+they are not ROCm benchmark results. Default async_chunk matched blocking
+output frame counts and Stage-1 patch counts for every case:
+
+| Case | Blocking frames / patches / sec | Async chunk frames / patches / sec |
+|---|---:|---:|
+| `style` | 409248 / 29 / 9.28 | 409248 / 29 / 9.28 |
+| `ip` | 183456 / 13 / 4.16 | 183456 / 13 / 4.16 |
+| `bgm` | 1326528 / 94 / 30.08 | 1326528 / 94 / 30.08 |
+| `tta` | 465696 / 33 / 10.56 | 465696 / 33 / 10.56 |
+| `emotion` | 324576 / 23 / 7.36 | 324576 / 23 / 7.36 |
+| `basic` | 211680 / 15 / 4.80 | 211680 / 15 / 4.80 |
+| `dialect` | 239904 / 17 / 5.44 | 239904 / 17 / 5.44 |
+| `zero_shot` | 409248 / 29 / 9.28 | 409248 / 29 / 9.28 |
+| `podcast` | 437472 / 31 / 9.92 | 437472 / 31 / 9.92 |
+| `speech_bgm` | 296352 / 21 / 6.72 | 296352 / 21 / 6.72 |
+| `speech_sound` | 352800 / 25 / 8.00 | 352800 / 25 / 8.00 |
+
+## Key Arguments
+
+| Argument | Description |
+|---|---|
+| `--model` | Hugging Face repo or local Ming checkpoint path |
+| `--deploy-config` | Deploy config YAML. Use `vllm_omni/deploy/ming_tts.yaml` |
+| `--case` | Built-in demo case |
+| `--ref-audio` | Single reference wav path for cloning-style cases |
+| `--ref-audio-paths` | Multiple reference wav paths, used by `podcast` |
+| `--ref-text` | Reference transcript. Required for `zero_shot` |
+| `--instructions` | Free-form Ming instruction string |
+| `--instruction-json` | Structured Ming instruction JSON |
+| `--speaker-embedding` | JSON file containing a 192-d speaker embedding |
+| `--extract-speaker-embeddings` | Force CampPlus speaker extraction from the provided reference audio paths |
+| `--max-decode-steps` | Override `ming_max_decode_steps` |
+| `--num-prompts` | Repeat the same case N times. Outputs are indexed when `N > 1` |
+| `--streaming` | Use `AsyncOmni` and async_chunk transport |
+| `--enforce-eager` | Recommended for Ming dense; non-eager is out of scope |
+| `--enable-stats` / `--log-stats` | Enable vLLM-Omni per-request stats logging |
+| `--stats-log-file` | Optional path for the stats log |
+| `--metadata-json` | Optional path for the run manifest JSON |
+| `--stage-init-timeout` | Per-stage initialization timeout in seconds |
+| `--init-timeout` | Total initialization timeout in seconds |
+| `--batch-timeout` | Batch timeout in seconds |
+| `--worker-backend` | `multi_process` or `ray` |
+| `--ray-address` | Ray cluster address when using `--worker-backend ray` |
+
+## Output
+
+- The script writes one mono 44.1 kHz WAV file per run
+- Default output directory: `output_audio/`
+- Default filename: `ming_<case>.wav`
+- When `--num-prompts > 1`, outputs are indexed as `ming_<case>_00000.wav`, `..._00001.wav`, etc.
+- When stats are enabled, the script can also write:
+  - a stats log file such as `ming_style_pipeline.log`
+  - a manifest JSON with per-output metadata, stage durations, peak memory info,
+    and streaming client latency metrics when `--streaming` is used
+
+## Notes
+
+- `style` and `ip` are zero-speaker paths and do not require a reference clip
+- `emotion`, `basic`, `dialect`, `speech_bgm`, and `speech_sound` require one reference clip
+- `zero_shot` requires both `--ref-audio` and `--ref-text`
+- `podcast` requires at least two reference clips via `--ref-audio-paths`
+- `podcast` automatically extracts one speaker embedding per reference clip
+- `--speaker-embedding` may contain either one 192-d vector or a list of 192-d vectors
+- `--enforce-eager` was used for the validated runs
+- The earlier L4 validation used SDPA for the Ming audio VAE instead of
+  FlashAttention2, which is the preferred default when available.

examples/offline_inference/text_to_speech/ming_tts/cases.yaml

@@ -0,0 +1,110 @@
+style:
+  prompt: "Please generate speech based on the following description.\n"
+  text: "我会一直在这里陪着你，直到你慢慢、慢慢地沉入那个最温柔的梦里……好吗？"
+  instruction:
+    风格: >-
+      这是一种ASMR耳语，属于一种旨在引发特殊感官体验的创意风格。这个女性使用轻柔的普通话进行耳语，声音气音成分重。音量极低，紧贴麦克风，语速极慢，旨在制造触发听者颅内快感的声学刺激。
+  use_zero_spk_emb: true
+  max_decode_steps: 200
+
+ip:
+  prompt: "Please generate speech based on the following description.\n"
+  text: "这款产品的名字，叫变态坑爹牛肉丸。"
+  instruction:
+    IP: "灵小甄"
+  use_zero_spk_emb: true
+  max_decode_steps: 200
+
+bgm:
+  prompt: "Please generate music based on the following description.\n"
+  text: " Genre: 电子舞曲. Mood: 自信 / 坚定. Instrument: 架子鼓. Theme: 节日. Duration: 30s."
+  instruction: null
+  use_zero_spk_emb: false
+  max_decode_steps: 400
+
+tta:
+  prompt: "Please generate audio events based on given text.\n"
+  text: "Thunder and a gentle rain"
+  instruction: null
+  use_zero_spk_emb: false
+  max_decode_steps: 200
+  cfg: 4.5
+  sigma: 0.3
+  temperature: 2.5
+
+emotion:
+  prompt: "Please generate speech based on the following description.\n"
+  text: "我竟然抢到了陈奕迅的演唱会门票！太棒了！终于可以现场听一听他的歌声了！"
+  instruction:
+    情感: "高兴"
+  requires_ref_audio: true
+  auto_extract_speaker_embeddings: true
+  max_decode_steps: 200
+
+basic:
+  prompt: "Please generate speech based on the following description.\n"
+  text: "简单地说，这相当于惠普把消费领域市场拱手相让了。"
+  instruction:
+    语速: "快速"
+    基频: "中"
+    音量: "中"
+  requires_ref_audio: true
+  auto_extract_speaker_embeddings: true
+  max_decode_steps: 200
+
+dialect:
+  prompt: "Please generate speech based on the following description.\n"
+  text: "我觉得社会企业同个人都有责任"
+  instruction:
+    方言: "广粤话"
+  requires_ref_audio: true
+  auto_extract_speaker_embeddings: true
+  max_decode_steps: 200
+
+zero_shot:
+  prompt: "Please generate speech based on the following description.\n"
+  text: "我们的愿景是构建未来服务业的数字化基础设施，为世界带来更多微小而美好的改变。"
+  instruction: null
+  requires_ref_audio: true
+  requires_ref_text: true
+  auto_extract_speaker_embeddings: true
+  max_decode_steps: 200
+
+podcast:
+  prompt: "Please generate speech based on the following description.\n"
+  text: " speaker_1:你可以说一下，就大概说一下，可能虽然我也不知道，我看过那部电影没有。\n speaker_2:就是那个叫什么，变相一节课的嘛。\n speaker_1:嗯。\n speaker_2:一部搞笑的电影。\n speaker_1:一部搞笑的。\n"
+  instruction: null
+  prompt_text: " speaker_1:并且我们还要进行每个月还要考核 笔试的话还要进行笔试，做个，当服务员还要去笔试了\n speaker_2:对啊，这真的很奇怪，就是 单纯的因，单纯自己工资不高，只是因为可能人家那个店比较出名一点，就对你苛刻要求\n"
+  requires_ref_audio_count: 2
+  auto_extract_speaker_embeddings: true
+  max_decode_steps: 200
+
+speech_bgm:
+  prompt: "Please generate speech based on the following description.\n"
+  text: "此次业绩下滑原因，可归结为企业停止服务某些品牌，而带来的负面影响。"
+  instruction:
+    BGM:
+      Genre: "当代古典音乐."
+      Mood: "温暖 / 友善."
+      Instrument: "电吉他"
+      Theme: "节日."
+      SNR: 10.0
+      ENV: null
+  requires_ref_audio: true
+  auto_extract_speaker_embeddings: true
+  max_decode_steps: 200
+
+speech_sound:
+  prompt: "Please generate speech based on the following description.\n"
+  text: "此次业绩下滑原因，可归结为企业停止服务某些品牌，而带来的负面影响。"
+  instruction:
+    BGM:
+      ENV: "Birds chirping"
+      SNR: 10.0
+      Genre: null
+      Mood: null
+      Instrument: null
+      Theme: null
+  requires_ref_audio: true
+  auto_extract_speaker_embeddings: true
+  max_decode_steps: 200

examples/offline_inference/text_to_speech/ming_tts/end2end.py

@@ -0,0 +1,200 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Offline inference demo for Ming-omni-tts via vLLM Omni."""
+
+import json
+import os
+from pathlib import Path
+
+import soundfile as sf
+import torch
+import torchaudio
+import yaml
+from transformers import AutoTokenizer
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+from vllm_omni.model_executor.models.ming_tts.config_ming_tts import (
+    KEY_CFG,
+    KEY_MAX_DECODE_STEPS,
+    KEY_SIGMA,
+    KEY_TEMPERATURE,
+    SAMPLE_RATE,
+)
+from vllm_omni.model_executor.models.ming_tts.prompt_assembly import build_ming_dense_prompt
+from vllm_omni.model_executor.models.ming_tts.speaker_extractor import MingSpeakerEmbeddingExtractor
+
+try:
+    from .runner import (
+        build_manifest,
+        build_sampling_params,
+        resolve_metadata_json,
+        resolve_stats_log_file,
+        run_generation,
+    )
+except ImportError:
+    from runner import (
+        build_manifest,
+        build_sampling_params,
+        resolve_metadata_json,
+        resolve_stats_log_file,
+        run_generation,
+    )
+
+_DEFAULT_MODEL = "inclusionAI/Ming-omni-tts-0.5B"
+_REPO_ROOT = Path(__file__).resolve().parents[4]
+_DEFAULT_DEPLOY_CONFIG = str(_REPO_ROOT / "vllm_omni/deploy/ming_tts.yaml")
+_CASES_FILE = Path(__file__).with_name("cases.yaml")
+
+CASE_DEFAULTS = yaml.safe_load(_CASES_FILE.read_text(encoding="utf-8")) or {}
+if not CASE_DEFAULTS:
+    raise RuntimeError(f"Empty or missing case definitions in {_CASES_FILE}")
+
+
+def _build_parser() -> FlexibleArgumentParser:
+    p = FlexibleArgumentParser(description="Offline Ming-omni-tts example")
+    p.add_argument("--model", default=_DEFAULT_MODEL, help="Model name or local path")
+    p.add_argument("--deploy-config", default=None, help="Deploy config YAML; auto-selected when omitted")
+    p.add_argument("--case", choices=sorted(CASE_DEFAULTS), default="style", help="Built-in demo case")
+    p.add_argument("--text", default=None, help="Override case text")
+    p.add_argument("--prompt", default=None, help="Override the system prompt prefix")
+    p.add_argument("--instructions", default=None, help="Free-form Ming instruction string")
+    p.add_argument(
+        "--instruction-json", default=None, help='Structured Ming instruction JSON, e.g. \'{"方言":"广粤话"}\''
+    )
+    p.add_argument("--ref-audio", default=None, help="Single reference audio path for cloning-style cases")
+    p.add_argument("--ref-audio-paths", nargs="+", default=None, help="Multiple reference audio paths (podcast)")
+    p.add_argument("--ref-text", default=None, help="Reference transcript; required for zero_shot")
+    p.add_argument("--speaker-embedding", default=None, help="Path to a JSON speaker embedding file")
+    p.add_argument(
+        "--extract-speaker-embeddings",
+        action="store_true",
+        help="Extract speaker embeddings from ref audio via campplus.onnx",
+    )
+    p.add_argument("--max-decode-steps", type=int, default=None, help="Override ming_max_decode_steps")
+    p.add_argument("--output-dir", default="output_audio", help="Directory for output wav files")
+    p.add_argument("--output-name", default=None, help="Output wav filename")
+    p.add_argument("--num-prompts", type=int, default=1, help="Repeat the same prompt N times")
+    p.add_argument("--streaming", action="store_true", help="Use AsyncOmni with async_chunk streaming")
+    p.add_argument("--trust-remote-code", action="store_true")
+    p.add_argument("--enforce-eager", action="store_true")
+    p.add_argument(
+        "--log-stats", "--enable-stats", dest="log_stats", action="store_true", help="Enable Omni stats logging"
+    )
+    p.add_argument("--stats-log-file", default=None, help="Path for the Omni stats log file")
+    p.add_argument("--metadata-json", default=None, help="Path for the run manifest JSON")
+    p.add_argument("--stage-init-timeout", type=int, default=300, help="Per-stage init timeout (s)")
+    p.add_argument("--init-timeout", type=int, default=600, help="Total init timeout (s)")
+    p.add_argument("--batch-timeout", type=int, default=5, help="Batch timeout (s)")
+    p.add_argument("--shm-threshold-bytes", type=int, default=65536)
+    p.add_argument("--worker-backend", default="multi_process", choices=["multi_process", "ray"])
+    p.add_argument("--ray-address", default=None, help="Ray cluster address (--worker-backend ray)")
+    return p
+
+
+def _finalize_args(args) -> None:
+    if args.instructions is not None and args.instruction_json is not None:
+        raise RuntimeError("Use either --instructions or --instruction-json, not both")
+    if args.num_prompts < 1:
+        raise RuntimeError("--num-prompts must be at least 1")
+    if args.streaming and args.num_prompts != 1:
+        raise RuntimeError("--streaming currently supports exactly one prompt")
+    if args.deploy_config is None:
+        args.deploy_config = _DEFAULT_DEPLOY_CONFIG
+
+
+def _load_waveform(path: str) -> torch.Tensor:
+    samples, sr = sf.read(path, dtype="float32")
+    wav = torch.as_tensor(samples, dtype=torch.float32)
+    if wav.ndim == 2:
+        wav = wav.mean(dim=1)
+    wav = wav.reshape(1, -1)
+    if int(sr) != SAMPLE_RATE:
+        wav = torchaudio.functional.resample(wav, int(sr), SAMPLE_RATE)
+    return wav
+
+
+def _load_speaker_embedding(path: str) -> torch.Tensor:
+    return torch.as_tensor(json.loads(Path(path).read_text(encoding="utf-8")), dtype=torch.float32)
+
+
+def _ref_audio_paths(args) -> list[str]:
+    if args.ref_audio is not None and args.ref_audio_paths is not None:
+        raise RuntimeError("Use either --ref-audio or --ref-audio-paths, not both")
+    if args.ref_audio_paths is not None:
+        return list(args.ref_audio_paths)
+    return [args.ref_audio] if args.ref_audio else []
+
+
+def _resolve_reference_inputs(args, case: dict, paths: list[str]):
+    required = int(case.get("requires_ref_audio_count", 0))
+    if required > 0 and len(paths) < required:
+        raise RuntimeError(f"Case '{args.case}' needs {required} ref audio paths via --ref-audio-paths")
+    if required <= 0 and case.get("requires_ref_audio") and not paths:
+        raise RuntimeError(f"--ref-audio required for case '{args.case}'")
+    if not paths:
+        return None
+    wavs = [_load_waveform(p) for p in paths]
+    return wavs[0] if len(wavs) == 1 else wavs
+
+
+def _resolve_speaker_embedding(args, case: dict, paths: list[str]):
+    if args.speaker_embedding:
+        return _load_speaker_embedding(args.speaker_embedding)
+    if not (case.get("auto_extract_speaker_embeddings") or args.extract_speaker_embeddings) or not paths:
+        return None
+    embs = MingSpeakerEmbeddingExtractor(args.model).extract_many(paths)
+    if not embs:
+        raise RuntimeError("Speaker extraction produced no embeddings")
+    return embs[0] if len(embs) == 1 else torch.stack(embs, dim=0)
+
+
+def _build_prompt_payload(tokenizer, args):
+    case = CASE_DEFAULTS[args.case]
+    paths = _ref_audio_paths(args)
+    prompt_text = args.ref_text if args.ref_text is not None else case.get("prompt_text")
+    if case.get("requires_ref_text") and not prompt_text:
+        raise RuntimeError(f"--ref-text required for case '{args.case}'")
+    prompt_waveform = _resolve_reference_inputs(args, case, paths) if prompt_text is not None else None
+    speaker_embedding = _resolve_speaker_embedding(args, case, paths)
+    use_zero_spk_emb = bool(case.get("use_zero_spk_emb")) and prompt_waveform is None and speaker_embedding is None
+    runtime_controls = {KEY_MAX_DECODE_STEPS: args.max_decode_steps or case["max_decode_steps"]}
+    for key, field in [(KEY_CFG, "cfg"), (KEY_SIGMA, "sigma"), (KEY_TEMPERATURE, "temperature")]:
+        if field in case:
+            runtime_controls[key] = case[field]
+    instruction = (
+        json.loads(args.instruction_json) if args.instruction_json else (args.instructions or case.get("instruction"))
+    )
+    return build_ming_dense_prompt(
+        tokenizer,
+        prompt=args.prompt or case["prompt"],
+        text=args.text or case["text"],
+        runtime_controls=runtime_controls,
+        instruction=instruction,
+        prompt_text=prompt_text,
+        prompt_waveform=prompt_waveform,
+        speaker_embedding=speaker_embedding,
+        use_zero_spk_emb=use_zero_spk_emb,
+    )
+
+
+def main():
+    args = _build_parser().parse_args()
+    _finalize_args(args)
+    tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=False)
+    prompt_payload = _build_prompt_payload(tokenizer, args)
+    case = CASE_DEFAULTS[args.case]
+    sampling_params_list = build_sampling_params(args.max_decode_steps or case["max_decode_steps"])
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    stats_log_file = resolve_stats_log_file(args)
+    summaries = run_generation(args, prompt_payload, sampling_params_list, output_dir, stats_log_file)
+    metadata_json = resolve_metadata_json(args)
+    manifest = build_manifest(args, prompt_payload, stats_log_file, summaries)
+    if metadata_json is not None:
+        Path(metadata_json).write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
+        print(f"Saved run manifest to {metadata_json}")
+
+
+if __name__ == "__main__":
+    os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
+    main()

examples/offline_inference/text_to_speech/ming_tts/runner.py

@@ -0,0 +1,277 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import asyncio
+import json
+import time
+import uuid
+import wave
+from pathlib import Path
+
+import torch
+from vllm import SamplingParams
+
+from vllm_omni import AsyncOmni, Omni
+from vllm_omni.model_executor.models.ming_tts.config_ming_tts import (
+    KEY_SPEAKER_EMBEDDING,
+    SAMPLE_RATE,
+    TEXT_EOS_TOKEN_ID,
+)
+
+
+def coerce_audio_tensor(audio, *, async_chunk: bool) -> torch.Tensor:
+    if isinstance(audio, list):
+        parts = []
+        for item in audio:
+            tensor = torch.as_tensor(item, dtype=torch.float32).reshape(-1)
+            if tensor.numel() > 0:
+                parts.append(tensor)
+        if not parts:
+            return torch.zeros((0,), dtype=torch.float32)
+        return torch.cat(parts, dim=0)
+
+    return torch.as_tensor(audio, dtype=torch.float32).reshape(-1)
+
+
+def resolve_sr(sr) -> int:
+    if isinstance(sr, list):
+        sr = sr[-1]
+    if hasattr(sr, "item"):
+        return int(sr.item())
+    return int(sr)
+
+
+def extract_sample_rate(multimodal_output: dict) -> int:
+    sr = multimodal_output.get("sr")
+    if sr is None:
+        raise RuntimeError("Expected multimodal_output['sr']")
+    return resolve_sr(sr)
+
+
+def write_wav(path: str, audio: torch.Tensor, sample_rate: int) -> None:
+    audio = audio.clamp(-1.0, 1.0)
+    pcm16 = (audio * 32767.0).round().to(torch.int16).cpu().numpy()
+    with wave.open(path, "wb") as wav_file:
+        wav_file.setnchannels(1)
+        wav_file.setsampwidth(2)
+        wav_file.setframerate(int(sample_rate))
+        wav_file.writeframes(pcm16.tobytes())
+
+
+def request_index(request_id: str | None, fallback: int) -> int:
+    try:
+        return int(request_id)
+    except (TypeError, ValueError):
+        if isinstance(request_id, str):
+            head = request_id.split("_", 1)[0]
+            if head.isdigit():
+                return int(head)
+    return fallback
+
+
+def audio_summary(audio: torch.Tensor, sample_rate: int) -> dict:
+    waveform = audio.detach().cpu().reshape(-1).to(torch.float32)
+    return {
+        "sample_rate": int(sample_rate),
+        "num_samples": int(waveform.numel()),
+        "duration_seconds": float(waveform.numel()) / float(sample_rate),
+        "max_abs_amplitude": float(waveform.abs().max().item()) if waveform.numel() > 0 else 0.0,
+    }
+
+
+def resolve_output_name(output_name: str | None, case: str, index: int, total: int) -> str:
+    if total == 1:
+        return output_name or f"ming_{case}.wav"
+    base = Path(output_name or f"ming_{case}.wav")
+    return f"{base.stem}_{index:05d}{base.suffix or '.wav'}"
+
+
+def resolve_stats_log_file(args) -> str | None:
+    if not args.log_stats:
+        return None
+    if args.stats_log_file:
+        return args.stats_log_file
+    base = Path(args.output_name or f"ming_{args.case}.wav").stem
+    return str(Path(args.output_dir) / f"{base}_pipeline.log")
+
+
+def resolve_metadata_json(args) -> str | None:
+    if args.metadata_json:
+        return args.metadata_json
+    if args.log_stats:
+        base = Path(args.output_name or f"ming_{args.case}.wav").stem
+        return str(Path(args.output_dir) / f"{base}_manifest.json")
+    return None
+
+
+def build_manifest(args, prompt_payload, stats_log_file: str | None, outputs: list[dict]) -> dict:
+    additional_information = {}
+    if isinstance(prompt_payload, dict):
+        additional_information = dict(prompt_payload.get("additional_information", {}))
+    return {
+        "model": args.model,
+        "case": args.case,
+        "streaming": bool(args.streaming),
+        "deploy_config": args.deploy_config,
+        "enforce_eager": bool(args.enforce_eager),
+        "num_prompts": int(args.num_prompts),
+        "log_stats": bool(args.log_stats),
+        "stats_log_file": stats_log_file,
+        "prompt_text": additional_information.get("prompt_text"),
+        "instruction": additional_information.get("instruction"),
+        "speaker_embedding_shape": (
+            list(additional_information[KEY_SPEAKER_EMBEDDING].shape)
+            if KEY_SPEAKER_EMBEDDING in additional_information
+            and hasattr(additional_information[KEY_SPEAKER_EMBEDDING], "shape")
+            else None
+        ),
+        "outputs": outputs,
+        "generated_at_unix": time.time(),
+    }
+
+
+def build_engine_kwargs(args, stats_log_file: str | None) -> dict:
+    kwargs = {
+        "model": args.model,
+        "deploy_config": args.deploy_config,
+        "enforce_eager": args.enforce_eager,
+        "trust_remote_code": args.trust_remote_code,
+        "log_stats": args.log_stats,
+        "stage_init_timeout": args.stage_init_timeout,
+        "init_timeout": args.init_timeout,
+        "batch_timeout": args.batch_timeout,
+        "shm_threshold_bytes": args.shm_threshold_bytes,
+        "worker_backend": args.worker_backend,
+    }
+    if stats_log_file is not None:
+        kwargs["log_file"] = stats_log_file
+    if args.ray_address is not None:
+        kwargs["ray_address"] = args.ray_address
+    return kwargs
+
+
+def build_sampling_params(max_decode_steps: int) -> list[SamplingParams]:
+    return [
+        SamplingParams(
+            temperature=0.0,
+            max_tokens=max_decode_steps + 1,
+            stop_token_ids=[int(TEXT_EOS_TOKEN_ID)],
+        ),
+        SamplingParams(temperature=0.0, max_tokens=1),
+    ]
+
+
+async def run_streaming(args, prompt_payload, sampling_params_list, output_dir: Path, stats_log_file: str | None):
+    engine = AsyncOmni(**build_engine_kwargs(args, stats_log_file))
+    try:
+        all_audio_chunks = []
+        accumulated_samples = 0
+        chunk_idx = 0
+        start_time = time.time()
+        chunk_times = []
+        ttfp_seconds = None
+        final_stage_output = None
+        async for stage_output in engine.generate(
+            prompt=prompt_payload,
+            request_id=str(uuid.uuid4()),
+            sampling_params_list=sampling_params_list,
+        ):
+            final_stage_output = stage_output
+            multimodal_output = stage_output.multimodal_output or {}
+            audio = multimodal_output.get("audio")
+            if audio is None:
+                continue
+
+            finished = stage_output.finished
+            if isinstance(audio, torch.Tensor):
+                if finished:
+                    audio_chunk = audio[accumulated_samples:].float().detach().cpu()
+                else:
+                    audio_chunk = audio.float().detach().cpu()
+            elif isinstance(audio, list):
+                audio_chunk = torch.as_tensor(audio[chunk_idx], dtype=torch.float32).reshape(-1).cpu()
+            else:
+                audio_chunk = torch.as_tensor(audio, dtype=torch.float32).reshape(-1).cpu()
+
+            accumulated_samples += int(audio_chunk.numel())
+            chunk_idx += 1
+            if audio_chunk.numel() > 0:
+                now = time.time()
+                if ttfp_seconds is None:
+                    ttfp_seconds = now - start_time
+                chunk_times.append(now)
+                all_audio_chunks.append(audio_chunk)
+
+        if not all_audio_chunks:
+            raise RuntimeError("Streaming Ming example produced no audio chunks")
+
+        waveform = torch.cat(all_audio_chunks, dim=0)
+        output_name = resolve_output_name(args.output_name, args.case, 0, 1)
+        output_path = str(output_dir / output_name)
+        write_wav(output_path, waveform, SAMPLE_RATE)
+        summary = {
+            "request_id": getattr(final_stage_output, "request_id", None),
+            "stage_id": getattr(final_stage_output, "stage_id", None),
+            "output_path": output_path,
+            "stage_durations": getattr(final_stage_output, "stage_durations", {}),
+            "peak_memory_mb": getattr(final_stage_output, "peak_memory_mb", 0.0),
+            "ttfp_seconds": ttfp_seconds,
+            "mean_inter_chunk_seconds": (
+                sum(t1 - t0 for t0, t1 in zip(chunk_times, chunk_times[1:])) / (len(chunk_times) - 1)
+                if len(chunk_times) > 1
+                else None
+            ),
+        }
+        summary.update(audio_summary(waveform, SAMPLE_RATE))
+        print(f"Saved streaming output to {output_path}")
+        print(json.dumps(summary, ensure_ascii=False, indent=2))
+        return [summary]
+    finally:
+        engine.shutdown()
+
+
+def run_non_streaming(args, prompt_payload, sampling_params_list, output_dir: Path, stats_log_file: str | None):
+    engine = Omni(**build_engine_kwargs(args, stats_log_file))
+    try:
+        outputs = engine.generate(
+            prompts=[prompt_payload for _ in range(args.num_prompts)],
+            sampling_params_list=sampling_params_list,
+            py_generator=False,
+        )
+        summaries = []
+        for fallback_index, output in enumerate(outputs):
+            if output.final_output_type != "audio":
+                continue
+            multimodal_output = output.multimodal_output or {}
+            waveform = coerce_audio_tensor(multimodal_output.get("audio"), async_chunk=False)
+            sample_rate = extract_sample_rate(multimodal_output)
+            output_name = resolve_output_name(
+                args.output_name,
+                args.case,
+                request_index(output.request_id, fallback_index),
+                args.num_prompts,
+            )
+            output_path = str(output_dir / output_name)
+            write_wav(output_path, waveform, sample_rate)
+            summary = {
+                "request_id": output.request_id,
+                "stage_id": output.stage_id,
+                "output_path": output_path,
+                "stage_durations": output.stage_durations,
+                "peak_memory_mb": output.peak_memory_mb,
+            }
+            summary.update(audio_summary(waveform, sample_rate))
+            summaries.append(summary)
+            print(f"Saved output to {output_path}")
+            print(json.dumps(summary, ensure_ascii=False, indent=2))
+        if not summaries:
+            raise RuntimeError("Non-streaming Ming example produced no audio outputs")
+        return summaries
+    finally:
+        engine.close()
+
+
+def run_generation(args, prompt_payload, sampling_params_list, output_dir: Path, stats_log_file: str | None):
+    if args.streaming:
+        return asyncio.run(run_streaming(args, prompt_payload, sampling_params_list, output_dir, stats_log_file))
+    return run_non_streaming(args, prompt_payload, sampling_params_list, output_dir, stats_log_file)

examples/offline_inference/text_to_speech/qwen3_tts/end2end.py

@@ -135,17 +135,18 @@ def _estimate_prompt_len(
         return 2048
 
 
-def get_custom_voice_query(use_batch_sample: bool = False) -> QueryResult:
+def get_custom_voice_query(use_batch_sample: bool = False, model_name: str | None = None) -> QueryResult:
     """Build CustomVoice sample inputs.
 
     Args:
         use_batch_sample: When True, return a batch of prompts; otherwise a single prompt.
+        model_name: Override the default HF model path (e.g. with a local path).
 
     Returns:
         QueryResult with Omni inputs and the CustomVoice model path.
     """
     task_type = "CustomVoice"
-    model_name = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
+    model_name = model_name or "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
     if use_batch_sample:
         texts = [
             "其实我真的有发现，我是一个特别善于观察别人情绪的人。",
@@ -196,17 +197,18 @@ def get_custom_voice_query(use_batch_sample: bool = False) -> QueryResult:
     )
 
 
-def get_voice_design_query(use_batch_sample: bool = False) -> QueryResult:
+def get_voice_design_query(use_batch_sample: bool = False, model_name: str | None = None) -> QueryResult:
     """Build VoiceDesign sample inputs.
 
     Args:
         use_batch_sample: When True, return a batch of prompts; otherwise a single prompt.
+        model_name: Override the default HF model path (e.g. with a local path).
 
     Returns:
         QueryResult with Omni inputs and the VoiceDesign model path.
     """
     task_type = "VoiceDesign"
-    model_name = "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign"
+    model_name = model_name or "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign"
     if use_batch_sample:
         texts = [
             "哥哥，你回来啦，人家等了你好久好久了，要抱抱！",
@@ -255,18 +257,19 @@ def get_voice_design_query(use_batch_sample: bool = False) -> QueryResult:
     )
 
 
-def get_base_query(use_batch_sample: bool = False, mode_tag: str = "icl") -> QueryResult:
+def get_base_query(use_batch_sample: bool = False, mode_tag: str = "icl", model_name: str | None = None) -> QueryResult:
     """Build Base (voice clone) sample inputs.
 
     Args:
         use_batch_sample: When True, return a batch of prompts (Case 2).
         mode_tag: "icl" or "xvec_only" to control x_vector_only_mode behavior.
+        model_name: Override the default HF model path (e.g. with a local path).
 
     Returns:
         QueryResult with Omni inputs and the Base model path.
     """
     task_type = "Base"
-    model_name = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
+    model_name = model_name or "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
     ref_audio_path_1 = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav"
     ref_audio_single = ref_audio_path_1
     ref_text_single = (
@@ -335,14 +338,17 @@ def _build_inputs(args) -> tuple[str, list]:
         )
 
     query_func = query_map[args.query_type]
+    model_override = args.model if args.model else None
     if args.query_type in {"CustomVoice", "VoiceDesign"}:
-        query_result = query_func(use_batch_sample=args.use_batch_sample)
+        query_result = query_func(use_batch_sample=args.use_batch_sample, model_name=model_override)
     elif args.query_type == "Base":
-        query_result = query_func(use_batch_sample=args.use_batch_sample, mode_tag=args.mode_tag)
+        query_result = query_func(
+            use_batch_sample=args.use_batch_sample, mode_tag=args.mode_tag, model_name=model_override
+        )
     else:
         query_result = query_func()
 
-    model_name = query_result.model_name
+    model_name = args.model if args.model else query_result.model_name
 
     if args.txt_prompts:
         with open(args.txt_prompts) as f:
@@ -359,7 +365,10 @@ def _build_inputs(args) -> tuple[str, list]:
             for t in lines
         ]
     else:
-        inputs = query_result.inputs if isinstance(query_result.inputs, list) else [query_result.inputs]
+        if isinstance(query_result.inputs, list):
+            inputs = query_result.inputs * args.num_prompts
+        else:
+            inputs = [query_result.inputs] * args.num_prompts
 
     return model_name, inputs
 
@@ -434,6 +443,12 @@ async def main_streaming(args):
 
 def parse_args():
     parser = TrackingArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
+    parser.add_argument(
+        "--model",
+        type=str,
+        default=None,
+        help="Override the default model path for the selected query type.",
+    )
     parser.add_argument(
         "--query-type",
         "-q",

examples/offline_inference/text_to_speech/qwen3_tts/end2end_benchmark_script.py

@@ -0,0 +1,640 @@
+"""Offline benchmark script for Qwen3 TTS via vLLM Omni.
+
+Supports warmup + test rounds with a single Omni instance to avoid
+repeated model loading. Prints per-round wall time summary at the end.
+"""
+
+import asyncio
+import logging
+import os
+import time
+from typing import Any, NamedTuple
+
+import soundfile as sf
+import torch
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+from vllm_omni import AsyncOmni, Omni
+
+logger = logging.getLogger(__name__)
+
+
+class QueryResult(NamedTuple):
+    """Container for a prepared Omni request."""
+
+    inputs: dict
+    model_name: str
+
+
+_model_cache: dict[str, Any] = {}
+
+
+def _estimate_prompt_len(
+    additional_information: dict[str, Any],
+    model_name: str,
+) -> int:
+    """Estimate prompt_token_ids placeholder length for the Talker stage."""
+    try:
+        from vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts import Qwen3TTSConfig
+        from vllm_omni.model_executor.models.qwen3_tts.qwen3_tts_talker import (
+            Qwen3TTSTalkerForConditionalGeneration,
+        )
+
+        if model_name not in _model_cache:
+            from transformers import AutoTokenizer
+
+            tok = AutoTokenizer.from_pretrained(
+                model_name, trust_remote_code=True, padding_side="left", fix_mistral_regex=True
+            )
+            cfg = Qwen3TTSConfig.from_pretrained(model_name, trust_remote_code=True)
+
+            speech_tok = None
+            try:
+                from transformers.utils import cached_file
+
+                from vllm_omni.model_executor.models.qwen3_tts.qwen3_tts_tokenizer import Qwen3TTSTokenizer
+
+                st_cfg_path = cached_file(model_name, "speech_tokenizer/config.json")
+                if st_cfg_path:
+                    speech_tok = Qwen3TTSTokenizer.from_pretrained(
+                        # os.path.dirname(st_cfg_path), torch_dtype=torch.bfloat16
+                        os.path.dirname(st_cfg_path),
+                        torch_dtype=torch.bfloat16,
+                        fix_mistral_regex=True,
+                    )
+                    logger.info("Loaded speech tokenizer for exact ref_code_len estimation")
+            except Exception as e:
+                logger.debug("Could not load speech tokenizer: %s", e)
+
+            _model_cache[model_name] = (tok, getattr(cfg, "talker_config", None), speech_tok)
+
+        tok, tcfg, speech_tok = _model_cache[model_name]
+        task_type = (additional_information.get("task_type") or ["CustomVoice"])[0]
+
+        def _estimate_ref_code_len(ref_audio: object) -> int | None:
+            if not isinstance(ref_audio, (str, list)):
+                return None
+            audio_path = ref_audio[0] if isinstance(ref_audio, list) else ref_audio
+            if not isinstance(audio_path, str) or not audio_path.strip():
+                return None
+            try:
+                from vllm.multimodal.media import MediaConnector
+
+                connector = MediaConnector(allowed_local_media_path="/")
+                audio, sr = connector.fetch_audio(audio_path)
+                import numpy as np
+
+                wav_np = np.asarray(audio, dtype=np.float32)
+
+                if speech_tok is not None:
+                    enc = speech_tok.encode(wav_np, sr=int(sr), return_dict=True)
+                    ref_code = getattr(enc, "audio_codes", None)
+                    if isinstance(ref_code, list):
+                        ref_code = ref_code[0] if ref_code else None
+                    if ref_code is not None and hasattr(ref_code, "shape"):
+                        shape = ref_code.shape
+                        return int(shape[0]) if len(shape) == 2 else int(shape[1]) if len(shape) == 3 else None
+
+                codec_hz = getattr(tcfg, "codec_frame_rate", None) or 12
+                return int(len(audio) / sr * codec_hz)
+            except Exception:
+                return None
+
+        return Qwen3TTSTalkerForConditionalGeneration.estimate_prompt_len_from_additional_information(
+            additional_information=additional_information,
+            task_type=task_type,
+            tokenize_prompt=lambda t: tok(t, padding=False)["input_ids"],
+            codec_language_id=getattr(tcfg, "codec_language_id", None),
+            spk_is_dialect=getattr(tcfg, "spk_is_dialect", None),
+            estimate_ref_code_len=_estimate_ref_code_len,
+        )
+    except Exception as exc:
+        logger.warning("Failed to estimate prompt length, using fallback 2048: %s", exc)
+        return 2048
+
+
+def get_custom_voice_query(use_batch_sample: bool = False) -> QueryResult:
+    task_type = "CustomVoice"
+    model_name = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
+    if use_batch_sample:
+        texts = [
+            "其实我真的有发现，我是一个特别善于观察别人情绪的人。",
+            "She said she would be here by noon.",
+            "I like you very much.",
+            "Really, you do?",
+            "Yes, absolutely.",
+        ]
+        instructs = ["", "Very happy.", "Very happy.", "Very happy.", "Very happy."]
+        languages = ["Chinese", "English", "English", "English", "English"]
+        speakers = ["Vivian", "Ryan", "Ryan", "Ryan", "Ryan"]
+        inputs = []
+        for text, instruct, language, speaker in zip(texts, instructs, languages, speakers):
+            additional_information = {
+                "task_type": [task_type],
+                "text": [text],
+                "instruct": [instruct],
+                "language": [language],
+                "speaker": [speaker],
+                "max_new_tokens": [2048],
+            }
+            inputs.append(
+                {
+                    "prompt_token_ids": [0] * _estimate_prompt_len(additional_information, model_name),
+                    "additional_information": additional_information,
+                }
+            )
+    else:
+        text = "其实我真的有发现，我是一个特别善于观察别人情绪的人。"
+        language = "Chinese"
+        speaker = "Vivian"
+        instruct = "用特别愤怒的语气说"
+        additional_information = {
+            "task_type": [task_type],
+            "text": [text],
+            "language": [language],
+            "speaker": [speaker],
+            "instruct": [instruct],
+            "max_new_tokens": [2048],
+        }
+        inputs = {
+            "prompt_token_ids": [0] * _estimate_prompt_len(additional_information, model_name),
+            "additional_information": additional_information,
+        }
+    return QueryResult(inputs=inputs, model_name=model_name)
+
+
+def get_voice_design_query(use_batch_sample: bool = False) -> QueryResult:
+    task_type = "VoiceDesign"
+    model_name = "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign"
+    if use_batch_sample:
+        texts = [
+            "哥哥，你回来啦，人家等了你好久好久了，要抱抱！",
+            "It's in the top drawer... wait, it's empty? No way, that's impossible! I'm sure I put it there!",
+        ]
+        instructs = [
+            "体现撒娇稚嫩的萝莉女声，音调偏高且起伏明显，营造出黏人、做作又刻意卖萌的听觉效果。",
+            "Speak in an incredulous tone, but with a hint of panic beginning to creep into your voice.",
+        ]
+        languages = ["Chinese", "English"]
+        inputs = []
+        for text, instruct, language in zip(texts, instructs, languages):
+            additional_information = {
+                "task_type": [task_type],
+                "text": [text],
+                "language": [language],
+                "instruct": [instruct],
+                "max_new_tokens": [2048],
+                "non_streaming_mode": [True],
+            }
+            inputs.append(
+                {
+                    "prompt_token_ids": [0] * _estimate_prompt_len(additional_information, model_name),
+                    "additional_information": additional_information,
+                }
+            )
+    else:
+        text = "哥哥，你回来啦，人家等了你好久好久了，要抱抱！"
+        instruct = "体现撒娇稚嫩的萝莉女声，音调偏高且起伏明显，营造出黏人、做作又刻意卖萌的听觉效果。"
+        language = "Chinese"
+        additional_information = {
+            "task_type": [task_type],
+            "text": [text],
+            "language": [language],
+            "instruct": [instruct],
+            "max_new_tokens": [2048],
+            "non_streaming_mode": [True],
+        }
+        inputs = {
+            "prompt_token_ids": [0] * _estimate_prompt_len(additional_information, model_name),
+            "additional_information": additional_information,
+        }
+    return QueryResult(inputs=inputs, model_name=model_name)
+
+
+def get_base_query(use_batch_sample: bool = False, mode_tag: str = "icl") -> QueryResult:
+    task_type = "Base"
+    model_name = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
+    ref_audio_path_1 = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav"
+    ref_audio_single = ref_audio_path_1
+    ref_text_single = (
+        "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
+    )
+    syn_text_single = "Good one. Okay, fine, I'm just gonna leave this sock monkey here. Goodbye."
+    syn_lang_single = "Auto"
+    x_vector_only_mode = mode_tag == "xvec_only"
+    if use_batch_sample:
+        syn_text_batch = [
+            "Good one. Okay, fine, I'm just gonna leave this sock monkey here. Goodbye.",
+            "其实我真的有发现，我是一个特别善于观察别人情绪的人。",
+        ]
+        syn_lang_batch = ["Chinese", "English"]
+        inputs = []
+        for text, language in zip(syn_text_batch, syn_lang_batch):
+            additional_information = {
+                "task_type": [task_type],
+                "ref_audio": [ref_audio_single],
+                "ref_text": [ref_text_single],
+                "text": [text],
+                "language": [language],
+                "x_vector_only_mode": [x_vector_only_mode],
+                "max_new_tokens": [2048],
+            }
+            inputs.append(
+                {
+                    "prompt_token_ids": [0] * _estimate_prompt_len(additional_information, model_name),
+                    "additional_information": additional_information,
+                }
+            )
+    else:
+        additional_information = {
+            "task_type": [task_type],
+            "ref_audio": [ref_audio_single],
+            "ref_text": [ref_text_single],
+            "text": [syn_text_single],
+            "language": [syn_lang_single],
+            "x_vector_only_mode": [x_vector_only_mode],
+            "max_new_tokens": [2048],
+        }
+        inputs = {
+            "prompt_token_ids": [0] * _estimate_prompt_len(additional_information, model_name),
+            "additional_information": additional_information,
+        }
+    return QueryResult(inputs=inputs, model_name=model_name)
+
+
+query_map = {
+    "CustomVoice": get_custom_voice_query,
+    "VoiceDesign": get_voice_design_query,
+    "Base": get_base_query,
+}
+
+
+def _build_inputs(args) -> tuple[str, list]:
+    """Resolve model name and inputs list from CLI args."""
+    if args.batch_size < 1 or (args.batch_size & (args.batch_size - 1)) != 0:
+        raise ValueError(
+            f"--batch-size must be a power of two (got {args.batch_size}); "
+            "non-power-of-two values do not align with CUDA graph capture sizes "
+            "of Code2Wav."
+        )
+
+    query_func = query_map[args.query_type]
+    if args.query_type in {"CustomVoice", "VoiceDesign"}:
+        query_result = query_func(use_batch_sample=args.use_batch_sample)
+    elif args.query_type == "Base":
+        query_result = query_func(use_batch_sample=args.use_batch_sample, mode_tag=args.mode_tag)
+    else:
+        query_result = query_func()
+
+    model_name = query_result.model_name
+
+    if args.txt_prompts:
+        with open(args.txt_prompts) as f:
+            lines = [line.strip() for line in f if line.strip()]
+        if not lines:
+            raise ValueError(f"No valid prompts found in {args.txt_prompts}")
+        template = query_result.inputs if not isinstance(query_result.inputs, list) else query_result.inputs[0]
+        template_info = template["additional_information"]
+        inputs = [
+            {
+                "prompt_token_ids": [0] * _estimate_prompt_len({**template_info, "text": [t]}, model_name),
+                "additional_information": {**template_info, "text": [t]},
+            }
+            for t in lines
+        ]
+    else:
+        inputs = query_result.inputs if isinstance(query_result.inputs, list) else [query_result.inputs]
+
+    return model_name, inputs
+
+
+def _save_wav(output_dir: str, request_id: str, mm: dict) -> None:
+    """Concatenate audio chunks and write to a wav file."""
+    audio_data = mm["audio"]
+    sr_raw = mm["sr"]
+    sr_val = sr_raw[-1] if isinstance(sr_raw, list) and sr_raw else sr_raw
+    sr = sr_val.item() if hasattr(sr_val, "item") else int(sr_val)
+    audio_tensor = torch.cat(audio_data, dim=-1) if isinstance(audio_data, list) else audio_data
+    out_wav = os.path.join(output_dir, f"output_{request_id}.wav")
+    sf.write(out_wav, audio_tensor.float().cpu().numpy().flatten(), samplerate=sr, format="WAV")
+    logger.info("Saved audio to %s", out_wav)
+
+
+class _SummaryCollector:
+    """Monkey-patches omni._log_summary_and_cleanup to capture summary dicts.
+
+    NOTE: This relies on a private method ``_log_summary_and_cleanup`` which may
+    break if the upstream Omni implementation is refactored.  If Omni exposes a
+    public hook for summary collection in the future, prefer that instead.
+    """
+
+    def __init__(self, omni):
+        self.omni = omni
+        self.summaries: list[dict[str, Any]] = []
+        self.collecting = False
+        self._orig = omni._log_summary_and_cleanup
+
+    def install(self):
+        from pprint import pformat as _pformat
+
+        collector = self
+
+        def _patched(request_id: str) -> None:
+            req_state = collector.omni.request_states.get(request_id)
+            try:
+                if req_state is None or req_state.metrics is None:
+                    return
+                summary = req_state.metrics.build_and_log_summary()
+                logger.info("[Summary] %s", _pformat(summary, sort_dicts=False))
+                if collector.collecting and summary:
+                    collector.summaries.append(summary)
+            except Exception:
+                logger.exception("Failed to build/log summary for req=%s", request_id)
+            finally:
+                collector.omni.request_states.pop(request_id, None)
+
+        collector.omni._log_summary_and_cleanup = _patched
+
+
+def _print_summary(round_times: list[float], summaries: list[dict[str, Any]] | None = None) -> None:
+    """Print per-round wall time and optional detailed pipeline summary."""
+    print("\n" + "=" * 60)
+    print("BENCHMARK SUMMARY")
+    print("=" * 60)
+    for i, t in enumerate(round_times):
+        print(f"  Test round {i}: {t:.3f}s")
+    print("-" * 60)
+    avg = sum(round_times) / len(round_times)
+    print(f"  Rounds:  {len(round_times)}")
+    print(f"  Total:   {sum(round_times):.3f}s")
+    print(f"  Average: {avg:.3f}s")
+    print(f"  Min:     {min(round_times):.3f}s")
+    print(f"  Max:     {max(round_times):.3f}s")
+    print("=" * 60)
+
+    if not summaries:
+        print()
+        return
+
+    # --- Aggregate from captured summaries ---
+    overall_list = [s["overall_summary"] for s in summaries if "overall_summary" in s]
+    if not overall_list:
+        print()
+        return
+
+    n = len(overall_list)
+
+    # Average Wall time
+    avg_wall = sum(d["e2e_wall_time_ms"] for d in overall_list) / n
+    print(f"\n{'=' * 60}")
+    print("DETAILED PIPELINE SUMMARY (test rounds average)")
+    print("=" * 60)
+    print(f"  Avg Wall Time:           {avg_wall:.3f} ms")
+
+    # Per-stage timing
+    stage_keys = sorted(k for k in overall_list[0] if k.startswith("e2e_stage_") and k.endswith("_wall_time_ms"))
+    if stage_keys:
+        print("-" * 60)
+        print("  Per-stage timing (avg):")
+        for sk in stage_keys:
+            stage_avg = sum(d.get(sk, 0.0) for d in overall_list) / n
+            # extract stage index from key like e2e_stage_0_wall_time_ms
+            stage_idx = sk.replace("e2e_stage_", "").replace("_wall_time_ms", "")
+            print(f"    Stage {stage_idx} wall time:   {stage_avg:.3f} ms")
+
+    # Per-stage gen time from stage_table
+    all_stage_gen: dict[int, list[float]] = {}
+    for s in summaries:
+        for req_entry in s.get("stage_table", []):
+            for stage_info in req_entry.get("stages", []):
+                sid = stage_info.get("stage_id", -1)
+                gen_ms = stage_info.get("stage_gen_time_ms", 0.0)
+                all_stage_gen.setdefault(sid, []).append(gen_ms)
+    if all_stage_gen:
+        print("  Per-stage gen time (avg):")
+        for sid in sorted(all_stage_gen):
+            vals = all_stage_gen[sid]
+            print(f"    Stage {sid} gen time:     {sum(vals) / len(vals):.3f} ms")
+
+    # Per-prompt throughput
+    avg_tokens_per_s = sum(d.get("e2e_avg_tokens_per_s", 0.0) for d in overall_list) / n
+    avg_time_per_req = sum(d.get("e2e_avg_time_per_request_ms", 0.0) for d in overall_list) / n
+    total_tokens_list = [d.get("e2e_total_tokens", 0) for d in overall_list]
+    avg_total_tokens = sum(total_tokens_list) / n
+    print("-" * 60)
+    print("  Per-prompt throughput (avg):")
+    print(f"    Tokens per request:    {avg_total_tokens:.1f}")
+    print(f"    Time per request:      {avg_time_per_req:.3f} ms")
+    print(f"    Tokens/s:              {avg_tokens_per_s:.3f}")
+    print("=" * 60 + "\n")
+
+
+def main(args):
+    """Run sync benchmark: warmup rounds then test rounds."""
+    model_name, inputs = _build_inputs(args)
+    output_dir = args.output_dir
+    os.makedirs(output_dir, exist_ok=True)
+
+    omni = Omni(
+        model=model_name,
+        stage_configs_path=args.stage_configs_path,
+        log_stats=args.log_stats,
+        stage_init_timeout=args.stage_init_timeout,
+        enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
+    )
+
+    collector = _SummaryCollector(omni)
+    collector.install()
+
+    batch_size = args.batch_size
+
+    # --- Warmup ---
+    for r in range(args.warmup_rounds):
+        print(f"[Warmup {r}/{args.warmup_rounds}] running...")
+        for batch_start in range(0, len(inputs), batch_size):
+            batch = inputs[batch_start : batch_start + batch_size]
+            for _ in omni.generate(batch):
+                pass
+        print(f"[Warmup {r}/{args.warmup_rounds}] done")
+
+    # --- Test ---
+    collector.collecting = True
+    round_times = []
+    for r in range(args.test_rounds):
+        print(f"[Test {r}/{args.test_rounds}] running...")
+        t_start = time.perf_counter()
+        for batch_start in range(0, len(inputs), batch_size):
+            batch = inputs[batch_start : batch_start + batch_size]
+            for stage_outputs in omni.generate(batch):
+                output = stage_outputs.request_output
+                _save_wav(
+                    output_dir,
+                    f"round{r}_{output.request_id}",
+                    output.outputs[0].multimodal_output,
+                )
+        elapsed = time.perf_counter() - t_start
+        round_times.append(elapsed)
+        print(f"[Test {r}/{args.test_rounds}] done in {elapsed:.3f}s")
+    collector.collecting = False
+
+    _print_summary(round_times, collector.summaries if args.log_stats else None)
+
+
+async def main_streaming(args):
+    """Run async streaming benchmark: warmup rounds then test rounds."""
+    model_name, inputs = _build_inputs(args)
+    output_dir = args.output_dir
+    os.makedirs(output_dir, exist_ok=True)
+
+    omni = AsyncOmni(
+        model=model_name,
+        stage_configs_path=args.stage_configs_path,
+        log_stats=args.log_stats,
+        stage_init_timeout=args.stage_init_timeout,
+        enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
+    )
+
+    collector = _SummaryCollector(omni)
+    collector.install()
+
+    # --- Warmup ---
+    for r in range(args.warmup_rounds):
+        print(f"[Warmup {r}/{args.warmup_rounds}] running...")
+        for i, prompt in enumerate(inputs):
+            async for _ in omni.generate(prompt, request_id=f"warmup_{r}_{i}"):
+                pass
+        print(f"[Warmup {r}/{args.warmup_rounds}] done")
+
+    # --- Test ---
+    collector.collecting = True
+    round_times = []
+    for r in range(args.test_rounds):
+        print(f"[Test {r}/{args.test_rounds}] running...")
+        t_start = time.perf_counter()
+        for i, prompt in enumerate(inputs):
+            request_id = f"round{r}_{i}"
+            t_req_start = time.perf_counter()
+            t_prev = t_req_start
+            chunk_idx = 0
+            async for stage_output in omni.generate(prompt, request_id=request_id):
+                mm = stage_output.request_output.outputs[0].multimodal_output
+                if not stage_output.finished:
+                    t_now = time.perf_counter()
+                    audio = mm.get("audio")
+                    n = len(audio) if isinstance(audio, list) else (0 if audio is None else 1)
+                    if chunk_idx == 0:
+                        ttfa_ms = (t_now - t_req_start) * 1000
+                        logger.info("Request %s: chunk %d samples=%d TTFA=%.1fms", request_id, chunk_idx, n, ttfa_ms)
+                    else:
+                        dt_ms = (t_now - t_prev) * 1000
+                        logger.info(
+                            "Request %s: chunk %d samples=%d inter_chunk=%.1fms", request_id, chunk_idx, n, dt_ms
+                        )
+                    t_prev = t_now
+                    chunk_idx += 1
+                else:
+                    t_end = time.perf_counter()
+                    total_ms = (t_end - t_req_start) * 1000
+                    logger.info("Request %s: done total=%.1fms chunks=%d", request_id, total_ms, chunk_idx)
+                    _save_wav(output_dir, request_id, mm)
+        elapsed = time.perf_counter() - t_start
+        round_times.append(elapsed)
+        print(f"[Test {r}/{args.test_rounds}] done in {elapsed:.3f}s")
+    collector.collecting = False
+
+    _print_summary(round_times, collector.summaries if args.log_stats else None)
+
+
+def parse_args():
+    parser = FlexibleArgumentParser(description="Benchmark script for Qwen3 TTS via vLLM Omni")
+    parser.add_argument(
+        "--query-type",
+        "-q",
+        type=str,
+        default="CustomVoice",
+        choices=query_map.keys(),
+        help="Query type.",
+    )
+    parser.add_argument(
+        "--warmup-rounds",
+        type=int,
+        default=3,
+        help="Number of warmup rounds (results discarded, default: 3).",
+    )
+    parser.add_argument(
+        "--test-rounds",
+        type=int,
+        default=7,
+        help="Number of test rounds (results saved and timed, default: 7).",
+    )
+    parser.add_argument(
+        "--log-stats",
+        action="store_true",
+        default=False,
+        help="Enable writing detailed statistics (default: disabled).",
+    )
+    parser.add_argument(
+        "--stage-init-timeout",
+        type=int,
+        default=300,
+        help="Timeout for initializing a single stage in seconds (default: 300).",
+    )
+    parser.add_argument(
+        "--output-dir",
+        default="output_audio",
+        help="Output directory for generated wav files (default: output_audio).",
+    )
+    parser.add_argument(
+        "--txt-prompts",
+        type=str,
+        default=None,
+        help="Path to a .txt file with one prompt per line.",
+    )
+    parser.add_argument(
+        "--stage-configs-path",
+        type=str,
+        default=None,
+        help="Path to a stage configs file.",
+    )
+    parser.add_argument(
+        "--use-batch-sample",
+        action="store_true",
+        default=False,
+        help="Use batch input sample for CustomVoice/VoiceDesign/Base query.",
+    )
+    parser.add_argument(
+        "--mode-tag",
+        type=str,
+        default="icl",
+        choices=["icl", "xvec_only"],
+        help="Mode tag for Base query x_vector_only_mode (default: icl).",
+    )
+    parser.add_argument(
+        "--streaming",
+        action="store_true",
+        default=False,
+        help="Stream audio chunks via AsyncOmni.",
+    )
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=1,
+        help="Number of prompts per batch (default: 1).",
+    )
+    parser.add_argument(
+        "--enable-diffusion-pipeline-profiler",
+        action="store_true",
+        help="Enable diffusion pipeline profiler to display stage durations.",
+    )
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    if args.streaming:
+        asyncio.run(main_streaming(args))
+    else:
+        main(args)

examples/offline_inference/text_to_speech/qwen3_tts/run_tests.sh

@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+# =============================================================================
+# Qwen3-TTS end2end functional test script
+# Iterates over query-type and mode combinations for qwen3_tts/end2end.py
+#
+# Usage:
+#   ./run_tests.sh --custom-voice-model /path/a --voice-design-model /path/b --base-model /path/c
+#   ./run_tests.sh --custom-voice-model /path/a                # only run CustomVoice tests
+#   ./run_tests.sh --base-model /path/c                        # only run Base tests
+# =============================================================================
+
+set -euo pipefail
+
+export VLLM_OMNI_USE_V2_RUNNER=1
+# export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
+
+# ---- Parse named arguments ----
+CUSTOM_VOICE_MODEL=""
+VOICE_DESIGN_MODEL=""
+BASE_MODEL=""
+
+usage() {
+    echo "Usage: $0 [--custom-voice-model PATH] [--voice-design-model PATH] [--base-model PATH]"
+    echo ""
+    echo "Pass one or more model paths. Only the query types with a model provided will be tested."
+    exit 1
+}
+
+if [[ $# -eq 0 ]]; then
+    usage
+fi
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --custom-voice-model) CUSTOM_VOICE_MODEL="$2"; shift 2 ;;
+        --voice-design-model) VOICE_DESIGN_MODEL="$2"; shift 2 ;;
+        --base-model)         BASE_MODEL="$2";         shift 2 ;;
+        -h|--help)            usage ;;
+        *) echo "Unknown option: $1"; usage ;;
+    esac
+done
+
+if [[ -z "$CUSTOM_VOICE_MODEL" && -z "$VOICE_DESIGN_MODEL" && -z "$BASE_MODEL" ]]; then
+    echo "Error: at least one model path must be provided."
+    usage
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+END2END="$SCRIPT_DIR/end2end.py"
+LOG_DIR="$SCRIPT_DIR/test_logs"
+SUMMARY_FILE="$LOG_DIR/summary.log"
+
+rm -rf "$LOG_DIR"
+mkdir -p "$LOG_DIR"
+> "$SUMMARY_FILE"
+
+PASS=0
+FAIL=0
+TOTAL=0
+
+run_test() {
+    local name="$1"
+    shift
+    local log_file="$LOG_DIR/${name}.log"
+    local output_dir="$LOG_DIR/${name}_output"
+
+    TOTAL=$((TOTAL + 1))
+    echo "========================================"
+    echo "[${TOTAL}] Running: ${name}"
+    echo "  Command: python $END2END $*"
+    echo "========================================"
+
+    local start_time
+    start_time=$(date +%s)
+
+    if python "$END2END" "$@" --output-dir "$output_dir" > "$log_file" 2>&1; then
+        local end_time
+        end_time=$(date +%s)
+        local elapsed=$((end_time - start_time))
+        echo "  => PASS (${elapsed}s)"
+        echo "[PASS] ${name} (${elapsed}s)" >> "$SUMMARY_FILE"
+        PASS=$((PASS + 1))
+    else
+        local exit_code=$?
+        local end_time
+        end_time=$(date +%s)
+        local elapsed=$((end_time - start_time))
+        echo "  => FAIL (exit code: ${exit_code}, ${elapsed}s)"
+        echo "[FAIL] ${name} (exit code: ${exit_code}, ${elapsed}s)" >> "$SUMMARY_FILE"
+        echo "  --- last 30 lines of log ---" >> "$SUMMARY_FILE"
+        tail -n 30 "$log_file" >> "$SUMMARY_FILE"
+        echo "  --- end ---" >> "$SUMMARY_FILE"
+        FAIL=$((FAIL + 1))
+    fi
+    echo ""
+}
+
+# =============================================================================
+# Test cases — only run sections whose model path was provided
+# =============================================================================
+
+# --- 1. CustomVoice ---
+if [[ -n "$CUSTOM_VOICE_MODEL" ]]; then
+    run_test "custom_voice" \
+        --model "$CUSTOM_VOICE_MODEL" --query-type CustomVoice
+
+    run_test "custom_voice_batch_sample" \
+        --model "$CUSTOM_VOICE_MODEL" --query-type CustomVoice --use-batch-sample
+
+    run_test "custom_voice_py_generator" \
+        --model "$CUSTOM_VOICE_MODEL" --query-type CustomVoice --py-generator
+
+    run_test "custom_voice_streaming" \
+        --model "$CUSTOM_VOICE_MODEL" --query-type CustomVoice --streaming
+
+    run_test "custom_voice_batch4" \
+        --model "$CUSTOM_VOICE_MODEL" --query-type CustomVoice --batch-size 4
+
+    run_test "custom_voice_batch4_batch_sample" \
+        --model "$CUSTOM_VOICE_MODEL" --query-type CustomVoice --batch-size 4 --use-batch-sample --stage-configs-path "$SCRIPT_DIR/../../../vllm_omni/model_executor/stage_configs/qwen3_tts_batch.yaml"
+
+    run_test "custom_voice_num_prompts3" \
+        --model "$CUSTOM_VOICE_MODEL" --query-type CustomVoice --num-prompts 3
+fi
+
+# --- 2. VoiceDesign ---
+if [[ -n "$VOICE_DESIGN_MODEL" ]]; then
+    run_test "voice_design" \
+        --model "$VOICE_DESIGN_MODEL" --query-type VoiceDesign
+
+    run_test "voice_design_batch_sample" \
+        --model "$VOICE_DESIGN_MODEL" --query-type VoiceDesign --use-batch-sample
+
+    run_test "voice_design_py_generator" \
+        --model "$VOICE_DESIGN_MODEL" --query-type VoiceDesign --py-generator
+
+    run_test "voice_design_streaming" \
+        --model "$VOICE_DESIGN_MODEL" --query-type VoiceDesign --streaming
+fi
+
+# --- 3. Base (icl mode) ---
+if [[ -n "$BASE_MODEL" ]]; then
+    run_test "base_icl" \
+        --model "$BASE_MODEL" --query-type Base --mode-tag icl
+
+    run_test "base_icl_batch_sample" \
+        --model "$BASE_MODEL" --query-type Base --mode-tag icl --use-batch-sample
+
+    run_test "base_icl_py_generator" \
+        --model "$BASE_MODEL" --query-type Base --mode-tag icl --py-generator
+
+    run_test "base_icl_streaming" \
+        --model "$BASE_MODEL" --query-type Base --mode-tag icl --streaming
+
+    # --- 4. Base (xvec_only mode) ---
+    run_test "base_xvec_only" \
+        --model "$BASE_MODEL" --query-type Base --mode-tag xvec_only
+
+    run_test "base_xvec_only_batch_sample" \
+        --model "$BASE_MODEL" --query-type Base --mode-tag xvec_only --use-batch-sample
+fi
+
+# =============================================================================
+# Summary
+# =============================================================================
+echo "========================================" | tee -a "$SUMMARY_FILE"
+echo "Tests done: total ${TOTAL}, passed ${PASS}, failed ${FAIL}" | tee -a "$SUMMARY_FILE"
+echo "========================================" | tee -a "$SUMMARY_FILE"
+echo ""
+echo "Detailed logs: $LOG_DIR"
+echo "Summary report: $SUMMARY_FILE"
+
+exit $FAIL

examples/offline_inference/text_to_speech/voxtral_tts/end2end.py

@@ -212,7 +212,7 @@ def run_non_streaming(inputs, sampling_params_list, model_name, args, output_dir
     output_audio_dur = 0.0
 
     for batch_idx, o in enumerate(outputs):
-        audio_tensor = torch.cat(o.multimodal_output["audio"])
+        audio_tensor = torch.cat((o.multimodal_output["audio"],))
         audio_array = audio_tensor.tolist()
         output_audio_dur += float(len(audio_array)) / 24000
         if args.write_audio:

examples/online_serving/bagel/README.md

@@ -1,404 +0,0 @@
-# BAGEL-7B-MoT
-
-## Installation
-
-Please refer to [README.md](../../../README.md)
-
-## Architecture
-
-BAGEL-7B-MoT is a Mixture-of-Transformers (MoT) model supporting both image generation and understanding. It offers two deployment topologies:
-
-| Topology | Stages | Description |
-| :------- | :----- | :---------- |
-| **Two-stage** (default) | Stage 0 (Thinker, AR) + Stage 1 (DiT, Diffusion) | Thinker handles text/understanding via vLLM AR engine; DiT handles image generation. KV cache is transferred between stages. |
-| **Single-stage** | Stage 0 (DiT, Diffusion) only | The DiT stage contains a full LLM, ViT, VAE, and tokenizer internally. All modalities are handled within a single diffusion process. |
-
-Both topologies support all four modalities: `text2img`, `img2img`, `img2text`, `text2text`.
-
-> **Note**: These examples work with the default configuration on an **NVIDIA A100 (80GB)**. We also tested on dual **NVIDIA RTX 5000 Ada (32GB each)**. For dual-GPU setups, modify the deploy YAML to distribute stages across devices.
-
-## Launch the Server
-
-### Two-Stage (Default)
-
-The default pipeline is auto-detected from the model. No extra flags needed:
-
-```bash
-vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091
-```
-
-Or use the convenience script:
-
-```bash
-cd examples/online_serving/bagel
-bash run_server.sh
-
-# Initialize each stage in a discrete isolated process terminal
-bash run_server_stage_cli.sh --stage 0
-bash run_server_stage_cli.sh --stage 1
-```
-
-To use a custom deploy YAML, pass it via `--deploy-config`:
-
-```bash
-vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 \
-    --deploy-config /path/to/deploy_config.yaml
-```
-
-See [`bagel.yaml`](../../../vllm_omni/deploy/bagel.yaml) for the default two-stage deploy configuration.
-
-### Single-Stage
-
-The DiT stage contains a full LLM, ViT, VAE, and tokenizer, so it can handle all modalities (text2img, img2img, img2text, text2text, think) without a separate Thinker stage:
-
-```bash
-vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 \
-    --deploy-config vllm_omni/deploy/bagel_single_stage.yaml
-```
-
-See [`bagel_single_stage.yaml`](../../../vllm_omni/deploy/bagel_single_stage.yaml) for configuration. The `pipeline: bagel_single_stage` field selects the single-stage topology from the pipeline registry.
-
-### Tensor Parallelism (TP)
-
-For larger models or multi-GPU environments, enable TP via CLI:
-
-```bash
-vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 --tensor-parallel-size 2
-```
-
-Or set `tensor_parallel_size` per stage in a custom deploy YAML.
-
-### VAE Patch Parallelism
-
-[VAE Patch Parallelism](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/parallelism/vae_patch_parallel.html) distributes Bagel VAE **decode/encode** across multiple GPUs by splitting latent tiles. It lowers **per-GPU peak memory during VAE decode**, which helps high-resolution `text2img` / `img2img` when VAE becomes a bottleneck.
-
-**Scope for Bagel:**
-
-| Topology | VAE patch parallel |
-| :------- | :----------------- |
-| **Single-stage** (DiT only) | Supported on stage 0 (`BagelPipeline` + `DistributedAutoEncoder`) |
-| **Two-stage** | Supported on **stage 1 (DiT)** only; stage 0 (Thinker) uses encoder-only VAE and is unrelated |
-
-**Requirements:**
-
-- `vae_patch_parallel_size > 1` and a distributed VAE (`DistributedAutoEncoder` on the DiT pipeline).
-- The DiT process group must have at least `vae_patch_parallel_size` ranks. In practice this means the diffusion stage `world_size` must be ≥ 2 (commonly `tensor_parallel_size=2` on that stage).
-- `vae_use_tiling` must be enabled. If you set `vae_patch_parallel_size > 1` and omit tiling, the registry auto-enables `vae_use_tiling` at startup.
-
-VAE patch parallel **reuses the DiT process group** (`dit_group`); it does not create a separate VAE-only worker pool. It is not a substitute for single-GPU VAE tiling (`vae_pp=1`).
-
-**Online serving (single-stage, 2 GPUs):**
-
-```bash
-CUDA_VISIBLE_DEVICES=0,1 vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 \
-    --deploy-config vllm_omni/deploy/bagel_single_stage.yaml \
-    --tensor-parallel-size 2 \
-    --vae-patch-parallel-size 2 \
-    --vae-use-tiling
-```
-
-**Online serving (two-stage, VAE PP on DiT stage 1):** use a custom deploy YAML, for example:
-
-```yaml
-stages:
-  - stage_id: 0
-    devices: "0"
-    # Thinker (AR) — no VAE patch parallel here
-
-  - stage_id: 1
-    devices: "0,1"
-    vae_use_tiling: true
-    parallel_config:
-      tensor_parallel_size: 2
-      vae_patch_parallel_size: 2
-```
-
-```bash
-vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 \
-    --deploy-config /path/to/bagel_vae_pp.yaml
-```
-
-**Verify it is active** (check server logs at startup):
-
-```text
-INFO ... vae_patch_parallel_size=2 requires vae_use_tiling; automatically enabling it.
-```
-
-| CLI flag | Default | Description |
-| :------- | :------ | :---------- |
-| `--vae-patch-parallel-size` | `1` | Number of DiT ranks used for VAE tile parallelism. Set to `2` or higher to enable. Should be ≤ DiT process group size (typically match `--tensor-parallel-size` on the diffusion stage). |
-| `--vae-use-tiling` | off | Enable VAE spatial tiling. Required for VAE patch parallel (auto-enabled when `vae_patch_parallel_size > 1`). |
-
-#### Hybrid Sharded Data Parallel (HSDP)
-
-For larger Bagel deployments on multiple GPUs, you can enable HSDP (Hybrid Sharded Data Parallel) by modifying the stage configuration (for example, [`bagel.yaml`](../../../vllm_omni/deploy/bagel.yaml)). HSDP shards transformer weights across GPUs to reduce per-GPU memory usage.
-
-1. **Enable HSDP**: Set `use_hsdp: true`.
-2. **Set shard size**: Set `hsdp_shard_size` to the number of GPUs used for sharding (for example, `4`).
-3. **Set replicate size**: Usually keep `hsdp_replicate_size: 1` unless you want replicated HSDP groups.
-4. **Set devices**: Specify the comma-separated GPU IDs used by the diffusion stage (for example, `"0,1,2,3"`).
-
-Example configuration for HSDP across 4 GPUs:
-```yaml
-  - stage_id: 1
-    devices: "0,1,2,3"
-    parallel_config:
-      use_hsdp: true
-      hsdp_shard_size: 4
-      hsdp_replicate_size: 1
-```
-
-### Multi-Node Deployment
-
-Deploy each stage on a **separate node** for better resource utilization. Replace `<ORCHESTRATOR_IP>` with the actual IP address of your orchestrator node.
-
-**1. Launch Stage 0 (Thinker / Orchestrator)** on the orchestrator node:
-
-```bash
-# API server port for client requests: 8000
-vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni \
-    --port 8000 \
-    --stage-id 0 \
-    --omni-master-address <ORCHESTRATOR_IP> \
-    --omni-master-port 8091
-```
-
-**2. Launch Stage 1 (DiT)** on the remote node in headless mode:
-
-```bash
-vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni \
-    --stage-id 1 \
-    --headless \
-    --omni-master-address <ORCHESTRATOR_IP> \
-    --omni-master-port 8091
-```
-
-Or use the convenience script:
-
-```bash
-# Terminal 1: Stage 0
-bash run_server_stage_cli.sh --stage 0
-
-# Terminal 2: Stage 1
-bash run_server_stage_cli.sh --stage 1
-
-# With extra args
-bash run_server_stage_cli.sh --stage 0 -- --tensor-parallel-size 2
-bash run_server_stage_cli.sh --stage 1 -- --gpu-memory-utilization 0.9
-```
-
-**vllm serve arguments:**
-
-| Argument | Description |
-| :------- | :---------- |
-| `--stage-id` | Which stage this process runs (0 = Thinker, 1 = DiT) |
-| `--headless` | Run without the API server (worker-only mode) |
-| `-oma` / `--omni-master-address` | Orchestrator master address |
-| `-omp` / `--omni-master-port` | Orchestrator master port |
-
-> [!IMPORTANT]
-> **Startup Order**: Stage 0 (orchestrator) must be launched **before** Stage 1 (headless).
-> Stage 0 will appear to hang on startup until Stage 1 (worker) connects — this is expected behavior.
-
-### Inter-Stage Connectors
-
-When deploying stages across nodes, configure the connector type in the deploy YAML:
-
-- **SharedMemoryConnector** (default): Used for single-node deployments. No explicit configuration needed.
-- **MooncakeTransferEngineConnector**: For multi-node setups with RDMA hardware. Defined in [`bagel.yaml`](../../../vllm_omni/deploy/bagel.yaml) under `connectors.rdma_connector`.
-
-To use Mooncake, create a custom deploy YAML that binds `output_connectors` / `input_connectors` on each stage to the `rdma_connector` defined in the `connectors` section.
-
-## Send Requests
-
-```bash
-cd examples/online_serving/bagel
-```
-
-### Text to Image (text2img)
-
-**Python client:**
-
-```bash
-python openai_chat_client.py \
-    --prompt "A beautiful sunset over mountains" \
-    --modality text2img \
-    --output sunset.png \
-    --steps 50
-```
-
-**curl:**
-
-```bash
-curl http://localhost:8091/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -d '{
-    "messages": [{"role": "user", "content": [{"type": "text", "text": "<|im_start|>A beautiful sunset over mountains<|im_end|>"}]}],
-    "modalities": ["image"],
-    "height": 512,
-    "width": 512,
-    "num_inference_steps": 50,
-    "seed": 42
-  }'
-```
-
-### Image to Image (img2img)
-
-**Python client:**
-
-```bash
-python openai_chat_client.py \
-    --prompt "Make the cat stand up" \
-    --modality img2img \
-    --image-url /path/to/input.jpg \
-    --output transformed.png
-```
-
-**curl:**
-
-```bash
-IMAGE_BASE64=$(base64 -w 0 cat.jpg)
-
-cat <<EOF > payload.json
-{
-    "messages": [{
-      "role": "user",
-      "content": [
-        {"type": "text", "text": "<|im_start|>Make the cat stand up<|im_end|>"},
-        {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,${IMAGE_BASE64}"}}
-      ]
-    }],
-    "modalities": ["image"],
-    "height": 512,
-    "width": 512,
-    "num_inference_steps": 50,
-    "seed": 42
-}
-EOF
-
-curl http://localhost:8091/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -d @payload.json
-```
-
-### Image to Text (img2text)
-
-**Python client:**
-
-```bash
-python openai_chat_client.py \
-    --prompt "Describe this image in detail" \
-    --modality img2text \
-    --image-url /path/to/image.jpg
-```
-
-**curl:**
-
-```bash
-IMAGE_BASE64=$(base64 -w 0 cat.jpg)
-
-cat <<EOF > payload.json
-{
-  "messages": [{
-    "role": "user",
-    "content": [
-      {"type": "text", "text": "<|im_start|>user\n<|image_pad|>\nDescribe this image in detail<|im_end|>\n<|im_start|>assistant\n"},
-      {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,${IMAGE_BASE64}"}}
-    ]
-  }],
-  "modalities": ["text"]
-}
-EOF
-
-curl http://localhost:8091/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -d @payload.json
-```
-
-### Text to Text (text2text)
-
-**Python client:**
-
-```bash
-python openai_chat_client.py \
-    --prompt "What is the capital of France?" \
-    --modality text2text
-```
-
-**curl:**
-
-```bash
-curl http://localhost:8091/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -d '{
-    "messages": [{"role": "user", "content": [{"type": "text", "text": "<|im_start|>user\nWhat is the capital of France?<|im_end|>\n<|im_start|>assistant\n"}]}],
-    "modalities": ["text"]
-  }'
-```
-
-### Python Client Arguments
-
-| Argument | Default | Description |
-| :------- | :------ | :---------- |
-| `--prompt` / `-p` | `A cute cat` | Text prompt |
-| `--output` / `-o` | `bagel_output.png` | Output file path |
-| `--server` / `-s` | `http://localhost:8091` | Server URL |
-| `--image-url` / `-i` | `None` | Input image URL or local path (img2img/img2text) |
-| `--modality` / `-m` | `text2img` | `text2img`, `img2img`, `img2text`, `text2text` |
-| `--height` | `512` | Image height in pixels |
-| `--width` | `512` | Image width in pixels |
-| `--steps` | `25` | Number of inference steps |
-| `--seed` | `42` | Random seed |
-| `--negative` | `None` | Negative prompt for CFG |
-
-Example with custom parameters:
-
-```bash
-python openai_chat_client.py \
-    --prompt "A futuristic city" \
-    --modality text2img \
-    --height 768 \
-    --width 768 \
-    --steps 50 \
-    --seed 42 \
-    --negative "blurry, low quality"
-```
-
-## Configuration Reference
-
-### Deploy YAML Files
-
-| File | Description |
-| :--- | :---------- |
-| [`bagel.yaml`](../../../vllm_omni/deploy/bagel.yaml) | Two-stage default (Thinker + DiT on GPU 0) |
-| [`bagel_single_stage.yaml`](../../../vllm_omni/deploy/bagel_single_stage.yaml) | Single-stage (DiT only) |
-
-### Key Deploy YAML Fields
-
-| Field | Scope | Description |
-| :---- | :---- | :---------- |
-| `pipeline` | top-level | Override auto-detected pipeline (e.g. `bagel_single_stage`) |
-| `stages[].stage_id` | per-stage | Stage identifier (0, 1, ...) |
-| `stages[].devices` | per-stage | GPU device IDs (e.g. `"0"`, `"0,1"`) |
-| `stages[].max_num_seqs` | per-stage | Maximum concurrent sequences |
-| `stages[].gpu_memory_utilization` | per-stage | Fraction of GPU memory to use |
-| `stages[].enforce_eager` | per-stage | Disable CUDA graphs |
-| `stages[].tensor_parallel_size` | per-stage | TP degree for this stage |
-| `stages[].parallel_config.vae_patch_parallel_size` | per-stage (DiT) | VAE tile parallelism degree (DiT stage only) |
-| `stages[].vae_use_tiling` | per-stage (DiT) | Enable VAE tiling (required for VAE patch parallel) |
-| `connectors` | top-level | Define available connector instances (SHM, Mooncake) |
-| `platforms` | top-level | Platform-specific overrides (e.g. `xpu`) |
-
-## FAQ
-
-- If you encounter OOM errors, try decreasing `max_model_len` or `gpu_memory_utilization` in the deploy YAML.
-
-**Two-stage VRAM usage:**
-
-| Stage | VRAM |
-| :---- | :--- |
-| Stage 0 (Thinker) | **15.04 GiB + KV Cache** |
-| Stage 1 (DiT) | **26.50 GiB** |
-| Total | **~42 GiB + KV Cache** |
-
-**Single-stage VRAM usage:** The DiT loads the full model (~42 GiB) in one process.

examples/online_serving/bagel/openai_chat_client.py

@@ -1,184 +0,0 @@
-#!/usr/bin/env python3
-"""
-Bagel OpenAI-compatible chat client for image generation and multimodal tasks.
-
-Usage:
-    python openai_chat_client.py --prompt "A cute cat" --output output.png
-    python openai_chat_client.py --prompt "Describe this image" --image-url https://example.com/image.png
-"""
-
-import argparse
-import base64
-from pathlib import Path
-
-import requests
-
-
-def generate_image(
-    prompt: str,
-    server_url: str = "http://localhost:8091",
-    image_url: str | None = None,
-    height: int | None = None,
-    width: int | None = None,
-    steps: int | None = None,
-    seed: int | None = None,
-    negative_prompt: str | None = None,
-    modality: str = "text2img",  # "text2img" (default), "img2img", "img2text", "text2text"
-) -> bytes | str | None:
-    """Generate an image or text using the chat completions API.
-
-    Args:
-        prompt: Text description or prompt
-        server_url: Server URL
-        image_url: URL or path to input image (for img2img/img2text)
-        height: Image height in pixels
-        width: Image width in pixels
-        steps: Number of inference steps
-        seed: Random seed
-        negative_prompt: Negative prompt
-        modality: Task modality hint
-
-    Returns:
-        Image bytes (for image outputs) or Text string (for text outputs) or None if failed
-    """
-
-    # Construct Message Content
-    content = [{"type": "text", "text": f"<|im_start|>{prompt}<|im_end|>"}]
-
-    if image_url:
-        # Check if local file
-        if Path(image_url).exists():
-            with open(image_url, "rb") as f:
-                b64_data = base64.b64encode(f.read()).decode("utf-8")
-                final_image_url = f"data:image/jpeg;base64,{b64_data}"
-        else:
-            final_image_url = image_url
-
-        content.append({"type": "image_url", "image_url": {"url": final_image_url}})
-
-    messages = [{"role": "user", "content": content}]
-
-    # Build request payload with all parameters at top level
-    # Note: vLLM ignores "extra_body", so we put parameters directly in the payload
-    payload = {"messages": messages}
-
-    # Set output modalities at top level
-    if modality == "text2img" or modality == "img2img":
-        payload["modalities"] = ["image"]
-    elif modality == "img2text" or modality == "text2text":
-        payload["modalities"] = ["text"]
-
-    # Add generation parameters directly to payload
-    if height is not None:
-        payload["height"] = height
-    if width is not None:
-        payload["width"] = width
-    if steps is not None:
-        payload["num_inference_steps"] = steps
-    if seed is not None:
-        payload["seed"] = seed
-    if negative_prompt:
-        payload["negative_prompt"] = negative_prompt
-
-    # Send request
-    try:
-        print(f"Sending request to {server_url} with modality {modality}...")
-        response = requests.post(
-            f"{server_url}/v1/chat/completions",
-            headers={"Content-Type": "application/json"},
-            json=payload,
-            timeout=300,
-        )
-        response.raise_for_status()
-        data = response.json()
-
-        # Extract content - check ALL choices since server may return multiple
-        # (e.g., text in choices[0], image in choices[1])
-        choices = data.get("choices", [])
-
-        # First pass: look for image output in any choice
-        for choice in choices:
-            choice_content = choice.get("message", {}).get("content")
-
-            # Handle Image Output
-            if isinstance(choice_content, list) and len(choice_content) > 0:
-                first_item = choice_content[0]
-                if isinstance(first_item, dict) and "image_url" in first_item:
-                    img_url_str = first_item["image_url"].get("url", "")
-                    if img_url_str.startswith("data:image"):
-                        _, b64_data = img_url_str.split(",", 1)
-                        return base64.b64decode(b64_data)
-
-        # Second pass: look for text output if no image found
-        for choice in choices:
-            choice_content = choice.get("message", {}).get("content")
-            if isinstance(choice_content, str) and choice_content:
-                return choice_content
-
-        print(f"Unexpected response format: {choices}")
-        return None
-
-    except Exception as e:
-        print(f"Error: {e}")
-        return None
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Bagel multimodal chat client")
-    parser.add_argument("--prompt", "-p", default="A cute cat", help="Text prompt")
-    parser.add_argument("--output", "-o", default="bagel_output.png", help="Output file (for image results)")
-    parser.add_argument("--server", "-s", default="http://localhost:8091", help="Server URL")
-
-    # Modality Control
-    parser.add_argument("--image-url", "-i", type=str, help="Input image URL or local path")
-    parser.add_argument(
-        "--modality",
-        "-m",
-        default="text2img",
-        choices=["text2img", "img2img", "img2text", "text2text"],
-        help="Task modality",
-    )
-
-    # Generation Params
-    parser.add_argument("--height", type=int, default=512, help="Image height")
-    parser.add_argument("--width", type=int, default=512, help="Image width")
-    parser.add_argument("--steps", type=int, default=25, help="Inference steps")
-    parser.add_argument("--seed", type=int, default=42, help="Random seed")
-    parser.add_argument("--negative", help="Negative prompt")
-
-    args = parser.parse_args()
-
-    print(f"Mode: {args.modality}")
-    if args.image_url:
-        print(f"Input Image: {args.image_url}")
-
-    result = generate_image(
-        prompt=args.prompt,
-        server_url=args.server,
-        image_url=args.image_url,
-        height=args.height,
-        width=args.width,
-        steps=args.steps,
-        seed=args.seed,
-        negative_prompt=args.negative,
-        modality=args.modality,
-    )
-
-    if result:
-        if isinstance(result, bytes):
-            # It's an image
-            output_path = Path(args.output)
-            output_path.write_bytes(result)
-            print(f"Image saved to: {output_path}")
-            print(f"Size: {len(result) / 1024:.1f} KB")
-        elif isinstance(result, str):
-            # It's text
-            print("Response:")
-            print(result)
-    else:
-        print("Failed to generate response")
-        exit(1)
-
-
-if __name__ == "__main__":
-    main()

examples/online_serving/bagel/run_server.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-# Bagel online serving startup script
-
-MODEL="${MODEL:-ByteDance-Seed/BAGEL-7B-MoT}"
-PORT="${PORT:-8091}"
-
-echo "Starting Bagel server..."
-echo "Model: $MODEL"
-echo "Port: $PORT"
-
-vllm serve "$MODEL" --omni \
-    --port "$PORT"

examples/online_serving/bagel/run_server_stage_cli.sh

@@ -1,164 +0,0 @@
-#!/bin/bash
-# Bagel multi-stage online serving startup script.
-#
-# Usage:
-#   ./run_server_stage_cli.sh --stage 0
-#   ./run_server_stage_cli.sh --stage 1
-#   ./run_server_stage_cli.sh --stage 0 -- --tensor-parallel-size 2
-#   ./run_server_stage_cli.sh --stage 1 -- --gpu-memory-utilization 0.9
-#
-# By default, `--stage all` keeps the old behavior and launches both stages in
-# one session. Use `--stage 0` / `--stage 1` to launch each stage separately in
-# different terminal sessions, with stage-specific extra CLI arguments passed
-# after `--`.
-
-set -euo pipefail
-
-MODEL="${MODEL:-ByteDance-Seed/BAGEL-7B-MoT}"
-PORT="${PORT:-8091}"
-MASTER_ADDRESS="${MASTER_ADDRESS:-127.0.0.1}"
-MASTER_PORT="${MASTER_PORT:-8092}"
-STAGE="all"
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-DEPLOY_CONFIG="${DEPLOY_CONFIG:-$SCRIPT_DIR/../../../vllm_omni/deploy/bagel.yaml}"
-EXTRA_ARGS=()
-
-usage() {
-    cat <<EOF
-Usage: $0 [OPTIONS] [-- EXTRA_VLLM_ARGS...]
-
-Options:
-  --stage {0|1|all}          Stage to launch (default: all)
-  --model MODEL              Model name/path (default: $MODEL)
-  --port PORT                API port for stage 0 (default: $PORT)
-  --master-address ADDRESS   Master/orchestrator address (default: $MASTER_ADDRESS)
-  --master-port PORT         Master/orchestrator port (default: $MASTER_PORT)
-  --deploy-config PATH       Deploy config YAML path (default: $DEPLOY_CONFIG)
-  --help                     Show this help message
-
-Examples:
-  $0 --stage 0
-  $0 --stage 1
-  $0 --stage 0 -- --tensor-parallel-size 2
-  $0 --stage 1 -- --gpu-memory-utilization 0.9
-
-Notes:
-  - Use different terminal sessions to launch stage 0 and stage 1 separately.
-  - Extra args after '--' are forwarded only to the selected stage.
-  - When using '--stage all', the extra args are forwarded to both stages.
-EOF
-}
-
-while [[ $# -gt 0 ]]; do
-    case "$1" in
-        --stage)
-            STAGE="$2"
-            shift 2
-            ;;
-        --model)
-            MODEL="$2"
-            shift 2
-            ;;
-        --port)
-            PORT="$2"
-            shift 2
-            ;;
-        --master-address)
-            MASTER_ADDRESS="$2"
-            shift 2
-            ;;
-        --master-port)
-            MASTER_PORT="$2"
-            shift 2
-            ;;
-        --deploy-config)
-            DEPLOY_CONFIG="$2"
-            shift 2
-            ;;
-        --help|-h)
-            usage
-            exit 0
-            ;;
-        --)
-            shift
-            EXTRA_ARGS=("$@")
-            break
-            ;;
-        *)
-            echo "Unknown option: $1" >&2
-            usage
-            exit 1
-            ;;
-    esac
-done
-
-if [[ "$STAGE" != "0" && "$STAGE" != "1" && "$STAGE" != "all" ]]; then
-    echo "Invalid --stage value: $STAGE" >&2
-    usage
-    exit 1
-fi
-
-print_config() {
-    echo "Model: $MODEL"
-    echo "API Port: $PORT"
-    echo "Master Address: $MASTER_ADDRESS"
-    echo "Master Port: $MASTER_PORT"
-    echo "Deploy Config: $DEPLOY_CONFIG"
-    echo "Selected Stage: $STAGE"
-    if [[ ${#EXTRA_ARGS[@]} -gt 0 ]]; then
-        echo "Extra Args: ${EXTRA_ARGS[*]}"
-    fi
-}
-
-run_stage_0() {
-    echo "Starting Stage 0 (Thinker) as master..."
-    vllm serve "$MODEL" --omni \
-        --port "$PORT" \
-        --deploy-config "$DEPLOY_CONFIG" \
-        --stage-id 0 \
-        --omni-master-address "$MASTER_ADDRESS" \
-        --omni-master-port "$MASTER_PORT" \
-        "${EXTRA_ARGS[@]}"
-}
-
-run_stage_1() {
-    echo "Starting Stage 1 (DiT) in headless mode..."
-    vllm serve "$MODEL" --omni \
-        --deploy-config "$DEPLOY_CONFIG" \
-        --stage-id 1 \
-        --headless \
-        --omni-master-address "$MASTER_ADDRESS" \
-        --omni-master-port "$MASTER_PORT" \
-        "${EXTRA_ARGS[@]}"
-}
-
-echo "Starting Bagel multi-stage server..."
-print_config
-
-case "$STAGE" in
-    0)
-        run_stage_0
-        ;;
-    1)
-        run_stage_1
-        ;;
-    all)
-        echo "Launching both stages in one session (legacy mode)..."
-        echo "Starting Stage 0 (Thinker) in background first..."
-        run_stage_0 &
-        STAGE_0_PID=$!
-
-        cleanup() {
-            if [[ -n "${STAGE_0_PID:-}" ]]; then
-                kill "$STAGE_0_PID" 2>/dev/null || true
-                wait "$STAGE_0_PID" 2>/dev/null || true
-            fi
-        }
-
-        trap cleanup EXIT INT TERM
-
-        echo "Waiting briefly for Stage 0 to initialize..."
-        sleep 2
-        run_stage_1
-        ;;
-esac

examples/online_serving/image_to_image/README.md

@@ -1,6 +1,8 @@
 # Image-To-Image
 
-This example demonstrates how to deploy Qwen-Image-Edit model for online image editing service using vLLM-Omni.
+This example demonstrates how to deploy image-to-image models for online image editing service using vLLM-Omni.
+
+Supported models include Qwen-Image-Edit, BAGEL, and other image-to-image pipelines.
 
 For **multi-image** input editing, use **Qwen-Image-Edit-2509** (QwenImageEditPlusPipeline) and send multiple images in the user message content.
 
@@ -21,6 +23,12 @@ vllm serve Qwen/Qwen-Image-Edit --omni --port 8092
 vllm serve Qwen/Qwen-Image-Edit-2509 --omni --port 8092
 ```
 
+### BAGEL
+
+```bash
+vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091
+```
+
 ### Start with Parameters
 
 
@@ -119,6 +127,16 @@ python openai_chat_client.py --input input.png --prompt "Convert to oil painting
 python openai_chat_client.py --input input1.png input2.png --prompt "Combine these images into a single scene" --output output.png
 ```
 
+Pass model-specific parameters through `--extra-body` (e.g. for BAGEL):
+
+```bash
+python openai_chat_client.py \
+  --input input.png \
+  --prompt "Make the scene look like a watercolor painting" \
+  --server http://localhost:8091 \
+  --extra-body '{"cfg_text_scale": 4.0, "cfg_img_scale": 1.5}'
+```
+
 ### Method 4: Using Gradio Demo
 
 ```bash
@@ -318,6 +336,9 @@ count, use `size` and `n` rather than `height`, `width`, or
 | `layers`                 | int   | 4       | Number of layers (Qwen-Image-Layered) |
 | `resolution`             | int   | 640     | Resolution, 640 or 1024 (Qwen-Image-Layered) |
 
+Models like BAGEL accept additional parameters via `extra_body` (e.g.
+`cfg_text_scale`, `cfg_img_scale`). See the [BAGEL recipe](../../../recipes/Bagel/BAGEL-7B-MoT.md) for the full list.
+
 ## Response Format
 
 ```json

examples/online_serving/image_to_image/openai_chat_client.py

@@ -38,6 +38,7 @@ def edit_image(
     guidance_scale: float | None = None,
     seed: int | None = None,
     negative_prompt: str | None = None,
+    extra_body: dict | None = None,
 ) -> bytes | None:
     """Edit an image using the chat completions API.
 
@@ -51,6 +52,7 @@ def edit_image(
         guidance_scale: CFG guidance scale
         seed: Random seed
         negative_prompt: Negative prompt
+        extra_body: Additional model-specific params (e.g. cfg_text_scale for BAGEL)
 
     Returns:
         Edited image bytes or None if failed
@@ -75,24 +77,26 @@ def edit_image(
     ]
 
     # Build extra_body with generation parameters
-    extra_body = {}
+    merged_extra_body: dict[str, object] = {}
     if height is not None:
-        extra_body["height"] = height
+        merged_extra_body["height"] = height
     if width is not None:
-        extra_body["width"] = width
+        merged_extra_body["width"] = width
     if steps is not None:
-        extra_body["num_inference_steps"] = steps
+        merged_extra_body["num_inference_steps"] = steps
     if guidance_scale is not None:
-        extra_body["guidance_scale"] = guidance_scale
+        merged_extra_body["guidance_scale"] = guidance_scale
     if seed is not None:
-        extra_body["seed"] = seed
+        merged_extra_body["seed"] = seed
     if negative_prompt:
-        extra_body["negative_prompt"] = negative_prompt
+        merged_extra_body["negative_prompt"] = negative_prompt
+    if extra_body:
+        merged_extra_body.update(extra_body)
 
     # Build request payload
-    payload = {"messages": messages}
-    if extra_body:
-        payload["extra_body"] = extra_body
+    payload: dict[str, object] = {"messages": messages}
+    if merged_extra_body:
+        payload["extra_body"] = merged_extra_body
 
     # Send request
     try:
@@ -121,6 +125,19 @@ def edit_image(
         return None
 
 
+def parse_extra_body(value: str) -> dict:
+    """Parse a JSON string into a dict for --extra-body."""
+    import json
+
+    try:
+        obj = json.loads(value)
+    except json.JSONDecodeError as e:
+        raise argparse.ArgumentTypeError(f"--extra-body must be valid JSON: {e}") from e
+    if not isinstance(obj, dict):
+        raise argparse.ArgumentTypeError("--extra-body must be a JSON object")
+    return obj
+
+
 def main():
     parser = argparse.ArgumentParser(description="Qwen-Image-Edit chat client")
     parser.add_argument("--input", "-i", required=True, nargs="+", help="Input image path(s)")
@@ -133,6 +150,12 @@ def main():
     parser.add_argument("--guidance", type=float, default=7.5, help="Guidance scale")
     parser.add_argument("--seed", type=int, default=0, help="Random seed")
     parser.add_argument("--negative", help="Negative prompt")
+    parser.add_argument(
+        "--extra-body",
+        type=parse_extra_body,
+        default=None,
+        help="JSON object merged into request extra_body, e.g. '{\"cfg_text_scale\": 4.0}'.",
+    )
 
     args = parser.parse_args()
 
@@ -152,6 +175,7 @@ def main():
         guidance_scale=args.guidance,
         seed=args.seed,
         negative_prompt=args.negative,
+        extra_body=args.extra_body,
     )
 
     if image_bytes:

examples/online_serving/image_to_video/README.md

@@ -120,14 +120,17 @@ Use `/v1/videos/sync` if you want to write the MP4 directly to a file. `POST /v1
 Generated video files are stored on local disk by the async video API.
 Local file storage behavior can be controlled via the following environment variables:
 
-- `VLLM_OMNI_STORAGE_PATH`: directory used for generated files (default: `/tmp/storage`)
-- `VLLM_OMNI_STORAGE_MAX_CONCURRENCY`: max concurrent save/delete operations (default: `4`)
+- `VLLM_OMNI_SERVER_STORAGE__PATH`: directory used for generated files (default: `/tmp/storage`)
+- `VLLM_OMNI_SERVER_STORAGE__FILE_CONCURRENCY`: max concurrent save/delete operations (default: `4`)
+
+`VLLM_OMNI_STORAGE_PATH` and `VLLM_OMNI_STORAGE_MAX_CONCURRENCY` are deprecated and will be
+removed in a future release; use the names above instead.
 
 Example:
 
 ```bash
-export VLLM_OMNI_STORAGE_PATH=/var/tmp/vllm-omni-videos
-export VLLM_OMNI_STORAGE_MAX_CONCURRENCY=8
+export VLLM_OMNI_SERVER_STORAGE__PATH=/var/tmp/vllm-omni-videos
+export VLLM_OMNI_SERVER_STORAGE__FILE_CONCURRENCY=8
 ```
 
 ## API Calls

examples/online_serving/qwen2_5_omni/run_curl_multimodal_generation.sh

@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+# Default model name
+MODEL="${MODEL:-Qwen/Qwen2.5-Omni-7B}"
+
 # Default query type
 QUERY_TYPE="${1:-mixed_modalities}"
 
@@ -162,7 +165,7 @@ echo ""
 
 request_body=$(cat <<EOF
 {
-  "model": "Qwen/Qwen2.5-Omni-7B",
+  "model": "$MODEL",
   "sampling_params_list": $sampling_params_list,
   "mm_processor_kwargs": $mm_processor_kwargs,
   "modalities": $MODALITIES,

examples/online_serving/qwen3_omni/README.md

@@ -24,12 +24,15 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
     --deploy-config /path/to/your_deploy_config.yaml
 ```
 
-For the bundled 3x-GPU multi-replica layout (talker/code2wav scale-out),
-use:
+For a 3x-GPU multi-replica layout (talker/code2wav scale-out on cuda:1,2),
+use `--stage-overrides` on top of the default config:
 
 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
-    --deploy-config vllm_omni/deploy/qwen3_omni_moe_multi_replicas.yaml
+    --stage-overrides '{
+        "1": {"num_replicas": 2, "devices": "1,2"},
+        "2": {"num_replicas": 2, "devices": "1,2"}
+    }'
 ```
 
 ### Launch individual stages (stage-based CLI)

examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml

@@ -20,7 +20,7 @@ stage_args:
       trust_remote_code: true
       engine_output_type: text
       distributed_executor_backend: "mp"
-      enable_prefix_caching: false
+      enable_prefix_caching: true
       hf_config_name: thinker_config
       tensor_parallel_size: 2
     final_output: true

examples/online_serving/sensenova_u1/openai_chat_client.py

@@ -48,7 +48,7 @@ def generate(
     forwarded as top-level fields in the request payload.  The serving layer
     maps standard keys (height, width, seed, num_inference_steps) to sampling
     params and forwards model-specific keys (think, cfg_norm, img_cfg_scale,
-    …) to ``extra_args`` based on the pipeline's ``EXTRA_BODY_PARAMS``.
+    …) to ``extra_args`` based on the model extra registry.
 
     Returns:
         Image bytes (for image outputs) or text string (for text outputs).
@@ -131,7 +131,7 @@ def main():
     parser.add_argument("--width", type=int, default=2048, help="Image width")
     parser.add_argument("--num-steps", type=int, default=50, help="Inference steps")
     parser.add_argument("--seed", type=int, default=42, help="Random seed")
-    # Model-specific parameters (forwarded via EXTRA_BODY_PARAMS)
+    # Model-specific parameters forwarded via the model extra registry.
     parser.add_argument("--cfg-scale", type=float, default=4.0, help="CFG scale")
     parser.add_argument("--img-cfg-scale", type=float, help="Image CFG scale (img2img)")
     parser.add_argument("--cfg-norm", type=str, help="CFG normalization mode")

examples/online_serving/speech_to_video/README.md

@@ -0,0 +1,115 @@
+# Speech-To-Video
+
+This example demonstrates how to deploy the Wan2.2 speech-to-video (S2V) model for online video generation using vLLM-Omni.
+
+## Supported Models
+
+| Model | Model ID |
+|-------|----------|
+| Wan2.2 S2V (14B) | `Wan-AI/Wan2.2-S2V-14B` |
+
+## Start Server
+
+### Basic Start
+
+```bash
+VLLM_WORKER_MULTIPROC_METHOD=spawn \
+vllm serve Wan-AI/Wan2.2-S2V-14B --omni \
+  --model-class-name WanS2VPipeline \
+  --tensor-parallel-size 2 \
+  --flow-shift 3.0 \
+  --vae-use-slicing --vae-use-tiling \
+  --cache-backend cache_dit \
+  --port 8091
+```
+
+### Start with Script
+
+```bash
+bash run_server.sh
+```
+
+The script allows overriding:
+- `MODEL` (default: `Wan-AI/Wan2.2-S2V-14B`)
+- `PORT` (default: `8091`)
+- `FLOW_SHIFT` (default: `3.0`)
+- `TP` (default: `2`)
+- `CACHE_BACKEND` (default: `cache_dit`)
+
+### Start with 4 GPUs
+
+```bash
+TP=4 bash run_server.sh
+```
+
+## Sync API (Recommended for Testing)
+
+`POST /v1/videos/sync` blocks until generation completes and returns the raw
+video bytes (`video/mp4`) directly in the response body.
+
+```bash
+bash run_curl_speech_to_video.sh
+```
+
+## Async Job API
+
+`POST /v1/videos` creates an asynchronous job and returns immediately.
+
+```bash
+no_proxy=127.0.0.1 \
+create_response=$(curl -sS -X POST http://127.0.0.1:8091/v1/videos \
+  -H "Accept: application/json" \
+  -F "prompt=A person singing" \
+  -F 'image_reference={"image_url": "https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/examples/Five%20Hundred%20Miles.png"}' \
+  -F 'audio_reference={"audio_url": "https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/examples/Five%20Hundred%20Miles.MP3"}' \
+  -F "width=832" -F "height=480" \
+  -F "num_inference_steps=40" \
+  -F "guidance_scale=4.5" \
+  -F "fps=16")
+
+video_id=$(echo "$create_response" | jq -r '.id')
+
+# Poll until complete
+while true; do
+  status=$(curl -s "http://127.0.0.1:8091/v1/videos/${video_id}" | jq -r '.status')
+  if [ "$status" = "completed" ]; then
+    break
+  fi
+  if [ "$status" = "failed" ]; then
+    echo "Video generation failed"
+    exit 1
+  fi
+  sleep 2
+done
+
+# Download
+curl -L "http://127.0.0.1:8091/v1/videos/${video_id}/content" -o s2v_output.mp4
+```
+
+## Request Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `prompt` | str | - | Text description of the desired video |
+| `image_reference` | JSON str | - | Image reference: `{"image_url": "..."}` — supports HTTP(s) URLs or base64 data URLs |
+| `audio_reference` | JSON str | - | Audio reference: `{"audio_url": "..."}` — supports HTTP(s) URLs or base64 data URLs |
+| `width` | int | None | Video width in pixels |
+| `height` | int | None | Video height in pixels |
+| `fps` | int | None | Output video frame rate |
+| `num_inference_steps` | int | None | Number of denoising steps (40 recommended) |
+| `guidance_scale` | float | None | CFG guidance scale |
+| `seed` | int | None | Random seed for reproducibility |
+
+## Notes
+
+- S2V requires both a reference image (`image_reference`) and an audio reference
+  (`audio_reference`). The generated video will show a person matching the reference
+  image with lip movements synchronized to the audio.
+- `audio_reference` accepts a JSON string: `{"audio_url": "..."}` where the URL can be:
+  - An HTTP/HTTPS URL (e.g., `https://example.com/audio.mp3`)
+  - A base64 data URL (e.g., `data:audio/mp3;base64,...`)
+- `--model-class-name WanS2VPipeline` is required on the server to select the
+  S2V pipeline (distinct from the T2V/I2V pipelines).
+- `--cache-backend cache_dit` enables DiT caching for ~2x speedup on cached steps.
+- Audio is muxed into the output MP4 automatically.
+- Always pass `fps=16` to ensure correct video/audio alignment.

examples/online_serving/speech_to_video/run_curl_speech_to_video.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Wan2.2 S2V (speech-to-video) curl example using the sync video API.
+
+set -euo pipefail
+
+BASE_URL="${BASE_URL:-http://localhost:8091}"
+OUTPUT_PATH="${OUTPUT_PATH:-s2v_480p_serve.mp4}"
+IMAGE_URL="${IMAGE_URL:-https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/examples/Five%20Hundred%20Miles.png}"
+AUDIO_URL="${AUDIO_URL:-https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/examples/Five%20Hundred%20Miles.MP3}"
+PROMPT="${PROMPT:-A person singing}"
+WIDTH="${WIDTH:-832}"
+HEIGHT="${HEIGHT:-480}"
+NUM_INFERENCE_STEPS="${NUM_INFERENCE_STEPS:-40}"
+GUIDANCE_SCALE="${GUIDANCE_SCALE:-4.5}"
+FPS="${FPS:-16}"
+
+echo "Sending S2V request..."
+echo "  Image URL: $IMAGE_URL"
+echo "  Audio URL: $AUDIO_URL"
+echo "  Prompt: $PROMPT"
+echo "  Resolution: ${WIDTH}x${HEIGHT}"
+echo "  Steps: $NUM_INFERENCE_STEPS"
+echo "  FPS: $FPS"
+
+IMAGE_REF_JSON="{\"image_url\": \"${IMAGE_URL}\"}"
+AUDIO_REF_JSON="{\"audio_url\": \"${AUDIO_URL}\"}"
+
+no_proxy=127.0.0.1 \
+curl -X POST "${BASE_URL}/v1/videos/sync" \
+  -F "prompt=${PROMPT}" \
+  -F "image_reference=${IMAGE_REF_JSON}" \
+  -F "audio_reference=${AUDIO_REF_JSON}" \
+  -F "width=${WIDTH}" -F "height=${HEIGHT}" \
+  -F "num_inference_steps=${NUM_INFERENCE_STEPS}" \
+  -F "guidance_scale=${GUIDANCE_SCALE}" \
+  -F "fps=${FPS}" \
+  --output "${OUTPUT_PATH}"
+
+if [ -f "$OUTPUT_PATH" ] && [ -s "$OUTPUT_PATH" ]; then
+    echo "Saved video to ${OUTPUT_PATH} ($(du -h "$OUTPUT_PATH" | cut -f1))"
+else
+    echo "ERROR: Output file is empty or missing"
+    exit 1
+fi

examples/online_serving/speech_to_video/run_server.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Wan2.2 S2V (speech-to-video) online serving startup script
+
+MODEL="${MODEL:-Wan-AI/Wan2.2-S2V-14B}"
+PORT="${PORT:-8091}"
+FLOW_SHIFT="${FLOW_SHIFT:-3.0}"
+TP="${TP:-2}"
+CACHE_BACKEND="${CACHE_BACKEND:-cache_dit}"
+
+echo "Starting Wan2.2 S2V server..."
+echo "Model: $MODEL"
+echo "Port: $PORT"
+echo "Flow shift: $FLOW_SHIFT"
+echo "Tensor parallel size: $TP"
+echo "Cache backend: $CACHE_BACKEND"
+
+CACHE_BACKEND_FLAG=""
+if [ "$CACHE_BACKEND" != "none" ]; then
+    CACHE_BACKEND_FLAG="--cache-backend $CACHE_BACKEND"
+fi
+
+VLLM_WORKER_MULTIPROC_METHOD=spawn \
+vllm serve "$MODEL" --omni \
+    --model-class-name WanS2VPipeline \
+    --tensor-parallel-size "$TP" \
+    --flow-shift "$FLOW_SHIFT" \
+    --vae-use-slicing --vae-use-tiling \
+    --port "$PORT" \
+    $CACHE_BACKEND_FLAG