Format diffusion RPC guard

Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>

.buildkite/pipeline-intel.yaml

@@ -10,7 +10,7 @@ steps:
           DOCKER_BUILDKIT: "1"
           # Buildkite will automatically replace this with the actual commit hash
           VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.23.0"
+          VLLM_VERSION: "v0.22.0"
         priority: 100
         timeout_in_minutes: 60
         soft_fail: false

.buildkite/pipeline.yml

@@ -1,12 +1,11 @@
-# Document 1: Buildkite loads only this block on first parse. The next step resolves docs-only,
-# pytest skip-mark-only, or combined skip-ci from git diff, then uploads document 2. When skip-ci
-# applies, image-build still runs for the nightly exception in upload_pipeline.py (nightly-test
-# label or main NIGHTLY=1); otherwise upload-nightly can be skipped together with test-ready/test-merge.
-# Weekly uploads also depend on image-build.
+# Document 1: Buildkite loads only this block on first parse. The next step resolves docs-only skip-ci
+# from git diff, then uploads document 2. When docs-only skip applies, image-build still runs for the nightly
+# exception in upload_pipeline.py (nightly-test label or main NIGHTLY=1); otherwise upload-nightly
+# can be skipped together with test-ready/test-merge. Weekly uploads also depend on image-build.
 
 # Document 2: appended after `---`; same file, read by upload_pipeline.py (not evaluated as a second pipeline by Buildkite). Child uploads include test-ready.yml, test-merge.yml, test-nightly.yml, and test-weekly.yml.
 steps:
-  - label: ":github: Resolve skip-ci (docs / skip marks) & upload pipeline"
+  - label: ":github: Resolve skip-ci & upload pipeline"
     key: upload-ci-pipeline
     commands:
       - "python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/pipeline.yml"

.buildkite/rebase-pipeline.yaml

@@ -14,7 +14,7 @@ steps:
     depends_on: image-build
     key: upload-ready-pipeline
     commands:
-      - python3 .buildkite/scripts/upload_pipeline.py --upload --all .buildkite/test-ready.yml
+      - python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/test-ready.yml
     agents:
       queue: "cpu_queue_premerge"
 
@@ -23,7 +23,7 @@ steps:
     depends_on: image-build
     key: upload-merge-pipeline
     commands:
-      - python3 .buildkite/scripts/upload_pipeline.py --upload --all .buildkite/test-merge.yml
+      - python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/test-merge.yml
     agents:
       queue: "cpu_queue_premerge"
 
@@ -33,6 +33,6 @@ steps:
     key: upload-nightly-pipeline
     if: build.env("NIGHTLY") == "1"
     commands:
-      - python3 .buildkite/scripts/upload_pipeline.py --upload --all .buildkite/test-nightly.yml
+      - buildkite-agent pipeline upload .buildkite/test-nightly.yml
     agents:
       queue: "cpu_queue_premerge"

.buildkite/scripts/hardware_ci/run-amd-test.sh

@@ -64,8 +64,7 @@ container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | hea
 docker pull "${image_name}"
 
 remove_docker_container() {
-   docker ps -aq --filter "name=${container_name}" | xargs -r docker rm -f || true
-   docker image rm -f "${image_name}" || true
+   docker rm -f "${container_name}" || docker image rm -f "${image_name}" || true
 }
 trap remove_docker_container EXIT
 
@@ -108,6 +107,7 @@ if [[ $commands == *"--shard-id="* ]]; then
         --network=host \
         --shm-size=16gb \
         --group-add "$render_gid" \
+        --rm \
         -e MIOPEN_DEBUG_CONV_DIRECT=0 \
         -e MIOPEN_DEBUG_CONV_GEMM=0 \
         -e VLLM_ROCM_USE_AITER=0 \
@@ -140,6 +140,7 @@ else
           --network=host \
           --shm-size=16gb \
           --group-add "$render_gid" \
+          --rm \
           -e MIOPEN_DEBUG_CONV_DIRECT=0 \
           -e MIOPEN_DEBUG_CONV_GEMM=0 \
           -e VLLM_ROCM_USE_AITER=0 \

.buildkite/scripts/upload_pipeline.py

@@ -3,8 +3,7 @@
 """Render and optionally upload Buildkite pipeline YAML with diff-aware logic.
 
 Bootstrap mode (pipeline.yml with __IMAGE_BUILD_IF__ placeholders):
-  - Detect docs-only, pytest skip-mark-only, or combined docs+skip-mark PR/main changes and
-    substitute skip-ci ``if`` expressions.
+  - Detect docs-only PR/main changes and substitute skip-ci ``if`` expressions.
 
 Test pipeline mode (e.g. test-merge.yml):
   - Drop steps or groups whose ``source_file_dependencies`` do not match changed files.
@@ -13,23 +12,19 @@ Test pipeline mode (e.g. test-merge.yml):
     recognize this uploader-only key).
 
 Usage:
-  python3 upload_pipeline.py [--upload] [--all] <pipeline.yml>
+  python3 upload_pipeline.py [--upload] <pipeline.yml>
 
   # Bootstrap (replaces upload_pipeline_with_skip_ci.sh):
   python3 upload_pipeline.py --upload .buildkite/pipeline.yml
 
   # Test pipeline (replaces upload_test_pipeline_with_diff_skip.py):
   python3 upload_pipeline.py --upload .buildkite/test-merge.yml
-
-  # Full suite (rebase pipeline): keep all steps, still strip the uploader-only key:
-  python3 upload_pipeline.py --upload --all .buildkite/test-merge.yml
 """
 
 from __future__ import annotations
 
 import argparse
 import os
-import re
 import subprocess
 import sys
 from pathlib import Path
@@ -62,8 +57,8 @@ def _git(*args: str) -> subprocess.CompletedProcess[str]:
     )
 
 
-def resolve_diff_range() -> str | None:
-    """Return a git diff range for PR or main builds, or None when unavailable."""
+def resolve_changed_files() -> list[str] | None:
+    """Return changed file paths, or None when diff cannot be resolved."""
     is_pr = os.environ.get("BUILDKITE_PULL_REQUEST", "false") != "false" and os.environ.get(
         "BUILDKITE_PULL_REQUEST", ""
     )
@@ -81,22 +76,14 @@ def resolve_diff_range() -> str | None:
             else:
                 _log(f"cannot resolve PR base {base_branch}; using safe defaults")
                 return None
-        return f"{base_ref}...{commit}"
-
-    if os.environ.get("BUILDKITE_BRANCH", "") == "main":
+        diff_range = f"{base_ref}...{commit}"
+    elif os.environ.get("BUILDKITE_BRANCH", "") == "main":
         if _git("rev-parse", "--verify", f"{commit}^").returncode != 0:
             _log("main commit has no parent; using safe defaults")
             return None
-        return f"{commit}^..{commit}"
-
-    _log("not PR/main build; using safe defaults")
-    return None
-
-
-def resolve_changed_files() -> list[str] | None:
-    """Return changed file paths, or None when diff cannot be resolved."""
-    diff_range = resolve_diff_range()
-    if diff_range is None:
+        diff_range = f"{commit}^..{commit}"
+    else:
+        _log("not PR/main build; using safe defaults")
         return None
 
     result = _git("diff", "--name-only", diff_range)
@@ -109,207 +96,30 @@ def resolve_changed_files() -> list[str] | None:
     return files
 
 
-def _is_doc_file(file_path: str) -> bool:
-    if not file_path:
-        return False
-    if file_path.startswith("docs/"):
-        return True
-    if file_path.endswith(".md"):
-        return True
-    return file_path == "mkdocs.yml"
-
-
 def is_docs_only_change(changed_files: list[str]) -> bool:
     has_any = False
     for file_path in changed_files:
         if not file_path:
             continue
         has_any = True
-        if not _is_doc_file(file_path):
-            return False
-    return has_any
-
-
-def _is_test_python_file(file_path: str) -> bool:
-    path = Path(file_path)
-    return path.suffix == ".py" and bool(path.parts) and path.parts[0] == "tests"
-
-
-_SKIP_MARK_RE = re.compile(r"pytest\.mark\.skip(?:if)?\b|pytest\.skip\s*\(")
-_PYTESTMARK_SKIP_RE = re.compile(r"pytest\.mark\.skip\b")
-_PYTEST_MARK_ONLY_RE = re.compile(r"pytest\.mark\.\w+")
-
-
-def _paren_balance(line: str) -> int:
-    return line.count("(") - line.count(")")
-
-
-def _is_skip_mark_related_content(line: str) -> bool:
-    stripped = line.strip()
-    if not stripped:
-        return True
-    if stripped.startswith("#"):
-        return True
-    return _SKIP_MARK_RE.search(stripped) is not None
-
-
-def _is_pytestmark_adjacent_content(line: str) -> bool:
-    """Allow pytestmark list refactors that only add skip/skipif alongside existing marks."""
-    stripped = line.strip().rstrip(",")
-    if not stripped:
-        return True
-    if stripped in {"[", "]"}:
-        return True
-    if stripped.startswith("#"):
-        return True
-    if stripped.startswith("pytestmark"):
-        return True
-    return _PYTEST_MARK_ONLY_RE.search(stripped) is not None
-
-
-def diff_only_contains_skip_mark_changes(diff_text: str) -> bool:
-    """True when diff only edits skip marks or reformats pytestmark to add them."""
-    pending_depth = 0
-    saw_change = False
-    has_skip_mark_edit = False
-    for raw_line in diff_text.splitlines():
-        if raw_line.startswith("@@"):
-            pending_depth = 0
+        if file_path.startswith("docs/"):
             continue
-        if not (raw_line.startswith("+") or raw_line.startswith("-")):
+        if file_path.endswith(".md"):
             continue
-        if raw_line.startswith("+++") or raw_line.startswith("---"):
+        if file_path == "mkdocs.yml":
             continue
-
-        saw_change = True
-        content = raw_line[1:]
-
-        if pending_depth > 0:
-            pending_depth += _paren_balance(content)
-            if pending_depth < 0:
-                pending_depth = 0
-            continue
-
-        if _is_skip_mark_related_content(content):
-            has_skip_mark_edit = True
-            pending_depth = max(0, _paren_balance(content))
-            continue
-
-        if _is_pytestmark_adjacent_content(content):
-            continue
-
-        return False
-
-    return saw_change and has_skip_mark_edit
-
-
-def _git_file_diff(file_path: str, diff_range: str) -> str | None:
-    result = _git("diff", diff_range, "--", file_path)
-    if result.returncode != 0:
-        _log(f"skip-mark-only: git diff failed for {file_path}")
-        return None
-    return result.stdout
-
-
-def _is_new_file_diff(diff_text: str) -> bool:
-    return "new file mode" in diff_text or "--- /dev/null" in diff_text
-
-
-def _file_has_module_level_pytest_skip(content: str) -> bool:
-    """True when ``pytestmark`` applies unconditional ``pytest.mark.skip`` to the module."""
-    match = re.search(r"^pytestmark\s*=\s*\[(.*?)^\s*\]", content, re.MULTILINE | re.DOTALL)
-    if match is not None:
-        return _PYTESTMARK_SKIP_RE.search(match.group(1)) is not None
-    match = re.search(r"^pytestmark\s*=\s*pytest\.mark\.skip\b", content, re.MULTILINE)
-    return match is not None
-
-
-def _file_content_at_commit(file_path: str, commit: str) -> str | None:
-    result = _git("show", f"{commit}:{file_path}")
-    if result.returncode != 0:
-        _log(f"skip-mark-only: cannot read {file_path} at {commit}")
-        return None
-    return result.stdout
-
-
-def _qualifies_as_skip_mark_test_change(file_path: str, diff_range: str) -> bool:
-    diff_text = _git_file_diff(file_path, diff_range)
-    if diff_text is None:
-        return False
-    if not diff_text.strip():
-        _log(f"skip-mark-only: empty diff for {file_path}")
-        return False
-
-    if diff_only_contains_skip_mark_changes(diff_text):
-        return True
-
-    if not _is_new_file_diff(diff_text):
-        _log(f"skip-mark-only: non-skip changes in {file_path}")
-        return False
-
-    commit = os.environ.get("BUILDKITE_COMMIT", "")
-    if not commit:
-        return False
-    content = _file_content_at_commit(file_path, commit)
-    if content is None:
-        return False
-    if _file_has_module_level_pytest_skip(content):
-        _log(f"skip-mark-only: new test file with module-level pytest.mark.skip: {file_path}")
-        return True
-
-    _log(f"skip-mark-only: new test file without module-level pytest.mark.skip: {file_path}")
-    return False
-
-
-def is_skip_mark_only_change(changed_files: list[str], *, diff_range: str) -> bool:
-    """True when every changed file is a tests/ module with only skip-mark edits."""
-    has_any = False
-    for file_path in changed_files:
-        if not file_path:
-            continue
-        has_any = True
-        if not _is_test_python_file(file_path):
-            return False
-        if not _qualifies_as_skip_mark_test_change(file_path, diff_range):
-            return False
-    return has_any
-
-
-def is_docs_or_skip_mark_only_change(changed_files: list[str], *, diff_range: str) -> bool:
-    """True when each changed file is a doc path or a qualifying skip-mark test change."""
-    has_any = False
-    for file_path in changed_files:
-        if not file_path:
-            continue
-        has_any = True
-        if _is_doc_file(file_path):
-            continue
-        if _is_test_python_file(file_path) and _qualifies_as_skip_mark_test_change(file_path, diff_range):
-            continue
-        _log(f"docs/skip-mark-only: rejecting {file_path}")
         return False
     return has_any
 
 
-def resolve_skip_ci(changed_files: list[str] | None, *, diff_range: str | None = None) -> bool:
+def resolve_skip_ci(changed_files: list[str] | None) -> bool:
     if changed_files is None:
         _log("skip-ci=0 (could not resolve changed files)")
         return False
-
-    if diff_range is not None and is_docs_or_skip_mark_only_change(changed_files, diff_range=diff_range):
-        if is_docs_only_change(changed_files):
-            _log("docs-only change detected; skip-ci=1")
-        elif is_skip_mark_only_change(changed_files, diff_range=diff_range):
-            _log("pytest skip-mark-only change detected; skip-ci=1")
-        else:
-            _log("docs + pytest skip-mark-only change detected; skip-ci=1")
-        return True
-
-    if diff_range is None and is_docs_only_change(changed_files):
+    if is_docs_only_change(changed_files):
         _log("docs-only change detected; skip-ci=1")
         return True
-
-    _log("non-doc/non-skip-mark changes detected; skip-ci=0")
+    _log("non-doc changes detected; skip-ci=0")
     return False
 
 
@@ -440,22 +250,12 @@ def resolve_pipeline_path(arg: str) -> Path:
     return ROOT / path
 
 
-def render_pipeline(path: Path, *, force_all: bool = False) -> str:
+def render_pipeline(path: Path) -> str:
     text = path.read_text(encoding="utf-8")
-    diff_range = resolve_diff_range()
     changed_files = resolve_changed_files()
 
-    # ``--all`` forces the keep-all-steps path (no diff-aware skipping) while still
-    # stripping ``source_file_dependencies``. Used by the rebase pipeline so main builds
-    # run the full e2e suite (see .buildkite/rebase-pipeline.yaml).
-    if force_all:
-        changed_files = None
-
     if BOOTSTRAP_MARKER in text:
-        rendered = render_bootstrap_pipeline(
-            text,
-            skip_ci=resolve_skip_ci(changed_files, diff_range=diff_range),
-        )
+        rendered = render_bootstrap_pipeline(text, skip_ci=resolve_skip_ci(changed_files))
         return rendered
 
     doc = yaml.safe_load(text)
@@ -489,11 +289,6 @@ def main() -> int:
         action="store_true",
         help="Pipe rendered YAML to buildkite-agent pipeline upload",
     )
-    parser.add_argument(
-        "--all",
-        action="store_true",
-        help="Keep all steps (disable diff-aware skipping); still strips source_file_dependencies",
-    )
     args = parser.parse_args()
 
     path = resolve_pipeline_path(args.pipeline)
@@ -501,7 +296,7 @@ def main() -> int:
         _log(f"missing pipeline file: {path}")
         return 1
 
-    rendered = render_pipeline(path, force_all=args.all)
+    rendered = render_pipeline(path)
     if args.upload:
         upload_to_buildkite(rendered)
     else:

.buildkite/test-amd-merge.yml

@@ -179,30 +179,17 @@ steps:
             pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model" --run-level "advanced_model"
           '
 
-    # It is the whisper ASR validation step that is slow, not the Voxtral TTS generation.
-    # whisper ASR runs on CPU.
-    - label: "Voxtral-TTS E2E Test (Online Serving)"
+    - label: "Voxtral-TTS E2E Test"
       agent_pool: mi325_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
         - |
-          timeout 40m bash -c '
+          timeout 20m bash -c '
+            export VLLM_LOGGING_LEVEL=DEBUG
             export VLLM_WORKER_MULTIPROC_METHOD=spawn
-            pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
-          '
-
-    - label: "Voxtral-TTS E2E Test (Offline Inference)"
-      agent_pool: mi325_1
-      depends_on: amd-build
-      mirror_hardwares: [amdproduction]
-      grade: Blocking
-      commands:
-        - |
-          timeout 30m bash -c '
-            export VLLM_WORKER_MULTIPROC_METHOD=spawn
-            pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
+            pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
           '
 
 # TODO: Bagel test on ROCm is very unstable. @tjtanaa

.buildkite/test-amd-ready.yaml

@@ -185,6 +185,20 @@ steps:
         - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
         - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"
 
+    - label: "Voxtral-TTS E2E Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - |
+          timeout 20m bash -c '
+            export VLLM_LOGGING_LEVEL=DEBUG
+            export VLLM_WORKER_MULTIPROC_METHOD=spawn
+            pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
+            pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
+          '
+
     - label: "CosyVoice3-TTS E2E Test"
       agent_pool: mi325_1
       depends_on: amd-build

.buildkite/test-ready.yml

@@ -61,7 +61,7 @@ steps:
     steps:
       - label: "Diffusion · Offloader Test"
         commands:
-          - timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
+          - timeout 10m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
         agents:
           queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
         plugins:

.claude/skills/add-diffusion-model/SKILL.md

@@ -24,7 +24,7 @@ Check the model's HF repo for `model_index.json`. This determines your path:
 
 | Scenario | How to identify | Migration path |
 |----------|----------------|----------------|
-| **Already supported** | `_class_name` in `model_index.json` matches a key in `_DIFFUSION_MODELS` in `registry.py` | Skip implementation, then validate model-specific examples, tests, and docs as needed |
+| **Already supported** | `_class_name` in `model_index.json` matches a key in `_DIFFUSION_MODELS` in `registry.py` | Skip to Step 5 (test) and Step 7 (docs) |
 | **Diffusers-based** | Has standard `model_index.json` with `_diffusers_version`, subfolders for `transformer/`, `vae/`, etc. | Follow **Path A** below |
 | **Custom/private repo** | No diffusers `model_index.json`, weights in non-standard format, custom model code in a separate git repo | Follow **Path B** below |
 | **Hybrid** | Has some diffusers components (VAE) but custom transformer/fusion | Mix of Path A and Path B |
@@ -52,7 +52,7 @@ vllm_omni/diffusion/models/your_model_name/
 2. Replace attention with `vllm_omni.diffusion.attention.layer.Attention` (QKV shape: `[B, seq, heads, head_dim]`).
 3. Add `od_config: OmniDiffusionConfig | None = None` to `__init__`.
 4. Add `load_weights()` method mapping diffusers weight names to vllm-omni names.
-5. Add class attributes for acceleration features such as `_repeated_blocks` and `_layerwise_offload_blocks_attrs` (see [references/transformer-adaptation.md](references/transformer-adaptation.md) for examples).
+5. Add class attributes: `_repeated_blocks`, `_layerwise_offload_blocks_attr`.
 
 ### A4. Adapt pipeline
 
@@ -78,34 +78,13 @@ class YourPipeline(nn.Module):
 
 Add post/pre-process functions in the same pipeline file. Register them in `registry.py`.
 
-### A4.1 Add progress bar support (recommended)
-
-For pipelines with a standard denoising loop, prefer the existing progress bar pattern instead of hand-rolled logging.
-
-```python
-from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
-
-class YourPipeline(nn.Module, ProgressBarMixin):
-    def forward(self, req: OmniDiffusionRequest) -> DiffusionOutput:
-        # ... prepare timesteps / latents ...
-        with self.progress_bar(total=len(timesteps)) as progress_bar:
-            for i, t in enumerate(timesteps):
-                # predict noise / scheduler step
-                latents = ...
-                progress_bar.update()
-
-        return DiffusionOutput(output=output)
-```
-
-For custom loop structures, follow `vllm_omni/diffusion/models/progress_bar.py` and existing pipelines using `ProgressBarMixin`.
-
 ### A5. Register, test, docs → continue at Step 4 below.
 
 ---
 
 ## Path B: Custom/Private Repo Model
 
-For models without a diffusers pipeline — weights in custom format, model code in a private repo. Real examples: DreamID-Omni, BAGEL, HunyuanImage3. See [references/custom-model-patterns.md](references/custom-model-patterns.md) for concrete custom-model integration patterns.
+For models without a diffusers pipeline — weights in custom format, model code in a private repo. Real examples: DreamID-Omni, BAGEL, HunyuanImage3.
 
 ### B1. Understand the reference repo
 
@@ -155,7 +134,7 @@ Copy the essential model files into `vllm_omni/diffusion/models/<name>/` and ada
 
 ### B4. Handle custom weight loading
 
-Custom models have two common patterns for weight loading:
+Custom models have two patterns for weight loading:
 
 **Pattern 1: Bypass standard loader** (DreamID-Omni style)
 
@@ -170,10 +149,8 @@ class CustomPipeline(nn.Module):
         self.vae = custom_init_vae(model, device=self.device)
         self.text_encoder = custom_init_text_encoder(model, device=self.device)
         self.transformer = CustomFusionModel(CONFIG)
-        load_custom_checkpoint(
-            self.transformer,
-            checkpoint_path=os.path.join(model, "model.safetensors"),
-        )
+        load_custom_checkpoint(self.transformer,
+            checkpoint_path=os.path.join(model, "model.safetensors"))
         # NO weights_sources defined — bypasses standard loader
 
     def load_weights(self, weights):
@@ -185,8 +162,6 @@ class CustomPipeline(nn.Module):
 When weights are in safetensors format but need name remapping:
 
 ```python
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
-
 class CustomPipeline(nn.Module):
     def __init__(self, *, od_config, prefix=""):
         super().__init__()
@@ -212,9 +187,7 @@ class CustomPipeline(nn.Module):
             # Remap original weight names to vllm-omni module names
             name = self._remap_weight_name(name)
             if name in params:
-                param = params[name]
-                weight_loader = getattr(param, "weight_loader", default_weight_loader)
-                weight_loader(param, tensor)
+                default_weight_loader(params[name], tensor)
                 loaded.add(name)
         return loaded
 ```
@@ -308,43 +281,16 @@ For Omni or custom models, create:
 ### Step 7: Update Documentation
 
 Required updates:
-1. `docs/user_guide/diffusion/parallelism/overview.md` — parallelism support overview/table
+1. `docs/user_guide/diffusion/parallelism_acceleration.md` — parallelism support table
 2. `docs/user_guide/diffusion/cpu_offload_diffusion.md` — if CPU offload supported (add to supported models table)
-3. `docs/user_guide/diffusion/cache_acceleration/teacache.md` — if TeaCache supported
-4. `docs/user_guide/diffusion/cache_acceleration/cache_dit.md` — if Cache-DiT supported
-5. Offline example docs under `examples/offline_inference/<name>/` (`README.md` or category-specific `.md`)
-6. `examples/online_serving/<name>/README.md` — online serving docs
+3. `docs/user_guide/diffusion/teacache.md` — if TeaCache supported
+4. `docs/user_guide/diffusion/cache_dit_acceleration.md` — if Cache-DiT supported
+5. `examples/offline_inference/xxx/README.md` — offline example docs
+6. `examples/online_serve/xxx/README.md` — online serve docs
 
-### Step 8: Add E2E Tests
+### Step 8: Add E2E Tests (Recommended)
 
-**Follow the [vllm-omni-test skill](../vllm-omni-test/SKILL.md)** for markers, file naming, Buildkite wiring, and run commands. Also read [l4_functionality_tests.inc.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_examples/l4_functionality_tests.inc.md) and [CI_5levels.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/CI_5levels.md).
-
-Classify the model's **CI priority** first:
-
-| Priority | Required test levels | Files & markers |
-|----------|---------------------|-----------------|
-| **High** (listed in [#1832](https://github.com/vllm-project/vllm-omni/issues/1832) or on the diffusion hot path) | **L1** · **L2** online · **L3** online + offline · **L4** feature + performance | See table below |
-| **Medium** (*normal priority* in L4 docs) | **L3** online + offline · **L4** feature only | Fewer L4 parametrized rows |
-| **Low** | **L4** feature only | One or two `*_expansion.py` cases |
-
-**Per-level deliverables (diffusion / `pytest.mark.diffusion`):**
-
-| Level | Location | Marker | CI pipeline | Notes |
-|-------|----------|--------|-------------|-------|
-| **L1** | `tests/diffusion/models/{slug}/`, `tests/diffusion/cache/`, transformer unit tests | `core_model` + `cpu` | `test-ready.yml` | Weight remap, `_sp_plan`, cache enabler registration, shape contracts |
-| **L2** | `tests/e2e/online_serving/test_{slug}.py` (and offline if the category is offline-first) | **`core_model` + `advanced_model`** (both on baseline smoke) + `diffusion` + `@hardware_test` / `hardware_marks` | `test-ready.yml` | Default deploy smoke — minimal `num_inference_steps`, single prompt |
-| **L3** | `tests/e2e/online_serving/test_{slug}.py` **and** `tests/e2e/offline_inference/test_{slug}.py` when offline matters | Baseline smoke: **`core_model` + `advanced_model`**; heavier cases: `advanced_model` only (+ `diffusion`) | `test-merge.yml` **or** merged into nightly diffusion function job | Real weights, streaming/API paths, LoRA/offload smoke |
-| **L4** | `tests/e2e/online_serving/test_{slug}_expansion.py` (+ offline expansion if needed) | `full_model` + `diffusion` | `test-nightly.yml` (X2I / X2V / X2A function groups) | Feature combos per [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); perf → `tests/dfx/perf/tests/test_{model}_vllm_omni.json` |
-
-**L2 & L3 online — same file, dual marks on the baseline smoke:** The **first / simplest** case in `test_{slug}.py` (default deploy, minimal steps, single prompt) should carry **both** `@pytest.mark.core_model` and `@pytest.mark.advanced_model` on the **same** function so L2 (`test-ready.yml`) and L3 (`test-merge.yml`) share one smoke test. Heavier deploy variants or API paths in the same file use **`advanced_model` only**. When L3 moves to nightly, migrate those heavier cases into `test_{slug}_expansion.py` with `full_model` and remove the dedicated `test-merge.yml` job (see `test_longcat_image_expansion.py`, `test_qwen_image_expansion.py`).
-
-**L4 design (high priority):** Combine multiple supported features (Cache-DiT, TP, USP, CFG, HSDP, CPU offload, quantization) into **few parametrized** `OmniServerParams` rows so each feature appears in at least one case without exploding GPU jobs. Shard single-GPU vs multi-GPU cases across the nightly X2I/X2V function steps (1-GPU vs `distributed_cuda`).
-
-**L4 design (medium / low):** One or two parametrized rows covering the best quality/perf trade-off; skip perf JSON unless the model is high priority.
-
-**Reference implementations:** `tests/e2e/online_serving/test_qwen_image_edit_expansion.py`, `tests/e2e/online_serving/test_longcat_image_expansion.py`, `tests/e2e/online_serving/test_hunyuan_video_15_expansion.py`.
-
-**Keep model-specific code inside test modules — not `tests/helpers/{slug}.py`:** deploy constants, prompts, sampling dicts, and inline `request_config` / `form_data` belong in each `test_{slug}.py` and `test_{slug}_expansion.py`. Do not add per-model files under `tests/helpers/`; reuse only repo-wide harness (`mark`, `media`, `runtime`, `stage_config`, `assertions`). **L2+ online/offline e2e:** reuse or add `send_*_request` in `tests/helpers/runtime.py` — tests call the handler, not raw `omni.generate` / HTTP. See [vllm-omni-test skill](../vllm-omni-test/SKILL.md) § **Runtime send helpers**.
+Create `tests/e2e/online_serving/test_your_model_expansion.py`.
 
 ### Step 9: Add Cache-DiT Acceleration
 
@@ -564,7 +510,7 @@ class YourTransformer(nn.Module):
 #### 10e. Update parallelism documentation
 
 After adding parallelism support, update:
-1. `docs/user_guide/diffusion/parallelism/overview.md` — add your model to the support overview/table
+1. `docs/user_guide/diffusion/parallelism_acceleration.md` — add your model to the support table
 2. Record which parallelism methods are supported (USP, Ring, CFG, TP, HSDP, VAE-Patch)
 
 ### Step 11: Add CPU Offload Support
@@ -622,7 +568,6 @@ See the [Profiling Single-Stage Diffusion](../../../docs/contributing/profiling.
 
 ## Reference Files
 
-- [vllm-omni-test skill](../vllm-omni-test/SKILL.md) — L1–L4 markers, naming, Buildkite wiring, run commands
 - [Transformer Adaptation](references/transformer-adaptation.md) — porting transformers from diffusers
 - [Custom Model Patterns](references/custom-model-patterns.md) — patterns for non-diffusers models
 - [Parallelism Patterns](references/parallelism-patterns.md) — TP, SP/USP, CFG parallel, HSDP implementation details

.claude/skills/add-diffusion-model/references/parallelism-patterns.md

@@ -569,4 +569,4 @@ Check which parallelism methods a model supports:
 | **TP** | Uses `ColumnParallelLinear` / `QKVParallelLinear`. Search: `grep -r 'ParallelLinear\|QKVParallel' vllm_omni/diffusion/models/<model>/` |
 | **HSDP** | Transformer defines `_hsdp_shard_conditions`. Search: `grep -r '_hsdp_shard_conditions' vllm_omni/diffusion/models/` |
 
-The canonical per-model support overview/table is in `docs/user_guide/diffusion/parallelism/overview.md`.
+The canonical per-model support table is in `docs/user_guide/diffusion/parallelism_acceleration.md`.

.claude/skills/add-diffusion-model/references/transformer-adaptation.md

@@ -219,14 +219,13 @@ def load_weights(self, weights):
 
 ### Class-Level Attributes for Features
 
-
 ```python
 class MyTransformer(nn.Module):
     # torch.compile: list block class names that repeat and can be compiled
     _repeated_blocks = ["MyTransformerBlock"]
 
     # CPU offload: attribute name of the nn.ModuleList containing blocks
-    _layerwise_offload_blocks_attrs = ["blocks"]
+    _layerwise_offload_blocks_attr = "blocks"
 
     # LoRA: mapping of fused param names to original param names
     packed_modules_mapping = {"to_qkv": ["to_q", "to_k", "to_v"]}

.claude/skills/add-tts-model/SKILL.md

@@ -305,72 +305,13 @@ def build_voice_clone_prompt(ref_audio_path: str, text: str, codec) -> list:
     ]
 ```
 
-### Test Case Writing (CI Levels)
-
-**Follow the [vllm-omni-test skill](../vllm-omni-test/SKILL.md)** for markers, file naming (`test_{slug}.py` / `test_{slug}_expansion.py`), Buildkite wiring, and copy-paste run commands. Also read [CI_5levels.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/CI_5levels.md) and [tests_style.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/tests_style.md).
-
-Classify the model's **CI priority** first (high / medium / low). High-priority TTS models are typically those on the integration hot path or listed in tracking issues such as [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); medium and low tiers cover the long tail. When unsure, ask the reviewer which tier applies.
-
-| Priority | Required test levels | Files & markers |
-|----------|---------------------|-----------------|
-| **High** | **L1** unit/logic · **L2** online smoke · **L3** online + offline integration · **L4** feature + performance | See table below |
-| **Medium** | **L3** online + offline · **L4** feature only | Skip dedicated L1/L2 unless fixing a logic bug |
-| **Low** | **L4** feature only | One or two `*_expansion.py` parametrized cases |
-
-**Per-level deliverables (TTS / `pytest.mark.tts`):**
-
-| Level | Location | Marker | CI pipeline | Notes |
-|-------|----------|--------|-------------|-------|
-| **L1** | `tests/model_executor/…`, `tests/entrypoints/openai_api/…`, stage-processor tests | `core_model` + `cpu` | `test-ready.yml` | Prompt assembly, async_chunk helpers, `serving_speech` validation — no GPU |
-| **L2** | `tests/e2e/online_serving/test_{slug}.py` | **`core_model` + `advanced_model`** (both on baseline smoke) + `tts` + `@hardware_test(...)` | `test-ready.yml` (`ready` label) | Default deploy smoke: single `/v1/audio/speech` or offline `OmniRunner` path |
-| **L3** | `tests/e2e/online_serving/test_{slug}.py` **and** `tests/e2e/offline_inference/test_{slug}.py` | Baseline smoke: **`core_model` + `advanced_model`**; heavier cases: `advanced_model` only (+ `tts`) | `test-merge.yml` **or** merged into nightly TTS function job | Streaming, voice clone, batch/queue, async_chunk |
-| **L4** | `tests/e2e/online_serving/test_{slug}_expansion.py`, optional offline expansion | `full_model` + `tts` | `test-nightly.yml` (`:full_moon: TTS · Function Test with L4`) | Feature matrix; perf → `tests/dfx/perf/tests/test_tts.json` |
-
-**L2 & L3 online — same file, dual marks on the baseline smoke:** The **first / simplest** case in `test_{slug}.py` (default deploy, single non-streaming `/v1/audio/speech` or equivalent offline path) should carry **both** `@pytest.mark.core_model` **and** `@pytest.mark.advanced_model` on the **same** function so it runs in L2 (`test-ready.yml`, `--run-level core_model`, basic validation) and L3 (`test-merge.yml`, `--run-level advanced_model`, deeper validation) without duplicating the test. In-tree examples: `test_voxcpm2_tts.py::test_text_to_audio_001`, `test_qwen3_tts_customvoice.py::test_text_to_audio_001`.
-
-Heavier scenarios in the same file use **`advanced_model` only** (streaming, extra languages, concurrency, async_chunk, batch). Example: `test_voice_clone_en_streaming_001` → `advanced_model` only. When migrating L3 to nightly, move those heavier cases into `test_{slug}_expansion.py` with `full_model` and drop the dedicated merge job (see `test_ming_tts_expansion.py`, `test_glm_tts_expansion.py`).
-
-```python
-@pytest.mark.core_model
-@pytest.mark.advanced_model
-@pytest.mark.tts
-@hardware_test(res={"cuda": "L4"}, num_cards=1)
-@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
-def test_voice_clone_en_non_streaming_001(omni_server, openai_client) -> None:
-    openai_client.send_audio_speech_request({...})
-```
-
-**L4 consolidation:** Prefer parametrized `OmniServerParams` rows (`default`, `async_chunk`, feature flags) in one expansion module rather than many merge-only files ([#1832](https://github.com/vllm-project/vllm-omni/issues/1832)).
-
-**L4 performance (high-priority models):** Add latency / throughput / stress rows in `tests/dfx/perf/tests/test_tts.json`, or a dedicated `tests/dfx/perf/tests/test_{slug}.json` when the model must not join the shared nightly server matrix before integration lands (see VoxCPM2 / Coqui XTTS pattern). Register the model in `benchmarks/tts/model_configs.yaml` for local `bench_tts.py`. Wire a **separate** `test-nightly.yml` Perf Test step when the JSON is not merged into `test_tts.json` yet.
-
-**Keep model-specific code inside test modules — not `tests/helpers/{slug}.py`:**
-
-- Put `MODEL`, deploy path, vendored `REF_AUDIO_URL`, `get_prompt()`, and inline `request_config` dicts **in each** `test_{slug}.py`, `test_{slug}_expansion.py`, offline `test_{slug}.py`, and L1 `test_{slug}_*.py` as needed.
-- **Do not** add `tests/helpers/{slug}.py` (or `tests/helpers/{model_name}.py`) to deduplicate constants or request builders across those files. A little duplication is intentional; follow in-tree references such as `tests/e2e/online_serving/test_glm_tts.py` and `tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py`.
-- `tests/helpers/` is for **repo-wide** harness code only (`mark.py`, `media.py`, `runtime.py`, `stage_config.py`, `assertions.py`, `fixtures/`). Import those; do not extend the tree with per-model modules.
-
-**Runtime send helpers (`tests/helpers/runtime.py`) — online and offline e2e:**
-
-| Path | Fixture | Call |
-|------|---------|------|
-| Online `/v1/*` | `openai_client` | `openai_client.send_*_request(request_config)` |
-| Offline inference | `omni_runner_handler` | `omni_runner_handler.send_*_request(request_config)` |
-
-1. **Grep `runtime.py` first** — reuse `send_omni_request`, `send_diffusion_request`, `send_audio_speech_request` (online + offline Qwen-style TTS), `send_single_stage_tts_request` (Coqui XTTS / MOSS-TTS-Nano offline), etc.
-2. **No matching helper** → add `send_<feature>_request` (or `send_<route>_http_request` for negative/dfx) in **`runtime.py`** with general `assert_*` bundled inside, **then** call it from the test.
-3. **Test file** holds `request_config` dicts only — not `omni.generate`, not `_collect_audio()`, not raw HTTP/SDK.
-4. Wire `tests/helpers/runtime.py` into Buildkite `source_file_dependencies` when you add helpers.
-
-See [vllm-omni-test skill](../vllm-omni-test/SKILL.md) § **Runtime send helpers** for full tables and exceptions.
-
 ### Deliverables
 
 - Updated `serving_speech.py` with all 5 integration points (single commit)
 - Client scripts and server launcher under `examples/online_serving/text_to_speech/<model>/`
 - Gradio demo with streaming and voice cloning UI in the same dir
-- E2E tests per **Test Case Writing (CI Levels)** above (priority tier determines L1–L4 scope)
-- Buildkite wired per level: `test-ready.yml` (L1/L2), `test-merge.yml` or nightly function job (L3), `test-nightly.yml` (L4) — see [vllm-omni-test skill](../vllm-omni-test/SKILL.md)
+- E2E online serving test (`tests/e2e/online_serving/test_<model>.py`)
+- Buildkite CI entry in `.buildkite/test-merge.yml`
 - New section in `examples/online_serving/text_to_speech/README.md` (table row + per-model section). Do **not** create a top-level `examples/online_serving/<model>/` dir or a per-model `README.md` inside `text_to_speech/<model>/`.
 
 ### E2E test pitfalls to avoid
@@ -385,9 +326,9 @@ See [vllm-omni-test skill](../vllm-omni-test/SKILL.md) § **Runtime send helpers
 - **Use the harness readiness gate.** The fixture waits for HTTP 200 on
   `/health`; don't add `time.sleep` in tests. If warmup is incomplete, make
   `/health` return non-200 until you're actually ready.
-- **Mark tests per the CI Levels table** — baseline smoke: `core_model` + `advanced_model`; heavier cases: `advanced_model` only; L4 expansion: `full_model`
-- **No per-model helper modules** — do not create `tests/helpers/{slug}.py`; keep constants and `request_config` payloads in the test file
-- **Online and offline e2e go through `runtime.py`** — `openai_client.send_audio_speech_request` (online); `omni_runner_handler.send_audio_speech_request` (Qwen-style offline) or `send_single_stage_tts_request` (single-stage offline). Add a new `send_*_request` in `runtime.py` when none fits; do not embed `omni.generate` or HTTP in tests
+- **Mark with `@pytest.mark.core_model` + `hardware_test(res={"cuda": "H100"})`**
+  so the test lands in `test-ready.yml` (triggered by the `ready` label) rather
+  than only nightly.
 
 ## Phase 4: Async Chunk (Streaming)
 
@@ -520,8 +461,8 @@ Use this checklist when integrating a new TTS model:
 - [ ] Voice cloning works (if supported)
 - [ ] All response formats work (wav, mp3, flac, pcm)
 - [ ] Client scripts and server launcher created
-- [ ] E2E tests added per model priority tier (see **Test Case Writing (CI Levels)**)
-- [ ] Buildkite entries match level: `test-ready.yml` / `test-merge.yml` or nightly TTS job / `test-nightly.yml`
+- [ ] E2E online serving test written (`tests/e2e/online_serving/test_<model>.py`)
+- [ ] Buildkite CI entry added to `.buildkite/test-merge.yml`
 - [ ] Gradio demo working
 - [ ] Documentation added (offline + online docs, nav, supported models)
 
@@ -559,6 +500,5 @@ Project docs and adjacent skills:
 - [TTS audio skill](../vllm-omni-audio-tts/SKILL.md) — supported models and usage
 - [Fish Speech integration](../vllm-omni-audio-tts/references/fish-speech.md) — complete example of Phases 1–3
 - [Qwen3-TTS reference](../vllm-omni-audio-tts/references/qwen-tts.md) — complete example of all 5 phases
-- [vllm-omni-test skill](../vllm-omni-test/SKILL.md) — L1–L4 markers, naming, Buildkite wiring, run commands
 - [Adding a TTS model (developer guide)](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/model/adding_tts_model.md)
 - `plan/voxcpm2_native_ar_design.md` — VoxCPM2's vLLM-native AR + side-computation pattern (distinct from the generator-based single-stage described above)

.claude/skills/readme.md

@@ -30,12 +30,6 @@ include:
 - `precheck-pr`: self-check a branch before creating a PR — validates PR title
   format, catches dead code, verifies accuracy/perf claims, and confirms merge
   readiness
-- `vllm-omni-test`: guides generation and execution of CI-aligned tests (L1–L4),
-  pytest marker selection (`core_model` / `advanced_model` / `full_model`,
-  `omni` / `tts` / `diffusion`), Buildkite wiring (`test-ready.yml`,
-  `test-merge.yml`, `test-nightly.yml`, `test-weekly.yml`), and copy-paste
-  local plus CI-like `pytest` commands; see `references/test-routing.md` for
-  level-to-command mapping
 - `review-pr`: provides a structured workflow for reviewing pull requests
 - `vllm-omni-npu-model-runner-upgrade`: upgrades NPU model runners to align with the
   latest vllm-ascend NPUModelRunner

.claude/skills/vllm-omni-test/SKILL.md

@@ -1,890 +0,0 @@
----
-name: vllm-omni-test
-description: Generate and run tests for vllm-project/vllm-omni with CI-aligned levels and markers; wire new tests into Buildkite (test-ready.yml for L1/L2, test-merge.yml for L3, test-nightly.yml for L4). On completion, always provide copy-paste local and CI-like pytest commands plus prerequisites. Use when creating regression tests, adding L1-L4 coverage, selecting pytest markers, or validating fixes from issues/PRs.
----
-
-# vLLM-Omni Test Generator & Runner
-
-## Purpose
-
-Use this skill to generate minimal, stable test cases and run them with the correct marker/level strategy for [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni).
-
-**Link convention:** Paths such as `.buildkite/` and `docs/contributing/` live at the **vllm-omni repo root**. Markdown links use repo-relative paths from this skill file (e.g. `../../../.buildkite/test-ready.yml`, `../../../docs/contributing/ci/CI_5levels.md`).
-
-Default priorities:
-
-1. Reproducible regression coverage for bug fixes
-2. Correct test level and marker selection
-3. Low flake, low dependency tests first
-4. CI-compatible run commands
-5. **Actionable run commands for the human**: whenever you add or change tests, always finish with **copy-paste-ready** `pytest` lines (local: single file and/or single test; CI-like: markers + `--run-level`), plus short **prerequisites** (GPU tier, HF cache, optional `model_prefix`). Do not assume the reader will infer commands from `test-routing.md` alone.
-
-## Inputs
-
-- Issue/PR link and summary
-- Changed files or suspected code path
-- Whether the user wants local quick validation or CI-equivalent validation
-- Hardware constraints (CPU only / CUDA / ROCm / NPU)
-
-## Workflow
-
-### Step 1: Classify Test Goal
-
-- **Bugfix regression**: start from a minimal failing scenario and add assertions that prevent recurrence. Before writing tests, output **`required` / `recommended` / `not_needed`**: **`required`** — stable logic/contract bug that should have been caught; **`recommended`** — environment-sensitive but a small regression still helps; **`not_needed`** — one-off external/config failure or existing tests already cover the path. Prefer the narrowest stable L1 (CPU) case; escalate to L2/L3 only when the bug needs real weights or serving.
-- **Feature coverage**: verify new behavior and one negative/boundary case.
-- **Perf/benchmark claim**: require benchmark-oriented tests and explicit metrics.
-
-### Step 2: Select Test Level
-
-- **L1**: unit/logic, deterministic, CPU-friendly, fastest feedback.
-- **L2**: basic e2e/integration and platform-dependent checks.
-- **L3/L4**: advanced model/integration/perf validation.
-
-Use [references/test-routing.md](references/test-routing.md) for level-to-marker and command mapping.
-
-### Step 3: Pick Markers
-
-Always attach markers deliberately:
-
-- **Level**: `core_model` (L1/L2) and/or `advanced_model` (L3) and/or `full_model` (L4 nightly)
-- **Model type** (required on model-centric e2e — pick exactly one):
-  - `omni` — end-to-end multimodal LLM pipelines (thinker/talker/stages; Qwen-Omni family)
-  - `tts` — speech synthesis / TTS-only models (`/v1/audio/speech`, voice clone, etc.)
-  - `diffusion` — generative diffusion models (image / audio / text / video from noise)
-- **Cross-cutting area** (when relevant): `parallel`, `cache`, `example`, `benchmark`
-- **Hardware**: `cpu`, `gpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `distributed_cuda`, …
-- Optional: `slow`, distributed markers when multi-card is required
-
-**Baseline smoke (L2 + L3):** The simplest e2e case per model — default deploy, minimal request — should usually carry **both** `@pytest.mark.core_model` and `@pytest.mark.advanced_model` on the **same** test function so `test-ready.yml` and `test-merge.yml` share one test. `send_*_request` picks validation depth from `--run-level`. References: `test_voxcpm2_tts.py::test_text_to_audio_001`, `test_qwen3_tts_customvoice.py::test_text_to_audio_001`. Heavier scenarios use **`advanced_model` only**; L4 expansion uses **`full_model`**.
-
-For hardware-aware tests, prefer `@hardware_test(...)` or `hardware_marks(...)` in `tests/helpers/mark.py`.
-
-**Diffusion nightly split** (`test-nightly.yml`): all diffusion tests use `pytest.mark.diffusion`, but CI groups them by **output modality**, not by separate pytest markers:
-
-| Nightly group | Scope | Typical models / paths |
-|---------------|--------|-------------------------|
-| **Diffusion X2I(&A&T) Model Test** | x2**i** (image), x2**a** (audio), x2**t** (text) and other **non-video** diffusion | Qwen-Image*, BAGEL, FLUX, SD3, Z-Image, LongCat, DreamZero, … — `test_*_expansion.py` under X2I shards; L4 sweep uses `-k "not test_wan and not test_bagel_expansion and not hunyuan"` for L4 |
-| **Diffusion X2V Model Test** | x2**v** (video) only | Wan2.2, HunyuanVideo 1.5, Wan VACE, … — e.g. `test_wan22_expansion.py`, `test_hunyuan_video_15_expansion.py` |
-
-Wire new **video** diffusion expansion tests into the **X2V** group; wire **image/audio/text** diffusion expansion tests into **X2I(&A&T)**. Do not place x2v modules in X2I shards (see comment in `test-nightly.yml` above the X2I group).
-
-### Naming: generated test module files (L2–L4, model-centric e2e)
-
-When adding **new** pytest modules whose primary scope is a **specific model** (typical under `tests/e2e/offline_inference/` or `tests/e2e/online_serving/`), use this filename pattern:
-
-| Level | Filename pattern | Example (model `Qwen/Qwen2.5-Omni-7B`) |
-|-------|------------------|----------------------------------------|
-| **L2**, **L3** | `test_{lowercase_model_slug}.py` | `test_qwen2_5_omni.py` |
-| **L4** | `test_{lowercase_model_slug}_expansion.py` | `test_qwen2_5_omni_expansion.py` |
-
-**Slug rules for `{lowercase_model_slug}`:**
-
-1. Start from the HuggingFace-style id (e.g. `Qwen/Qwen2.5-Omni-7B`), but **do not** put the org into the filename: use the **repo segment only** (`Qwen2.5-Omni-7B`), not `Qwen_...` / `qwen_qwen2_5_...`.
-2. **Lowercase**; replace `.`, `-`, and whitespace with a single `_` (e.g. `Qwen2.5-Omni-7B` → `qwen2_5_omni`). **Omit** trailing size tokens such as `7b` / `30b` in the basename when a single file covers that model line in the directory (matches `test_qwen2_5_omni.py` in-tree).
-3. If two checkpoints in the same folder need separate modules, add a **minimal** disambiguator (e.g. `_7b` vs `_3b`) only then.
-4. **L1** unit tests are **not** bound to this pattern; use `tests/<area>/test_<feature>.py` as today.
-
-Routing tables and commands: [references/test-routing.md](references/test-routing.md) § *Model-centric e2e filename convention*.
-
-Existing references: `tests/e2e/offline_inference/test_qwen2_5_omni.py` (L2-style omni), `tests/e2e/offline_inference/test_qwen3_5_9b.py` (L2-style omni, single-stage VL), `tests/e2e/online_serving/test_qwen3_omni_expansion.py` (L4-style omni), `tests/e2e/online_serving/test_qwen_image_edit_expansion.py` / `test_qwen_image_expansion.py` (L4-style diffusion).
-
-### Step 4: Generate Test Case Skeleton
-
-**1. Pick the functional scenario** (then choose directory, fixtures, and markers):
-
-| Scenario | Typical location | Fixtures / runner pattern | Baseline markers & level |
-|----------|------------------|---------------------------|---------------------------|
-| **Offline inference e2e** | `tests/e2e/offline_inference/` | **Module (default):** `omni_runner` + `omni_runner_handler`. **Function (isolation only):** `omni_runner_function` + `omni_runner_handler_function`. Diffusion/TTS may use `Omni(...).generate` directly | L2: `core_model` + **one of** `omni` / `tts` / `diffusion`; `@hardware_test(...)` when GPU/NPU is required |
-| **Online serving e2e** | `tests/e2e/online_serving/` | **Module (default):** `omni_server` + `openai_client`. **Function (isolation only):** `omni_server_function` + `openai_client_function`. Clients: `send_omni_request` (omni), `send_audio_speech_request` (tts), `send_diffusion_request` / `send_video_diffusion_request` / `send_images_generations_request` (diffusion) | Baseline smoke: **`core_model` + `advanced_model`**; heavier paths: `advanced_model` only; L4 expansion: `full_model` |
-| **Documentation / runnable examples** | `tests/examples/offline_inference/`, `tests/examples/online_serving/` | **Offline docs (preferred):** extract Python/Bash blocks from the doc README (e.g. `ReadmeSnippet.extract_readme_snippets`), `pytest.mark.parametrize` each snippet, run via `example_runner.run` with a stable `output_subfolder`. **Online docs:** copy client/request scripts into dedicated tests and keep them in sync with the doc page. | Usually **L4**: `advanced_model`, often `example` plus hardware marks matching the nightly docs-example job (see `.buildkite/test-nightly.yml`). Full conventions: [docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md) (introduced in [PR #1910](https://github.com/vllm-project/vllm-omni/pull/1910): naming, output directory layout, skip rules, avoid trimming `num_inference_steps` without a strong CI reason). |
-| **Performance / benchmark** | `tests/dfx/perf/tests/*.json` + `run_*_benchmark.py` | JSON or script-driven server + load config; assert explicit metrics / baselines | L4 Perf: `full_model` + `benchmark`; wire `test-nightly.yml` Perf steps |
-| **Invalid parameter / negative HTTP validation** | `tests/dfx/reliability/invalid_param_test/` | Live `omni_server` + low-level `send_*_http_request` with `err_code` / `err_message` | `pytest.mark.slow` + `omni` / `tts` / `diffusion` + `@hardware_marks` (`H100` or `L4`); CI in **`test-weekly.yml`** (not ready/merge/nightly) |
-
-**If the user’s test plan includes invalid parameter validation / invalid params / negative HTTP / 400 validation:** do **not** add those `test_*` functions to `tests/e2e/online_serving/test_*.py` or `*_expansion.py`. **Move or author them** under `tests/dfx/reliability/invalid_param_test/` in the **endpoint-matching script** (see **Invalid parameter validation** below). Success-path e2e and invalid-param dfx tests must stay in separate modules.
-
-**1b. Model type — `omni` vs `tts` vs `diffusion`**
-
-After choosing offline/online/docs/perf, classify the **product under test** and attach **exactly one** model-type marker. All three can live under the same `tests/e2e/` trees; conventions diverge:
-
-| Dimension | **Omni** (`pytest.mark.omni`) | **TTS** (`pytest.mark.tts`) | **Diffusion** (`pytest.mark.diffusion`) |
-|-----------|------------------------------|----------------------------|----------------------------------------|
-| **What it is** | Multimodal LLM pipeline (thinker/talker/stages; text + vision + audio I/O) | Speech synthesis / voice models | Generative diffusion (noise → image, audio, text, or video) |
-| **Examples** | Qwen2.5-Omni, Qwen3-Omni | Qwen3-TTS, VoxCPM2, Higgs-Audio, Step-Audio2 | Qwen-Image, BAGEL, Wan2.2, HunyuanVideo |
-| **Offline runner** | `omni_runner` + `omni_runner_handler`, `generate_multimodal` | `Omni(...).generate` or stage YAML + TTS params | `Omni(...).generate` + `OmniDiffusionSamplingParams` |
-| **Online client** | `openai_client.send_omni_request` (chat completions, modalities) | `openai_client.send_audio_speech_request` (`/v1/audio/speech`) | `send_diffusion_request` (chat/T2I), `send_video_diffusion_request` (`/v1/videos`, X2V), or `send_images_generations_http_request` / `send_images_edits_http_request` (DALL-E routes) |
-| **Typical assertions** | Stage outputs, text/audio keywords via `OmniRunnerHandler` / response handler | WAV bytes, stream chunks, speech endpoint contract | Image/video dimensions, `final_output_type`, `assert_diffusion_response` |
-| **Stage / deploy YAML** | Per-model omni stage configs (`ci/qwen3_omni_moe.yaml`, …) | `qwen3_tts.yaml`, `voxcpm2.yaml`, … | Often default serve; parallel/offload YAML for heavy DiT |
-| **Nightly group** (`test-nightly.yml`) | **Omni Model Test** — `-m "full_model and omni"` | **TTS Model Test** — `-m "full_model and tts"` | **Diffusion X2I(&A&T)** *or* **Diffusion X2V** (see Step 3 table; same `diffusion` marker, different YAML group / file shard) |
-| **L4 pressure** | Expansion per modality/model as needed | Expansion + accuracy/perf in TTS group | X2I: merge feature combos per [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); X2V: separate nightly group |
-
-Do **not** mix fixtures across types (e.g. do not use `omni_runner` layout for a pure diffusion or TTS model without mirroring an in-tree test in that family).
-
-**Diffusion only — X2I(&A&T) vs X2V (nightly routing, not extra markers):**
-
-- **X2I(&A&T)**: image / audio / text generation — Qwen-Image*, FLUX, SD3, Z-Image, BAGEL (expansion), LongCat, audio diffusion, etc.
-- **X2V**: video generation only — Wan2.2, HunyuanVideo 1.5, Wan VACE, LTX video similarity paths.
-
-When adding a new `test_*_expansion.py`, place it in the matching **nightly group** step (explicit file list in `test-nightly.yml`), not only by marker expression.
-
-**2. Use the narrowest deterministic skeleton for the scenario**
-
-*L1 unit / logic (CPU-first):*
-
-**Mocking rule (L1 only):** use **pytest** integration — **`mocker`** (`pytest-mock`) or **`monkeypatch`** (built-in). **Do not** import or call **`unittest.mock`** (`patch`, `MagicMock`, `@patch`, `with patch(...)`, etc.) in L1 tests; patches must auto-revert with the test lifecycle.
-
-```python
-import pytest
-
-pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
-
-
-def test_<scenario_name>(mocker):
-    # Prefer mocker.patch / mocker.spy / mocker.Mock — not unittest.mock.patch
-    fake_fn = mocker.patch("vllm_omni.some.module.expensive_call", return_value=...)
-    # Act
-    # Assert
-    fake_fn.assert_called_once()
-
-
-def test_<env_or_attr>(monkeypatch):
-    # For simple env / attribute substitution without a Mock object
-    monkeypatch.setenv("SOME_FLAG", "1")
-    monkeypatch.setattr("vllm_omni.some.module.CONST", 42)
-    ...
-```
-
-**Avoid in L1:**
-
-```python
-# BAD — do not use in L1 unit tests
-from unittest.mock import patch, MagicMock
-
-@patch("vllm_omni.some.module.fn")
-def test_bad(mock_fn): ...
-
-def test_also_bad():
-    with patch("...") as m: ...
-```
-
-See **L1 unit test constraints (mocking)** below for the full do/don't list.
-
-*Offline multimodal e2e — **Omni** (representative):*
-
-```python
-@pytest.mark.core_model
-@pytest.mark.omni
-@hardware_test(...)
-@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
-def test_<scenario>(omni_runner, omni_runner_handler) -> None:
-    request_config = {"prompts": ..., "modalities": [...]}  # optional: images, videos, audios
-    omni_runner_handler.send_omni_request(request_config)
-```
-
-*Offline generative e2e — **Diffusion** (representative):*
-
-```python
-@pytest.mark.core_model
-@pytest.mark.diffusion
-@hardware_test(...)
-@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
-def test_text_to_image_001(omni_runner_handler) -> None:
-    omni_runner_handler.send_diffusion_request({"prompt": "...", "extra_body": {"num_inference_steps": 4, ...}})
-```
-
-*Offline TTS e2e — **Qwen3-TTS** (two-stage; representative):*
-
-```python
-@pytest.mark.advanced_model
-@pytest.mark.tts
-@hardware_test(...)
-@pytest.mark.parametrize("omni_runner", tts_server_params, indirect=True)
-def test_text_to_audio_001(omni_runner, omni_runner_handler) -> None:
-    omni_runner_handler.send_audio_speech_request({
-        "input": "...",
-        "task_type": "Base",
-        "ref_audio": REF_AUDIO_URL,
-        "ref_text": REF_TEXT,
-    })
-```
-
-*Offline TTS e2e — **single-stage** (Coqui XTTS, MOSS-TTS-Nano; representative):*
-
-```python
-@pytest.mark.advanced_model
-@pytest.mark.tts
-@hardware_test(...)
-@pytest.mark.parametrize("omni_runner", tts_server_params, indirect=True)
-def test_voice_clone_001(omni_runner_handler) -> None:
-    omni_runner_handler.send_single_stage_tts_request({
-        "input": "...",
-        "language": "en",
-        "prompt_audio_path": REF_AUDIO_PATH,
-        "response_format": "wav",
-        "run_level": "advanced_model",
-    })
-```
-
-*Online serving e2e — **Omni** (representative):*
-
-```python
-@pytest.mark.core_model
-@pytest.mark.omni
-@hardware_test(...)
-@pytest.mark.parametrize("omni_server", test_params, indirect=True)
-def test_text_to_text_001(omni_server, openai_client) -> None:
-    request_config = {"model": omni_server.model, "messages": ..., "modalities": ["text"]}
-    openai_client.send_omni_request(request_config)
-```
-
-*Online serving e2e — **TTS** (representative):*
-
-```python
-@pytest.mark.core_model
-@pytest.mark.advanced_model
-@pytest.mark.tts
-@hardware_test(...)
-@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
-def test_text_to_audio_001(omni_server, openai_client) -> None:
-    request_config = {"model": omni_server.model, "input": "...", "response_format": "wav", ...}
-    openai_client.send_audio_speech_request(request_config)
-```
-
-*Online serving e2e — **Diffusion** X2I (representative):*
-
-```python
-@pytest.mark.core_model
-@pytest.mark.diffusion
-@pytest.mark.parametrize("omni_server", _get_default_case(MODEL), indirect=True)
-def test_text_to_image_001(omni_server, openai_client) -> None:
-    openai_client.send_diffusion_request({...})  # chat completions + extra_body
-```
-
-*Online serving e2e — **Diffusion** X2V (representative):*
-
-```python
-@pytest.mark.core_model
-@pytest.mark.diffusion
-def test_text_to_video_001(omni_server, openai_client) -> None:
-    openai_client.send_video_diffusion_request({"model": ..., "form_data": {...}})  # /v1/videos
-```
-
-*Documentation example tests:* follow the **Preferred Test Strategy** in [l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md): dynamic extraction for offline READMEs; explicit copied client code for online pages until extraction is justified; use the documented **naming**, **output directory** (page folder + case id), and **skipping** rules (e.g. Gradio-only scripts).
-
-*Performance tests:* add or extend entries under `tests/perf/` (and JSON configs where the project uses them), with **explicit baselines** and the same marker/run-level pairing as the CI step that will execute them.
-
-**3. Cross-cutting rules**
-
-- Reuse existing fixtures for the chosen scenario; do not mix “online client” assumptions into offline `OmniRunner` tests without a clear reason.
-- Avoid external network dependency in assertions unless the scenario is explicitly “online serving” or doc examples that require a model hub (then align with CI secrets/cache).
-- Keep **one test function = one intent** (one modality combo, one endpoint contract, or one acceleration combo).
-- **E2E test function layout**: **one case → one `test_<scenario>` function** with a name that states what is validated (endpoint, size/`n`, server flag, or route). **Do not** merge multiple cases into a single test that branches on `request.node.callspec.id`, `param.id`, or `if case_id == ...`. Use `@pytest.mark.parametrize("omni_server", [...], indirect=True)` per function (usually **one** `OmniServerParams` per test). A loop inside one test is OK only when it serves **that function’s single intent** (e.g. three standard sizes in `test_*_sizes_256_512_1024`).
-- **Runtime fixture scope** (`tests/helpers/fixtures/runtime.py`): default **`omni_server` / `omni_runner`** (module) + matching client/handler; use **`omni_server_function` / `omni_runner_function`** only when each `test_*` must start a fresh instance (see below).
-- **L1 mocks**: never `unittest.mock`; use `mocker` or `monkeypatch` only (see below).
-- **API calls (L2+ e2e, online and offline)**: **reuse** `send_*_request` in `tests/helpers/runtime.py` when it exists; **otherwise add** the helper in `runtime.py` first, then call it from the test. General `assert_*` inside `send_*_request`; special `assert_*` only in the test. See **Runtime send helpers** below — do not call `omni.generate`, raw HTTP, or SDK clients from `test_*.py`.
-- **Response assertions**: reusable checks on API bodies / decoded media belong in **`tests/helpers/assertions.py`** — not as private `_assert_*` helpers inside `test_*.py` (see below).
-- **Model-specific payloads stay in test modules** — per-model `MODEL`, deploy path, `REF_AUDIO_URL`, `get_prompt()`, `_build_request()`, and inline `request_config` dicts live in `test_{slug}.py` / `test_{slug}_expansion.py` (and offline/L1 siblings). **Do not** create `tests/helpers/{slug}.py` to deduplicate them; a little copy across files is preferred (see `test_glm_tts.py`, `test_cosyvoice3_tts_expansion.py`). `tests/helpers/` is repo-wide harness only (`mark`, `media`, `runtime`, `stage_config`, `assertions`, `fixtures/`).
-
-#### L1 unit test constraints (mocking)
-
-L1 tests (`core_model and cpu`, under `tests/diffusion/`, `tests/engine/`, `tests/model_executor/`, etc.) must follow the repo’s **pytest-mock** convention (`pytest-mock>=3.10.0` in `pyproject.toml` `[project.optional-dependencies] dev`).
-
-| Do | Don't |
-|----|-------|
-| `def test_foo(mocker):` + `mocker.patch(...)`, `mocker.spy(...)`, `mocker.Mock()`, `mocker.MagicMock()`, `mocker.AsyncMock()` | `from unittest.mock import patch, MagicMock, Mock, AsyncMock` |
-| `def test_bar(monkeypatch):` + `monkeypatch.setattr` / `setenv` / `delenv` / `setitem` | `@patch(...)` decorator |
-| Let `mocker` auto-stop patches after the test | `with patch(...):` / `patch.object(...)` context managers |
-| Mirror neighboring L1 tests in the same directory | `unittest.mock.create_autospec` unless an existing file already documents an exception |
-
-**Rationale:** `mocker` ties patch lifecycle to pytest fixtures (no leaked patches across tests). `unittest.mock` decorators/context managers are easy to compose incorrectly with parametrized or async tests and are inconsistent with in-tree L1 style.
-
-**Minimal patterns:**
-
-```python
-def test_returns_cached_config(mocker):
-    loader = mocker.patch(
-        "vllm_omni.foo.load_yaml",
-        return_value={"stages": []},
-    )
-    result = get_deploy_config("ci/foo.yaml")
-    assert result["stages"] == []
-    loader.assert_called_once_with("ci/foo.yaml")
-
-
-def test_skips_when_env_unset(monkeypatch):
-    monkeypatch.delenv("VLLM_OMNI_FEATURE", raising=False)
-    assert should_enable_feature() is False
-```
-
-E2E levels (L2+) generally **avoid mocks**; if a rare L2 stub is unavoidable, still prefer `mocker` over `unittest.mock` for consistency.
-
-#### Runtime fixtures — scope (`tests/helpers/fixtures/runtime.py`)
-
-vLLM-Omni e2e tests start a real **OmniServer** (online) or **OmniRunner** (offline). Pick scope by **how often the process must be recreated**, not by test level alone.
-
-| Scope | Online fixtures | Offline fixtures | When to use |
-|-------|-----------------|------------------|-------------|
-| **Module** (default) | `omni_server` → `openai_client` | `omni_runner` → `omni_runner_handler` | **Default for L2/L3/L4 expansion** — amortize model/server init across `test_*` in the same module. Same `OmniServerParams` / runner config can be reused by multiple tests. |
-| **Function** | `omni_server_function` → `openai_client_function` | `omni_runner_function` → `omni_runner_handler_function` | **Only when required** — each `test_*` must get a **clean** server/runner (no shared engine/GPU state). Typical: `tests/dfx/reliability/`, sleep/wakeup, crash/restart, tests that mutate global server state. |
-
-**Rules:**
-
-1. **Default to module scope** (`omni_server` / `omni_runner`) unless the scenario **explicitly** needs a fresh instance per test function.
-2. **Indirect parametrize name must match the fixture name**: `@pytest.mark.parametrize("omni_server", ...)` with `omni_server` + `openai_client`; `@pytest.mark.parametrize("omni_server_function", ...)` with `omni_server_function` + `openai_client_function`. Do not mix module fixture with function client (or vice versa).
-3. Different `OmniServerParams` per `test_*` (e.g. default vs `--enable-cpu-offload`) is still OK with **module** `omni_server` — pytest parametrizes per test node; only switch to `_function` when isolation between tests matters, not merely because `server_args` differ.
-4. One `test_*` with **many** server configs (expansion matrix) → single function + `@pytest.mark.parametrize("omni_server", [...], indirect=True)` + module `omni_server` (see in-tree `test_qwen_image_expansion.py`).
-
-```python
-# Default — module-scoped server (L4 expansion)
-@pytest.mark.parametrize("omni_server", [pytest.param(OmniServerParams(model=MODEL), marks=H100)], indirect=True)
-def test_foo_images_generations_default_1024(omni_server, openai_client) -> None:
-    openai_client.send_images_generations_request({...})
-
-# Function-scoped — reliability / per-test clean state only
-@pytest.mark.parametrize(
-    "omni_server_function",
-    [pytest.param(OmniServerParams(model=MODEL), marks=H100)],
-    indirect=True,
-)
-def test_foo_sleep_wakeup_cycle(omni_server_function, openai_client_function) -> None:
-    openai_client_function.send_omni_sleep_http_request({...})
-    openai_client_function.send_omni_wakeup_http_request({...})
-```
-
-#### Runtime send helpers — online **and** offline (`tests/helpers/runtime.py`)
-
-**L2+ e2e (online serving and offline inference) must call APIs through `tests/helpers/runtime.py`.** Fixtures live in `tests/helpers/fixtures/runtime.py`; **send/assert implementation** lives in `tests/helpers/runtime.py` + `tests/helpers/assertions.py`.
-
-| Principle | Action |
-|-----------|--------|
-| **Reuse first** | Grep `runtime.py` for an existing **`send_*_request`** on `OpenAIClientHandler` (online) or `OmniRunnerHandler` (offline) that matches the endpoint / pipeline shape. |
-| **Extend when close** | If an existing `send_*_request` almost fits (e.g. missing one optional field), extend it in `runtime.py` — do not fork logic in the test file. |
-| **Add when missing** | No suitable helper → add **`send_<feature>_request`** (high-level: call + general `assert_*`) or **`send_<route>_<verb>_http_request`** (low-level HTTP for negative/dfx) in **`runtime.py` first**, then call it from tests. |
-| **Test module owns payload only** | In `test_*.py`: `MODEL`, deploy path, vendored media, `get_prompt()`, and inline **`request_config` dicts** only. **No** `omni.generate(...)`, raw `requests.post`, OpenAI SDK calls, or `_collect_audio()` / `_process_output()` in e2e tests. |
-
-**Online** (`openai_client` from `omni_server`): `OpenAIClientHandler.send_*_request`.
-
-**Offline** (`omni_runner_handler` from `omni_runner`): `OmniRunnerHandler.send_*_request`.
-
-| Do | Don't |
-|----|-------|
-| Online: `openai_client.send_omni_request`, `send_diffusion_request`, `send_audio_speech_request`, `send_video_diffusion_request`, `send_images_generations_request`, … | `requests.post(f"{base_url}/v1/...", json=…)` or `client.chat.completions.create(...)` inside a test |
-| Offline: `omni_runner_handler.send_omni_request`, `send_diffusion_request`, `send_audio_speech_request`, `send_single_stage_tts_request`, `send_single_stage_tts_batch_request`, … | `omni_runner.omni.generate(...)` + hand-rolled tensor/WAV extraction in `test_*.py` |
-| Add missing `send_*` to **`runtime.py`** first; bundle general `assert_*` inside the send helper | A one-off `def _post_*` / `def _collect_audio` at the bottom of a test module |
-| Mirror naming/style of neighboring `send_*` (docstring, `request_config` dict, optional `run_level`, `err_code` / `err_message` for negative cases) | Different parameter shapes per test file for the same endpoint |
-
-**Workflow when generating online/offline e2e tests:**
-
-1. Decide whether the needed check is **general** (every success call of this `send_*`) or **special** (one case / one parameter combo only). See **General vs special assert placement** below.
-2. **Search `runtime.py`** for a matching **`send_*_request`**; if missing, add it (with general `assert_*` inside) before writing the test body.
-3. In the test module: build `request_config` → call **`send_*_request` only** for the general contract; call **`assert_*` in the test file only** for special, case-specific checks (import from `assertions.py`).
-4. Reserve **low-level** `send_*_http_request` for negative/dfx tests (`err_code` / `err_message`) — not for ordinary L2+ success-path e2e.
-5. When wiring Buildkite `source_file_dependencies`, include **`tests/helpers/runtime.py`** and/or **`tests/helpers/assertions.py`** when new helpers were added.
-
-**Exceptions (document in the test docstring why):** models whose offline prompt path cannot go through existing handlers yet (e.g. `test_higgs_audio_v2.py`, `test_voxtral_tts.py` with custom tokenizer compose) may call `omni.generate` directly until a `send_*_request` is added to `runtime.py` — treat as **debt**, not the default for new TTS/diffusion/omni e2e.
-
-**General vs special assert placement:**
-
-| Kind | Definition | Where it lives | Test file calls |
-|------|------------|----------------|-----------------|
-| **General** | Default success contract for this endpoint/client method — e.g. HTTP 200, `data[]` shape, `n` count, `size` dimensions, decodable image/audio, bundled omni/diffusion fields | Implement in `assertions.py`, invoke **inside** `send_*_request` in `runtime.py` (after low-level send / SDK call, when `err_code` is not set) | **`send_*_request` only** — do **not** repeat the same `assert_*` |
-| **Special** | Extra check tied to **this test case only** — e.g. seed byte-identical replay, accuracy/CLIP threshold, perf ceiling, model-specific optional field | Add or reuse `assert_*` in `assertions.py` (never inline in test) | `send_*_request` **then** `assert_<special>(...)` **once** for that case |
-
-Changing `request_config` fields (`size`, `n`, `seed`, server flags) is **not** special validation — the general `assert_*` should read those from `request_config` inside `send_*`.
-
-**`send_*` ↔ `assert_*` pairing (online `OpenAIClientHandler`):**
-
-| High-level `send_*` (prefer in L2+ e2e) | Assert already invoked inside `runtime.py` | Low-level HTTP-only sibling |
-|----------------------------------------|--------------------------------------------|-----------------------------|
-| `send_omni_request` | `assert_omni_response` | `send_chat_completions_http_request` → `assert_http_error` only |
-| `send_diffusion_request` | `assert_diffusion_response` | — |
-| `send_audio_speech_request` | `assert_audio_speech_response` | `send_audio_speech_http_request` → `assert_http_error` only |
-| `send_video_diffusion_request` | `assert_diffusion_response` | `send_videos_*_http_request` → `assert_http_error` only |
-| `send_images_generations_request` *(add when needed)* | **`assert_images_generations_response`** *(general — inside send)* | `send_images_generations_http_request` → `assert_http_error` only |
-| `send_images_edits_request` *(add when needed)* | **`assert_images_edits_response`** *(general — inside send)* | `send_images_edits_http_request` → `assert_http_error` only |
-
-**Rule:** Tests call high-level `send_*_request` for the general contract. **Never** call the same bundled `assert_*` again. Call an extra `assert_*` in the test **only** for special, case-specific validation.
-
-**Common `OpenAIClientHandler` entry points** (non-exhaustive — grep `runtime.py` before adding):
-
-| Area | High-level (SDK + assert) | Low-level HTTP (`*_http_request`) |
-|------|---------------------------|-----------------------------------|
-| Omni chat | `send_omni_request` | `send_chat_completions_http_request` |
-| Diffusion T2I (chat route) | `send_diffusion_request` | — |
-| Diffusion X2V | `send_video_diffusion_request` | `send_videos_create_http_request`, `send_video_content_http_request`, … |
-| DALL-E T2I / edit | `send_images_generations_request`, `send_images_edits_request` | `send_images_generations_http_request`, `send_images_edits_http_request` |
-| TTS (online) | `send_audio_speech_request` | `send_audio_speech_http_request`, `send_audio_generate_http_request`, … |
-| Ops / meta | — | `send_health_http_request`, `send_models_http_request`, `send_omni_sleep_http_request`, … |
-
-**Common `OmniRunnerHandler` entry points** (offline — grep `runtime.py` before adding):
-
-| Area | High-level (`send_*_request` + assert) | Notes |
-|------|----------------------------------------|-------|
-| Omni multimodal | `send_omni_request` | `generate_multimodal` path |
-| Diffusion offline | `send_diffusion_request` | chat-route / `OmniTextPrompt` offline |
-| Qwen-style TTS | `send_audio_speech_request` | two-stage, `generate_multimodal` + `mm_processor_kwargs` |
-| Single-stage TTS (Coqui XTTS, MOSS-TTS-Nano, …) | `send_single_stage_tts_request`, `send_single_stage_tts_batch_request` | `prompt` + `additional_information` + `omni.generate` — **do not** duplicate in tests |
-| New model family | **Add** `send_<family>_request` here first | Then call from `tests/e2e/offline_inference/test_*.py` |
-
-L1 tests under `tests/entrypoints/` may use **FastAPI `TestClient`** or direct handler calls with mocks; they do **not** need `OpenAIClientHandler` / `OmniRunnerHandler`. **L2+ online and offline e2e** must use `runtime.py` `send_*` helpers.
-
-#### Invalid parameter validation (`tests/dfx/reliability/invalid_param_test/`)
-
-When the user asks for **invalid parameter validation**, **invalid request bodies**, **HTTP 4xx contract tests**, or any case that sends **malformed / out-of-range / mismatched** API payloads against a **live** server:
-
-1. **Do not** place these in `tests/e2e/online_serving/` or `*_expansion.py`. If already drafted there, **move** the `test_*` into the correct `invalid_param_test` script and delete the duplicate from e2e.
-2. **Pick the script by HTTP route** (extend an existing file; add a new `test_invalid_<area>.py` only when no in-tree script covers that route family):
-
-| API / area | Script |
-|------------|--------|
-| Omni chat completions, WebSocket video/realtime paths | `test_invalid_omni_chat.py` |
-| `POST /v1/audio/speech`, stream, batch, voices | `test_invalid_audio_speech.py` |
-| Audio diffusion endpoints | `test_invalid_audio_diffusion.py` |
-| `POST /v1/images/generations` | `test_invalid_image_generation.py` |
-| `POST /v1/images/edits` | `test_invalid_image_editing.py` |
-| `POST/GET/DELETE /v1/videos*` | `test_invalid_video_generation.py` |
-| Sleep / wakeup / server control | `test_invalid_server_control.py` |
-
-3. **Match in-tree style** in the chosen script:
-
-| Element | Convention |
-|---------|------------|
-| **Module markers** | `pytestmark = [pytest.mark.slow, pytest.mark.<omni or tts or diffusion>]` |
-| **Server fixture** | `_PARAMS` / `_QWEN3_TTS_SPEECH`-style list of `pytest.param(OmniServerParams(...), id="...", marks=hardware_marks(...))`; `@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)` |
-| **Hardware** | `hardware_marks(res={"cuda": "H100"})` for heavy diffusion/omni/video; `hardware_marks(res={"cuda": "L4"})` for smaller TTS models (must match weekly `-m "slow and L4"` step) |
-| **HTTP client** | **Low-level** `openai_client.send_*_http_request({..., "err_code": 400, "err_message": (...)} )` — **not** `send_*_request` (success path) |
-| **Case shape** | Prefer **one `test_*` per route family** + `@pytest.mark.parametrize("body_spec, err_message", [...])` with stable `id=` per case; or dedicated `test_<route>_malformed_json` when not parametrized |
-| **Body helpers** | `_minimal_<endpoint>_json(omni_server)` / `_minimal_*_form_data()`; merge overrides with `body.update(body_spec)` |
-| **Known gaps** | `pytest.mark.skip(reason="…#3649")` as `_SKIP_ISSUE_3649` when server validation is not yet strict (mirror neighboring cases) |
-| **Sections** | Route banner comments (`# ─── POST /v1/images/generations ───`) like existing files |
-| **Shared fixtures** | Reuse `tests/dfx/reliability/invalid_param_test/conftest.py` (`tiny_png_bytes`, env defaults) |
-
-4. **Adding a new model** to invalid-param coverage: append a `pytest.param(OmniServerParams(model="...", stage_config_path=..., server_args=...), ...)` entry to the script’s `_PARAMS` list — do **not** create `test_invalid_<model>.py` unless the route is new.
-
-5. **CI — `.buildkite/test-weekly.yml` only** (not `test-ready.yml` / `test-merge.yml` / `test-nightly.yml`):
-
-| Weekly step | Command | When your cases run |
-|-------------|---------|---------------------|
-| **Invalid parameters Test · H100** | `pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"` | Diffusion / omni / video invalid-param tests with `H100` hardware mark |
-| **Invalid parameters Test · L4** | `pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"` | TTS / lighter models marked `L4` |
-
-- **Trigger:** `build.env("WEEKLY") == "1"` or PR label `weekly-test`.
-- **Default:** extending an existing `invalid_param_test` script needs **no YAML edit** — the weekly steps already sweep the whole directory.
-- **Edit YAML only** when adding a **new hardware queue**, a **new top-level script** that must run in isolation, or a **model-specific weekly shard** (mirror neighboring reliability steps).
-- Weekly steps **do not** use `source_file_dependencies`.
-
-**Example — append to `test_invalid_image_generation.py`:**
-
-```python
-@pytest.mark.parametrize(
-    "body_spec, err_message",
-    [
-        pytest.param({"seed": -1}, ("seed", "greater_than_equal", "0"), id="seed_negative"),
-    ],
-)
-@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
-def test_images_generations_invalid_requests(
-    omni_server: OmniServer,
-    openai_client: OpenAIClientHandler,
-    body_spec: dict[str, object],
-    err_message: str | tuple[str, ...],
-) -> None:
-    body = _minimal_images_gen_json(omni_server)
-    body.update(body_spec)
-    openai_client.send_images_generations_http_request(
-        {"json": body, "timeout": 300, "err_code": 400, "err_message": err_message}
-    )
-```
-
-See [references/test-routing.md](references/test-routing.md) **Invalid parameter / weekly CI**.
-
-#### Assertion helpers (`tests/helpers/assertions.py`)
-
-**Do not** add module-local helpers such as `_assert_images_generations_payload` or `_send_and_assert_*` in e2e test files. Response/media validation belongs in **`tests/helpers/assertions.py`**, grouped by **category**:
-
-| Category | Existing anchors | When to extend vs add |
-|----------|------------------|------------------------|
-| **Image** (chat diffusion + DALL-E JSON) | `assert_image_diffusion_response`, `assert_image_valid` | DALL-E `/v1/images/generations` JSON → add **`assert_images_generations_response`** beside image helpers (reuses `assert_image_valid`); do not fork decode logic into tests |
-| **Video** | `assert_video_diffusion_response`, `assert_video_valid` | Extend these for new video contracts |
-| **Audio** | `assert_audio_diffusion_response`, `assert_audio_speech_response`, `assert_audio_valid` | Extend for new TTS/audio endpoints |
-| **Omni multimodal** | `assert_omni_response` | Extend for new modality combos |
-| **HTTP errors** | `assert_http_error`, `assert_err_message_in_text` | Used inside low-level `send_*_http_request` and negative tests |
-
-| Do | Don't |
-|----|-------|
-| Put **general** `assert_*` inside `send_*_request` in `runtime.py`; tests call `send_*_request` only | Call `assert_diffusion_response` after `send_diffusion_request` (general assert already bundled) |
-| Put **special** `assert_*` in `assertions.py` and call it **in the test** after `send_*_request` when that case needs extra checks | Put general decode/count/size logic in the test because “this case uses `n=4`” (that belongs in general assert reading `request_config`) |
-| Extend the matching category function, or add `assert_<endpoint>_response` next to its category | `_send_and_assert_*` or PIL/base64 loops in `test_*.py` |
-| Use low-level `send_*_http_request` in dfx/negative tests (`err_code`) | Use low-level HTTP send + manual general assert in every L2+ expansion file |
-
-**Workflow when generating tests:**
-
-1. Implement or reuse **general** `assert_*` in `assertions.py` (by category).
-2. Wire it into **`send_*_request`** in `runtime.py` if not already bundled.
-3. In the test: `request_config` → `send_*_request`. Add a separate `assert_*` import/call **only** for special case validation.
-4. Update `source_file_dependencies` when `runtime.py` or `assertions.py` changes.
-
-**Example — DALL-E `/v1/images/generations`:**
-
-```python
-# tests/helpers/assertions.py — general contract (image category)
-def assert_images_generations_response(resp_body: dict, request_config: dict, *, run_level: str | None = None) -> None:
-    ...  # data[], n, size → assert_image_valid
-
-# tests/helpers/runtime.py — general assert INSIDE send
-def send_images_generations_request(self, request_config: dict[str, Any], ...) -> list[HttpResponse]:
-    responses = self.send_images_generations_http_request(request_config, ...)
-    if request_config.get("err_code") is None:
-        body = responses[0].json_body
-        assert isinstance(body, dict)
-        assert_images_generations_response(body, request_config, run_level=self.run_level)
-    return responses
-
-# tests/e2e/.../test_foo_expansion.py — one case per test_* (no case_id branching)
-@pytest.mark.parametrize("omni_server", [pytest.param(OmniServerParams(model=MODEL), marks=SINGLE_L4)], indirect=True)
-def test_foo_images_generations_default_1024(omni_server, openai_client) -> None:
-    openai_client.send_images_generations_request({"json": body, "timeout": 300})
-
-@pytest.mark.parametrize("omni_server", [pytest.param(OmniServerParams(model=MODEL), marks=SINGLE_L4)], indirect=True)
-def test_foo_images_generations_sizes_256_512_1024(omni_server, openai_client) -> None:
-    for size in ("256x256", "512x512", "1024x1024"):
-        openai_client.send_images_generations_request({"json": {**body, "size": size}, "timeout": 300})
-
-# chat diffusion — separate test; send_diffusion_request bundles assert_diffusion_response
-@pytest.mark.parametrize("omni_server", [pytest.param(OmniServerParams(model=MODEL), marks=SINGLE_L4)], indirect=True)
-def test_foo_chat_completions_t2i_fallback(omni_server, openai_client) -> None:
-    openai_client.send_diffusion_request({...})
-```
-
-L1 tests under `tests/entrypoints/` may keep **minimal** asserts next to the handler under test when validating a single branch; still prefer `assertions.py` when the same JSON/media check appears in more than one file or level (e.g. L1 protocol test + L4 e2e).
-
-#### Omni Test Writing Guidance (L1-L4 Layering)
-
-When the goal is a general Omni/multimodal test case, prioritize mapping the test to the correct purpose, directory, and resource assumptions aligned with the layers (see [CI_5levels.md](../../../docs/contributing/ci/CI_5levels.md)):
-
-- **L1**: Unit/logic validation on CPU (`core_model and cpu`). Cover input validation, branches, and exception paths (`tests/<component>/test_*.py`). **Mock with `mocker` / `monkeypatch` only — not `unittest.mock`.**
-- **L2**: Basic e2e (online/offline basic scenarios). Prefer dummy/lightweight models to validate the end-to-end request-to-output-structure/streaming chain (typically `tests/e2e/online_serving/` and `tests/e2e/offline_inference/`).
-- **L3/L4**: Important integration, performance, and accuracy validation. L4 emphasizes “full functional scenarios + performance/stress + runnable doc examples” (typically `*_expansion.py` plus related expansion cases).
-
-Also keep markers consistent with the run level: use `core_model` for L1/L2 and `advanced_model` for L3/L4, and pair with `--run-level` to select the intended CI strategy.
-
-#### Diffusion Test Writing Guidance (L4 Coverage Combinations)
-
-When the task involves diffusion models/features, organize L4 test cases following [`#1832`](https://github.com/vllm-project/vllm-omni/issues/1832): combine multiple diffusion features into as few test cases as possible to fit limited CI GPU resources.
-
-Implementation strategy:
-
-1. If “full L4 is too heavy”: first provide a **reduced local validation case in L1/L2** (so the key assertion and the contract fix point are covered deterministically).
-2. Then provide **CI/nightly-ready L3/L4 high-marked cases** (e.g. `advanced_model`) to broaden coverage under resource constraints.
-
-#### L4 nightly sub-pillars — Function vs Accuracy vs Perf
-
-**L4 is not only `test_*_expansion.py`.** In `test-nightly.yml`, each model-type group typically runs **separate jobs**:
-
-| Sub-pillar | Nightly step label pattern | What you add | Typical paths |
-|------------|----------------------------|--------------|---------------|
-| **Function** | `· Function Test with …` | E2e expansion / feature matrix | `tests/e2e/online_serving/test_<model>_expansion.py` (or offline) |
-| **Accuracy** | `· Accuracy Test` | Quality / similarity vs baseline | `tests/e2e/accuracy/test_<model>*.py` |
-| **Perf** | `· Perf Test · <Model>` | Throughput / latency / memory benchmarks | `tests/dfx/perf/tests/test_<model>_vllm_omni.json` + runner script |
-| **Doc** (optional) | `· Doc Test` | Runnable doc examples | `tests/examples/*/test_text_to_image.py`, … |
-
-**Default when the user asks for “L4 functional cases”:** deliver **Function** pillar only (`*_expansion.py` + Function Test shard in `test-nightly.yml`). **Do not** silently add Perf or Accuracy unless the user also asks for **performance** / **benchmark** / **accuracy** / **full L4** / **full L4 coverage**.
-
-**When the user explicitly asks for L4 perf (or full L4 including perf):**
-
-1. **Do not** put throughput/latency assertions inside `test_*_expansion.py` — perf uses the **dfx benchmark harness**, not `omni_server` e2e fixtures.
-2. Add or extend a **JSON config** under `tests/dfx/perf/tests/` (mirror in-tree names: `test_qwen_image_vllm_omni.json`, `test_bagel_vllm_omni.json`).
-3. Each JSON entry: `test_name`, `server_params` (`model`, `serve_args`), `benchmark_params[]` with explicit **`baseline`** (`throughput_qps`, `latency_mean`, `peak_memory_mb_mean`, …).
-4. Run locally via the matching script (model type → runner):
-
-| Model type | Runner script | Example config |
-|------------|---------------|----------------|
-| **Diffusion X2I/X2V** | `tests/dfx/perf/scripts/run_diffusion_benchmark.py` | `tests/dfx/perf/tests/test_qwen_image_vllm_omni.json` |
-| **TTS** | `tests/dfx/perf/scripts/run_benchmark.py` | `tests/dfx/perf/tests/test_tts.json` (shared) or `test_{slug}.json` (dedicated; use when the model must not join the shared nightly matrix before integration — e.g. `test_voxcpm2.json`, `test_coqui_tts.json`) |
-| **Omni** | (see existing Omni Perf steps — `run_benchmark.py` / dedicated omni configs in-tree) | `tests/dfx/perf/tests/test_omni*.json` |
-
-5. Wire **`test-nightly.yml` · Perf Test** step for that model (separate from Function Test): export `DIFFUSION_BENCHMARK_DIR` / `BENCHMARK_DIR`, run pytest on the script with `--test-config-file`, **upload artifacts** (`buildkite-agent artifact upload`), often **multi-GPU H100** for diffusion perf.
-6. **One benchmark scenario → one `test_name` block** in JSON (same spirit as one `test_*` per function case). Combine server `serve_args` + workload in one entry; use multiple `benchmark_params` rows for size/step sweeps under the same server config.
-
-**Perf local command (diffusion example):**
-
-```bash
-cd tests
-export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
-pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py \
-  --test-config-file dfx/perf/tests/test_<model>_vllm_omni.json
-```
-
-**Clarify in the test plan** which L4 pillars you deliver: `Function only` | `Function + Perf` | `Function + Accuracy + Perf`.
-
-See [references/test-routing.md](references/test-routing.md) **L4 nightly pillars** for Buildkite step patterns.
-
-### Step 5: Wire Buildkite (when CI must run the new test)
-
-If the test is not already collected by an existing pipeline command (for example, L1 tests marked `core_model and cpu` are already covered by the **Simple Unit Test** step in ready/merge), update the appropriate pipeline under `.buildkite/`:
-
-| Test level | Edit this file | Typical trigger / intent |
-|------------|----------------|---------------------------|
-| **L1** and **L2** | [`.buildkite/test-ready.yml`](../../../.buildkite/test-ready.yml) | PR **ready** label; L1 CPU + L2 GPU/basic e2e (steps labeled **Omni ·**, **TTS ·**, **Diffusion ·**) |
-| **L3** | [`.buildkite/test-merge.yml`](../../../.buildkite/test-merge.yml) | Post-merge; `advanced_model` integration per model type |
-| **L4** | [`.buildkite/test-nightly.yml`](../../../.buildkite/test-nightly.yml) | Nightly; grouped by model type (see below) |
-| **Invalid param / reliability (weekly)** | [`.buildkite/test-weekly.yml`](../../../.buildkite/test-weekly.yml) | Weekly; `tests/dfx/reliability/invalid_param_test/` — **not** L1–L4 e2e pipelines |
-
-**Level-specific Buildkite delivery (follow the user’s requested level):**
-
-| User asks for | Document / edit **only** | Do **not** add by default |
-|---------------|---------------------------|---------------------------|
-| **L4** (`*_expansion.py`, `full_model`) | `test-nightly.yml` — append file to the matching nightly shard (X2I / X2V / Omni / TTS) **or** note it is already collected by an existing `-m` / `-k` sweep | `test-merge.yml` (L3 / `advanced_model`) or `test-ready.yml` (L2) |
-| **L3** | `test-merge.yml` with `advanced_model` + `source_file_dependencies` | `test-nightly.yml` unless user also wants nightly |
-| **L2** | `test-ready.yml` with `core_model` + `source_file_dependencies` | merge / nightly pipelines |
-| **Invalid param** | `test-weekly.yml` — **Invalid parameters Test** group (`-m "slow and H100"` / `-m "slow and L4"`). Usually **no YAML edit** when appending cases to existing `invalid_param_test` scripts | `test-ready.yml`, `test-merge.yml`, `test-nightly.yml` |
-
-`test-nightly.yml` shards use **explicit pytest file paths** in `commands` (and PR labels like `diffusion-x2iat-test`); they generally **do not** use `source_file_dependencies`. Only suggest merge/ready YAML when the requested level is L3/L2 (or the user explicitly asks for multi-level CI).
-
-**`test-nightly.yml` top-level groups** (each uses `full_model` + type marker in pytest `-m`):
-
-| Group | Type marker | Notes |
-|-------|-------------|--------|
-| **Omni Model Test** | `omni` | Function / doc / accuracy / **perf** for Qwen-Omni family |
-| **TTS Model Test** | `tts` | `-m "full_model and L4 and tts"` (and H100 variants when enabled); **Perf Test** uses `run_benchmark.py` + `test_tts.json` |
-| **Diffusion X2I(&A&T) Model Test** | `diffusion` | Image/audio/text diffusion; explicit file shards for **Function**; **Perf Test · &lt;Model&gt;** uses `run_diffusion_benchmark.py` + JSON config |
-| **Diffusion X2V Model Test** | `diffusion` | Video-only (Wan, HunyuanVideo, …); PR label `diffusion-x2v-test`; separate **Perf** steps when in-tree |
-
-Guidelines when editing YAML:
-
-- **Match markers and `--run-level`** to the test: L1/L2 in `test-ready.yml` use `core_model` + `omni` / `tts` / `diffusion` as appropriate; L3 merge uses `advanced_model`; L4 nightly uses `full_model` with the same type marker.
-- **Diffusion L4**: add x2i/x2a/x2t expansion files to an **X2I(&A&T)** step; add x2v expansion files to **X2V** — never rely on a broad `tests/e2e/` sweep alone (shards exist to save GPU and avoid wrong queue).
-- **`source_file_dependencies` (required for new E2E jobs in `test-ready.yml` and `test-merge.yml`)**: every step under the **`:card_index_dividers: E2E Test`** group **must** declare `source_file_dependencies` so Buildkite only schedules the job when relevant paths change. Omitting it breaks path-based CI skipping for GPU e2e jobs.
-- **Reuse or extend an existing step** when the new test shares the same marker expression, queue, and timeout; otherwise add a new `steps:` entry with the correct `agents.queue`, `timeout_in_minutes`, and docker/kubernetes plugin blocks consistent with neighboring jobs.
-- **Job timeout — use `timeout_in_minutes` on the step, not `timeout` before pytest**: when adding or documenting a new E2E job in `test-ready.yml` / `test-merge.yml`, set `timeout_in_minutes` on the step and run `pytest` directly in `commands`. **Do not** wrap pytest in `timeout 40m bash -c "..."` / `timeout 20m bash -c "..."`; Buildkite already enforces the step deadline via `timeout_in_minutes`.
-- **Never omit `plugins` in skill examples or PR YAML snippets**: H100 E2E steps on `mithril-h100-pool` use the full `kubernetes` block (`resources`, `volumeMounts`, `env` with `HF_TOKEN` secret, `nodeSelector: gpu-h100-sxm`, `devshm` + `hf-cache` volumes). L4/docker steps on `gpu_1_queue` use the full `docker#v5.2.0` block (`shm-size`, `HF_HOME`, volumes). Do not write `# ... plugins unchanged` — paste the complete block from a sibling step.
-- **Platform forks** (e.g. AMD-ready / AMD-merge) live alongside these files; apply the same level → file mapping for those pipelines when the test is platform-specific.
-
-#### `source_file_dependencies` for E2E Test jobs (`test-ready.yml`, `test-merge.yml`)
-
-When adding or splitting a step inside the **E2E Test** group, **always** include `source_file_dependencies` on that step. List paths the job actually depends on — at minimum:
-
-1. **Test module(s)** — every `tests/e2e/...` file the `pytest` command runs (online and offline if both are in one step).
-2. **Model implementation** — `vllm_omni/model_executor/models/<family>/` and/or `vllm_omni/diffusion/models/<family>/` as applicable.
-3. **Stage / input plumbing** (omni & multi-stage TTS) — `vllm_omni/model_executor/stage_input_processors/<family>.py` when present.
-4. **Deploy / stage YAML** — `vllm_omni/deploy/<config>.yaml` or `vllm_omni/deploy/ci/<config>.yaml` referenced by the test’s `OmniServerParams` / `get_deploy_config_path(...)`.
-5. **Test helpers** (when changed in the same PR) — `tests/helpers/runtime.py` for new `send_*` client methods; `tests/helpers/assertions.py` for new shared `assert_*` helpers.
-
-Use **repo-relative paths** (no leading `./`). Prefer **directory** entries (`.../models/bagel/`) when the whole package matters; use **file** entries for single test modules and YAML configs. Mirror a neighboring step for the same model family (e.g. **Diffusion · Bagel Test** in `test-ready.yml`).
-
-**Example** (L2 ready — diffusion online smoke, H100 kubernetes):
-
-```yaml
-- label: "Diffusion · Bagel Test"
-  source_file_dependencies:
-    - tests/e2e/online_serving/test_bagel.py
-    - vllm_omni/model_executor/models/bagel/
-    - vllm_omni/diffusion/models/bagel/
-    - vllm_omni/model_executor/stage_input_processors/bagel.py
-    - vllm_omni/deploy/bagel.yaml
-  timeout_in_minutes: 40
-  commands:
-    - |
-      export VLLM_IMAGE_FETCH_TIMEOUT=60
-      pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model'
-  agents:
-    queue: "mithril-h100-pool"
-  plugins:
-    - kubernetes:
-        podSpec:
-          containers:
-            - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              resources:
-                limits:
-                  nvidia.com/gpu: 1
-              volumeMounts:
-                - name: devshm
-                  mountPath: /dev/shm
-                - name: hf-cache
-                  mountPath: /root/.cache/huggingface
-              env:
-                - name: HF_HOME
-                  value: /root/.cache/huggingface
-                - name: HF_TOKEN
-                  valueFrom:
-                    secretKeyRef:
-                      name: hf-token-secret
-                      key: token
-          nodeSelector:
-            node.kubernetes.io/instance-type: gpu-h100-sxm
-          volumes:
-            - name: devshm
-              emptyDir:
-                medium: Memory
-            - name: hf-cache
-              hostPath:
-                path: /mnt/hf-cache
-                type: DirectoryOrCreate
-```
-
-**Example** (L3 merge — TTS online + offline, L4 docker):
-
-```yaml
-- label: "TTS · Qwen3-TTS Base Test"
-  source_file_dependencies:
-    - tests/e2e/online_serving/test_qwen3_tts_base.py
-    - tests/e2e/offline_inference/test_qwen3_tts_base.py
-    - vllm_omni/model_executor/models/qwen3_tts/
-    - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
-    - vllm_omni/deploy/qwen3_tts.yaml
-  timeout_in_minutes: 20
-  commands:
-    - |
-      export VLLM_LOGGING_LEVEL=DEBUG
-      export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
-      pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-  agents:
-    queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-  plugins:
-    - docker#v5.2.0:
-        image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-        always-pull: true
-        propagate-environment: true
-        shm-size: "8gb"
-        environment:
-          - "HF_HOME=/fsx/hf_cache"
-        volumes:
-          - "/fsx/hf_cache:/fsx/hf_cache"
-```
-
-**Example** (L3 merge — diffusion online + offline, H100 kubernetes):
-
-```yaml
-- label: "Diffusion · Qwen Image Test"
-  source_file_dependencies:
-    - tests/e2e/online_serving/test_qwen_image.py
-    - tests/e2e/offline_inference/test_qwen_image.py
-    - vllm_omni/diffusion/models/qwen_image/
-  timeout_in_minutes: 40
-  commands:
-    - |
-      export VLLM_IMAGE_FETCH_TIMEOUT=60
-      pytest -s -v \
-        tests/e2e/online_serving/test_qwen_image.py \
-        tests/e2e/offline_inference/test_qwen_image.py \
-        -m 'advanced_model and cuda' --run-level 'advanced_model'
-  agents:
-    queue: "mithril-h100-pool"
-  plugins:
-    - kubernetes:
-        podSpec:
-          containers:
-            - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              resources:
-                limits:
-                  nvidia.com/gpu: 1
-              volumeMounts:
-                - name: devshm
-                  mountPath: /dev/shm
-                - name: hf-cache
-                  mountPath: /root/.cache/huggingface
-              env:
-                - name: HF_HOME
-                  value: /root/.cache/huggingface
-                - name: HF_TOKEN
-                  valueFrom:
-                    secretKeyRef:
-                      name: hf-token-secret
-                      key: token
-          nodeSelector:
-            node.kubernetes.io/instance-type: gpu-h100-sxm
-          volumes:
-            - name: devshm
-              emptyDir:
-                medium: Memory
-            - name: hf-cache
-              hostPath:
-                path: /mnt/hf-cache
-                type: DirectoryOrCreate
-```
-
-When extending an **existing** E2E step to run an additional test file, **append** that file (and any new model/deploy paths) to `source_file_dependencies` on the same step. When authoring a PR, mention the dependency list in the **Buildkite change** section of your test plan.
-
-#### L3 / L4: validating Buildkite from a feature branch (root `pipeline.yml`)
-
-Production behavior is driven by [`.buildkite/pipeline.yml`](../../../.buildkite/pipeline.yml): it builds the CI image, then uploads **L2** (`test-ready.yml` on non-`main`), **L3** (`test-merge.yml` when `build.branch == "main"`), and **L4** (`test-nightly.yml` when `build.env("NIGHTLY") == "1"`).
-
-When you add or debug **L3** or **L4** tests and need those child pipelines to run **off `main`** or **without** `NIGHTLY=1`, apply **temporary** edits (revert before merge):
-
-1. **Point the upload at the right file (optional shortcut)**  
-   In the **Upload Ready Pipeline** step, the command is `buildkite-agent pipeline upload .buildkite/test-ready.yml`. For a one-off run you may change that path to `.buildkite/test-merge.yml` (L3) or `.buildkite/test-nightly.yml` (L4), and adjust the step’s `if:` so it does not conflict with the other upload steps (avoid double-uploading unless intentional).
-
-2. **L3 — merge pipeline**  
-   The gate `if: build.branch == "main"` lives on the **Upload Merge Pipeline** step in `pipeline.yml` (not inside `test-merge.yml`). **Comment out that `if` line** so `test-merge.yml` is uploaded on your feature branch. Alternatively rely on step (1) instead of the dedicated merge upload step.
-
-3. **L4 — nightly pipeline**  
-   **Comment out** `if: build.env("NIGHTLY") == "1"` on the **Upload Nightly Pipeline** step in `pipeline.yml` so the nightly definition is uploaded without setting the env var.  
-   Child steps in [`test-nightly.yml`](../../../.buildkite/test-nightly.yml) often repeat `if: build.env("NIGHTLY") == "1"`; **comment out those lines on the steps you need to run**, otherwise they will still be skipped after upload.
-
-### Step 6: Run Tests
-
-Pick one command path:
-
-- **Quick local regression (preferred first)**: single file or `file.py::test_name`
-- **CI-like level run**: marker expression + `--run-level`
-
-Full templates live in [references/test-routing.md](references/test-routing.md). **After authoring tests, always emit concrete commands** (see **Output Format**).
-
-**Typical copy-paste examples** (run from repo `tests/` directory; requires matching vLLM/vllm-omni install and hardware):
-
-| Area | Example |
-|------|---------|
-| **Omni offline L2** | `pytest -s -v e2e/offline_inference/test_qwen2_5_omni.py -m "core_model and omni and not cpu" --run-level=core_model` |
-| **Omni online L2** | `pytest -s -v e2e/online_serving/test_qwen3_omni.py -m "core_model and omni" --run-level=core_model` |
-| **TTS online L2** | `pytest -s -v e2e/online_serving/test_qwen3_tts_base.py -m "core_model and tts" --run-level=core_model` |
-| **Diffusion X2I L2 online** | `pytest -s -v e2e/online_serving/test_qwen_image.py -m "core_model and diffusion" --run-level=core_model` |
-| **Diffusion X2V L2 online** | `pytest -s -v e2e/online_serving/test_wan22_t2v.py -m "core_model and diffusion" --run-level=core_model` |
-| **Diffusion X2I L4 nightly** | `pytest -s -v e2e/online_serving/test_qwen_image_expansion.py -m "full_model and diffusion and H100" --run-level=full_model` |
-| **Diffusion X2I L4 perf (nightly)** | `pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file dfx/perf/tests/test_qwen_image_vllm_omni.json` |
-| **Diffusion X2V L4 nightly** | `pytest -s -v e2e/online_serving/test_wan22_expansion.py -m "full_model and cuda" --run-level=full_model` |
-| **Invalid param (weekly H100)** | `pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and H100"` |
-| **Invalid param (weekly L4)** | `pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and L4"` |
-| **L1 CPU** | `pytest -s -v -m "core_model and cpu"` |
-
-**Prerequisites to mention when relevant**: GPU model (e.g. L4 vs H100), `HF_HOME` / token for hub weights, module-level `skipif` (NPU/XPU-only gaps), and whether CI already collects the path (e.g. `test_*_expansion.py` glob in `test-nightly.yml`).
-
-### Step 7: Validate Result Quality
-
-Before finishing:
-
-- Is the new assertion directly tied to the bug/feature contract?
-- Are API calls made via **`runtime.py` `send_*_request`** (not inline HTTP/SDK in the test file)?
-- Is the **general** `assert_*` bundled inside `send_*_request` (not duplicated in the test)?
-- Is each scenario a **separate `test_*` function** (no `case_id` / `if param.id` branching across cases)?
-- **Fixture scope**: default **`omni_server` / `omni_runner`** (module); **`_function`** variants only when each test must restart the instance?
-- Is the test deterministic (no fragile timing/network coupling)?
-- Is runtime appropriate for the selected level?
-- Are markers and `--run-level` consistent?
-- **Invalid-param cases** in `tests/dfx/reliability/invalid_param_test/` (not e2e), using `send_*_http_request` + `err_code`, with `pytest.mark.slow` and correct `H100` / `L4` hardware mark?
-
-## Output Format
-
-When completing a request, return:
-
-1. **Test plan** (level, markers, file target, **L4 pillar(s)** if applicable: Function / Accuracy / Perf / Doc; **or** **Invalid param** pillar with target `invalid_param_test/test_invalid_<area>.py`; module basename for e2e: `test_{slug}.py` / `test_{slug}_expansion.py`)
-2. **Generated/updated test file(s)** — model-specific constants and request payloads stay **inside** those test modules (**never** `tests/helpers/{slug}.py`). Extend **`tests/helpers/runtime.py`** / **`tests/helpers/assertions.py`** only when new **repo-wide** `send_*` or shared assert helpers are required (do not leave ad-hoc HTTP or `_assert_*` in test modules). **If L4 Perf was requested**, also list **`tests/dfx/perf/tests/*.json`**. **If invalid-param cases were requested**, list the **`invalid_param_test/` script** and parametrized `test_*` / new `id=` rows — **not** e2e paths.
-3. **Buildkite change** — **match the requested level and pillar**:
-   - **L4 Function**: `test-nightly.yml` **Function Test** shard (file list / note existing sweep). **No** `test-merge.yml` unless user also asked for L3.
-   - **L4 Perf**: `test-nightly.yml` **Perf Test · &lt;Model&gt;** step (new step or extend commands); include env exports + artifact upload pattern from a sibling perf job.
-   - **L4 Accuracy**: `test-nightly.yml` **Accuracy Test** step under the same model-type group.
-   - **L3**: `test-merge.yml` + full `source_file_dependencies` + `agents` + `plugins`.
-   - **L2**: `test-ready.yml` + same E2E block requirements.
-   - **Invalid param**: `test-weekly.yml` — note **Invalid parameters Test · H100/L4** group; usually **no YAML change** when only extending existing scripts.
-   If the user asked only for **L4 functional cases**, state explicitly that **Perf / Accuracy / Invalid param** were not included (offer to add if needed). If the plan mixes **success e2e** and **invalid param**, split deliverables across e2e vs `invalid_param_test/` and call out both CI files.
-4. **Run commands (required)** — always include, in fenced `bash` blocks:
-   - **Local — whole file**: `cd tests` then `pytest -s -v <path> …`
-   - **Local — single test** (optional but preferred when the change is one function): `pytest -s -v path::test_func …`
-   - **CI-like** (when not L1 CPU): the same **marker + `--run-level`** pairing the level uses (see [references/test-routing.md](references/test-routing.md) and **Step 6** table above)
-   - **Prerequisites** (one line): e.g. “Requires CUDA L4 + weights cached locally”; “Requires H100 + `HF_TOKEN`”; “Syntax-only validation when vllm is not installed locally”
-5. **Result summary** (pass/fail if you executed tests; if not executed, state that explicitly and what the user should run)
-
-## Additional Resources
-
-- Marker and command routing: [references/test-routing.md](references/test-routing.md)
-- CI pipelines (vllm-omni): [test-ready.yml](../../../.buildkite/test-ready.yml) (L1/L2), [test-merge.yml](../../../.buildkite/test-merge.yml) (L3), [test-nightly.yml](../../../.buildkite/test-nightly.yml) (L4), [test-weekly.yml](../../../.buildkite/test-weekly.yml) (invalid param / reliability)
-- L4 documentation example tests (naming, extraction vs copied scripts, output dirs, skips): [docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md) — see also [PR #1910](https://github.com/vllm-project/vllm-omni/pull/1910)

.claude/skills/vllm-omni-test/references/test-routing.md

@@ -1,372 +0,0 @@
-# Test Routing Reference
-
-Use this reference to map testing goals to levels, markers, and runnable commands.
-
-**Repo paths** (`.buildkite/`, `docs/contributing/…`): link with repo-relative paths from this file (e.g. `../../../../.buildkite/test-ready.yml`, `../../../../docs/contributing/ci/CI_5levels.md`).
-
-## Model-centric e2e filename convention (L2–L4)
-
-When generating a **new** test module tied to a **specific model** under `tests/e2e/offline_inference/` or `tests/e2e/online_serving/`:
-
-| Level | Pattern | Example (`Qwen/Qwen2.5-Omni-7B`) |
-|-------|---------|----------------------------------|
-| **L2**, **L3** | `test_{lowercase_model_slug}.py` | `test_qwen2_5_omni.py` |
-| **L4** | `test_{lowercase_model_slug}_expansion.py` | `test_qwen2_5_omni_expansion.py` |
-
-**Slug rules for `{lowercase_model_slug}`** (canonical: [SKILL.md](../SKILL.md) § *Naming: generated test module files*):
-
-1. Start from the HuggingFace-style id (e.g. `Qwen/Qwen2.5-Omni-7B`), but **do not** put the org into the filename: use the **repo segment only** (`Qwen2.5-Omni-7B`), not `Qwen_...` / `qwen_qwen2_5_...`.
-2. **Lowercase**; replace `.`, `-`, and whitespace with a single `_` (e.g. `Qwen2.5-Omni-7B` → `qwen2_5_omni`). **Omit** trailing size tokens such as `7b` / `30b` in the basename when a single file covers that model line in the directory (matches `test_qwen2_5_omni.py` in-tree).
-3. If two checkpoints in the same folder need separate modules, add a **minimal** disambiguator (e.g. `_7b` vs `_3b`) **only then**.
-4. **L1** unit tests are **not** bound to this pattern; use `tests/<area>/test_<feature>.py` as today.
-
-## Model type markers (`omni` / `tts` / `diffusion`)
-
-Every model-centric e2e test must declare **exactly one** type marker:
-
-| Marker | Model family | Typical APIs |
-|--------|--------------|--------------|
-| `pytest.mark.omni` | Multimodal LLM (Qwen-Omni, …) | `send_omni_request`, `omni_runner`, stage YAML |
-| `pytest.mark.tts` | TTS / speech synthesis | `send_audio_speech_request`, `/v1/audio/speech` |
-| `pytest.mark.diffusion` | Diffusion generative models | `send_diffusion_request`, `send_video_diffusion_request`, `send_images_generations_http_request`, `OmniDiffusionSamplingParams` |
-
-**Diffusion** uses one marker for all output modalities. Nightly CI splits diffusion by **group**, not by extra markers:
-
-| Nightly group (`test-nightly.yml`) | Diffusion scope | Examples |
-|-----------------------------------|-----------------|----------|
-| **Diffusion X2I(&A&T) Model Test** | x2**i** / x2**a** / x2**t** — image, audio, text (non-video) | Qwen-Image*, BAGEL, FLUX, SD3, Z-Image, LongCat, DreamZero |
-| **Diffusion X2V Model Test** | x2**v** — video only | Wan2.2, HunyuanVideo 1.5, Wan VACE |
-
-PR labels for selective nightly runs: `diffusion-x2iat-test`, `diffusion-x2v-test` (plus `nightly-test` / `NIGHTLY=1`).
-
-## Level and Marker Mapping
-
-| Goal | Suggested Level | Marker baseline | Typical location |
-|------|------------------|-----------------|------------------|
-| Unit logic, regression on pure code path | L1 | `core_model and cpu` | `tests/<component>/test_*.py` |
-
-### L1 unit tests — mocking (`mocker`, not `unittest.mock`)
-
-L1 modules run in **Simple Test** (`test-ready.yml` / `test-merge.yml`: `-m 'core_model and cpu'`). When isolation requires doubles:
-
-- **Use** the **`mocker`** fixture (`pytest-mock`): `mocker.patch`, `mocker.spy`, `mocker.Mock`, `mocker.MagicMock`, `mocker.AsyncMock`.
-- **Use** **`monkeypatch`** for env vars and simple `setattr` / `delattr` without mock objects.
-- **Do not use** `unittest.mock` — no `from unittest.mock import ...`, no `@patch`, no `with patch(...)`.
-
-```python
-# Good (L1)
-def test_handler(mocker):
-    mocker.patch("vllm_omni.pkg.fn", return_value=0)
-
-# Bad (L1)
-from unittest.mock import patch
-
-@patch("vllm_omni.pkg.fn")
-def test_handler(mock_fn): ...
-```
-
-E2E (L2+) should not rely on mocks unless documenting a rare exception; prefer real (or lightweight) execution paths.
-| Basic integration/e2e | L2 | `core_model` + **one of** `omni` / `tts` / `diffusion` | `tests/e2e/...` |
-| Advanced integration | L3 | `advanced_model` + type marker | `tests/e2e/...` |
-| Full function / nightly | L4 | `full_model` (nightly) + type marker | **Function:** `tests/e2e/*_expansion.py`; **Perf:** `tests/dfx/perf/tests/*.json`; **Accuracy:** `tests/e2e/accuracy/` |
-| Invalid HTTP / param validation | Weekly (dfx) | `pytest.mark.slow` + type marker + `H100` or `L4` | `tests/dfx/reliability/invalid_param_test/test_invalid_*.py` |
-
-## Marker Selection Rules
-
-1. **Level** (pick one):
-   - `core_model` — L1/L2 (`test-ready.yml`)
-   - `advanced_model` — L3 (`test-merge.yml`)
-   - `full_model` — L4 nightly (`test-nightly.yml`); some expansion tests still carry both `advanced_model` and `full_model` during migration
-2. **Model type** (pick one): `omni`, `tts`, or `diffusion`
-3. **Cross-cutting** when relevant: `parallel`, `cache`, `example`, `benchmark`
-4. **Hardware**: `cpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `distributed_cuda`, …
-5. Multi-card: `@hardware_test(...)` in `tests/helpers/mark.py`
-
-## Command Templates
-
-### Quick local checks
-
-```bash
-cd tests
-pytest -s -v test_xxxx.py
-```
-
-### L1
-
-```bash
-cd tests
-pytest -s -v -m "core_model and cpu"
-```
-
-### L2
-
-```bash
-cd tests
-pytest -s -v -m "core_model and not cpu" --run-level=core_model
-```
-
-### L3 (merge)
-
-```bash
-cd tests
-pytest -s -v -m "advanced_model" --run-level=advanced_model
-```
-
-### L4 (nightly)
-
-**Function** (e2e expansion — default when user asks for “L4 functional cases”):
-
-```bash
-cd tests
-pytest -s -v e2e/online_serving/test_qwen_image_expansion.py -m "full_model and diffusion and H100" --run-level=full_model
-```
-
-**Perf** (dfx benchmark harness — separate pillar; not `omni_server` fixtures):
-
-```bash
-cd tests
-export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
-pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py \
-  --test-config-file dfx/perf/tests/test_qwen_image_vllm_omni.json
-```
-
-Broad marker sweep (when no explicit file shard):
-
-```bash
-cd tests
-pytest -s -v -m "full_model" --run-level=full_model
-```
-
-## L4 nightly pillars (Function / Accuracy / Perf)
-
-Nightly **L4** for a model is often **multiple jobs** in `test-nightly.yml`, not a single pytest module:
-
-| Pillar | User says | Deliver | Nightly step |
-|--------|-----------|---------|--------------|
-| **Function** | “L4 functional cases” / “functional” / `*_expansion.py` | `tests/e2e/.../test_<model>_expansion.py` | `· Function Test with H100/L4` |
-| **Perf** | “performance” / “perf” / “benchmark” / “full L4” | `tests/dfx/perf/tests/test_<model>_vllm_omni.json` + runner | `· Perf Test · <Model>` |
-| **Accuracy** | “accuracy” / “similarity” | `tests/e2e/accuracy/test_<model>*.py` | `· Accuracy Test` |
-
-**Rule:** “L4 functional cases” → **Function only** by default. State that Perf/Accuracy are separate pillars; add them only when requested.
-
-**Diffusion perf JSON** (`tests/dfx/perf/tests/test_<slug>_vllm_omni.json`): array of objects with `test_name`, `server_params` (`model`, `serve_args`), `benchmark_params[]`, and per-workload **`baseline`** (`throughput_qps`, `latency_mean`, `peak_memory_mb_mean`). Copy structure from `test_qwen_image_vllm_omni.json` in-tree.
-
-**Diffusion perf nightly step** (under **Diffusion X2I(&A&T)**; copy full `kubernetes` plugins from `· Perf Test · Qwen-Image`):
-
-```yaml
-      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · <Model-Display-Name>"
-        key: nightly-diffusion-x2iat-performance-<slug>
-        timeout_in_minutes: 180
-        commands:
-          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
-          - export CACHE_DIT_VERSION=1.3.0
-          - |
-            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_<slug>_vllm_omni.json
-            EXIT=$$?
-            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
-            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
-            exit $$EXIT
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          # … full kubernetes block (often multi-GPU) — copy from Perf Test · Qwen-Image
-```
-
-| Model type | Perf runner | Config example |
-|------------|-------------|----------------|
-| Diffusion X2I/X2V | `dfx/perf/scripts/run_diffusion_benchmark.py` | `test_qwen_image_vllm_omni.json` |
-| TTS | `dfx/perf/scripts/run_benchmark.py` | `test_tts.json` (`BENCHMARK_DIR`) |
-| Omni | `run_benchmark.py` / omni configs in-tree | `test_omni*.json` |
-
-Do **not** put throughput/latency baselines inside `test_*_expansion.py` — that belongs in the dfx perf JSON + nightly Perf job.
-
-## Invalid parameter validation (weekly / dfx)
-
-When the user’s test plan includes **invalid parameter validation**, **invalid request bodies**, or **HTTP 4xx** validation against a live server:
-
-1. **Do not** author these in `tests/e2e/online_serving/test_*.py` or `*_expansion.py`. **Move** any drafted `test_*` into `tests/dfx/reliability/invalid_param_test/`.
-2. **Pick script by route** (extend in-tree file):
-
-| Route family | Script |
-|--------------|--------|
-| Omni chat / WS video·realtime | `test_invalid_omni_chat.py` |
-| `/v1/audio/speech` (+ stream / batch / voices) | `test_invalid_audio_speech.py` |
-| Audio diffusion | `test_invalid_audio_diffusion.py` |
-| `/v1/images/generations` | `test_invalid_image_generation.py` |
-| `/v1/images/edits` | `test_invalid_image_editing.py` |
-| `/v1/videos*` | `test_invalid_video_generation.py` |
-| Sleep / wakeup / server control | `test_invalid_server_control.py` |
-
-3. **Style:** `pytestmark = [pytest.mark.slow, pytest.mark.<type>]`; `_PARAMS` + `hardware_marks`; `send_*_http_request` with `err_code` + `err_message`; parametrized `body_spec` rows with `id=`; `_minimal_*_json()` helpers; `_SKIP_ISSUE_3649` when tracked in [#3649](https://github.com/vllm-project/vllm-omni/issues/3649).
-4. **CI:** [`.buildkite/test-weekly.yml`](../../../../.buildkite/test-weekly.yml) group **Reliability Test - Invalid parameters Test** — **not** ready/merge/nightly.
-
-```bash
-# Weekly H100 (diffusion / omni / video invalid-param)
-cd tests
-pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and H100"
-
-# Weekly L4 (e.g. Qwen3-TTS speech invalid-param)
-pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and L4"
-
-# Single script / case
-pytest -s -v dfx/reliability/invalid_param_test/test_invalid_image_generation.py::test_images_generations_invalid_requests -m "slow and H100"
-```
-
-**Trigger:** `WEEKLY=1` or PR label `weekly-test`. Appending cases to existing scripts usually needs **no YAML edit** (directory sweep). No `source_file_dependencies` on weekly steps.
-
-### Platform-targeted examples
-
-```bash
-cd tests
-pytest -s -v -m "core_model and distributed_cuda and L4" --run-level=core_model
-```
-
-### Concrete e2e paths (common in-tree)
-
-Paths are relative to `tests/` after `cd tests`.
-
-| Scenario | Example command |
-|----------|------------------|
-| **Omni** offline L2 | `pytest -s -v e2e/offline_inference/test_qwen2_5_omni.py -m "core_model and omni and not cpu" --run-level=core_model` |
-| **Omni** offline L2 — one test | `pytest -s -v e2e/offline_inference/test_qwen2_5_omni.py::test_text_to_text -m "core_model and omni and not cpu" --run-level=core_model` |
-| **Omni** online L2 | `pytest -s -v e2e/online_serving/test_qwen3_omni.py -m "core_model and omni" --run-level=core_model` |
-| **Omni** offline L2 (Qwen3.5-9B VL) | `pytest -s -v e2e/offline_inference/test_qwen3_5_9b.py -m "core_model and omni and not cpu" --run-level=core_model` |
-| **TTS** online L2 | `pytest -s -v e2e/online_serving/test_qwen3_tts_base.py -m "core_model and tts" --run-level=core_model` |
-| **TTS** online L2 — one test | `pytest -s -v e2e/online_serving/test_qwen3_tts_base.py::test_text_to_audio_001 -m "core_model and tts" --run-level=core_model` |
-| **Diffusion X2I** offline L2 | `pytest -s -v e2e/offline_inference/test_t2i_model.py -m "core_model and diffusion and not cpu" --run-level=core_model` |
-| **Diffusion X2I** L2 online smoke | `pytest -s -v e2e/online_serving/test_qwen_image.py e2e/online_serving/test_bagel.py -m "core_model and diffusion" --run-level=core_model` |
-| **Diffusion X2V** L2 online smoke | `pytest -s -v e2e/online_serving/test_wan22_t2v.py -m "core_model and diffusion" --run-level=core_model` |
-| **Diffusion X2I** L4 online expansion (H100) | `pytest -s -v e2e/online_serving/test_qwen_image_expansion.py -m "full_model and diffusion and H100" --run-level=full_model` |
-| **Diffusion X2I** L4 perf (nightly) | `pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file dfx/perf/tests/test_qwen_image_vllm_omni.json` |
-| **Diffusion X2I** L4 online expansion (Edit) | `pytest -s -v e2e/online_serving/test_qwen_image_edit_expansion.py -m "full_model and diffusion and H100" --run-level=full_model` |
-| **Diffusion X2V** L4 nightly | `pytest -s -v e2e/online_serving/test_wan22_expansion.py -m "full_model and cuda" --run-level=full_model` |
-| **Invalid param** weekly H100 | `pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and H100"` |
-| **Invalid param** weekly L4 | `pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and L4"` |
-
-### Agent / author completion checklist
-
-When adding or modifying tests, do not stop at “where the file lives” — also deliver:
-
-1. **Local**: `cd tests` + `pytest -s -v <path>` (and `path::test_func` when a single case is enough).
-2. **CI-like**: marker string + `--run-level` matching the pipeline (`core_model` / `advanced_model` / `full_model`).
-3. **L1 mocks**: `mocker` / `monkeypatch` only; no `unittest.mock`.
-4. **API client + assert placement**: **General** validation → implement in `assertions.py`, call **inside** `send_*_request` in `runtime.py`; tests call **`send_*_request` only**. **Special** case validation → `assert_*` in `assertions.py`, called **in the test** after `send_*_request`. Low-level `send_*_http_request` is for negative/dfx tests (`err_code`), not ordinary L2+ success e2e.
-5. **Shared assertions**: logic in **`tests/helpers/assertions.py`**; general checks inside `send_*_request`; special checks only in the test. No `_assert_*` in `test_*.py`.
-6. **One case → one `test_*`**: function name reflects what is validated; no `if case_id == ...` mega-test merging multiple scenarios.
-7. **Fixture scope**: default **`omni_server` + `openai_client`** / **`omni_runner` + `omni_runner_handler`** (module). Use **`omni_server_function` + `openai_client_function`** / **`omni_runner_function` + `omni_runner_handler_function`** only when each `test_*` must spawn a fresh instance. Parametrize name must match fixture (`omni_server` vs `omni_server_function`).
-8. **Type marker**: `omni`, `tts`, or `diffusion` on every model e2e module.
-9. **Diffusion L4 Function**: wire `*_expansion.py` into **X2I(&A&T)** or **X2V** **Function Test** in **`test-nightly.yml` only** — do not add `test-merge.yml` unless the user also requested L3.
-10. **Diffusion L4 Perf** (only when requested): add `tests/dfx/perf/tests/test_<slug>_vllm_omni.json` + **Perf Test · &lt;Model&gt;** step (artifact upload); not part of “L4 functional cases” by default.
-11. **E2E Buildkite (L2/L3 only)**: `test-ready.yml` / `test-merge.yml` steps need **`source_file_dependencies`** + full **`agents` + `plugins`**. **L4 nightly** uses explicit file lists or perf scripts in `test-nightly.yml` (no merge job).
-12. **Invalid param**: cases in **`tests/dfx/reliability/invalid_param_test/`** (route-matching script), `send_*_http_request` + `err_code`, `pytest.mark.slow` + `H100`/`L4`; CI = **`test-weekly.yml`** only — do not put in e2e or nightly.
-13. **Prerequisites**: GPU tier, HF cache/token, and any module `skipif` / platform-only YAML.
-
-## Buildkite pipeline mapping
-
-| Level | Repo file | Model-type grouping |
-|-------|-----------|---------------------|
-| L1, L2 | [`.buildkite/test-ready.yml`](../../../../.buildkite/test-ready.yml) | Steps prefixed **Omni ·**, **TTS ·**, **Diffusion ·** under **E2E Test** — **`source_file_dependencies` required** |
-| L3 | [`.buildkite/test-merge.yml`](../../../../.buildkite/test-merge.yml) | Per-model E2E steps; `-m "advanced_model and …"` — **`source_file_dependencies` required** |
-| L4 | [`.buildkite/test-nightly.yml`](../../../../.buildkite/test-nightly.yml) | **Omni / TTS / Diffusion X2I(&A&T) / X2V** — each group may have **Function**, **Accuracy**, **Perf**, **Doc** steps |
-| Invalid param (weekly) | [`.buildkite/test-weekly.yml`](../../../../.buildkite/test-weekly.yml) | **Invalid parameters Test · H100** / **· L4** — sweeps `tests/dfx/reliability/invalid_param_test/` |
-
-### `source_file_dependencies` (E2E Test only — ready & merge)
-
-Required on each step inside the **E2E Test** group in `test-ready.yml` and `test-merge.yml`. Typical entries:
-
-| Category | Path pattern |
-|----------|----------------|
-| Test module | `tests/e2e/online_serving/test_<slug>.py`, `tests/e2e/offline_inference/test_<slug>.py` |
-| API client helpers | `tests/helpers/runtime.py` (when the e2e job uses newly added `send_*` methods) |
-| Shared assert helpers | `tests/helpers/assertions.py` (when the e2e job uses newly added `assert_*` helpers) |
-| AR / omni model | `vllm_omni/model_executor/models/<family>/` |
-| Diffusion model | `vllm_omni/diffusion/models/<family>/` |
-| Stage processor | `vllm_omni/model_executor/stage_input_processors/<family>.py` |
-| Deploy config | `vllm_omni/deploy/<name>.yaml` or `vllm_omni/deploy/ci/<name>.yaml` |
-
-Copy the dependency block from the closest in-tree E2E step for the same model family. Update the list whenever the pytest command or server YAML changes.
-
-### E2E `agents` + `plugins` blocks (required — do not truncate)
-
-When generating or documenting E2E steps, **always include the complete `plugins` section**. Set **`timeout_in_minutes`** on the step for the job deadline; run **`pytest` directly** in `commands` (no `timeout 40m bash -c` wrapper around pytest).
-
-Common patterns in `test-ready.yml` / `test-merge.yml`:
-
-| Queue | Plugin | Required fields |
-|-------|--------|-----------------|
-| `mithril-h100-pool` | `kubernetes` | `resources.limits.nvidia.com/gpu`, `volumeMounts` (`devshm`, `hf-cache`), `env` (`HF_HOME`, `HF_TOKEN` via `secretKeyRef`), `nodeSelector: gpu-h100-sxm`, `volumes` (`devshm` emptyDir, `hf-cache` hostPath `/mnt/hf-cache`) |
-| `gpu_1_queue` / `gpu_4_queue` | `docker#v5.2.0` | `image`, `always-pull`, `propagate-environment`, `shm-size: "8gb"`, `environment: HF_HOME`, `volumes: /fsx/hf_cache` (add `HF_TOKEN` when the step needs hub access) |
-
-**H100 kubernetes template** (copy from **Diffusion · Bagel Test** / **Diffusion · Qwen Image Test** in-tree):
-
-```yaml
-  agents:
-    queue: "mithril-h100-pool"
-  plugins:
-    - kubernetes:
-        podSpec:
-          containers:
-            - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              resources:
-                limits:
-                  nvidia.com/gpu: 1
-              volumeMounts:
-                - name: devshm
-                  mountPath: /dev/shm
-                - name: hf-cache
-                  mountPath: /root/.cache/huggingface
-              env:
-                - name: HF_HOME
-                  value: /root/.cache/huggingface
-                - name: HF_TOKEN
-                  valueFrom:
-                    secretKeyRef:
-                      name: hf-token-secret
-                      key: token
-          nodeSelector:
-            node.kubernetes.io/instance-type: gpu-h100-sxm
-          volumes:
-            - name: devshm
-              emptyDir:
-                medium: Memory
-            - name: hf-cache
-              hostPath:
-                path: /mnt/hf-cache
-                type: DirectoryOrCreate
-```
-
-**L4 docker template** (copy from **TTS · Qwen3-TTS Base Test** in `test-merge.yml`):
-
-```yaml
-  agents:
-    queue: "gpu_1_queue"
-  plugins:
-    - docker#v5.2.0:
-        image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-        always-pull: true
-        propagate-environment: true
-        shm-size: "8gb"
-        environment:
-          - "HF_HOME=/fsx/hf_cache"
-        volumes:
-          - "/fsx/hf_cache:/fsx/hf_cache"
-```
-
-When extending an existing step (e.g. add `tests/e2e/offline_inference/test_qwen_image.py` to **Diffusion · Qwen Image Test**), update `source_file_dependencies` and `commands` only; **keep the existing `agents` + `plugins` block unchanged** unless the hardware tier changes.
-
-The root [`.buildkite/pipeline.yml`](../../../../.buildkite/pipeline.yml) decides **which** child file is uploaded. To run **L3** on a feature branch, comment out `if: build.branch == "main"` on the merge upload step. To run **L4** without `NIGHTLY=1`, comment out the nightly upload step’s `if: build.env("NIGHTLY") == "1"` and the same `if` on relevant steps inside `test-nightly.yml`. Revert such edits before merging.
-
-Platform-specific pipelines (e.g. AMD) follow the same level → file pairing under `.buildkite/`.
-
-## Diffusion RFC (#1832) Alignment Tips
-
-For **X2I(&A&T)** coverage planning:
-
-- Prioritize high-value feature combinations with minimal case count.
-- Split into lightweight L1/L2 validation plus L4 `*_expansion.py` under the **X2I** nightly shards.
-- **X2V** models (Wan, HunyuanVideo) stay in the **X2V** group — do not merge into X2I sweeps.
-
-If hardware is insufficient, provide an executable reduced case plus a deferred full CI/nightly plan.

README.md

@@ -68,7 +68,6 @@ Visit our [documentation](https://vllm-omni.readthedocs.io/en/latest/) to learn
 - [Installation](https://vllm-omni.readthedocs.io/en/latest/getting_started/installation/)
 - [Quickstart](https://vllm-omni.readthedocs.io/en/latest/getting_started/quickstart/)
 - [List of Supported Models](https://vllm-omni.readthedocs.io/en/latest/models/supported_models/)
-- [Deployment Recipes](https://recipes.vllm.ai) for vLLM-Omni model serving
 
 ## Contributing
 

benchmarks/diffusion/diffusion_benchmark_serving.py

@@ -79,7 +79,6 @@ Usage:
 import argparse
 import ast
 import asyncio
-import base64
 import glob
 import json
 import logging
@@ -107,9 +106,6 @@ from tqdm.asyncio import tqdm
 
 logger = logging.getLogger(__name__)
 
-_STAGE_METRICS_ENDPOINTS = {"/v1/chat/completions"}
-_RETURN_STAGE_METRICS_FIELD = "return_stage_metrics"
-
 
 class BaseDataset(ABC):
     def __init__(self, args, api_url: str, model: str):
@@ -1017,81 +1013,6 @@ def calculate_metrics(
     return metrics
 
 
-def _save_generated_outputs(
-    outputs: list[RequestFuncOutput],
-    requests_list: list[RequestFuncInput],
-    save_dir: str,
-) -> None:
-    """Decode and save base64 images/videos from successful responses."""
-    os.makedirs(save_dir, exist_ok=True)
-    saved = 0
-    failed = 0
-
-    for idx, (req, out) in enumerate(zip(requests_list, outputs)):
-        if not out.success or not out.response_body:
-            continue
-
-        if isinstance(out.response_body, bytes):
-            fname = f"req_{idx:04d}.mp4"
-            fpath = os.path.join(save_dir, fname)
-            with open(fpath, "wb") as f:
-                f.write(out.response_body)
-            saved += 1
-            continue
-
-        media_urls: list[str] = []
-
-        # Chat-completions style: choices[*].message.content[*].image_url.url
-        choices = out.response_body.get("choices", [])
-        if isinstance(choices, list):
-            for choice in choices:
-                content = (choice or {}).get("message", {}).get("content")
-                if not isinstance(content, list):
-                    continue
-                for item in content:
-                    if not isinstance(item, dict) or item.get("type") != "image_url":
-                        continue
-                    url = (item.get("image_url") or {}).get("url", "")
-                    if isinstance(url, str) and url.startswith("data:"):
-                        media_urls.append(url)
-
-        # Images endpoint style: data[*].b64_json
-        data_items = out.response_body.get("data", [])
-        if isinstance(data_items, list):
-            for data_item in data_items:
-                if not isinstance(data_item, dict):
-                    continue
-                b64_json = data_item.get("b64_json", "")
-                if isinstance(b64_json, str) and b64_json:
-                    media_urls.append(f"data:image/png;base64,{b64_json}")
-
-        for img_idx, url in enumerate(media_urls):
-            if "," not in url:
-                continue
-
-            try:
-                header, b64_data = url.split(",", 1)
-                ext = "png"
-                if "image/jpeg" in header:
-                    ext = "jpg"
-                elif "image/webp" in header:
-                    ext = "webp"
-                elif "video/mp4" in header:
-                    ext = "mp4"
-
-                img_bytes = base64.b64decode(b64_data)
-                fname = f"req_{idx:04d}_{img_idx}.{ext}"
-                fpath = os.path.join(save_dir, fname)
-                with open(fpath, "wb") as f:
-                    f.write(img_bytes)
-                saved += 1
-            except Exception as e:
-                failed += 1
-                logger.warning(f"Failed to save image for request {idx}: {e}", exc_info=True)
-
-    logger.info(f"Saved {saved} generated image(s) to {save_dir}. Failed to save {failed} image(s).")
-
-
 def wait_for_service(base_url: str, timeout: int = 120) -> None:
     print(f"Waiting for service at {base_url}...")
     start_time = time.time()
@@ -1174,10 +1095,6 @@ async def benchmark(args):
     requests_list = dataset.get_requests()
     print(f"Prepared {len(requests_list)} requests from {args.dataset} dataset.")
 
-    if args.return_stage_metrics and args.endpoint in _STAGE_METRICS_ENDPOINTS:
-        for req in requests_list:
-            req.extra_body.setdefault(_RETURN_STAGE_METRICS_FIELD, True)
-
     if args.endpoint == "/v1/images/edits":
         for req in requests_list:
             req.default_bot_task = args.bot_task
@@ -1285,9 +1202,6 @@ async def benchmark(args):
 
     print("\n" + "=" * 60)
 
-    if args.save_dir:
-        _save_generated_outputs(outputs, requests_list, args.save_dir)
-
     if args.output_file:
         with open(args.output_file, "w") as f:
             json.dump(metrics, f, indent=2)
@@ -1364,18 +1278,11 @@ if __name__ == "__main__":
         default=1,
         help="Number of warmup requests to run before measurement.",
     )
-    # NOTE Changed default from 1 to 2 because some models (e.g., Bagel) run
-    # `num_timesteps - 1` denoising iterations. A default of 1 results in 0 steps,
-    # which causes errors.
-    # TODO If this slightly longer warmup causes regression issues for other
-    # diffusion pipelines in the future, consider implementing model-specific
-    # overrides instead of a global default.
     parser.add_argument(
         "--warmup-num-inference-steps",
         type=int,
-        default=2,
-        help="Number of inference steps used for warmup requests. "
-        "Default is 2 to ensure at least one denoising step is executed.",
+        default=1,
+        help="num_inference_steps used for warmup requests.",
     )
     parser.add_argument(
         "--warmup-concurrency",
@@ -1416,13 +1323,6 @@ if __name__ == "__main__":
         default=3.0,
         help="SLO target multiplier: slo_ms = estimated_exec_time_ms * slo_scale (default: 3).",
     )
-    parser.add_argument(
-        "--save-dir",
-        type=str,
-        default=None,
-        help="Directory to save generated images/outputs for visual inspection. "
-        "If not set, generated outputs are discarded after metric collection.",
-    )
     parser.add_argument("--disable-tqdm", action="store_true", help="Disable progress bar.")
     parser.add_argument(
         "--enable-negative-prompt",
@@ -1458,11 +1358,6 @@ if __name__ == "__main__":
         default="think",
         help=("bot_task form field for --endpoint /v1/images/edits (think, recaption, think_recaption, vanilla)."),
     )
-    parser.add_argument(
-        "--return-stage-metrics",
-        action="store_true",
-        help="Request stage duration metrics from endpoints that support return_stage_metrics.",
-    )
 
     args = parser.parse_args()
     asyncio.run(benchmark(args))

docker/Dockerfile.ci

@@ -2,7 +2,7 @@
 # The pinned VLLM_PRECOMPILED_WHEEL_COMMIT and VLLM_BASE_TAG below must
 # be updated on every rebase to match the new upstream vllm commit/tag.
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai
-ARG VLLM_BASE_TAG=v0.23.0
+ARG VLLM_BASE_TAG=v0.22.0
 FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
 ARG APP_DIR=/workspace/vllm-omni
 WORKDIR ${APP_DIR}
@@ -16,7 +16,6 @@ RUN apt-get update && \
 
 RUN uv pip install --system ".[dev]"
 
-
 RUN ln -sf /usr/bin/python3 /usr/bin/python
 
 ENTRYPOINT []

docker/Dockerfile.cuda

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai:v0.23.0
+ARG BASE_IMAGE=vllm/vllm-openai:v0.22.0
 FROM ${BASE_IMAGE}
 
 ARG COMMON_WORKDIR=/app

docker/Dockerfile.npu

@@ -1,5 +1,5 @@
-ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.23.0-4885d8e
+ARG VLLM_ASCEND_IMAGE=quay.io/ascend/vllm-ascend
+ARG VLLM_ASCEND_TAG=v0.18.0rc1
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
 
 # WORKDIR /vllm-workspace/vllm

docker/Dockerfile.npu.a3

@@ -1,5 +1,5 @@
-ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.23.0-4885d8e-a3
+ARG VLLM_ASCEND_IMAGE=quay.io/ascend/vllm-ascend
+ARG VLLM_ASCEND_TAG=v0.18.0rc1-a3
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
 
 # WORKDIR /vllm-workspace/vllm

docker/Dockerfile.npu.ci

@@ -1,5 +1,5 @@
-ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.23.0-4885d8e
+ARG VLLM_ASCEND_IMAGE=quay.io/ascend/vllm-ascend
+ARG VLLM_ASCEND_TAG=nightly-main
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
 
 ARG APP_DIR=/vllm-workspace/vllm-omni

docker/Dockerfile.npu.ci.a3

@@ -1,5 +1,5 @@
-ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.23.0-4885d8e-a3
+ARG VLLM_ASCEND_IMAGE=quay.io/ascend/vllm-ascend
+ARG VLLM_ASCEND_TAG=nightly-main-a3
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
 
 ARG APP_DIR=/vllm-workspace/vllm-omni

docker/Dockerfile.rocm

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.23.0
+ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.22.0
 FROM ${BASE_IMAGE} AS base
 
 # Declare a variable to know if we want to use the nightly build or the stable build.

docker/Dockerfile.xpu

@@ -88,7 +88,7 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
 ENV UV_INDEX_STRATEGY="unsafe-best-match"
 ENV UV_LINK_MODE="copy"
 
-ARG VLLM_VERSION=v0.23.0
+ARG VLLM_VERSION=v0.22.0
 RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
 WORKDIR /workspace/vllm
 

docs/.nav.yml

@@ -18,6 +18,7 @@ nav:
     - Offline Inference:
       - BAGEL-7B-MoT: user_guide/examples/offline_inference/bagel.md
       - GLM-Image Multistage End-to-End Inference: user_guide/examples/offline_inference/glm_image.md
+      - Helios Video Generation: user_guide/examples/offline_inference/helios.md
       - HunyuanImage-3.0 Image-to-Text Inference: user_guide/examples/offline_inference/hunyuan_image3.md
       - Image-To-Image: user_guide/examples/offline_inference/image_to_image.md
       - Image-To-Video: user_guide/examples/offline_inference/image_to_video.md

docs/contributing/README.md

@@ -129,27 +129,6 @@ The precheck covers five PR types: Bug Fix, Performance, New Model, Diffusion Mo
 ### Local Test
 Please run the L1 and L2 test cases locally first and attach the results before contacting us to add the "ready" label. Please refer to the [test instructions](./ci/test_guide.md) for running the test cases.
 
-### Automatic skip-ci (docs and pytest skip marks)
-
-On pull requests and `main` pushes, the bootstrap step in [`.buildkite/pipeline.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/pipeline.yml) runs [`.buildkite/scripts/upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/scripts/upload_pipeline.py) against the git diff. When every changed file qualifies, **L2 (`ready`) and L3 (`merge-test`) pipelines are not uploaded**, so the default GPU CI jobs are skipped.
-
-| Change per file | Examples |
-| --- | --- |
-| Documentation | `docs/**`, any `*.md`, `mkdocs.yml` |
-| Pytest skip marks only (under `tests/`) | Add/remove/edit `@pytest.mark.skip`, `@pytest.mark.skipif`, or `pytest.skip(...)`; reformat `pytestmark` only to add a skip/skipif alongside existing `pytest.mark.*` entries |
-| New skipped test module | New `tests/**/*.py` whose `pytestmark` includes unconditional `pytest.mark.skip` |
-
-These PR shapes all trigger skip-ci:
-
-- Documentation only
-- Qualifying skip-mark edits in `tests/**/*.py` only
-- **A mix of documentation and qualifying skip-mark test edits**
-
-Skip-ci does **not** apply when the diff also touches product code (for example `vllm_omni/`), or when test files change assertions, imports, fixtures, or other non-skip logic. If the diff cannot be resolved (non-PR branches outside `main`), CI runs as usual.
-
-!!! note
-    Skipping L2/L3 does **not** disable the Docker image build step. Nightly (L4) upload can still run when the PR has a `nightly-test` label or on scheduled `main` builds with `NIGHTLY=1`. See [CI levels](./ci/CI_5levels.md) for how bootstrap skip-ci relates to diff-gated E2E jobs.
-
 ### Code Quality
 
 The PR needs to meet the following code quality standards:

docs/contributing/ci/CI_5levels.md

@@ -11,10 +11,6 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
     <tr>
       <th>Level</th>
       <th>Scope & Focus</th>
-      <th>Model Coverage Strategy</th>
-      <th>Feature Coverage Strategy</th>
-      <th>Interface Coverage Strategy</th>
-      <th>Tags</th>
       <th>Time Cost</th>
       <th>Test Dir</th>
       <th>Doc</th>
@@ -28,10 +24,6 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
       <td>Contribution Guideline & PR checklist</td>
       <td>/</td>
       <td>/</td>
-      <td>/</td>
-      <td>/</td>
-      <td>/</td>
-      <td>/</td>
       <td>.github/PULL_REQUEST_TEMPLATE.md <a href="../tests_style/"> Test Style (PR Checklist)</a></td>
       <td>/</td>
       <td>/</td>
@@ -40,10 +32,6 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
       <td>CI Failure Description</td>
       <td>/</td>
       <td>/</td>
-      <td>/</td>
-      <td>/</td>
-      <td>/</td>
-      <td>/</td>
       <td><a href="../failures/"> CI Failures</a></td>
       <td>/</td>
       <td>/</td>
@@ -51,10 +39,6 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
     <tr>
       <td><strong>L1</strong><br>(Unit & Logic)</td>
       <td>Unit tests for components like entrypoints, models</td>
-      <td>/</td>
-      <td>/</td>
-      <td>/</td>
-      <td><code>core_model and cpu</code></td>
       <td rowspan="2">&lt;15min</td>
       <td>/tests/{component_name}/test_xxx</td>
       <td>
@@ -66,18 +50,10 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
     </tr>
     <tr>
       <td><strong>L2</strong><br>(E2E across models & GPU-required UT)</td>
-      <td>Online (basic deployment scenarios):<br>dummy, normal inference function (output format, stream), some instance startup UT</td>
-      <td>High-priority models + online basic scenarios + request success validation</td>
-      <td>High-priority features (using random lightweight models)</td>
-      <td>High-priority interfaces (using random lightweight models)</td>
-      <td><code>core_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
+      <td>Online & Offline (basic deployment scenarios):<br>dummy, normal inference function (output format, stream), some instance startup UT</td>
       <td>
-        <strong>Model tests:</strong><br>
         /tests/e2e/online_serving/test_{model_name}.py<br>
-        <strong>Feature tests:</strong><br>
-        /tests/{component_name}/test_xxx<br>
-        <strong>Interface tests:</strong><br>
-        /tests/entrypoints/test_xxx
+        /tests/e2e/offline_inference/test_{model_name}.py
       </td>
       <td>
         <a href="#chapter-1-l1-l2-level-testing-unit-testing-and-basic-end-to-end-verification">Chapter 1</a><br>
@@ -89,19 +65,10 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
     <tr>
       <td><strong>L3</strong><br>(Important Perf & Integration & Accuracy)</td>
       <td>Online & Offline (multiple deployment scenarios):<br>real model, normal inference function, normal accuracy</td>
-      <td>High/medium-priority models with real weights + online/offline key scenarios + basic accuracy validation</td>
-      <td>Medium-priority features (using random lightweight models)</td>
-      <td>Medium-priority interfaces (using random lightweight models)</td>
-      <td><code>advanced_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
       <td>&lt;30min</td>
       <td>
-        <strong>Model tests:</strong><br>
         /tests/e2e/online_serving/test_{model_name}.py<br>
-        /tests/e2e/offline_inference/test_{model_name}.py<br>
-        <strong>Feature tests:</strong><br>
-        /tests/{component_name}/test_xxx<br>
-        <strong>Interface tests:</strong><br>
-        /tests/entrypoints/test_xxx
+        /tests/e2e/offline_inference/test_{model_name}.py
       </td>
       <td>
         <a href="#chapter-2-l3-level-testing-core-integration-performance-and-accuracy-verification">Chapter 2</a><br>
@@ -112,27 +79,18 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
     </tr>
     <tr>
       <td><strong>L4</strong><br>(Perf & Integration & Accuracy)</td>
-      <td>Online: full functional scenarios + performance test + doc test + accuracy test</td>
-      <td>High-priority models: function, performance, accuracy, and doc testing<br>Medium/low-priority models: function and doc testing</td>
-      <td>Low-priority features (using random lightweight models)</td>
-      <td>Low-priority interfaces (using random lightweight models)</td>
-      <td><code>full_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
+      <td>Online & Offline: full functional scenarios + performance test + doc test</td>
       <td>&lt;3 hour</td>
       <td>
-        <strong>Model tests:</strong><br>
+        <strong>Full Function:</strong><br>
         /tests/e2e/online_serving/test_{model_name}_expansion.py<br>
-        <strong>Feature tests:</strong><br>
-        /tests/{component_name}/test_xxx<br>
-        <strong>Interface tests:</strong><br>
-        /tests/entrypoints/test_xxx<br>
+        /tests/e2e/offline_inference/test_{model_name}_expansion.py<br>
         <strong>Performance:</strong><br>
         /tests/dfx/perf/tests/test_qwen_omni.json (Omni), test_tts.json (TTS),<br>
         and /tests/dfx/perf/tests/test_{diffusion_model}_vllm_omni.json (Diffusion)<br>
         <strong>Doc Test:</strong><br>
         tests/example/online_serving/test_{model_name}.py<br>
-        tests/example/offline_inference/test_{model_name}.py<br>
-        <strong>Accuracy Test:</strong><br>
-        /tests/e2e/accuracy/test_{model_name}.py
+        tests/example/offline_inference/test_{model_name}.py
       </td>
       <td>
         <a href="#chapter-3-l4-level-testing-full-functionality-performance-and-documentation-testing">Chapter 3</a><br>
@@ -143,11 +101,7 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
     </tr>
     <tr>
       <td><strong>L5</strong><br>(Stability & Reliability)</td>
-      <td>Online: long-term stability test + reliability test</td>
-      <td>Long-term stability and reliability testing for high-priority models</td>
-      <td>/</td>
-      <td>Invalid-parameter validation for high-priority interfaces</td>
-      <td><code>slow and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
+      <td>Online & Offline: long-term stability test + reliability test</td>
       <td> Depends on reality </td>
       <td>
         <strong>Stability:</strong><br>

docs/getting_started/installation/gpu.md

@@ -34,7 +34,7 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
 
 ### Pre-built wheels
 
-Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, and 0.23.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
+Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, and 0.22.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
 
 === "NVIDIA CUDA"
 
@@ -94,10 +94,6 @@ Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc
 
 ### Build your own docker image
 
-=== "NVIDIA CUDA"
-
-    --8<-- "docs/getting_started/installation/gpu/cuda.inc.md:build-docker"
-
 === "AMD ROCm"
 
     --8<-- "docs/getting_started/installation/gpu/rocm.inc.md:build-docker"

docs/getting_started/installation/gpu/cuda.inc.md

@@ -20,7 +20,7 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new**
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm==0.23.0 --torch-backend=auto
+uv pip install vllm==0.22.0 --torch-backend=auto
 ```
 
 #### Installation of vLLM-Omni
@@ -39,13 +39,13 @@ uv pip install 'vllm-omni[demo]'
 # --8<-- [start:build-wheel-from-source]
 
 #### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.23.0 release version of the library
+If you do not need to modify source code of vLLM, you can directly install the stable 0.22.0 release version of the library
 
 ```bash
-uv pip install vllm==0.23.0 --torch-backend=auto
+uv pip install vllm==0.22.0 --torch-backend=auto
 ```
 
-The 0.23.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The 0.22.0 release of vLLM ships CUDA 12.9-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
 
 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -66,12 +66,12 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.23.0
+git checkout v0.22.0
 ```
 Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
 ```bash
-#For CUDA 13.0 (the default for v0.23.0; the wheel filename has no `+cu130` suffix)
-export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.23.0/vllm-0.23.0-cp38-abi3-manylinux_2_28_x86_64.whl
+#For CUDA 12.9 (the default for v0.22.0; the wheel filename has no `+cu129` suffix)
+export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.22.0/vllm-0.22.0-cp38-abi3-manylinux_2_28_x86_64.whl
 ```
 Install vllm with command below (If you have no existing PyTorch).
 ```bash
@@ -102,7 +102,7 @@ docker run --runtime nvidia --gpus 2 \
     --env "HF_TOKEN=$HF_TOKEN" \
     -p 8091:8091 \
     --ipc=host \
-    vllm/vllm-omni:v0.23.0 \
+    vllm/vllm-omni:v0.22.0 \
     vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```
 
@@ -110,53 +110,3 @@ docker run --runtime nvidia --gpus 2 \
     The CUDA image does not define a default entrypoint, so include `vllm serve ... --omni` after the image name.
 
 # --8<-- [end:pre-built-images]
-
-# --8<-- [start:build-docker]
-
-#### Build docker image
-
-```bash
-DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.cuda -t vllm-omni-cuda .
-```
-
-If you want to specify the base vLLM version:
-
-```bash
-DOCKER_BUILDKIT=1 docker build \
-  -f docker/Dockerfile.cuda \
-  --build-arg BASE_IMAGE=vllm/vllm-openai:v0.22.1 \
-  -t vllm-omni-cuda .
-```
-
-#### Launch the docker image
-
-##### Launch with OpenAI API Server
-
-!!! note
-    The model `Qwen/Qwen3-Omni-30B-A3B-Instruct` requires significant GPU memory. The example below has been verified on 2 x H100's.
-
-```bash
-docker run --runtime nvidia --gpus 2 \
-  -v ${HF_HOME:-$HOME/.cache/huggingface}:/root/.cache/huggingface \
-  --env "HF_TOKEN=$HF_TOKEN" \
-  -p 8091:8091 \
-  --ipc=host \
-  vllm-omni-cuda \
-  vllm serve --omni --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 8091
-```
-
-By default, this mounts `$HOME/.cache/huggingface` as the model cache directory. To use a custom location, set the `HF_HOME` environment variable before running the command (e.g., `export HF_HOME=/data/models`).
-
-##### Launch with interactive session for development
-
-```bash
-docker run --runtime nvidia --gpus all -it --rm \
-  -v ${HF_HOME:-$HOME/.cache/huggingface}:/root/.cache/huggingface \
-  --env "HF_TOKEN=$HF_TOKEN" \
-  -p 8091:8091 \
-  --ipc=host \
-  --entrypoint bash \
-  vllm-omni-cuda
-```
-
-# --8<-- [end:build-docker]

docs/getting_started/installation/gpu/rocm.inc.md

@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm==0.23.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.23.0/rocm722
+uv pip install vllm==0.22.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.22.0/rocm722
 ```
 
 #### Installation of vLLM-Omni
@@ -34,13 +34,13 @@ uv pip install onnxruntime-rocm
 # --8<-- [start:build-wheel-from-source]
 
 #### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.23.0 release version of the library
+If you do not need to modify source code of vLLM, you can directly install the stable 0.22.0 release version of the library
 
 ```bash
-uv pip install vllm==0.23.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.23.0/rocm722
+uv pip install vllm==0.22.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.22.0/rocm722
 ```
 
-The pre-built 0.23.0 vLLM wheel targets ROCm 7.2.2. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The pre-built 0.22.0 vLLM wheel targets ROCm 7.2.2. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
 
 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -58,7 +58,7 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.23.0
+git checkout v0.22.0
 python3 -m pip install -r requirements/rocm.txt
 python3 setup.py develop
 ```
@@ -130,7 +130,7 @@ docker run --rm \
   -v ~/.cache/huggingface:/root/.cache/huggingface \
   --env "HF_TOKEN=$HF_TOKEN" \
   -p 8091:8091 \
-  vllm/vllm-omni-rocm:v0.23.0 \
+  vllm/vllm-omni-rocm:v0.22.0 \
   --model Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```
 
@@ -149,7 +149,7 @@ docker run --rm -it \
   -v ~/.cache/huggingface:/root/.cache/huggingface \
   --env "HF_TOKEN=$HF_TOKEN" \
   --entrypoint bash \
-  vllm/vllm-omni-rocm:v0.23.0
+  vllm/vllm-omni-rocm:v0.22.0
 ```
 
 # --8<-- [end:pre-built-images]

docs/getting_started/quickstart.md

@@ -19,10 +19,10 @@ uv venv --python 3.12 --seed
 source .venv/bin/activate
 
 # On CUDA
-uv pip install vllm==0.23.0 --torch-backend=auto
+uv pip install vllm==0.22.0 --torch-backend=auto
 
 # On ROCm
-uv pip install vllm==0.23.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.23.0/rocm722
+uv pip install vllm==0.22.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.22.0/rocm722
 
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
@@ -35,7 +35,7 @@ For additional installation methods — please see the [installation guide](inst
 !!! note
     It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.
 
-    If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.23.0` and vLLM Omni `0.23.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
+    If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.22.0` and vLLM Omni `0.22.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
 
 ## Offline Inference
 

docs/mkdocs/hooks/generate_api_readme.py

@@ -144,7 +144,6 @@ def scan_package(package_name: str = "vllm_omni") -> dict[str, list[str]]:
                 "vllm_omni.entrypoints.async_diffusion",
                 "vllm_omni.entrypoints.openai",
                 "vllm_omni.model_executor.models.voxtral_tts.configuration_voxtral_tts",
-                "vllm_omni.experimental",  # optional serving deps not installed in docs build
             ]
             if any(module_path.startswith(prefix) for prefix in excluded_prefixes):
                 continue

docs/models/supported_models.md

@@ -6,8 +6,6 @@ vLLM-Omni supports unified multimodal comprehension and generation models across
 
 If vLLM-Omni natively supports a model, its implementation can be found in <gh-file:vllm_omni/model_executor/models> and <gh-file:vllm_omni/diffusion/models>.
 
-For deployment recipes (TTS and multimodal), see [recipes.vllm.ai](https://recipes.vllm.ai) and the in-repo [`recipes/`](https://github.com/vllm-project/vllm-omni/tree/main/recipes) directory.
-
 ## List of Supported Models
 
 <style>
@@ -24,7 +22,6 @@ th {
 | `MingFlashOmniForConditionalGeneration` + `MingImagePipeline` | Ming-flash-omni-2.0 (omni-speech + imagegen<sup>1</sup>) | `Jonathan1909/Ming-flash-omni-2.0` | ✅︎ |   |   |   |
 | `BagelForConditionalGeneration` | BAGEL (DiT-only) | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ | | ✅︎ |
 | `InternVLAA1Pipeline` | InternVLA-A1 | `InternRobotics/InternVLA-A1-3B` | ✅︎ | ✅︎ | | |
-| `Gr00tN1d7Pipeline` | GR00T N1.7 | `nvidia/GR00T-N1.7-3B` | ✅︎ | | | |
 | `HunyuanImage3ForCausalMM` | HunyuanImage3.0 (DiT-only) | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `QwenImagePipeline` | Qwen-Image-2512 | `Qwen/Qwen-Image-2512` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
@@ -36,7 +33,7 @@ th {
 | `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V, V2V, T2V with sound, action policy | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
+| `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V, V2V, T2V with sound, action policy | `nvidia/Cosmos3-Nano` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `WanSpeechToVideoPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `LTX2Pipeline` | LTX-2-T2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
@@ -45,13 +42,12 @@ th {
 | `LTX2ImageToVideoTwoStagesPipeline` | LTX-2-I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
 | `LTX23Pipeline` | LTX-2.3-T2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
 | `LTX23ImageToVideoPipeline` | LTX-2.3-I2V | `dg845/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
-| `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
+| `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | | | |
 | `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | |
 | `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | |
 | `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ |
 | `LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `StableDiffusionXLPipeline` | Stable-Diffusion-XL | `stabilityai/stable-diffusion-xl-base-1.0` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ |
 | `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ |
 | `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | |
@@ -66,14 +62,13 @@ th {
 | `AudioXPipeline` | AudioX | `zhangj1an/AudioX` | ✅︎ | ✅︎ | | |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
+| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `MingTTSForConditionalGeneration` | Ming-omni-tts dense 0.5B | `inclusionAI/Ming-omni-tts-0.5B` | ✅︎ | | | |
 | `GLMTTSForConditionalGeneration` | GLM-TTS | `zai-org/GLM-TTS` | ✅︎ | | | |
 | `MossTTSNanoForCausalLM` | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✅︎ | | | |
 | `MossTTSDelayModel` | MOSS-TTS, MOSS-TTSD, MOSS-SoundEffect, MOSS-VoiceGenerator | `OpenMOSS-Team/MOSS-VoiceGenerator` | ✅︎ | | | |
 | `MossTTSRealtime` | MOSS-TTS-Realtime | `OpenMOSS-Team/MOSS-TTS-Realtime` | ✅︎ | | | |
 | `HiggsAudioV2ForConditionalGeneration` | Higgs-Audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` | ✅︎ | | | |
-| `HiggsMultimodalQwen3ForConditionalGeneration` | Higgs-Audio v3 (TTS) | `bosonai/higgs-audio-v3-tts-4b` | ✅︎ | | | |
 | `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ |
 | `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | |
 | `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | |

docs/serving/speech_api.md

@@ -9,13 +9,6 @@ vLLM-Omni provides an OpenAI-compatible API for text-to-speech (TTS) generation.
 
 See the [Supported Models](#supported-models) section below for the full list, including OmniVoice, VoxCPM2, and MOSS-TTS-Nano.
 
-!!! tip "Deployment recipes"
-    TTS deployment recipes are published at
-    [recipes.vllm.ai](https://recipes.vllm.ai) (e.g.
-    [Qwen3-TTS](https://recipes.vllm.ai/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice),
-    [Higgs-Audio v3](https://recipes.vllm.ai/bosonai/higgs-audio-v3-tts-4b)).
-    The in-repo runbooks live under [`recipes/`](https://github.com/vllm-project/vllm-omni/tree/main/recipes).
-
 Each server instance runs a single model (specified at startup via `vllm serve <model> --omni`).
 
 ## Quick Start

docs/serving/videos_api.md

@@ -160,12 +160,7 @@ JSON references currently support `image_url`/`video_url`; `file_id` references
 are not implemented yet. Models may expose additional V2V controls through
 `extra_params`. For example, Cosmos3 supports
 `condition_frame_indexes_vision` and `condition_video_keep` to select which
-decoded reference frames are used as clean conditioning. Cosmos3 transfer mode
-also accepts `edge`, `blur`, `depth`, `seg`, or `wsm` control hints plus
-transfer options such as `control_path`, `control_guidance`,
-`control_guidance_interval`, `num_video_frames_per_chunk`,
-`num_conditional_frames`, `show_control_condition`, and `show_input`; see the
-Cosmos3 recipe for complete examples.
+decoded reference frames are used as clean conditioning.
 
 ### Speech-to-Video
 
@@ -200,16 +195,13 @@ curl -X POST http://localhost:8091/v1/videos/sync \
 
 ## Storage
 
-Set `VLLM_OMNI_SERVER_STORAGE__PATH` to control where asynchronous video outputs are
+Set `VLLM_OMNI_STORAGE_PATH` to control where asynchronous video outputs are
 stored:
 
 ```bash
-export VLLM_OMNI_SERVER_STORAGE__PATH=/var/tmp/vllm-omni-videos
+export VLLM_OMNI_STORAGE_PATH=/var/tmp/vllm-omni-videos
 ```
 
-> `VLLM_OMNI_STORAGE_PATH` is deprecated and will be removed in a future release;
-> use `VLLM_OMNI_SERVER_STORAGE__PATH` instead.
-
 ## Model-Specific Examples
 
 For complete text-to-video, image-to-video, and model-specific video-to-video

docs/user_guide/diffusion/cpu_offload_diffusion.md

@@ -193,7 +193,6 @@ Factory function `get_offload_backend()` selects the appropriate backend based o
 | NextStep11Pipeline | `stepfun-ai/NextStep-1.1` | `NextStepModel` | - | ✓ | `"layers"` |
 | OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | - | ✓ | `"transformer"` |
 | QwenImagePipeline | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | ✓ | ✓ | `"transformer_blocks"` |
-| StableDiffusionXLPipeline | `stabilityai/stable-diffusion-xl-base-1.0` | `SDXLUNet2DConditionModel` | ✓ | ✓ | `"down_blocks"`, `"up_blocks"` |
 | StableDiffusion3Pipeline | `stabilityai/stable-diffusion-3.5-medium` | `SD3Transformer2DModel` | - | ✓ | `"transformer_blocks"` |
 | Wan22I2VPipeline | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | `"blocks"` |
 | Wan22Pipeline | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | `"blocks"` |

docs/user_guide/diffusion/mot_config.md

@@ -1,50 +0,0 @@
-This directory contains auto-tuned Triton kernel configurations for the
-MoT (Mixture-of-Tokens) GEMM and RMSNorm operators used by BAGEL and other
-MoT-architecture diffusion models.
-
-File naming convention:
-    device_name=<GPU_NAME>,dtype=<DTYPE>.json
-
-For example:
-    device_name=NVIDIA_A100-SXM4-80GB,dtype=w16a16.json
-
-Each JSON file maps (K, N) matrix shapes to a dictionary of batch sizes (M)
-and their optimal Triton tile configurations:
-
-    {
-        "3584_4608": {              // K=3584, N=4608 (QKV projection)
-            "1024": {
-                "BLOCK_SIZE_M": 64,
-                "BLOCK_SIZE_N": 128,
-                "BLOCK_SIZE_K": 64,
-                "GROUP_SIZE_M": 8,
-                "num_warps": 4,
-                "num_stages": 3
-            },
-            ...
-        },
-        "3584_3584": { ... }        // K=3584, N=3584 (output projection)
-    }
-
-Config loading order (3-tier, see ops/mot_gemm.py):
-    1. $VLLM_TUNED_CONFIG_FOLDER/<filename>  (env override)
-    2. This directory: vllm_omni/diffusion/layers/mot/configs/<filename>
-    3. Conservative default config (compiles everywhere, sub-optimal perf)
-
-If no config file matches the current device, a warning is printed with
-instructions to run the auto-tuning benchmark.
-
-To generate configs for your hardware:
-
-    python benchmarks/kernels/mot_linear_benchmarks.py \
-        --model ByteDance-Seed/BAGEL-7B-MoT \
-        --tp-size 1 --dtype w16a16 --tune \
-        --save-dir vllm_omni/diffusion/layers/mot/configs/
-
-For multi-GPU tuning (uses Ray for parallel search):
-
-    python benchmarks/kernels/mot_linear_benchmarks.py \
-        --model ByteDance-Seed/BAGEL-7B-MoT \
-        --tp-size 2 --tune
-
-See benchmarks/kernels/mot_linear_benchmarks.py for full options.

docs/user_guide/diffusion_features.md

@@ -130,7 +130,6 @@ The following tables show which models support each feature:
 | **Qwen-Image-Edit-2509** |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |        ✅ (decode)         |          ✅           |       ❌        |        ❌         |
 | **Qwen-Image-Layered**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **SenseNova-U1**         |     ❌     |     ✅      |           ❌           |       ❌        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
-| **Stable-Diffusion-XL**  |     ❌     |     ❌      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Stable-Diffusion3.5**  |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Z-Image**              |     ✅     |     ✅      |           ✅           |       ❓        |   ✅ (TP=2 only)   |          ❌          |   ✅    |             ❌             |      ✅ (decode)      |       ✅        |        ❌         |
 | **ERNIE-Image**          |     ❌     |     ✅      |           ✅           |       ❓        |         ✅         |          ❌          |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |

docs/user_guide/examples/offline_inference/helios.md

@@ -0,0 +1,203 @@
+# Helios Video Generation
+
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/helios>.
+
+
+Helios is a text-to-video (T2V), image-to-video (I2V), and video-to-video (V2V) diffusion model. This example demonstrates end-to-end video generation using vLLM-Omni with three model variants:
+
+| Variant | Description | Key Features |
+|---------|-------------|--------------|
+| **Helios-Base** | Base model, Stage 1 only | Single-stage denoising, `guidance_scale=5.0` |
+| **Helios-Mid** | Mid model, Stage 2 pyramid | Multi-stage pyramid denoising, CFG-Zero* support |
+| **Helios-Distilled** | Distilled model, Stage 2+3 | Few-step inference with DMD, `guidance_scale=1.0` |
+
+## Setup
+
+Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
+
+## Run Examples
+
+Get into the example folder:
+```bash
+cd examples/offline_inference/helios
+```
+
+### Text-to-Video (T2V)
+
+**Helios-Base** (Stage 1 only):
+```bash
+python end2end.py \
+    --model BestWishYsh/Helios-Base \
+    --sample-type t2v \
+    --prompt "A dynamic time-lapse video showing the rapidly moving scenery from the window of a speeding train." \
+    --guidance-scale 5.0 \
+    --output helios_t2v_base.mp4
+```
+
+**Helios-Mid** (Stage 2 + CFG-Zero*):
+```bash
+python end2end.py \
+    --model BestWishYsh/Helios-Mid \
+    --sample-type t2v \
+    --prompt "A dynamic time-lapse video showing the rapidly moving scenery from the window of a speeding train." \
+    --guidance-scale 5.0 \
+    --is-enable-stage2 \
+    --pyramid-num-inference-steps-list 20 20 20 \
+    --use-cfg-zero-star --use-zero-init --zero-steps 1 \
+    --output helios_t2v_mid.mp4
+```
+
+**Helios-Distilled** (Stage 2 pyramid + DMD):
+```bash
+python end2end.py \
+    --model BestWishYsh/Helios-Distilled \
+    --sample-type t2v \
+    --prompt "A dynamic time-lapse video showing the rapidly moving scenery from the window of a speeding train." \
+    --num-frames 240 \
+    --guidance-scale 1.0 \
+    --is-enable-stage2 \
+    --pyramid-num-inference-steps-list 2 2 2 \
+    --is-amplify-first-chunk \
+    --output helios_t2v_distilled.mp4
+```
+
+### Image-to-Video (I2V)
+
+**Helios-Base**:
+```bash
+python end2end.py \
+    --model BestWishYsh/Helios-Base \
+    --sample-type i2v \
+    --image-path /path/to/image.jpg \
+    --prompt "A towering emerald wave surges forward, its crest curling with raw power and energy." \
+    --guidance-scale 5.0 \
+    --output helios_i2v_base.mp4
+```
+
+**Helios-Mid** (Stage 2 + CFG-Zero*):
+```bash
+python end2end.py \
+    --model BestWishYsh/Helios-Mid \
+    --sample-type i2v \
+    --image-path /path/to/image.jpg \
+    --prompt "A towering emerald wave surges forward, its crest curling with raw power and energy." \
+    --guidance-scale 5.0 \
+    --is-enable-stage2 \
+    --pyramid-num-inference-steps-list 20 20 20 \
+    --use-cfg-zero-star --use-zero-init --zero-steps 1 \
+    --output helios_i2v_mid.mp4
+```
+
+**Helios-Distilled**:
+```bash
+python end2end.py \
+    --model BestWishYsh/Helios-Distilled \
+    --sample-type i2v \
+    --image-path /path/to/image.jpg \
+    --prompt "A towering emerald wave surges forward, its crest curling with raw power and energy." \
+    --num-frames 240 \
+    --guidance-scale 1.0 \
+    --is-enable-stage2 \
+    --pyramid-num-inference-steps-list 2 2 2 \
+    --is-amplify-first-chunk \
+    --output helios_i2v_distilled.mp4
+```
+
+### Video-to-Video (V2V)
+
+**Helios-Base**:
+```bash
+python end2end.py \
+    --model BestWishYsh/Helios-Base \
+    --sample-type v2v \
+    --video-path /path/to/video.mp4 \
+    --prompt "A bright yellow Lamborghini speeds along a curving mountain road." \
+    --guidance-scale 5.0 \
+    --output helios_v2v_base.mp4
+```
+
+**Helios-Mid** (Stage 2 + CFG-Zero*):
+```bash
+python end2end.py \
+    --model BestWishYsh/Helios-Mid \
+    --sample-type v2v \
+    --video-path /path/to/video.mp4 \
+    --prompt "A bright yellow Lamborghini speeds along a curving mountain road." \
+    --guidance-scale 5.0 \
+    --is-enable-stage2 \
+    --pyramid-num-inference-steps-list 20 20 20 \
+    --use-cfg-zero-star --use-zero-init --zero-steps 1 \
+    --output helios_v2v_mid.mp4
+```
+
+**Helios-Distilled**:
+```bash
+python end2end.py \
+    --model BestWishYsh/Helios-Distilled \
+    --sample-type v2v \
+    --video-path /path/to/video.mp4 \
+    --prompt "A bright yellow Lamborghini speeds along a curving mountain road." \
+    --num-frames 240 \
+    --guidance-scale 1.0 \
+    --is-enable-stage2 \
+    --pyramid-num-inference-steps-list 2 2 2 \
+    --is-amplify-first-chunk \
+    --output helios_v2v_distilled.mp4
+```
+
+## Common Parameters
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `--model` | `BestWishYsh/Helios-Base` | Model ID or local path |
+| `--sample-type` | `t2v` | Generation mode: `t2v`, `i2v`, or `v2v` |
+| `--prompt` | — | Text prompt describing the video |
+| `--negative-prompt` | *(see source)* | Negative prompt for CFG (includes anti-static terms) |
+| `--image-path` | — | Input image (required for `i2v`) |
+| `--video-path` | — | Input video (required for `v2v`) |
+| `--height` | `384` | Video height in pixels |
+| `--width` | `640` | Video width in pixels |
+| `--num-frames` | `99` | Number of output frames |
+| `--num-inference-steps` | `50` | Denoising steps (Stage 1 only) |
+| `--guidance-scale` | `5.0` | CFG scale (`1.0` for Distilled) |
+| `--seed` | `42` | Random seed |
+| `--fps` | `16` | Output video frame rate |
+| `--output` | `helios_output.mp4` | Output file path |
+
+### Stage 2 / Pyramid Parameters
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `--is-enable-stage2` | off | Enable pyramid multi-stage denoising |
+| `--pyramid-num-stages` | `3` | Number of pyramid stages |
+| `--pyramid-num-inference-steps-list` | `10 10 10` | Steps per pyramid stage |
+| `--is-amplify-first-chunk` | off | DMD amplification (Distilled) |
+
+### CFG-Zero* Parameters
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `--use-cfg-zero-star` | off | Enable CFG-Zero* guidance (Mid) |
+| `--use-zero-init` | off | Zero init for first steps |
+| `--zero-steps` | `1` | Number of zero-init steps |
+
+### Memory & Parallelism
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `--vae-use-slicing` | off | Enable VAE slicing |
+| `--vae-use-tiling` | off | Enable VAE tiling |
+| `--enforce-eager` | off | Disable torch.compile |
+| `--enable-cpu-offload` | off | CPU offloading |
+| `--enable-layerwise-offload` | off | Layerwise offloading |
+| `--ulysses-degree` | `1` | Ulysses SP degree |
+| `--ring-degree` | `1` | Ring SP degree |
+| `--cfg-parallel-size` | `1` | CFG parallel size (1 or 2) |
+| `--tensor-parallel-size` | `1` | Tensor parallelism size |
+
+## Example materials
+
+??? abstract "end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/helios/end2end.py"
+    ``````

docs/user_guide/examples/online_serving/aura_omni.md

@@ -1,65 +0,0 @@
-# AURA Omni Pipeline
-
-`aura_omni` wires ASR, AURA, and Qwen3-TTS into one vLLM-Omni pipeline:
-
-```text
-ASR -> AURA -> Qwen3-TTS Talker -> Code2Wav
-```
-
-Qwen3-TTS remains two engine stages so the pipeline reuses the existing native
-Talker and Code2Wav implementation.
-
-```bash
-vllm serve aurateam/AURA \
-  --omni \
-  --deploy-config vllm_omni/deploy/aura_omni.yaml \
-  --served-model-name aurateam/AURA \
-  --trust-remote-code
-```
-
-Configure local checkpoints by editing per-stage `model` values in
-`vllm_omni/deploy/aura_omni.yaml`. The deploy file sets
-`pipeline: aura_omni`, so the four-stage topology is used even if the
-command-line `--model` points at one of the component checkpoints.
-
-Send requests with `"model": "aurateam/AURA"`. The ASR, AURA, and Qwen3-TTS
-checkpoint paths are internal stage models from the deploy YAML, not the
-OpenAI-facing served model name.
-
-The AURA stage can emit `<|silent|>`. Silent outputs are treated as a gate:
-they produce no Qwen3-TTS Talker input, so no audio is synthesized for that
-turn.
-
-## GPU Utilization Recommendation
-
-`gpu_memory_utilization` in `vllm_omni/deploy/aura_omni.yaml` controls how much
-VRAM each stage can reserve. Start with this split for a single GPU:
-
-- Stage 0 (ASR): `0.10`
-- Stage 1 (AURA): `0.40`
-- Stage 2 (Qwen3-TTS Talker): `0.20`
-- Stage 3 (Qwen3-TTS Code2Wav): `0.20`
-
-## TTS Modes
-
-`aura_omni` can pass AURA text to Qwen3-TTS in two task modes:
-
-- `Base`: voice clone from `tts_ref_audio` with ICL enabled in the AURA
-  pipeline. Provide both `tts_ref_audio` and `tts_ref_text`. Set
-  `tts_x_vector_only_mode=true` to disable ICL and use speaker embedding only.
-- `CustomVoice`: predefined speaker mode. Use a Qwen3-TTS CustomVoice
-  checkpoint for stages 2 and 3 in `aura_omni.yaml`, then pass
-  `tts_task_type=CustomVoice` and `tts_speaker`.
-
-By default, AURA responses are passed to Qwen3-TTS as text. Set
-`additional_information.tts_pass_token_ids=true` to pass AURA-generated
-assistant token ids directly instead. Even when token passthrough is disabled,
-the stage processor uses AURA token ids when available to estimate the Talker
-prefill length, so scheduling does not rely on raw character length.
-
-The example client exposes this as:
-
-```bash
-python examples/online_serving/aura_omni/openai_chat_completion_client.py \
-  --tts-pass-token-ids
-```

docs/user_guide/examples/online_serving/glm_image.md

@@ -19,7 +19,7 @@ both stages on a single GPU, override per stage:
 
 ```bash
 vllm serve zai-org/GLM-Image --omni --port 8091 \
-    --stage-overrides '{"0": {"devices": "0"}, "1": {"devices": "0"}}'
+    --stage-0-devices 0 --stage-1-devices 0
 ```
 
 ## API Calls

docs/user_guide/examples/online_serving/image_to_video.md

@@ -84,26 +84,16 @@ curl -X POST http://localhost:8091/v1/videos/sync \
 Generated video files are stored on local disk by the async video API.
 Local file storage behavior can be controlled via the following environment variables:
 
-- `VLLM_OMNI_SERVER_STORAGE__PATH`: directory used for generated files
-  (default: `/tmp/storage`)
-- `VLLM_OMNI_SERVER_STORAGE__FILE_CONCURRENCY`: max concurrent save/delete/open
-  operations (default: `4`)
-- `VLLM_OMNI_SERVER_STORAGE__FILE_TTL`: optional TTL for generated files in
-  seconds
-- `VLLM_OMNI_SERVER_STORAGE__TTL_SWEEP_INTERVAL`: optional sweep interval in
-  seconds for enforcing file TTL; defaults to `300` when TTL is set
+- `VLLM_OMNI_STORAGE_PATH`: directory used for generated files (default: `/tmp/storage`)
+- `VLLM_OMNI_STORAGE_MAX_CONCURRENCY`: max concurrent save/delete operations (default: `4`)
 
 Example:
 
 ```bash
-export VLLM_OMNI_SERVER_STORAGE__PATH=/var/tmp/vllm-omni-videos
-export VLLM_OMNI_SERVER_STORAGE__FILE_CONCURRENCY=8
-export VLLM_OMNI_SERVER_STORAGE__FILE_TTL=86400
-export VLLM_OMNI_SERVER_STORAGE__TTL_SWEEP_INTERVAL=300
+export VLLM_OMNI_STORAGE_PATH=/var/tmp/vllm-omni-videos
+export VLLM_OMNI_STORAGE_MAX_CONCURRENCY=8
 ```
 
-See also: [Configuration Options](../../../configuration/README.md)
-
 ## API Calls
 
 ### Method 1: Using curl

docs/user_guide/examples/online_serving/text_to_video.md

@@ -90,26 +90,16 @@ curl -X POST http://localhost:8091/v1/videos/sync \
 Generated video files are stored on local disk by the async video API.
 Local file storage behavior can be controlled via the following environment variables:
 
-- `VLLM_OMNI_SERVER_STORAGE__PATH`: directory used for generated files
-  (default: `/tmp/storage`)
-- `VLLM_OMNI_SERVER_STORAGE__FILE_CONCURRENCY`: max concurrent save/delete/open
-  operations (default: `4`)
-- `VLLM_OMNI_SERVER_STORAGE__FILE_TTL`: optional TTL for generated files in
-  seconds
-- `VLLM_OMNI_SERVER_STORAGE__TTL_SWEEP_INTERVAL`: optional sweep interval in
-  seconds for enforcing file TTL; defaults to `300` when TTL is set
+- `VLLM_OMNI_STORAGE_PATH`: directory used for generated files (default: `/tmp/storage`)
+- `VLLM_OMNI_STORAGE_MAX_CONCURRENCY`: max concurrent save/delete operations (default: `4`)
 
 Example:
 
 ```bash
-export VLLM_OMNI_SERVER_STORAGE__PATH=/var/tmp/vllm-omni-videos
-export VLLM_OMNI_SERVER_STORAGE__FILE_CONCURRENCY=8
-export VLLM_OMNI_SERVER_STORAGE__FILE_TTL=86400
-export VLLM_OMNI_SERVER_STORAGE__TTL_SWEEP_INTERVAL=300
+export VLLM_OMNI_STORAGE_PATH=/var/tmp/vllm-omni-videos
+export VLLM_OMNI_STORAGE_MAX_CONCURRENCY=8
 ```
 
-See also: [Configuration Options](../../../configuration/README.md)
-
 ## API Calls
 
 ### Method 1: Using curl

docs/user_guide/quantization/fp8.md

@@ -85,7 +85,6 @@ warmup_quack_fp8([(14040, 2048, 6144), (14040, 2048, 2048)])
 | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B` | Yes | Yes | All layers | None | |
 | HunyuanImage-3.0 | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | Yes | Yes | All layers; use the Hunyuan stage config for multi-stage runs | None | |
 | HunyuanVideo-1.5 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `720p_t2v`, `480p_i2v` | Yes | Yes | All layers | None | |
-| Cosmos3 | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | Yes | Not validated | All layers | None | |
 
 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
 

docs/user_guide/quantization/modelopt.md

@@ -89,54 +89,11 @@ and
 | Model | Scope | Status |
 |-------|-------|--------|
 | Qwen3-Omni | Thinker language-model stage | ModelOpt FP8 checkpoint path |
-| Qwen3-Omni | Thinker language-model stage (W4A4 NVFP4) | Validated; see [Qwen3-Omni NVFP4 W4A4](#qwen3-omni-nvfp4-w4a4-thinker) below |
 | Qwen3-TTS | TTS language-model stage | Not validated |
 
 Audio encoder, vision encoder, talker, and code2wav stages stay in BF16 unless
 a model-specific guide documents otherwise.
 
-#### Qwen3-Omni NVFP4 W4A4 (thinker)
-
-vLLM-Omni serves ModelOpt NVFP4 W4A4 quantizations of the
-Qwen3-Omni-30B-A3B-Instruct thinker language model. The thinker text body
-(attention + MoE experts) is quantized to NVFP4 with FP8 per-tensor input
-scales; the audio encoder, vision encoder, talker, and code2wav stay in BF16.
-
-| Variant | HF checkpoint | Hardware |
-|---------|---------------|----------|
-| W4A4 NVFP4 (full thinker) | `YihongJin/Qwen3-Omni-30B-A3B-Instruct-NVFP4-W4A4-full-thinker-awqclip` | sm_100+ (Blackwell, FlashInfer FP4 GEMM) |
-
-Calibration uses ModelOpt `mtq.NVFP4_DEFAULT_CFG` with the `awq_clip` algorithm
-on 1024 ultrachat samples chat-templated through the Qwen3-Omni tokenizer.
-Excluded modules: `*audio_tower*`, `*visual*`, `*talker*`, `*code2wav*`,
-`*lm_head*`, `*mlp.gate*`. See `scripts/nvfp4/calibrate.py` for the reference
-recipe.
-
-!!! note "ModelOpt 0.44 NaN regression workaround"
-    ModelOpt 0.44's float32 -> FP8 E4M3 cast of per-block weight scales
-    occasionally emits literal NaN bytes (E4M3 encoding 0x7F / 0xFF) for
-    blocks whose pre-cast scale rounds above the FP8 max of 448 after the
-    global-scale division. A single NaN byte in any `weight_scale`
-    propagates through the FP4 GEMM and collapses the served model output
-    to `!!!!`. vLLM-Omni's `vllm_omni.patch` installs a defensive override
-    of `ModelOptNvFp4LinearMethod.process_weights_after_loading` that
-    clamps these bytes to the FP8 E4M3 max at load time. The override
-    self-extinguishes once vllm-omni's vllm pin moves to a release
-    containing the upstream fix. Set `VLLM_OMNI_SKIP_NVFP4_NAN_CLAMP=1`
-    to disable the override for diagnostics.
-
-Serving:
-
-```bash
-vllm serve YihongJin/Qwen3-Omni-30B-A3B-Instruct-NVFP4-W4A4-full-thinker-awqclip \
-    --omni --port 8000
-```
-
-> Do **not** pass `--enforce-eager` for production / benchmarks. CUDA graphs
-> amortize launch overhead and unlock the FP4 throughput wins; with
-> `--enforce-eager` set, W4A4 TPOT degrades ~10x relative to the CUDA-graph
-> configuration.
-
 ### Multi-Stage Diffusion Model
 
 ModelOpt checkpoints must be routed to the stage whose checkpoint contains the

examples/offline_inference/aura_omni/README.md

@@ -1,61 +0,0 @@
-# AURA Omni
-
-This example runs the native AURA Omni pipeline offline:
-
-```text
-Qwen3-ASR -> AURA/Qwen3-VL -> Qwen3-TTS Talker -> Qwen3-TTS Code2Wav
-```
-
-The first stage consumes speech and produces a transcript. The AURA stage then combines the transcript with the video frames from the original request and returns text or `<|silent|>`. Non-silent responses are passed to Qwen3-TTS as AURA-generated token ids.
-
-## Run
-
-```bash
-cd examples/offline_inference/aura_omni
-bash run_single_prompt.sh
-```
-
-Use local media:
-
-```bash
-python end2end.py \
-  --audio-path /path/to/input.wav \
-  --video-path /path/to/video.mp4 \
-  --modalities text,audio
-```
-
-Base voice clone mode is the default. It uses the AURA reference audio unless
-you override it:
-
-```bash
-python end2end.py \
-  --tts-task-type Base \
-  --tts-ref-audio /data/yrr/rein_test/shuhan.mp3 \
-  --tts-x-vector-only-mode
-```
-
-CustomVoice mode requires stages 2 and 3 in `vllm_omni/deploy/aura_omni.yaml`
-to point at a Qwen3-TTS CustomVoice checkpoint:
-
-```bash
-python end2end.py \
-  --tts-task-type CustomVoice \
-  --tts-speaker Vivian
-```
-
-For local checkpoints, edit the stage `model` entries in
-`vllm_omni/deploy/aura_omni.yaml` or pass a copied deploy config with
-`--deploy-config`.
-
-## GPU Utilization Recommendation
-
-Set `gpu_memory_utilization` in `vllm_omni/deploy/aura_omni.yaml` by stage.
-Suggested starting point:
-
-- Stage 0 (ASR): `0.10`
-- Stage 1 (AURA): `0.4`
-- Stage 2 (Qwen3-TTS Talker): `0.20`
-- Stage 3 (Qwen3-TTS Code2Wav): `0.20`
-
-Generated text and audio are written to `--output-dir`
-(default: `output_aura_omni`).

examples/offline_inference/aura_omni/end2end.py

@@ -1,236 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Offline AURA Omni example.
-
-The native AURA pipeline is:
-
-    Qwen3-ASR -> AURA/Qwen3-VL -> Qwen3-TTS Talker -> Code2Wav
-
-Stage 0 consumes audio and emits a transcript. The stage input processor then
-combines that transcript with the original visual payload for AURA.
-"""
-
-from __future__ import annotations
-
-import os
-from typing import NamedTuple
-
-import numpy as np
-import soundfile as sf
-import vllm
-from vllm import SamplingParams
-from vllm.assets.audio import AudioAsset
-from vllm.assets.video import VideoAsset, video_to_ndarrays
-from vllm.multimodal.media.audio import load_audio
-from vllm.utils.argparse_utils import FlexibleArgumentParser
-
-from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
-from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.model_executor.stage_input_processors.aura_omni import (
-    DEFAULT_QWEN3_TTS_REF_TEXT,
-    default_qwen3_tts_ref_audio_path,
-)
-
-SEED = 42
-DEFAULT_MODEL = "aurateam/AURA"
-DEFAULT_DEPLOY_CONFIG = "vllm_omni/deploy/aura_omni.yaml"
-
-
-class QueryResult(NamedTuple):
-    inputs: dict
-    limit_mm_per_prompt: dict[str, int]
-
-
-def _load_audio(audio_path: str | None, sampling_rate: int) -> tuple[np.ndarray, int]:
-    if audio_path:
-        if not os.path.exists(audio_path):
-            raise FileNotFoundError(f"Audio file not found: {audio_path}")
-        audio_signal, sr = load_audio(audio_path, sr=sampling_rate)
-        return audio_signal.astype(np.float32), int(sr)
-    audio_signal, sr = AudioAsset("mary_had_lamb").audio_and_sample_rate
-    return audio_signal.astype(np.float32), int(sr)
-
-
-def _load_video(video_path: str | None, num_frames: int) -> list[np.ndarray]:
-    if video_path:
-        if not os.path.exists(video_path):
-            raise FileNotFoundError(f"Video file not found: {video_path}")
-        return video_to_ndarrays(video_path, num_frames=num_frames)
-    return VideoAsset(name="baby_reading", num_frames=num_frames).np_ndarrays
-
-
-def build_query(args) -> QueryResult:
-    prompt = (
-        "<|im_start|>user\n"
-        "<|audio_start|><|audio_pad|><|audio_end|>"
-        "Please transcribe this speech and use the video context to decide whether a reply is needed.<|im_end|>\n"
-        "<|im_start|>assistant\n"
-    )
-    audio_data = _load_audio(args.audio_path, args.sampling_rate)
-    video_frames = _load_video(args.video_path, args.num_frames)
-    return QueryResult(
-        inputs={
-            "prompt": prompt,
-            "multi_modal_data": {
-                "audio": audio_data,
-                "video": video_frames,
-            },
-            "additional_information": {
-                "aura_system_prompt": args.aura_system_prompt,
-                "tts_task_type": args.tts_task_type,
-                "tts_language": args.tts_language,
-                "tts_speaker": args.tts_speaker,
-                "tts_instruct": args.tts_instruct,
-                "tts_ref_audio": args.tts_ref_audio,
-                "tts_ref_text": args.tts_ref_text,
-                "tts_x_vector_only_mode": args.tts_x_vector_only_mode,
-                "tts_pass_token_ids": args.tts_pass_token_ids,
-            },
-        },
-        limit_mm_per_prompt={"audio": 1, "video": 1},
-    )
-
-
-def build_sampling_params() -> list[SamplingParams]:
-    asr_sampling = SamplingParams(
-        temperature=0.0,
-        top_p=1.0,
-        top_k=-1,
-        max_tokens=256,
-        detokenize=True,
-        seed=SEED,
-    )
-    aura_sampling = SamplingParams(
-        temperature=0.5,
-        top_p=1.0,
-        top_k=-1,
-        max_tokens=256,
-        detokenize=True,
-        repetition_penalty=1.0,
-        seed=SEED,
-    )
-    talker_sampling = SamplingParams(
-        temperature=0.9,
-        top_k=50,
-        max_tokens=4096,
-        detokenize=False,
-        repetition_penalty=1.05,
-        stop_token_ids=[2150],
-        seed=SEED,
-    )
-    code2wav_sampling = SamplingParams(
-        temperature=0.0,
-        top_p=1.0,
-        top_k=-1,
-        max_tokens=65536,
-        detokenize=True,
-        repetition_penalty=1.0,
-        seed=SEED,
-    )
-    return [asr_sampling, aura_sampling, talker_sampling, code2wav_sampling]
-
-
-def save_stage_output(stage_outputs, output_dir: str) -> None:
-    output = stage_outputs.request_output
-    request_id = output.request_id
-    if stage_outputs.final_output_type == "text":
-        text_output = output.outputs[0].text
-        out_txt = os.path.join(output_dir, f"{request_id}.txt")
-        with open(out_txt, "w", encoding="utf-8") as f:
-            f.write(str(text_output).strip() + "\n")
-        print(f"Request ID: {request_id}, text saved to {out_txt}")
-        print(text_output)
-        return
-
-    if stage_outputs.final_output_type == "audio":
-        mm = output.outputs[0].multimodal_output
-        if not mm or "audio" not in mm:
-            print(f"Request ID: {request_id}, no audio output")
-            return
-        audio = mm["audio"]
-        if isinstance(audio, list):
-            import torch
-
-            audio = torch.cat([(item if hasattr(item, "detach") else torch.tensor(item)).flatten() for item in audio])
-        sample_rate = int(mm.get("sr", 24000))
-        audio_numpy = audio.float().detach().cpu().numpy().flatten()
-        out_wav = os.path.join(output_dir, f"{request_id}.wav")
-        sf.write(out_wav, audio_numpy, samplerate=sample_rate, format="WAV")
-        print(f"Request ID: {request_id}, audio saved to {out_wav}")
-
-
-def main(args) -> None:
-    print("=" * 20, "\n", f"vllm version: {vllm.__version__}", "\n", "=" * 20)
-    os.makedirs(args.output_dir, exist_ok=True)
-
-    query = build_query(args)
-    prompts = [query.inputs for _ in range(args.num_prompts)]
-    if args.modalities:
-        modalities = [item.strip() for item in args.modalities.split(",") if item.strip()]
-        for prompt in prompts:
-            prompt["modalities"] = modalities
-
-    omni_kwargs = vars(args).copy()
-    omni_kwargs["model"] = args.model
-    omni = Omni(**omni_kwargs)
-    sampling_params_list = build_sampling_params()[: omni.num_stages]
-
-    try:
-        for stage_outputs in omni.generate(prompts, sampling_params_list):
-            save_stage_output(stage_outputs, args.output_dir)
-    finally:
-        omni.close()
-
-
-def parse_args():
-    parser = FlexibleArgumentParser(description="AURA Omni offline inference example")
-    parser.add_argument("--model", default=DEFAULT_MODEL, help="Top-level model name/path.")
-    parser.add_argument("--deploy-config", default=DEFAULT_DEPLOY_CONFIG, help="AURA Omni deploy config.")
-    parser.add_argument("--audio-path", "-a", default=None, help="Local audio file. Uses a built-in asset by default.")
-    parser.add_argument("--video-path", "-v", default=None, help="Local video file. Uses a built-in asset by default.")
-    parser.add_argument("--num-frames", type=int, default=16, help="Number of video frames to sample.")
-    parser.add_argument("--sampling-rate", type=int, default=16000, help="Audio sampling rate for ASR input.")
-    parser.add_argument("--num-prompts", type=int, default=1, help="Number of repeated prompts.")
-    parser.add_argument("--modalities", default="text,audio", help="Comma-separated final modalities.")
-    parser.add_argument("--output-dir", default="output_aura_omni", help="Directory for generated files.")
-    parser.add_argument(
-        "--aura-system-prompt",
-        default=(
-            "You are receiving a live video stream where the final frame is the present moment. "
-            "Respond only when a response is needed. Otherwise output '<|silent|>'. Respond in Chinese."
-        ),
-        help="System prompt for the AURA stage.",
-    )
-    parser.add_argument("--tts-task-type", default="Base", choices=["Base", "CustomVoice"], help="Qwen3-TTS task.")
-    parser.add_argument("--tts-language", default="Chinese", help="Qwen3-TTS language.")
-    parser.add_argument("--tts-speaker", default="Vivian", help="Qwen3-TTS speaker.")
-    parser.add_argument("--tts-instruct", default="", help="Optional Qwen3-TTS style instruction.")
-    parser.add_argument(
-        "--tts-ref-audio",
-        default=default_qwen3_tts_ref_audio_path(),
-        help="Base-mode reference audio path/URL.",
-    )
-    parser.add_argument(
-        "--tts-ref-text",
-        default=DEFAULT_QWEN3_TTS_REF_TEXT,
-        help="Base-mode reference audio transcript.",
-    )
-    parser.add_argument(
-        "--tts-x-vector-only-mode",
-        action="store_true",
-        help="Use speaker embedding only for Base mode (disable ICL ref_text conditioning).",
-    )
-    parser.add_argument(
-        "--tts-pass-token-ids",
-        action="store_true",
-        help="Pass AURA-generated assistant token ids directly to Qwen3-TTS. Defaults to sending text.",
-    )
-    parser.add_argument("--init-timeout", type=int, default=2000, help="Overall initialization timeout.")
-    parser.add_argument("--stage-init-timeout", type=int, default=2000, help="Per-stage initialization timeout.")
-    parser.add_argument("--log-stats", action="store_true", default=False, help="Enable stats logging.")
-    nullify_stage_engine_defaults(parser)
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    main(parse_args())

examples/offline_inference/aura_omni/run_single_prompt.sh

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-python end2end.py \
-  --model aurateam/AURA \
-  --deploy-config vllm_omni/deploy/aura_omni.yaml \
-  --modalities text,audio \
-  "$@"

examples/offline_inference/bagel/README.md

@@ -0,0 +1,391 @@
+# BAGEL-7B-MoT
+
+## Setup
+
+Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
+
+## Architecture
+
+BAGEL-7B-MoT is a Mixture-of-Transformers (MoT) model supporting both image generation and understanding. It offers two deployment topologies:
+
+| Topology | Stages | Description |
+| :------- | :----- | :---------- |
+| **Two-stage** (default) | Stage 0 (Thinker, AR) + Stage 1 (DiT, Diffusion) | Thinker handles text/understanding via vLLM AR engine; DiT handles image generation. KV cache is transferred between stages. |
+| **Single-stage** | Stage 0 (DiT, Diffusion) only | The DiT stage contains a full LLM, ViT, VAE, and tokenizer internally. All modalities are handled within a single diffusion process. |
+
+Both topologies support all four modalities: `text2img`, `img2img`, `img2text`, `text2text`.
+
+## Quick Start
+
+```bash
+cd examples/offline_inference/bagel
+
+# Default two-stage mode (auto-detected)
+python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
+                  --modality text2img \
+                  --prompts "A cute cat"
+
+# Single-stage mode
+python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
+                  --modality text2img \
+                  --prompts "A cute cat" \
+                  --deploy-config vllm_omni/deploy/bagel_single_stage.yaml
+```
+
+> **Note**: These examples work with the default configuration on an **NVIDIA A100 (80GB)**. For dual-GPU setups, modify the deploy YAML to distribute stages across devices.
+
+## Modality Control
+
+Control the mode using the `--modality` argument:
+
+| Modality | Input | Output | Description |
+| :------- | :---- | :----- | :---------- |
+| `text2img` | Text | Image | Generate images from text prompts |
+| `img2img` | Image + Text | Image | Transform images using text guidance |
+| `img2text` | Image + Text | Text | Generate text descriptions from images |
+| `text2text` | Text | Text | Pure text generation (language model mode) |
+
+### Text to Image (text2img)
+
+```bash
+python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
+                  --modality text2img \
+                  --prompts "A cute cat" \
+                  --steps 50
+```
+
+### Image to Image (img2img)
+
+```bash
+python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
+                  --modality img2img \
+                  --image-path /path/to/image.jpg \
+                  --prompts "Let the woman wear a blue dress"
+```
+
+### Image to Text (img2text)
+
+```bash
+python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
+                  --modality img2text \
+                  --image-path /path/to/image.jpg \
+                  --prompts "Describe this image in detail"
+```
+
+### Text to Text (text2text)
+
+```bash
+python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
+                  --modality text2text \
+                  --prompts "What is the capital of France?"
+
+# Load prompts from a text file (one prompt per line):
+python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
+                  --modality text2text \
+                  --txt-prompts /path/to/prompts.txt
+```
+
+## Think Mode
+
+Think mode enables the model to generate `<think>...</think>` planning/reasoning tokens before producing the final output. This improves generation quality for complex prompts.
+
+- **Two-stage**: The Thinker (AR) stage decodes think tokens, then transfers the augmented KV cache to the DiT stage for image generation.
+- **Single-stage**: The DiT's internal LLM generates think tokens in-place before proceeding to denoise.
+
+```bash
+# Think + text2img: plan before generating
+python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
+                  --modality text2img \
+                  --prompts "A futuristic city with flying cars" \
+                  --think \
+                  --max-think-tokens 1000
+
+# Think + img2img: reason about the edit
+python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
+                  --modality img2img \
+                  --image-path /path/to/image.jpg \
+                  --prompts "Make it look like a watercolor painting" \
+                  --think
+
+# Think + img2text: reason before describing
+python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
+                  --modality img2text \
+                  --image-path /path/to/image.jpg \
+                  --prompts "What is happening in this image?" \
+                  --think
+
+# Think + text2text: chain-of-thought reasoning
+python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
+                  --modality text2text \
+                  --prompts "Solve: 23 * 47" \
+                  --think
+```
+
+Think mode parameters:
+
+| Argument | Default | Description |
+| :------- | :------ | :---------- |
+| `--think` | `False` | Enable thinking mode |
+| `--max-think-tokens` | `1000` | Maximum tokens for think generation |
+| `--do-sample` | `False` | Enable sampling (vs. greedy) for text generation |
+| `--text-temperature` | `0.3` | Temperature for text generation sampling |
+
+## Classifier-Free Guidance (CFG)
+
+CFG controls the trade-off between prompt fidelity and diversity. These parameters apply to image generation modalities (`text2img`, `img2img`).
+
+```bash
+python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
+                  --modality text2img \
+                  --prompts "A photorealistic portrait" \
+                  --cfg-text-scale 6.0 \
+                  --cfg-img-scale 2.0 \
+                  --negative-prompt "blurry, low quality, distorted" \
+                  --cfg-interval 0.4 1.0 \
+                  --cfg-renorm-type global \
+                  --cfg-renorm-min 0.0
+```
+
+| Argument | Default | Description |
+| :------- | :------ | :---------- |
+| `--cfg-text-scale` | `4.0` | Text CFG scale (higher = more prompt-adherent) |
+| `--cfg-img-scale` | `1.5` | Image CFG scale (for img2img) |
+| `--negative-prompt` | `None` | Negative prompt for CFG conditioning |
+| `--cfg-interval` | pipeline default | CFG active interval `[start, end]` as fractions of total timesteps |
+| `--cfg-renorm-type` | `None` | Renormalization type: `global`, `text_channel`, `channel` |
+| `--cfg-renorm-min` | `None` | Minimum renormalization value |
+| `--cfg-parallel-size` | `1` | CFG parallel size: `1` = batched (single GPU), `2` = 2-branch parallel, `3` = full 3-GPU parallel |
+
+## Deployment Topologies
+
+### Two-Stage (Default)
+
+The default topology auto-detected from the model. No extra flags needed.
+
+```bash
+python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
+                  --modality text2img \
+                  --prompts "A cute cat"
+```
+
+The pipeline is defined in [`bagel.yaml`](../../../vllm_omni/deploy/bagel.yaml). Stage 0 (Thinker) and Stage 1 (DiT) share GPU 0 by default. For dual-GPU setups, customize the deploy YAML and set `devices: "1"` for stage 1.
+
+### Single-Stage
+
+Pass the single-stage deploy config via `--deploy-config`:
+
+```bash
+python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
+                  --modality text2img \
+                  --prompts "A cute cat" \
+                  --deploy-config vllm_omni/deploy/bagel_single_stage.yaml
+```
+
+See [`bagel_single_stage.yaml`](../../../vllm_omni/deploy/bagel_single_stage.yaml) for configuration details. The `pipeline: bagel_single_stage` field selects the single-stage topology from the pipeline registry.
+
+### Tensor Parallelism (TP)
+
+For larger models or multi-GPU environments, customize the deploy YAML (see [`bagel.yaml`](../../../vllm_omni/deploy/bagel.yaml)) and set per-stage `tensor_parallel_size` and `devices`:
+
+```yaml
+# Example: TP=2 on GPUs 0,1 for the Thinker stage
+stages:
+  - stage_id: 0
+    tensor_parallel_size: 2
+    devices: "0,1"
+```
+
+### VAE Patch Parallelism
+
+[VAE Patch Parallelism](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/parallelism/vae_patch_parallel.html) splits Bagel VAE **decode/encode** tiles across multiple GPUs on the **DiT stage**, reducing **per-GPU peak memory during VAE decode**. Use it when high-resolution `text2img` or `img2img` hits VAE OOM or large decode spikes.
+
+**Bagel-specific notes:**
+
+- Implemented in `BagelPipeline` via `DistributedAutoEncoder` (DiT stage only).
+- **Single-stage** is the simplest path: one DiT process with TP + VAE patch parallel.
+- **Two-stage**: enable on **stage 1 (DiT)** only; stage 0 (Thinker) keeps encoder-only `VAEEncoder` and does not use VAE patch parallel.
+- You need a DiT `world_size` ≥ `vae_patch_parallel_size` (typically `tensor_parallel_size=2` on that stage). VAE PP reuses the DiT process group; it is not a standalone second-GPU VAE worker.
+
+**Single-stage via deploy YAML** (recommended for `end2end.py`):
+
+```yaml
+pipeline: bagel_single_stage
+async_chunk: false
+
+stages:
+  - stage_id: 0
+    max_num_batched_tokens: 32768
+    max_num_seqs: 1
+    enforce_eager: true
+    trust_remote_code: true
+    enable_prefix_caching: false
+    devices: "0,1"
+    vae_use_tiling: true
+    parallel_config:
+      tensor_parallel_size: 2
+      vae_patch_parallel_size: 2
+    default_sampling_params:
+      seed: 52
+```
+
+```bash
+cd examples/offline_inference/bagel
+
+CUDA_VISIBLE_DEVICES=0,1 python end2end.py \
+    --model /path/to/BAGEL-7B-MoT \
+    --deploy-config /path/to/bagel_single_stage_vae_pp.yaml \
+    --modality text2img \
+    --prompts "A cute cat" \
+    --steps 10 \
+    --output ./out_vae_pp
+```
+
+**Single-stage via `Omni` kwargs** (same flags as online serving):
+
+```python
+from vllm_omni.entrypoints.omni import Omni
+
+omni = Omni(
+    model="ByteDance-Seed/BAGEL-7B-MoT",
+    deploy_config="vllm_omni/deploy/bagel_single_stage.yaml",
+    tensor_parallel_size=2,
+    vae_patch_parallel_size=2,
+    vae_use_tiling=True,
+)
+# Then call omni.generate(...) as in end2end.py
+```
+
+**Two-stage (VAE PP on DiT only):**
+
+```yaml
+stages:
+  - stage_id: 0
+    devices: "0"
+    # AR Thinker — no vae_patch_parallel here
+
+  - stage_id: 1
+    devices: "0,1"
+    vae_use_tiling: true
+    parallel_config:
+      tensor_parallel_size: 2
+      vae_patch_parallel_size: 2
+```
+
+```bash
+python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
+    --deploy-config /path/to/bagel_vae_pp.yaml \
+    --modality text2img \
+    --prompts "A cute cat"
+```
+
+**Startup log checks:**
+
+```text
+INFO ... vae_patch_parallel_size=2 requires vae_use_tiling; automatically enabling it.
+```
+
+| Setting | Role |
+| :------ | :--- |
+| `parallel_config.tensor_parallel_size` | DiT world size / TP (must be ≥ `vae_patch_parallel_size`) |
+| `parallel_config.vae_patch_parallel_size` | Number of ranks for distributed VAE tiles (`1` = off) |
+| `vae_use_tiling` | Enable spatial tiling (auto-enabled when `vae_patch_parallel_size > 1`) |
+
+#### Hybrid Sharded Data Parallel (HSDP)
+
+For larger Bagel deployments on multiple GPUs, you can enable HSDP (Hybrid Sharded Data Parallel) by modifying the stage configuration (for example, [`bagel.yaml`](../../../vllm_omni/deploy/bagel.yaml)). HSDP shards transformer weights across GPUs to reduce per-GPU memory usage.
+
+1. **Enable HSDP**: Set `use_hsdp: true`.
+2. **Set shard size**: Set `hsdp_shard_size` to the number of GPUs used for sharding (for example, `4`).
+3. **Set replicate size**: Usually keep `hsdp_replicate_size: 1` unless you want replicated HSDP groups.
+4. **Set devices**: Specify the comma-separated GPU IDs used by the diffusion stage (for example, `"0,1,2,3"`).
+
+Example configuration for HSDP across 4 GPUs:
+```yaml
+  - stage_id: 1
+    devices: "0,1,2,3"
+    parallel_config:
+      use_hsdp: true
+      hsdp_shard_size: 4
+      hsdp_replicate_size: 1
+```
+
+Then pass the custom deploy YAML:
+
+```bash
+python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
+                  --modality text2img \
+                  --prompts "A cute cat" \
+                  --deploy-config /path/to/custom_bagel.yaml
+```
+
+### FP8 Quantization
+
+```bash
+python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
+                  --modality text2img \
+                  --prompts "A cute cat" \
+                  --quantization fp8
+```
+
+## Command Line Reference
+
+### Core Arguments
+
+| Argument | Type | Default | Description |
+| :------- | :--- | :------ | :---------- |
+| `--model` | string | `ByteDance-Seed/BAGEL-7B-MoT` | Model path or HuggingFace name |
+| `--modality` | choice | `text2img` | `text2img`, `img2img`, `img2text`, `text2text` |
+| `--prompts` | list | `None` | Input text prompts |
+| `--txt-prompts` | string | `None` | Path to text file with one prompt per line |
+| `--image-path` | string | `None` | Input image path (required for `img2img`/`img2text`) |
+| `--output` | string | `.` | Output directory for saved images |
+| `--steps` | int | `50` | Number of diffusion inference steps |
+| `--seed` | int | `None` | Random seed for reproducibility |
+
+### Think Mode Arguments
+
+| Argument | Type | Default | Description |
+| :------- | :--- | :------ | :---------- |
+| `--think` | flag | `False` | Enable `<think>...</think>` planning/reasoning |
+| `--max-think-tokens` | int | `1000` | Maximum tokens for think generation |
+| `--do-sample` | flag | `False` | Use sampling instead of greedy decoding |
+| `--text-temperature` | float | `0.3` | Sampling temperature for text generation |
+
+### CFG Arguments
+
+| Argument | Type | Default | Description |
+| :------- | :--- | :------ | :---------- |
+| `--cfg-text-scale` | float | `4.0` | Text CFG guidance scale |
+| `--cfg-img-scale` | float | `1.5` | Image CFG guidance scale |
+| `--negative-prompt` | string | `None` | Negative prompt for CFG |
+| `--cfg-parallel-size` | int | `1` | CFG parallel GPU count (1, 2, or 3) |
+| `--cfg-interval` | float[2] | pipeline default | CFG active window `[start, end]` |
+| `--cfg-renorm-type` | string | `None` | `global`, `text_channel`, or `channel` |
+| `--cfg-renorm-min` | float | `None` | Minimum renormalization value |
+
+### Engine Arguments
+
+| Argument | Type | Default | Description |
+| :------- | :--- | :------ | :---------- |
+| `--deploy-config` | string | `None` | Path to deploy YAML (auto-detected if omitted) |
+| `--worker-backend` | choice | `process` | `process` or `ray` |
+| `--ray-address` | string | `None` | Ray cluster address |
+| `--quantization` | string | `None` | Quantization method (e.g. `fp8`) |
+| `--log-stats` | flag | `False` | Enable statistics logging |
+| `--init-timeout` | int | `300` | Initialization timeout (seconds) |
+| `--batch-timeout` | int | `5` | Batch timeout (seconds) |
+| `--enable-diffusion-pipeline-profiler` | flag | `False` | Profile diffusion stage durations |
+
+## FAQ
+
+- If you encounter OOM errors, try decreasing `max_model_len` or `gpu_memory_utilization` in the deploy YAML.
+
+**Two-stage VRAM usage:**
+
+| Stage | VRAM |
+| :---- | :--- |
+| Stage 0 (Thinker) | **15.04 GiB + KV Cache** |
+| Stage 1 (DiT) | **26.50 GiB** |
+| Total | **~42 GiB + KV Cache** |
+
+**Single-stage VRAM usage:** The DiT loads the full model (~42 GiB) in one process.

examples/offline_inference/bagel/end2end.py

@@ -0,0 +1,278 @@
+import argparse
+import os
+
+from vllm_omni.inputs.data import OmniPromptType
+from vllm_omni.model_executor.stage_input_processors.bagel import (
+    GEN_THINK_SYSTEM_PROMPT,
+    VLM_THINK_SYSTEM_PROMPT,
+)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model",
+        default="ByteDance-Seed/BAGEL-7B-MoT",
+        help="Path to merged model directory.",
+    )
+    parser.add_argument("--prompts", nargs="+", default=None, help="Input text prompts.")
+    parser.add_argument(
+        "--txt-prompts",
+        type=str,
+        default=None,
+        help="Path to a .txt file with one prompt per line (preferred).",
+    )
+    parser.add_argument("--prompt-type", default="text", choices=["text"])
+
+    parser.add_argument(
+        "--modality",
+        default="text2img",
+        choices=["text2img", "img2img", "img2text", "text2text"],
+        help="Modality mode to control stage execution.",
+    )
+
+    parser.add_argument(
+        "--image-path",
+        type=str,
+        default=None,
+        help="Path to input image for img2img.",
+    )
+
+    parser.add_argument(
+        "--output",
+        type=str,
+        default=".",
+        help="Output directory to save images.",
+    )
+
+    # OmniLLM init args
+    parser.add_argument("--log-stats", action="store_true", default=False)
+    parser.add_argument("--init-sleep-seconds", type=int, default=20)
+    parser.add_argument("--batch-timeout", type=int, default=5)
+    parser.add_argument("--init-timeout", type=int, default=300)
+    parser.add_argument("--shm-threshold-bytes", type=int, default=65536)
+    parser.add_argument("--worker-backend", type=str, default="process", choices=["process", "ray"])
+    parser.add_argument("--ray-address", type=str, default=None)
+    parser.add_argument(
+        "--deploy-config",
+        type=str,
+        default=None,
+        help="Path to deploy YAML. If unset, auto-loads vllm_omni/deploy/bagel.yaml based on the HF model_type.",
+    )
+    parser.add_argument("--steps", type=int, default=50, help="Number of inference steps.")
+
+    parser.add_argument("--cfg-text-scale", type=float, default=4.0, help="Text CFG scale (default: 4.0)")
+    parser.add_argument("--cfg-img-scale", type=float, default=1.5, help="Image CFG scale (default: 1.5)")
+    parser.add_argument(
+        "--negative-prompt", type=str, default=None, help="Negative prompt for CFG (default: empty prompt)"
+    )
+    parser.add_argument(
+        "--cfg-parallel-size",
+        type=int,
+        default=1,
+        choices=[1, 2, 3],
+        help="CFG parallel size: 1=batched (single GPU), 2=parallel with 2 branches (text CFG only), 3=parallel (3 GPUs).",
+    )
+    parser.add_argument("--seed", type=int, default=None, help="Random seed for generation.")
+    parser.add_argument(
+        "--cfg-interval",
+        type=float,
+        nargs=2,
+        default=None,
+        help="CFG interval [start, end] (default: pipeline default)",
+    )
+    parser.add_argument(
+        "--cfg-renorm-type", type=str, default=None, help="CFG renorm type: global, text_channel, channel"
+    )
+    parser.add_argument("--cfg-renorm-min", type=float, default=None, help="CFG renorm min")
+    parser.add_argument(
+        "--enable-diffusion-pipeline-profiler",
+        action="store_true",
+        help="Enable diffusion pipeline profiler to display stage durations.",
+    )
+    parser.add_argument(
+        "--quantization",
+        type=str,
+        default=None,
+        help="Quantization method (e.g. 'fp8').",
+    )
+    parser.add_argument(
+        "--think",
+        action="store_true",
+        default=False,
+        help="Enable thinking mode: AR stage decodes <think>...</think> planning tokens before image generation.",
+    )
+    parser.add_argument(
+        "--max-think-tokens",
+        type=int,
+        default=1000,
+        help="Maximum number of tokens for thinking text generation (default: 1000).",
+    )
+    parser.add_argument(
+        "--do-sample",
+        action="store_true",
+        default=False,
+        help="Enable sampling for text generation (default: greedy).",
+    )
+    parser.add_argument(
+        "--text-temperature",
+        type=float,
+        default=0.3,
+        help="Temperature for text generation sampling (default: 0.3).",
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = parse_args()
+    os.makedirs(args.output, exist_ok=True)
+    model_name = args.model
+    prompts: list[OmniPromptType] = []
+    try:
+        if getattr(args, "txt_prompts", None) and args.prompt_type == "text":
+            with open(args.txt_prompts, encoding="utf-8") as f:
+                lines = [ln.strip() for ln in f.readlines()]
+            prompts = [ln for ln in lines if ln != ""]
+            print(f"[Info] Loaded {len(prompts)} prompts from {args.txt_prompts}")
+        else:
+            prompts = args.prompts
+    except Exception as e:
+        print(f"[Error] Failed to load prompts: {e}")
+        raise
+
+    if not prompts:
+        prompts = ["A cute cat"]
+        print(f"[Info] No prompts provided, using default: {prompts}")
+
+    from PIL import Image
+
+    from vllm_omni.entrypoints.omni import Omni
+
+    omni_kwargs = vars(args).copy()
+    omni_kwargs.pop("model", None)
+    deploy_config = args.deploy_config
+    if args.think and deploy_config is None:
+        deploy_config = "vllm_omni/deploy/bagel_think.yaml"
+        print(f"[Info] Think mode enabled, using deploy config: {deploy_config}")
+    if deploy_config:
+        omni_kwargs["deploy_config"] = deploy_config
+
+    if args.quantization:
+        omni_kwargs["quantization_config"] = args.quantization
+
+    omni = Omni(model=model_name, **omni_kwargs)
+
+    formatted_prompts = []
+    for p in prompts:
+        if args.modality == "img2img":
+            if not args.image_path or not os.path.exists(args.image_path):
+                raise ValueError(f"img2img requires --image-path pointing to an existing file, got: {args.image_path}")
+            loaded_image = Image.open(args.image_path).convert("RGB")
+            think_prefix = f"<|im_start|>{GEN_THINK_SYSTEM_PROMPT}<|im_end|>" if args.think else ""
+            final_prompt_text = f"{think_prefix}<|fim_middle|><|im_start|>{p}<|im_end|>"
+            prompt_dict = {
+                "prompt": final_prompt_text,
+                "multi_modal_data": {"img2img": loaded_image},
+                "modalities": ["img2img"],
+            }
+            if args.negative_prompt is not None:
+                prompt_dict["negative_prompt"] = args.negative_prompt
+            formatted_prompts.append(prompt_dict)
+        elif args.modality == "img2text":
+            if args.image_path:
+                loaded_image = Image.open(args.image_path).convert("RGB")
+                think_prefix = f"<|im_start|>system\n{VLM_THINK_SYSTEM_PROMPT}<|im_end|>\n" if args.think else ""
+                final_prompt_text = (
+                    f"{think_prefix}<|im_start|>user\n<|image_pad|>\n{p}<|im_end|>\n<|im_start|>assistant\n"
+                )
+                prompt_dict = {
+                    "prompt": final_prompt_text,
+                    "multi_modal_data": {"image": loaded_image},
+                    "modalities": ["text"],
+                }
+                formatted_prompts.append(prompt_dict)
+        elif args.modality == "text2text":
+            think_prefix = f"<|im_start|>{VLM_THINK_SYSTEM_PROMPT}<|im_end|>" if args.think else ""
+            final_prompt_text = f"{think_prefix}<|im_start|>{p}<|im_end|><|im_start|>"
+            prompt_dict = {"prompt": final_prompt_text, "modalities": ["text"]}
+            formatted_prompts.append(prompt_dict)
+        else:
+            think_prefix = f"<|im_start|>{GEN_THINK_SYSTEM_PROMPT}<|im_end|>" if args.think else ""
+            final_prompt_text = f"{think_prefix}<|im_start|>{p}<|im_end|>"
+            prompt_dict = {"prompt": final_prompt_text, "modalities": ["image"]}
+            if args.negative_prompt is not None:
+                prompt_dict["negative_prompt"] = args.negative_prompt
+            formatted_prompts.append(prompt_dict)
+
+    params_list = omni.default_sampling_params_list
+    # Bagel exposes 1 sampling param set for single-stage (DiT-only) and
+    # 2 for two-stage (Thinker + DiT).  This heuristic may need updating
+    # if future pipelines break that 1:1 mapping.
+    is_single_stage = len(params_list) == 1
+
+    diffusion_params_idx = 0 if is_single_stage else (1 if len(params_list) > 1 else 0)
+    diffusion_params = params_list[diffusion_params_idx]
+
+    if args.modality in ("text2img", "img2img"):
+        diffusion_params.num_inference_steps = args.steps  # type: ignore
+        diffusion_params.cfg_parallel_size = args.cfg_parallel_size  # type: ignore
+        if args.seed is not None:
+            diffusion_params.seed = args.seed  # type: ignore
+
+    extra = getattr(diffusion_params, "extra_args", {}) or {}
+    extra["cfg_text_scale"] = args.cfg_text_scale
+    extra["cfg_img_scale"] = args.cfg_img_scale
+    if args.cfg_interval is not None:
+        extra["cfg_interval"] = tuple(args.cfg_interval)
+    if args.cfg_renorm_type is not None:
+        extra["cfg_renorm_type"] = args.cfg_renorm_type
+    if args.cfg_renorm_min is not None:
+        extra["cfg_renorm_min"] = args.cfg_renorm_min
+    if args.negative_prompt is not None:
+        extra["negative_prompt"] = args.negative_prompt
+
+    needs_text_gen = is_single_stage and (args.think or args.modality in ("text2text", "img2text"))
+    if needs_text_gen:
+        if args.think:
+            extra["think"] = True
+        extra["max_think_tokens"] = args.max_think_tokens
+        extra["do_sample"] = args.do_sample
+        extra["text_temperature"] = args.text_temperature
+    diffusion_params.extra_args = extra  # type: ignore
+
+    omni_outputs = list(omni.generate(prompts=formatted_prompts, sampling_params_list=params_list))
+
+    img_idx = 0
+    for req_output in omni_outputs:
+        # 2-stage think mode: text output from thinker stage
+        ro = getattr(req_output, "request_output", None)
+        if ro and getattr(ro, "outputs", None):
+            txt = "".join(getattr(o, "text", "") or "" for o in ro.outputs)
+            if txt:
+                if args.think:
+                    print(f"[Think]\n{txt}")
+                else:
+                    print(f"[Output] Text:\n{txt}")
+
+        # Single-stage DiT: text from custom_output
+        custom = getattr(req_output, "_custom_output", {}) or {}
+        if custom.get("think_text"):
+            print(f"[Think]\n{custom['think_text']}")
+        if custom.get("text_output"):
+            print(f"[Output] Text:\n{custom['text_output']}")
+
+        images = getattr(req_output, "images", None)
+        if not images:
+            continue
+
+        for j, img in enumerate(images):
+            save_path = os.path.join(args.output, f"output_{img_idx}_{j}.png")
+            img.save(save_path)
+            print(f"[Output] Saved image to {save_path}")
+        img_idx += 1
+
+
+if __name__ == "__main__":
+    main()

examples/offline_inference/dreamzero/client_schedule.py

@@ -1,129 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-"""DreamZero AR client frame schedule aligned with upstream ``test_client_AR.py``."""
-
-from __future__ import annotations
-
-import logging
-from typing import Any
-
-import numpy as np
-
-# Frame schedule constants (matching upstream debug_inference.py / test_client_AR.py).
-ACTION_HORIZON = 24
-DEFAULT_ACTION_DIM = 8
-RELATIVE_OFFSETS = [-23, -16, -8, 0]
-DEFAULT_NUM_AR_CHUNKS = 15
-
-CAMERA_FILES = {
-    "observation/exterior_image_0_left": "exterior_image_1_left.mp4",
-    "observation/exterior_image_1_left": "exterior_image_2_left.mp4",
-    "observation/wrist_image_left": "wrist_image_left.mp4",
-}
-
-
-def build_frame_schedule(
-    total_frames: int,
-    num_chunks: int,
-    *,
-    logger: logging.Logger | None = None,
-) -> list[list[int]]:
-    """Build the frame index schedule for multi-frame chunks.
-
-    Args:
-        total_frames: Number of frames available in each camera video.
-        num_chunks: Number of 4-frame chunks to schedule after the initial frame.
-
-    Returns:
-        A list of frame-index lists. Each inner list has four indices. The
-        returned list may be shorter than ``num_chunks`` when the videos run
-        out of frames (upstream stops early instead of erroring).
-    """
-    chunks: list[list[int]] = []
-    current_frame = 23
-    for _ in range(num_chunks):
-        indices = [max(current_frame + offset, 0) for offset in RELATIVE_OFFSETS]
-        if indices[-1] >= total_frames:
-            if logger is not None:
-                logger.info(
-                    "Frame %s >= %s, stopping at %s chunks",
-                    indices[-1],
-                    total_frames,
-                    len(chunks),
-                )
-            break
-        chunks.append(indices)
-        current_frame += ACTION_HORIZON
-    return chunks
-
-
-def make_obs_from_video(
-    camera_frames: dict[str, np.ndarray],
-    frame_indices: list[int],
-    *,
-    prompt: str,
-    session_id: str,
-) -> dict[str, Any]:
-    """Build an observation dict from real video frames.
-
-    For one frame each image key is ``(H, W, 3)``. For four frames each key is
-    ``(4, H, W, 3)``.
-    """
-    obs: dict[str, Any] = {}
-    for camera_key, all_frames in camera_frames.items():
-        selected = all_frames[frame_indices]
-        obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected
-
-    obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
-    obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
-    obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
-    obs["prompt"] = prompt
-    obs["session_id"] = session_id
-    return obs
-
-
-def build_ar_observations(
-    camera_frames: dict[str, np.ndarray],
-    *,
-    prompt: str,
-    session_id: str,
-    num_chunks: int = DEFAULT_NUM_AR_CHUNKS,
-    repeat_chunk_observations: bool = False,
-    logger: logging.Logger | None = None,
-) -> list[dict[str, Any]]:
-    """Build the AR observation sequence used by upstream ``test_client_AR.py``.
-
-    Step 0 sends frame ``[0]``. Each subsequent step sends one 4-frame chunk.
-
-    ``num_chunks`` counts only the 4-frame chunks after the initial frame, not
-    the total number of inferences.
-    """
-    if num_chunks < 0:
-        raise ValueError("num_chunks must be non-negative")
-
-    total_frames = min(frames.shape[0] for frames in camera_frames.values())
-    observations = [
-        make_obs_from_video(
-            camera_frames,
-            [0],
-            prompt=prompt,
-            session_id=session_id,
-        )
-    ]
-
-    chunk_schedule = build_frame_schedule(total_frames, num_chunks, logger=logger)
-    if repeat_chunk_observations and chunk_schedule and len(chunk_schedule) < num_chunks:
-        while len(chunk_schedule) < num_chunks:
-            chunk_schedule.append(chunk_schedule[-1])
-
-    for frame_indices in chunk_schedule:
-        observations.append(
-            make_obs_from_video(
-                camera_frames,
-                frame_indices,
-                prompt=prompt,
-                session_id=session_id,
-            )
-        )
-    return observations

examples/offline_inference/dreamzero/export_prediction_video.py

@@ -11,11 +11,6 @@ from pathlib import Path
 import cv2
 import numpy as np
 import torch
-from client_schedule import (
-    CAMERA_FILES,
-    DEFAULT_NUM_AR_CHUNKS,
-    build_ar_observations,
-)
 from PIL import Image
 
 from vllm_omni import Omni
@@ -25,6 +20,13 @@ from vllm_omni.outputs import OmniRequestOutput
 WORKER_EXTENSION = "vllm_omni.diffusion.models.dreamzero.video_export_worker.DreamZeroVideoExportWorkerExtension"
 ASSET_REPO_ID = "YangshenDeng/vllm-omni-dreamzero-assets"
 DEFAULT_SESSION_PREFIX = "dreamzero-export"
+RELATIVE_OFFSETS = [-23, -16, -8, 0]
+ACTION_HORIZON = 24
+CAMERA_FILES = {
+    "observation/exterior_image_0_left": "exterior_image_1_left.mp4",
+    "observation/exterior_image_1_left": "exterior_image_2_left.mp4",
+    "observation/wrist_image_left": "wrist_image_left.mp4",
+}
 
 
 def _parse_args() -> argparse.Namespace:
@@ -47,17 +49,6 @@ def _parse_args() -> argparse.Namespace:
         default="Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan",
     )
     parser.add_argument("--session-id", default=None)
-    parser.add_argument(
-        "--num-chunks",
-        type=int,
-        default=DEFAULT_NUM_AR_CHUNKS,
-        help="Number of 4-frame chunks to send after the initial frame (default: 15).",
-    )
-    parser.add_argument(
-        "--repeat-chunk-observations",
-        action="store_true",
-        help="Repeat the last valid chunk observation when bundled assets run out of frames.",
-    )
     parser.add_argument("--save-input-video", action="store_true")
     parser.add_argument("--save-gif", action="store_true")
     parser.add_argument("--save-actions", action="store_true")
@@ -93,25 +84,57 @@ def _load_camera_frames(video_dir: Path) -> dict[str, np.ndarray]:
     return camera_frames
 
 
-def _build_observations(
-    video_dir: Path,
+def _build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
+    chunks: list[list[int]] = []
+    current_frame = 23
+    for _ in range(num_chunks):
+        indices = [max(current_frame + offset, 0) for offset in RELATIVE_OFFSETS]
+        if indices[-1] >= total_frames:
+            break
+        chunks.append(indices)
+        current_frame += ACTION_HORIZON
+    return chunks
+
+
+def _make_obs_from_video(
+    camera_frames: dict[str, np.ndarray],
+    frame_indices: list[int],
+    *,
     prompt: str,
     session_id: str,
-    *,
-    num_chunks: int,
-    repeat_chunk_observations: bool,
-) -> tuple[dict[str, np.ndarray], list[dict]]:
+) -> dict:
+    obs: dict = {}
+    for camera_key, all_frames in camera_frames.items():
+        selected = all_frames[frame_indices]
+        obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected
+
+    obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
+    obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
+    obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
+    obs["prompt"] = prompt
+    obs["session_id"] = session_id
+    return obs
+
+
+def _build_observations(video_dir: Path, prompt: str, session_id: str) -> tuple[dict[str, np.ndarray], list[dict]]:
     camera_frames = _load_camera_frames(video_dir)
-    observations = build_ar_observations(
-        camera_frames,
-        prompt=prompt,
-        session_id=session_id,
-        num_chunks=num_chunks,
-        repeat_chunk_observations=repeat_chunk_observations,
-    )
+    total_frames = min(frames.shape[0] for frames in camera_frames.values())
+    chunks = _build_frame_schedule(total_frames, 1)
+    observations = [
+        _make_obs_from_video(camera_frames, [0], prompt=prompt, session_id=session_id),
+    ]
+    if chunks:
+        observations.append(
+            _make_obs_from_video(
+                camera_frames,
+                chunks[0],
+                prompt=prompt,
+                session_id=session_id,
+            )
+        )
     if len(observations) < 2:
-        raise RuntimeError("Need at least two DreamZero observations to export a prediction video.")
-    return camera_frames, observations
+        raise RuntimeError("Need at least two DreamZero example observations to export a prediction video.")
+    return camera_frames, observations[:2]
 
 
 def _extract_latents(output: OmniRequestOutput) -> torch.Tensor:
@@ -206,26 +229,6 @@ def _run_generation(
     return omni, outputs
 
 
-def _decode_accumulated_with_worker(omni: Omni, session_id: str) -> np.ndarray:
-    """Decode server-side accumulated latents when the pipeline supports it."""
-    stage_client = omni.engine.stage_clients[0]
-    engine = getattr(stage_client, "_engine", None)
-    if engine is None:
-        raise RuntimeError("DreamZero export requires inline diffusion stage access.")
-
-    decoded = engine.executor.collective_rpc(
-        "decode_accumulated_video_latents_to_uint8",
-        args=(session_id,),
-        unique_reply_rank=0,
-        exec_all_ranks=True,
-    )
-    if isinstance(decoded, torch.Tensor):
-        decoded = decoded.numpy()
-    if not isinstance(decoded, np.ndarray):
-        raise TypeError(f"Unexpected decoded output type: {type(decoded)!r}")
-    return decoded
-
-
 def _decode_with_worker(omni: Omni, full_latents: torch.Tensor) -> np.ndarray:
     stage_client = omni.engine.stage_clients[0]
     engine = getattr(stage_client, "_engine", None)
@@ -245,15 +248,6 @@ def _decode_with_worker(omni: Omni, full_latents: torch.Tensor) -> np.ndarray:
     return decoded
 
 
-def _decode_video(omni: Omni, session_id: str, outputs: list[OmniRequestOutput]) -> np.ndarray:
-    """Decode full AR rollout by concatenating per-step latents from all outputs."""
-    latent_steps = [_extract_latents(output) for output in outputs]
-    latent_t = [step.shape[2] for step in latent_steps]
-    print(f"Decoding {len(latent_steps)} latent steps, latent_t={latent_t}, total_t={sum(latent_t)}")
-    full_latents = torch.cat(latent_steps, dim=2)
-    return _decode_with_worker(omni, full_latents)
-
-
 def main() -> None:
     args = _parse_args()
     session_id = args.session_id or f"{DEFAULT_SESSION_PREFIX}-{uuid.uuid4()}"
@@ -262,12 +256,6 @@ def main() -> None:
         video_dir=args.video_dir,
         prompt=args.prompt,
         session_id=session_id,
-        num_chunks=args.num_chunks,
-        repeat_chunk_observations=args.repeat_chunk_observations,
-    )
-    print(
-        f"Running {len(observations)} inferences "
-        f"(1 initial + {len(observations) - 1} chunks; requested {args.num_chunks} post-initial chunks)"
     )
 
     args.output_dir.mkdir(parents=True, exist_ok=True)
@@ -285,19 +273,17 @@ def main() -> None:
             deploy_config_path=args.deploy_config,
             observations=observations,
         )
-        frames = _decode_video(omni, session_id, outputs)
+        latent_steps = [_extract_latents(output) for output in outputs]
+        full_latents = torch.cat(latent_steps, dim=2)
+        frames = _decode_with_worker(omni, full_latents)
     finally:
         if omni is not None:
             omni.close()
 
-    expected_frames = 8 * len(observations) + 1
     mp4_path = args.output_dir / f"{args.output_stem}.mp4"
 
     _write_mp4(mp4_path, frames, fps=args.fps)
-    print(
-        f"SAVED_MP4={mp4_path} frames={len(frames)} "
-        f"expected≈{expected_frames} duration≈{len(frames) / max(args.fps, 1):.1f}s"
-    )
+    print(f"SAVED_MP4={mp4_path}")
 
     if args.save_gif:
         gif_path = args.output_dir / f"{args.output_stem}.gif"
@@ -308,10 +294,8 @@ def main() -> None:
         npz_path = args.output_dir / f"{args.output_stem}_actions.npz"
         np.savez(
             npz_path,
-            **{
-                f"step{index}": np.asarray(output.multimodal_output.get("actions"))
-                for index, output in enumerate(outputs)
-            },
+            step0=np.asarray(outputs[0].multimodal_output.get("actions")),
+            step1=np.asarray(outputs[1].multimodal_output.get("actions")),
         )
         print(f"SAVED_ACTIONS={npz_path}")
 

examples/offline_inference/helios/README.md

@@ -0,0 +1,193 @@
+# Helios Video Generation
+
+Helios is a text-to-video (T2V), image-to-video (I2V), and video-to-video (V2V) diffusion model. This example demonstrates end-to-end video generation using vLLM-Omni with three model variants:
+
+| Variant | Description | Key Features |
+|---------|-------------|--------------|
+| **Helios-Base** | Base model, Stage 1 only | Single-stage denoising, `guidance_scale=5.0` |
+| **Helios-Mid** | Mid model, Stage 2 pyramid | Multi-stage pyramid denoising, CFG-Zero* support |
+| **Helios-Distilled** | Distilled model, Stage 2+3 | Few-step inference with DMD, `guidance_scale=1.0` |
+
+## Setup
+
+Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
+
+## Run Examples
+
+Get into the example folder:
+```bash
+cd examples/offline_inference/helios
+```
+
+### Text-to-Video (T2V)
+
+**Helios-Base** (Stage 1 only):
+```bash
+python end2end.py \
+    --model BestWishYsh/Helios-Base \
+    --sample-type t2v \
+    --prompt "A dynamic time-lapse video showing the rapidly moving scenery from the window of a speeding train." \
+    --guidance-scale 5.0 \
+    --output helios_t2v_base.mp4
+```
+
+**Helios-Mid** (Stage 2 + CFG-Zero*):
+```bash
+python end2end.py \
+    --model BestWishYsh/Helios-Mid \
+    --sample-type t2v \
+    --prompt "A dynamic time-lapse video showing the rapidly moving scenery from the window of a speeding train." \
+    --guidance-scale 5.0 \
+    --is-enable-stage2 \
+    --pyramid-num-inference-steps-list 20 20 20 \
+    --use-cfg-zero-star --use-zero-init --zero-steps 1 \
+    --output helios_t2v_mid.mp4
+```
+
+**Helios-Distilled** (Stage 2 pyramid + DMD):
+```bash
+python end2end.py \
+    --model BestWishYsh/Helios-Distilled \
+    --sample-type t2v \
+    --prompt "A dynamic time-lapse video showing the rapidly moving scenery from the window of a speeding train." \
+    --num-frames 240 \
+    --guidance-scale 1.0 \
+    --is-enable-stage2 \
+    --pyramid-num-inference-steps-list 2 2 2 \
+    --is-amplify-first-chunk \
+    --output helios_t2v_distilled.mp4
+```
+
+### Image-to-Video (I2V)
+
+**Helios-Base**:
+```bash
+python end2end.py \
+    --model BestWishYsh/Helios-Base \
+    --sample-type i2v \
+    --image-path /path/to/image.jpg \
+    --prompt "A towering emerald wave surges forward, its crest curling with raw power and energy." \
+    --guidance-scale 5.0 \
+    --output helios_i2v_base.mp4
+```
+
+**Helios-Mid** (Stage 2 + CFG-Zero*):
+```bash
+python end2end.py \
+    --model BestWishYsh/Helios-Mid \
+    --sample-type i2v \
+    --image-path /path/to/image.jpg \
+    --prompt "A towering emerald wave surges forward, its crest curling with raw power and energy." \
+    --guidance-scale 5.0 \
+    --is-enable-stage2 \
+    --pyramid-num-inference-steps-list 20 20 20 \
+    --use-cfg-zero-star --use-zero-init --zero-steps 1 \
+    --output helios_i2v_mid.mp4
+```
+
+**Helios-Distilled**:
+```bash
+python end2end.py \
+    --model BestWishYsh/Helios-Distilled \
+    --sample-type i2v \
+    --image-path /path/to/image.jpg \
+    --prompt "A towering emerald wave surges forward, its crest curling with raw power and energy." \
+    --num-frames 240 \
+    --guidance-scale 1.0 \
+    --is-enable-stage2 \
+    --pyramid-num-inference-steps-list 2 2 2 \
+    --is-amplify-first-chunk \
+    --output helios_i2v_distilled.mp4
+```
+
+### Video-to-Video (V2V)
+
+**Helios-Base**:
+```bash
+python end2end.py \
+    --model BestWishYsh/Helios-Base \
+    --sample-type v2v \
+    --video-path /path/to/video.mp4 \
+    --prompt "A bright yellow Lamborghini speeds along a curving mountain road." \
+    --guidance-scale 5.0 \
+    --output helios_v2v_base.mp4
+```
+
+**Helios-Mid** (Stage 2 + CFG-Zero*):
+```bash
+python end2end.py \
+    --model BestWishYsh/Helios-Mid \
+    --sample-type v2v \
+    --video-path /path/to/video.mp4 \
+    --prompt "A bright yellow Lamborghini speeds along a curving mountain road." \
+    --guidance-scale 5.0 \
+    --is-enable-stage2 \
+    --pyramid-num-inference-steps-list 20 20 20 \
+    --use-cfg-zero-star --use-zero-init --zero-steps 1 \
+    --output helios_v2v_mid.mp4
+```
+
+**Helios-Distilled**:
+```bash
+python end2end.py \
+    --model BestWishYsh/Helios-Distilled \
+    --sample-type v2v \
+    --video-path /path/to/video.mp4 \
+    --prompt "A bright yellow Lamborghini speeds along a curving mountain road." \
+    --num-frames 240 \
+    --guidance-scale 1.0 \
+    --is-enable-stage2 \
+    --pyramid-num-inference-steps-list 2 2 2 \
+    --is-amplify-first-chunk \
+    --output helios_v2v_distilled.mp4
+```
+
+## Common Parameters
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `--model` | `BestWishYsh/Helios-Base` | Model ID or local path |
+| `--sample-type` | `t2v` | Generation mode: `t2v`, `i2v`, or `v2v` |
+| `--prompt` | — | Text prompt describing the video |
+| `--negative-prompt` | *(see source)* | Negative prompt for CFG (includes anti-static terms) |
+| `--image-path` | — | Input image (required for `i2v`) |
+| `--video-path` | — | Input video (required for `v2v`) |
+| `--height` | `384` | Video height in pixels |
+| `--width` | `640` | Video width in pixels |
+| `--num-frames` | `99` | Number of output frames |
+| `--num-inference-steps` | `50` | Denoising steps (Stage 1 only) |
+| `--guidance-scale` | `5.0` | CFG scale (`1.0` for Distilled) |
+| `--seed` | `42` | Random seed |
+| `--fps` | `16` | Output video frame rate |
+| `--output` | `helios_output.mp4` | Output file path |
+
+### Stage 2 / Pyramid Parameters
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `--is-enable-stage2` | off | Enable pyramid multi-stage denoising |
+| `--pyramid-num-stages` | `3` | Number of pyramid stages |
+| `--pyramid-num-inference-steps-list` | `10 10 10` | Steps per pyramid stage |
+| `--is-amplify-first-chunk` | off | DMD amplification (Distilled) |
+
+### CFG-Zero* Parameters
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `--use-cfg-zero-star` | off | Enable CFG-Zero* guidance (Mid) |
+| `--use-zero-init` | off | Zero init for first steps |
+| `--zero-steps` | `1` | Number of zero-init steps |
+
+### Memory & Parallelism
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `--vae-use-slicing` | off | Enable VAE slicing |
+| `--vae-use-tiling` | off | Enable VAE tiling |
+| `--enforce-eager` | off | Disable torch.compile |
+| `--enable-cpu-offload` | off | CPU offloading |
+| `--enable-layerwise-offload` | off | Layerwise offloading |
+| `--ulysses-degree` | `1` | Ulysses SP degree |
+| `--ring-degree` | `1` | Ring SP degree |
+| `--cfg-parallel-size` | `1` | CFG parallel size (1 or 2) |
+| `--tensor-parallel-size` | `1` | Tensor parallelism size |

examples/offline_inference/helios/end2end.py

@@ -0,0 +1,355 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""
+Helios video generation example supporting T2V, I2V, and V2V.
+
+Usage (T2V, Helios-Base, Stage 1 only):
+    python end2end.py \
+        --model /path/to/Helios-Base --sample-type t2v \
+        --prompt "A serene lakeside sunrise with mist over the water." \
+        --height 384 --width 640 --num-frames 99 \
+        --num-inference-steps 50 --guidance-scale 5.0
+
+Usage (I2V, Helios-Base):
+    python end2end.py \
+        --model /path/to/Helios-Base --sample-type i2v \
+        --image-path /path/to/image.jpg \
+        --prompt "Description of desired animation." \
+        --guidance-scale 5.0
+
+Usage (V2V, Helios-Base):
+    python end2end.py \
+        --model /path/to/Helios-Base --sample-type v2v \
+        --video-path /path/to/video.mp4 \
+        --prompt "Description of desired transformation." \
+        --guidance-scale 5.0
+
+Usage (Helios-Mid, Stage 2 + CFG-Zero*):
+    python end2end.py \
+        --model /path/to/Helios-Mid --sample-type t2v \
+        --prompt "A serene lakeside sunrise with mist over the water." \
+        --guidance-scale 5.0 \
+        --is-enable-stage2 \
+        --pyramid-num-inference-steps-list 20 20 20 \
+        --use-cfg-zero-star --use-zero-init --zero-steps 1
+
+Usage (Helios-Distilled, Stage 2 pyramid + DMD):
+    python end2end.py \
+        --model /path/to/Helios-Distilled --sample-type t2v \
+        --prompt "A serene lakeside sunrise with mist over the water." \
+        --guidance-scale 1.0 \
+        --is-enable-stage2 \
+        --pyramid-num-inference-steps-list 2 2 2 \
+        --is-amplify-first-chunk
+"""
+
+import argparse
+import time
+from pathlib import Path
+
+import numpy as np
+import torch
+
+from vllm_omni.diffusion.data import DiffusionParallelConfig
+from vllm_omni.entrypoints.omni import Omni
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+from vllm_omni.outputs import OmniRequestOutput
+from vllm_omni.platforms import current_omni_platform
+
+
+def load_image_as_tensor(image_path: str, height: int, width: int) -> torch.Tensor:
+    """Load an image and return a (1, C, H, W) tensor normalized to [-1, 1]."""
+    from PIL import Image
+    from torchvision import transforms
+
+    image = Image.open(image_path).convert("RGB").resize((width, height))
+    transform = transforms.Compose(
+        [
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
+        ]
+    )
+    return transform(image).unsqueeze(0)
+
+
+def load_video_as_tensor(video_path: str, height: int, width: int) -> torch.Tensor:
+    """Load a video and return a (1, C, T, H, W) tensor normalized to [-1, 1]."""
+    from torchvision import transforms
+    from torchvision.io import read_video
+
+    video_frames, _, _ = read_video(video_path, output_format="TCHW")
+    video_frames = video_frames.float() / 255.0
+
+    transform = transforms.Compose(
+        [
+            transforms.Resize((height, width), antialias=True),
+            transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
+        ]
+    )
+    video_frames = torch.stack([transform(f) for f in video_frames])
+    return video_frames.permute(1, 0, 2, 3).unsqueeze(0)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Generate a video with Helios (T2V / I2V / V2V).")
+    parser.add_argument(
+        "--model",
+        default="BestWishYsh/Helios-Base",
+        help="Helios model ID or local path (e.g. Helios-Base, Helios-Mid, Helios-Distilled).",
+    )
+    parser.add_argument(
+        "--sample-type",
+        choices=["t2v", "i2v", "v2v"],
+        default="t2v",
+        help="Generation mode: t2v (text-to-video), i2v (image-to-video), v2v (video-to-video).",
+    )
+    parser.add_argument("--prompt", default="A serene lakeside sunrise with mist over the water.", help="Text prompt.")
+    parser.add_argument(
+        "--negative-prompt",
+        default=(
+            "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, "
+            "images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, "
+            "incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, "
+            "misshapen limbs, fused fingers, still picture, messy background, three legs, many people "
+            "in the background, walking backwards"
+        ),
+        help="Negative prompt.",
+    )
+    parser.add_argument("--seed", type=int, default=42, help="Random seed.")
+    parser.add_argument("--guidance-scale", type=float, default=5.0, help="CFG scale.")
+    parser.add_argument("--height", type=int, default=384, help="Video height.")
+    parser.add_argument("--width", type=int, default=640, help="Video width.")
+    parser.add_argument("--num-frames", type=int, default=99, help="Number of video frames.")
+    parser.add_argument("--num-inference-steps", type=int, default=50, help="Sampling steps (Stage 1 only).")
+    parser.add_argument("--output", type=str, default="helios_output.mp4", help="Output video path.")
+    parser.add_argument("--fps", type=int, default=16, help="Frames per second for the output video.")
+
+    # I2V / V2V
+    parser.add_argument("--image-path", type=str, default=None, help="Input image path for I2V mode.")
+    parser.add_argument("--video-path", type=str, default=None, help="Input video path for V2V mode.")
+
+    # Stage 2 (pyramid multi-stage denoising)
+    parser.add_argument(
+        "--is-enable-stage2",
+        action="store_true",
+        help="Enable pyramid multi-stage denoising (Stage 2). Required for Helios-Distilled.",
+    )
+    parser.add_argument(
+        "--pyramid-num-stages",
+        type=int,
+        default=3,
+        help="Number of pyramid stages for Stage 2.",
+    )
+    parser.add_argument(
+        "--pyramid-num-inference-steps-list",
+        type=int,
+        nargs="+",
+        default=[10, 10, 10],
+        help="Inference steps per pyramid stage.",
+    )
+
+    # DMD
+    parser.add_argument(
+        "--is-amplify-first-chunk",
+        action="store_true",
+        help="Enable DMD amplification for the first chunk (Helios-Distilled).",
+    )
+
+    # CFG Zero Star
+    parser.add_argument(
+        "--use-cfg-zero-star",
+        action="store_true",
+        help="Enable CFG Zero Star guidance (recommended for Helios-Mid).",
+    )
+    parser.add_argument(
+        "--use-zero-init",
+        action="store_true",
+        help="Use zero initialization for the first denoising steps with CFG-Zero*.",
+    )
+    parser.add_argument(
+        "--zero-steps",
+        type=int,
+        default=1,
+        help="Number of initial denoising steps using zero prediction (default: 1).",
+    )
+    parser.add_argument(
+        "--enable-diffusion-pipeline-profiler",
+        action="store_true",
+        help="Enable diffusion pipeline profiler to display stage durations.",
+    )
+    parser.add_argument(
+        "--quantization",
+        type=str,
+        default=None,
+        help="Quantization method (e.g. 'fp8').",
+    )
+
+    # Memory & parallelism
+    parser.add_argument("--vae-use-slicing", action="store_true", help="Enable VAE slicing.")
+    parser.add_argument("--vae-use-tiling", action="store_true", help="Enable VAE tiling.")
+    parser.add_argument("--enforce-eager", action="store_true", help="Disable torch.compile.")
+    parser.add_argument("--enable-cpu-offload", action="store_true", help="Enable CPU offloading.")
+    parser.add_argument("--enable-layerwise-offload", action="store_true", help="Enable layerwise offloading.")
+    parser.add_argument("--ulysses-degree", type=int, default=1, help="Ulysses SP degree.")
+    parser.add_argument("--ring-degree", type=int, default=1, help="Ring SP degree.")
+    parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
+    parser.add_argument("--tensor-parallel-size", type=int, default=1, help="Tensor parallelism size.")
+
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
+
+    parallel_config = DiffusionParallelConfig(
+        ulysses_degree=args.ulysses_degree,
+        ring_degree=args.ring_degree,
+        cfg_parallel_size=args.cfg_parallel_size,
+        tensor_parallel_size=args.tensor_parallel_size,
+    )
+
+    omni = Omni(
+        model=args.model,
+        enable_layerwise_offload=args.enable_layerwise_offload,
+        vae_use_slicing=args.vae_use_slicing,
+        vae_use_tiling=args.vae_use_tiling,
+        enable_cpu_offload=args.enable_cpu_offload,
+        parallel_config=parallel_config,
+        enforce_eager=args.enforce_eager,
+        enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
+        quantization=args.quantization,
+    )
+
+    # Validate I2V / V2V arguments
+    if args.sample_type == "i2v" and not args.image_path:
+        raise ValueError("--image-path is required for I2V mode.")
+    if args.sample_type == "v2v" and not args.video_path:
+        raise ValueError("--video-path is required for V2V mode.")
+
+    # Build extra_args for Helios-specific parameters
+    extra_args = {}
+    if args.is_enable_stage2:
+        extra_args["is_enable_stage2"] = True
+        extra_args["pyramid_num_stages"] = args.pyramid_num_stages
+        extra_args["pyramid_num_inference_steps_list"] = args.pyramid_num_inference_steps_list
+    if args.is_amplify_first_chunk:
+        extra_args["is_amplify_first_chunk"] = True
+    if args.use_cfg_zero_star:
+        extra_args["use_cfg_zero_star"] = True
+    if args.use_zero_init:
+        extra_args["use_zero_init"] = True
+        extra_args["zero_steps"] = args.zero_steps
+
+    if args.sample_type == "i2v":
+        image_tensor = load_image_as_tensor(args.image_path, args.height, args.width)
+        extra_args["image"] = image_tensor
+    elif args.sample_type == "v2v":
+        video_tensor = load_video_as_tensor(args.video_path, args.height, args.width)
+        extra_args["video"] = video_tensor
+
+    # Print generation configuration
+    print(f"\n{'=' * 60}")
+    print("Helios Generation Configuration:")
+    print(f"  Model: {args.model}")
+    print(f"  Sample type: {args.sample_type.upper()}")
+    if args.sample_type == "i2v":
+        print(f"  Image: {args.image_path}")
+    elif args.sample_type == "v2v":
+        print(f"  Video: {args.video_path}")
+    print(f"  Prompt: {args.prompt[:80]}{'...' if len(args.prompt) > 80 else ''}")
+    print(f"  Video size: {args.width}x{args.height}, {args.num_frames} frames")
+    print(f"  Inference steps: {args.num_inference_steps}")
+    print(f"  Guidance scale: {args.guidance_scale}")
+    if args.is_enable_stage2:
+        print(f"  Stage 2: enabled (stages={args.pyramid_num_stages}, steps={args.pyramid_num_inference_steps_list})")
+        if args.is_amplify_first_chunk:
+            print("  DMD amplify first chunk: enabled")
+        if args.use_cfg_zero_star:
+            print(f"  CFG Zero Star: enabled (zero_init={args.use_zero_init}, zero_steps={args.zero_steps})")
+    else:
+        if args.use_cfg_zero_star:
+            print(f"  CFG Zero Star: enabled (zero_init={args.use_zero_init}, zero_steps={args.zero_steps})")
+        print("  Stage 2: disabled (Stage 1 only)")
+    print(f"{'=' * 60}\n")
+
+    generation_start = time.perf_counter()
+    frames = omni.generate(
+        {
+            "prompt": args.prompt,
+            "negative_prompt": args.negative_prompt,
+        },
+        OmniDiffusionSamplingParams(
+            height=args.height,
+            width=args.width,
+            generator=generator,
+            guidance_scale=args.guidance_scale,
+            num_inference_steps=args.num_inference_steps,
+            num_frames=args.num_frames,
+            extra_args=extra_args,
+        ),
+    )
+    generation_end = time.perf_counter()
+    generation_time = generation_end - generation_start
+
+    print(f"Total generation time: {generation_time:.4f} seconds ({generation_time * 1000:.2f} ms)")
+
+    # Extract video frames from OmniRequestOutput
+    if isinstance(frames, list) and len(frames) > 0:
+        first_item = frames[0]
+
+        if hasattr(first_item, "final_output_type"):
+            if first_item.final_output_type != "image":
+                raise ValueError(
+                    f"Unexpected output type '{first_item.final_output_type}', expected 'image' for video generation."
+                )
+
+            if hasattr(first_item, "is_pipeline_output") and first_item.is_pipeline_output:
+                inner_output = first_item.request_output
+                if isinstance(inner_output, OmniRequestOutput) and hasattr(inner_output, "images"):
+                    frames = inner_output.images[0] if inner_output.images else None
+                    if frames is None:
+                        raise ValueError("No video frames found in output.")
+            elif hasattr(first_item, "images") and first_item.images:
+                frames = first_item.images
+            else:
+                raise ValueError("No video frames found in OmniRequestOutput.")
+
+    # Unwrap batch list from postprocess_video: [numpy(T,H,W,C)] -> numpy(T,H,W,C)
+    if isinstance(frames, list) and len(frames) > 0 and isinstance(frames[0], np.ndarray):
+        frames = frames[0]
+
+    output_path = Path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        from diffusers.utils import export_to_video
+    except ImportError:
+        raise ImportError("diffusers is required for export_to_video.")
+
+    if isinstance(frames, torch.Tensor):
+        video_tensor = frames.detach().cpu()
+        if video_tensor.dim() == 5:
+            if video_tensor.shape[1] in (3, 4):
+                video_tensor = video_tensor[0].permute(1, 2, 3, 0)
+            else:
+                video_tensor = video_tensor[0]
+        elif video_tensor.dim() == 4 and video_tensor.shape[0] in (3, 4):
+            video_tensor = video_tensor.permute(1, 2, 3, 0)
+        if video_tensor.is_floating_point():
+            video_tensor = video_tensor.clamp(-1, 1) * 0.5 + 0.5
+        video_array = video_tensor.float().numpy()
+    elif isinstance(frames, np.ndarray):
+        video_array = frames
+    else:
+        video_array = frames
+
+    if isinstance(video_array, np.ndarray) and video_array.ndim == 5:
+        video_array = video_array[0]
+
+    export_to_video(video_array, str(output_path), fps=args.fps)
+    print(f"Saved generated video to {output_path}")
+
+
+if __name__ == "__main__":
+    main()

examples/offline_inference/image_to_image/image_edit.py

@@ -97,16 +97,9 @@ import torch
 from PIL import Image
 
 from vllm_omni.diffusion.data import DiffusionParallelConfig
-from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
 from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.entrypoints.openai.stage_params import clone_sampling_params
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-from vllm_omni.model_extras import (
-    build_image_to_image_prompt,
-    get_extra_body_params,
-    get_model_class_name,
-    should_init_extra_args_for_non_diffusion_stages,
-)
+from vllm_omni.outputs import OmniRequestOutput
 from vllm_omni.platforms import current_omni_platform
 
 
@@ -194,12 +187,6 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--guidance-scale-2", type=float, default=None, help="image guidance scale for image-to-image generation."
     )
-    parser.add_argument(
-        "--extra-args",
-        type=parse_profiler_config,
-        default=None,
-        help="JSON object copied to OmniDiffusionSamplingParams.extra_args, e.g. '{\"cfg_text_scale\": 4.0}'.",
-    )
     parser.add_argument(
         "--output",
         type=str,
@@ -481,8 +468,6 @@ def main():
         enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
         profiler_config=args.profiler_config,
     )
-    model_class_name = get_model_class_name(omni)
-    declared_extra_body_params = get_extra_body_params(model_class_name)
     print("Pipeline loaded")
 
     profiler_enabled = args.profiler_config is not None
@@ -493,8 +478,6 @@ def main():
     print(f"  Model: {args.model}")
     print(f"  Inference steps: {args.num_inference_steps}")
     print(f"  Cache backend: {args.cache_backend if args.cache_backend else 'None (no acceleration)'}")
-    if args.height is not None or args.width is not None:
-        print(f"  Output size: {args.width or 'auto'}x{args.height or 'auto'}")
     if isinstance(input_image, list):
         print(f"  Number of input images: {len(input_image)}")
         for idx, img in enumerate(input_image):
@@ -504,6 +487,7 @@ def main():
     print(
         f"  Parallel configuration: ulysses_degree={args.ulysses_degree}, ring_degree={args.ring_degree}, cfg_parallel_size={args.cfg_parallel_size}, tensor_parallel_size={args.tensor_parallel_size}, enable_expert_parallel: {args.enable_expert_parallel}"
     )
+    print(f"  Output image size: {args.width or 'default'}x{args.height or 'default'}")
     print(f"{'=' * 60}\n")
 
     generation_start = time.perf_counter()
@@ -512,20 +496,10 @@ def main():
         print("[Profiler] Starting profiling...")
         omni.start_profile()
 
-    prompt_dict = build_image_to_image_prompt(
-        model_class_name=model_class_name,
-        prompt=args.prompt,
-        negative_prompt=args.negative_prompt,
-        input_image=input_image,
-        height=args.height,
+    # Generate edited image
+    sampling_params = OmniDiffusionSamplingParams(
         width=args.width,
-    )
-
-    extra_args_from_cli = dict(args.extra_args or {})
-    if args.negative_prompt is not None:
-        extra_args_from_cli.setdefault("negative_prompt", args.negative_prompt)
-
-    diffusion_params = OmniDiffusionSamplingParams(
+        height=args.height,
         generator=generator,
         true_cfg_scale=args.cfg_scale,
         guidance_scale=args.guidance_scale,
@@ -533,44 +507,17 @@ def main():
         num_inference_steps=args.num_inference_steps,
         num_outputs_per_prompt=args.num_outputs_per_prompt,
         layers=args.layers,
-        resolution=args.resolution,
-        height=args.height,
-        width=args.width,
     )
-    if declared_extra_body_params:
-        apply_declared_extra_args(
-            diffusion_params,
-            declared_extra_body_params,
-            extra_args_from_cli,
-        )
-    else:
-        diffusion_params.extra_args.update({k: v for k, v in extra_args_from_cli.items() if v is not None})
-
-    # Build per-stage sampling params for multi-stage models
-    init_non_diffusion = should_init_extra_args_for_non_diffusion_stages(
-        model_class_name,
+    if args.resolution:
+        sampling_params.resolution = args.resolution
+    outputs = omni.generate(
+        {
+            "prompt": args.prompt,
+            "negative_prompt": args.negative_prompt,
+            "multi_modal_data": {"image": input_image},
+        },
+        sampling_params,
     )
-    defaults = list(omni.default_sampling_params_list or [])
-    sampling_params_list = [clone_sampling_params(p) for p in defaults]
-    if not sampling_params_list:
-        sampling_params_list = [diffusion_params]
-
-    diffusion_replaced = False
-    for idx, params in enumerate(sampling_params_list):
-        if isinstance(params, OmniDiffusionSamplingParams):
-            merged_extra = dict(getattr(params, "extra_args", {}) or {})
-            merged_extra.update(diffusion_params.extra_args)
-            diffusion_params.extra_args = merged_extra
-            sampling_params_list[idx] = diffusion_params
-            diffusion_replaced = True
-        elif init_non_diffusion and hasattr(params, "extra_args"):
-            if params.extra_args is None:
-                params.extra_args = {}
-
-    if not diffusion_replaced and len(sampling_params_list) == 1:
-        sampling_params_list = [diffusion_params]
-
-    outputs = omni.generate(prompt_dict, sampling_params_list=sampling_params_list)
     generation_end = time.perf_counter()
     generation_time = generation_end - generation_start
 
@@ -597,16 +544,17 @@ def main():
     if not outputs:
         raise ValueError("No output generated from omni.generate()")
 
-    images = None
-    for output in outputs:
-        images = getattr(output, "images", None)
-        if images:
-            break
-        req_out = getattr(output, "request_output", None)
-        images = getattr(req_out, "images", None) if req_out is not None else None
-        if images:
-            break
+    # Extract images from OmniRequestOutput
+    # omni.generate() returns list[OmniRequestOutput], extract images from request_output.images
+    first_output = outputs[0]
+    if not hasattr(first_output, "request_output") or not first_output.request_output:
+        raise ValueError("No request_output found in OmniRequestOutput")
 
+    req_out = first_output.request_output
+    if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"):
+        raise ValueError("Invalid request_output structure or missing 'images' key")
+
+    images = req_out.images
     if not images:
         raise ValueError("No images found in request_output")
 

examples/offline_inference/magi_human/README.md

@@ -0,0 +1,72 @@
+# MagiHuman Generation
+
+MagiHuman is an advanced, omni-modality model that generates both high-quality video and lip-synced audio from a text prompt.
+
+Because MagiHuman is a very large model featuring a powerful DiT MoE backbone and a ~9B parameter T5Gemma text encoder, it natively supports **Tensor Parallelism (TP)** in vLLM-Omni to run efficiently across multi-GPU setups, reducing device memory bottlenecks.
+
+## Setup
+
+### Install MagiCompiler (recommended)
+
+MagiHuman relies on [MagiCompiler](https://github.com/SandAI-org/MagiCompiler) for custom-op registration used by the DiT attention kernels. While the pipeline can fall back to stub implementations, installing MagiCompiler is **strongly recommended** for correct behaviour.
+
+```bash
+# Clone the repo
+git clone https://github.com/SandAI-org/MagiCompiler.git
+cd MagiCompiler
+
+# System dependencies (optional, for FX graph visualization; Debian/Ubuntu)
+sudo apt update && sudo apt install -y graphviz
+
+# Python dependencies
+pip install -r requirements.txt
+
+# Install MagiCompiler
+pip install .          # end users (recommended)
+# pip install -e .     # developers (editable install)
+```
+
+### Hardware requirements
+
+Ensure your hardware has enough VRAM. For a standard node with 80GB GPUs, running with `--tensor-parallel-size 4` is recommended to shard both the MoE weights and the T5Gemma text encoder across 4 GPUs, reducing the per-GPU peak VRAM overhead significantly (by roughly ~13.5GB per GPU compared to single-device inference).
+
+Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) for further details on allocating memory.
+
+## Run Examples
+
+Get into the example folder:
+```bash
+cd examples/offline_inference/magi_human
+```
+
+### End-to-End Generation (Text to Video+Audio)
+
+Generate a video with synchronized speech natively generated by the model.
+
+```bash
+python end2end.py \
+    --model /proj-tango-pvc/users/zhipeng.wang/workspace/models/daVinci-MagiHuman \
+    --prompt "A young woman with long, wavy golden blonde hair..." \
+    --tensor-parallel-size 4 \
+    --output output_magihuman.mp4
+```
+
+## Common Parameters
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `--model` | *(Required)* | Local model path or HuggingFace ID |
+| `--prompt` | *(built-in demo prompt)* | Highly detailed text prompt dictating visual look and dialogue text |
+| `--tensor-parallel-size` | `4` | Tensor parallelism size (Number of GPUs) |
+| `--height` | `256` | Initial resolution height |
+| `--width` | `448` | Initial resolution width |
+| `--num-inference-steps` | `8` | Denoising steps |
+| `--seed` | `52` | Random seed |
+| `--output` | `output_magihuman.mp4` | Output video with audio path |
+
+## Example materials
+
+??? abstract "end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/magi_human/end2end.py"
+    ``````

examples/offline_inference/magi_human/end2end.py

@@ -0,0 +1,122 @@
+import argparse
+
+from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes
+from vllm_omni.entrypoints.omni import Omni
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="End-to-end inference script for MagiHuman.")
+    parser.add_argument("--model", type=str, required=True, help="Path or ID of the MagiHuman model.")
+    parser.add_argument(
+        "--prompt",
+        type=str,
+        default="",
+        help="Text prompt containing visual description, dialogue, and background sound.",
+    )
+    parser.add_argument(
+        "--tensor-parallel-size", "-tp", type=int, default=4, help="Tensor parallel size (number of GPUs)."
+    )
+    parser.add_argument(
+        "--output", type=str, default="output_magihuman.mp4", help="Path to save the generated mp4 file."
+    )
+    parser.add_argument("--height", type=int, default=256, help="Video height.")
+    parser.add_argument("--width", type=int, default=448, help="Video width.")
+    parser.add_argument("--num-inference-steps", type=int, default=8, help="Number of denoising steps.")
+    parser.add_argument("--seed", type=int, default=52, help="Random seed for generation.")
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+
+    print(f"Initializing MagiHuman pipeline with TP={args.tensor_parallel_size}...")
+    omni = Omni(
+        model=args.model,
+        init_timeout=1200,
+        tensor_parallel_size=args.tensor_parallel_size,
+        devices=list(range(args.tensor_parallel_size)),
+    )
+
+    prompt = args.prompt
+    if not prompt:
+        prompt = (
+            "A young woman with long, wavy golden blonde hair and bright blue eyes, "
+            "wearing a fitted ivory silk blouse with a delicate lace collar, sits "
+            "stationary in front of a softly lit, blurred warm-toned interior. Her "
+            "overall disposition is warm, composed, and gently confident. The camera "
+            "holds a static medium close-up, framing her from the shoulders up, "
+            "with shallow depth of field keeping her face in sharp focus. Soft "
+            "directional key light falls from the upper left, casting a gentle "
+            "highlight along her cheekbone and nose bridge. She draws a quiet breath, "
+            "the levator labii superiors relaxing as her lips part. She speaks in "
+            "clear, warm, unhurried American English: "
+            "\"The most beautiful things in life aren't things at all — "
+            "they're moments, feelings, and the people who make you feel truly alive.\" "
+            "Her jaw descends smoothly on each stressed syllable; the orbicularis oris "
+            "shapes each vowel with precision. A faint, genuine smile engages the "
+            "zygomaticus major, lifting her lip corners fractionally. Her brows rest "
+            "in a soft, neutral arch throughout. She maintains steady, forward-facing "
+            "eye contact. Head position remains level; no torso displacement occurs.\n\n"
+            "Dialogue:\n"
+            "<Young blonde woman, American English>: "
+            "\"The most beautiful things in life aren't things at all — "
+            "they're moments, feelings, and the people who make you feel truly alive.\"\n\n"
+            "Background Sound:\n"
+            "<Soft, warm indoor ambience with a faint distant piano melody>"
+        )
+
+    sampling_params = OmniDiffusionSamplingParams(
+        height=args.height,
+        width=args.width,
+        num_inference_steps=args.num_inference_steps,
+        seed=args.seed,
+        extra_args={
+            "seconds": 5,
+            "sr_height": 1080,
+            "sr_width": 1920,
+            "sr_num_inference_steps": 5,
+        },
+    )
+
+    print(f"Generating with prompt: {prompt[:80]}...")
+    outputs = omni.generate(
+        prompts=[prompt],
+        sampling_params_list=[sampling_params],
+    )
+
+    print(f"Generation complete. Output type: {type(outputs)}")
+    if outputs:
+        first = outputs[0]
+
+        if hasattr(first, "images") and first.images:
+            video_frames = first.images[0]
+            print(f"Video frames: shape={video_frames.shape}, dtype={video_frames.dtype}")
+
+            audio_waveform = None
+            mm = first.multimodal_output or {}
+            if mm:
+                audio_waveform = mm.get("audio")
+                if audio_waveform is not None:
+                    print(f"Audio waveform: shape={audio_waveform.shape}, dtype={audio_waveform.dtype}")
+
+            output_fps = float(mm.get("fps", 25))
+            output_sr = int(mm.get("audio_sample_rate", 24000))
+            print(f"Using fps={output_fps}, audio_sample_rate={output_sr} from model output")
+
+            video_bytes = mux_video_audio_bytes(
+                video_frames,
+                audio_waveform,
+                fps=output_fps,
+                audio_sample_rate=output_sr,
+            )
+            with open(args.output, "wb") as f:
+                f.write(video_bytes)
+            print(f"Saved MP4 ({len(video_bytes)} bytes) to {args.output}")
+        print("SUCCESS: MagiHuman pipeline generation completed.")
+    else:
+        print("WARNING: No outputs returned.")
+
+
+if __name__ == "__main__":
+    main()

examples/offline_inference/sensenova_u1/README.md

@@ -145,7 +145,7 @@ python end2end.py \
 ## Reducing GPU Memory Usage
 
 For hardware with limited VRAM, enable **module-wise CPU offload** with
-`--enable-cpu-offload`. The pipeline implements `SupportsComponentDiscovery`, so the
+`--enable-cpu-offload`. The pipeline implements `SupportsModuleOffload`, so the
 vision encoder (`vision_model`) and the Qwen3 LLM (`language_model`) are
 swapped between CPU and GPU on demand:
 

examples/offline_inference/text_to_image/text_to_image.py

@@ -10,18 +10,10 @@ from typing import Any
 import torch
 
 from vllm_omni.diffusion.data import logger
-from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
 from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.entrypoints.openai.stage_params import clone_sampling_params
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 from vllm_omni.lora.request import LoRARequest
 from vllm_omni.lora.utils import stable_lora_int_id
-from vllm_omni.model_extras import (
-    build_text_to_image_prompt,
-    get_extra_body_params,
-    get_model_class_name,
-    should_init_extra_args_for_non_diffusion_stages,
-)
 from vllm_omni.platforms import current_omni_platform
 
 
@@ -267,19 +259,6 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="[NextStep-1.1 only] Apply layer normalization to sampled tokens.",
     )
-    parser.add_argument(
-        "--extra-body",
-        type=parse_profiler_config,
-        default=None,
-        help=(
-            "Model-specific generation params as a JSON object, e.g. "
-            '\'{"timestep_shift": 3.0, "cfg_text_scale": 4.0, "cfg_interval": [0.4, 1.0]}\'. '
-            "Each key is filtered against the model's declared extra_body_params "
-            "(see vllm_omni/model_extras), so unknown keys for the chosen model are "
-            "silently dropped. Values here take precedence over the equivalent "
-            "model-specific flags above."
-        ),
-    )
     parser.add_argument(
         "--enable-diffusion-pipeline-profiler",
         action="store_true",
@@ -426,8 +405,6 @@ def main():
         # NextStep-1.1 requires explicit pipeline class
         omni_kwargs["model_class_name"] = "NextStep11Pipeline"
     omni = Omni(**omni_kwargs)
-    model_class_name = get_model_class_name(omni)
-    declared_extra_body_params = get_extra_body_params(model_class_name)
 
     if profiler_enabled:
         print("[Profiler] Starting profiling...")
@@ -468,79 +445,34 @@ def main():
         )
 
     generation_start = time.perf_counter()
-
-    prompt_dict = build_text_to_image_prompt(
-        model_class_name=model_class_name,
-        prompt=args.prompt,
-        negative_prompt=args.negative_prompt,
-        height=args.height,
-        width=args.width,
-    )
-
-    diffusion_params = OmniDiffusionSamplingParams(
-        height=args.height,
-        width=args.width,
-        seed=args.seed,
-        generator=generator,
-        true_cfg_scale=args.cfg_scale,
-        guidance_scale=args.guidance_scale,
-        guidance_scale_2=args.guidance_scale_2,
-        num_inference_steps=args.num_inference_steps,
-        num_outputs_per_prompt=args.num_images_per_prompt,
-    )
-
-    # Base layer: backward-compatible model-specific flags. New model params should
-    # instead be declared in vllm_omni/model_extras and passed via --extra-body, so
-    # this dict does not need to grow per-model.
-    user_extra = {
-        "cfg_scale": args.cfg_scale,
-        "cfg_text_scale": args.cfg_scale,
-        "negative_prompt": args.negative_prompt,
-        "timestep_shift": args.timesteps_shift,
+    extra_args = {
         "timesteps_shift": args.timesteps_shift,
         "cfg_schedule": args.cfg_schedule,
         "use_norm": args.use_norm,
         "use_system_prompt": args.use_system_prompt,
         "system_prompt": args.system_prompt,
     }
-    # Override layer: generic JSON passthrough wins over the flags above. Keys are
-    # still filtered against the model's declared extra_body_params downstream.
-    if args.extra_body:
-        user_extra.update(args.extra_body)
-    if declared_extra_body_params:
-        apply_declared_extra_args(diffusion_params, declared_extra_body_params, user_extra)
-    else:
-        diffusion_params.extra_args.update({k: v for k, v in user_extra.items() if v is not None})
-
     if lora_request:
-        diffusion_params.extra_args["lora_request"] = lora_request
-        diffusion_params.extra_args["lora_scale"] = args.lora_scale
+        extra_args["lora_request"] = lora_request
+        extra_args["lora_scale"] = args.lora_scale
 
-    # Build per-stage sampling params for multi-stage models (e.g. BAGEL),
-    # or wrap single diffusion params for single-stage models.
-    init_non_diffusion = should_init_extra_args_for_non_diffusion_stages(
-        model_class_name,
+    outputs = omni.generate(
+        {
+            "prompt": args.prompt,
+            "negative_prompt": args.negative_prompt,
+        },
+        OmniDiffusionSamplingParams(
+            height=args.height,
+            width=args.width,
+            generator=generator,
+            true_cfg_scale=args.cfg_scale,
+            guidance_scale=args.guidance_scale,
+            guidance_scale_2=args.guidance_scale_2,
+            num_inference_steps=args.num_inference_steps,
+            num_outputs_per_prompt=args.num_images_per_prompt,
+            extra_args=extra_args,
+        ),
     )
-    defaults = list(omni.default_sampling_params_list or [])
-    sampling_params_list = [clone_sampling_params(p) for p in defaults]
-    if not sampling_params_list:
-        sampling_params_list = [diffusion_params]
-
-    diffusion_replaced = False
-    for idx, params in enumerate(sampling_params_list):
-        if isinstance(params, OmniDiffusionSamplingParams):
-            sampling_params_list[idx] = diffusion_params
-            diffusion_replaced = True
-        elif init_non_diffusion and hasattr(params, "extra_args"):
-            if params.extra_args is None:
-                params.extra_args = {}
-            if args.seed is not None and hasattr(params, "seed"):
-                params.seed = args.seed
-
-    if not diffusion_replaced and len(sampling_params_list) == 1:
-        sampling_params_list = [diffusion_params]
-
-    outputs = omni.generate(prompt_dict, sampling_params_list=sampling_params_list)
 
     generation_end = time.perf_counter()
     generation_time = generation_end - generation_start
@@ -570,16 +502,15 @@ def main():
         raise ValueError("No output generated from omni.generate()")
     logger.info(f"Outputs: {outputs}")
 
-    images = None
-    for output in outputs:
-        images = getattr(output, "images", None)
-        if images:
-            break
-        req_out = getattr(output, "request_output", None)
-        images = getattr(req_out, "images", None) if req_out is not None else None
-        if images:
-            break
+    first_output = outputs[0]
+    if not hasattr(first_output, "request_output") or not first_output.request_output:
+        raise ValueError("No request_output found in OmniRequestOutput")
 
+    req_out = first_output.request_output
+    if not hasattr(req_out, "images"):
+        raise ValueError("Invalid request_output structure or missing 'images'.")
+
+    images = req_out.images
     if not images:
         raise ValueError("No images found in request_output")
 

examples/offline_inference/text_to_speech/README.md

@@ -369,20 +369,6 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
 ```
 Streaming requires `async_chunk: true` in the stage config.
 
-### Word Timestamps
-Generate a WAV offline and a JSON sidecar with word-level timestamps from
-`Qwen/Qwen3-ForcedAligner-0.6B`:
-```bash
-python examples/offline_inference/text_to_speech/qwen3_tts/word_timestamps.py \
-    --model Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
-    --forced-aligner Qwen/Qwen3-ForcedAligner-0.6B \
-    --text "Hello world." \
-    --output-dir /tmp/qwen3_tts_timestamps
-```
-The script writes `qwen3_tts_word_timestamps.wav` and
-`qwen3_tts_word_timestamps.json`. On machines without a local CUDA toolkit,
-set `VLLM_USE_FLASHINFER_SAMPLER=0` to avoid FlashInfer sampler JIT.
-
 ### Batched decoding
 The Code2Wav stage supports batched decoding through the SpeechTokenizer. Pass multiple prompts via `--txt-prompts` and set `--batch-size` accordingly. To raise `max_num_seqs` on either stage, point `--stage-configs-path` at a stage configs YAML with the desired values (see `vllm_omni/model_executor/stage_configs/` for templates):
 ```bash

examples/offline_inference/text_to_speech/qwen3_tts/word_timestamps.py

@@ -1,172 +0,0 @@
-"""Offline Qwen3-TTS example with word-level timestamps.
-
-This demo runs Qwen3-TTS offline, aligns the synthesized audio in memory with
-the same shared forced aligner used by the streaming server path, then writes
-the WAV and JSON sidecar:
-
-    python examples/offline_inference/text_to_speech/qwen3_tts/word_timestamps.py \
-        --model Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
-        --forced-aligner Qwen/Qwen3-ForcedAligner-0.6B
-
-On machines without a local CUDA toolkit, set ``VLLM_USE_FLASHINFER_SAMPLER=0``
-to avoid FlashInfer sampler JIT during warmup.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import json
-import os
-from pathlib import Path
-from typing import Any
-
-os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
-# The in-process offline aligner consumes token_classify outputs. Keeping V1
-# multiprocessing off avoids msgspec serialization issues for token-wise logits.
-os.environ.setdefault("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
-
-import numpy as np
-import soundfile as sf
-import torch
-from end2end import _estimate_prompt_len
-from vllm.utils.argparse_utils import FlexibleArgumentParser
-
-from vllm_omni import Omni
-from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
-from vllm_omni.utils.forced_aligner import align, build_forced_aligner_config
-
-
-def _default_stage_config() -> str:
-    repo_root = Path(__file__).resolve().parents[4]
-    return str(repo_root / "vllm_omni" / "deploy" / "qwen3_tts.yaml")
-
-
-def _build_custom_voice_input(args: Any) -> dict[str, Any]:
-    additional_information = {
-        "task_type": ["CustomVoice"],
-        "text": [args.text],
-        "language": [args.language],
-        "speaker": [args.speaker],
-        "instruct": [args.instructions],
-        "max_new_tokens": [args.max_new_tokens],
-    }
-    return {
-        "prompt_token_ids": [0] * _estimate_prompt_len(additional_information, args.model),
-        "additional_information": additional_information,
-    }
-
-
-def _audio_tensor_and_sample_rate(mm: dict[str, Any]) -> tuple[torch.Tensor, int]:
-    audio_data = mm["audio"]
-    sr_raw = mm["sr"]
-    sr_val = sr_raw[-1] if isinstance(sr_raw, list) and sr_raw else sr_raw
-    sample_rate = int(sr_val.item()) if hasattr(sr_val, "item") else int(sr_val)
-    audio_tensor = torch.cat(audio_data, dim=-1) if isinstance(audio_data, list) else audio_data
-    return audio_tensor.float().detach().cpu().flatten(), sample_rate
-
-
-def _float_audio_to_pcm16_bytes(audio: torch.Tensor) -> bytes:
-    samples = audio.numpy()
-    pcm = (np.clip(samples, -1.0, 1.0) * 32767.0).astype("<i2")
-    return pcm.tobytes()
-
-
-async def _run_alignment(args: Any, audio: torch.Tensor, sample_rate: int) -> list[dict[str, Any]] | None:
-    aligner_config = build_forced_aligner_config(args)
-    if aligner_config is None:
-        raise ValueError("--forced-aligner or --forced-aligner-config is required")
-    timestamps = await align(
-        audio=_float_audio_to_pcm16_bytes(audio),
-        text=args.text,
-        sample_rate=sample_rate,
-        config=aligner_config,
-        language=args.language,
-    )
-    if timestamps is None:
-        return None
-    return [
-        {
-            "word": item.word,
-            "start_ms": item.start_ms,
-            "end_ms": item.end_ms,
-        }
-        for item in timestamps
-    ]
-
-
-def main(args: Any) -> None:
-    output_dir = Path(args.output_dir)
-    output_dir.mkdir(parents=True, exist_ok=True)
-
-    omni_kwargs = vars(args).copy()
-    for key in (
-        "forced_aligner",
-        "text",
-        "language",
-        "speaker",
-        "instructions",
-        "max_new_tokens",
-        "output_dir",
-    ):
-        omni_kwargs.pop(key, None)
-    omni_kwargs["stage_configs_path"] = args.stage_configs_path or _default_stage_config()
-    omni_kwargs["log_stats"] = args.log_stats
-    omni = Omni(**omni_kwargs)
-
-    prompt = _build_custom_voice_input(args)
-    final_output = None
-    for stage_outputs in omni.generate([prompt]):
-        final_output = stage_outputs.request_output
-    if final_output is None:
-        raise RuntimeError("Qwen3-TTS did not produce an output.")
-
-    mm = final_output.outputs[0].multimodal_output
-    audio, sample_rate = _audio_tensor_and_sample_rate(mm)
-
-    wav_path = output_dir / "qwen3_tts_word_timestamps.wav"
-    # Align the in-memory tensor, not the saved WAV. The WAV is only a demo
-    # artifact so users can listen to the same audio referenced by the sidecar.
-    timestamps = asyncio.run(_run_alignment(args, audio, sample_rate))
-    sf.write(wav_path, audio.numpy(), samplerate=sample_rate, format="WAV")
-
-    sidecar = {
-        "text": args.text,
-        "sample_rate": sample_rate,
-        "audio_path": str(wav_path),
-        "timestamps": timestamps,
-    }
-    json_path = output_dir / "qwen3_tts_word_timestamps.json"
-    json_path.write_text(json.dumps(sidecar, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
-
-    print(f"Saved audio: {wav_path}")
-    print(f"Saved timestamps: {json_path}")
-    print(json.dumps(sidecar["timestamps"], ensure_ascii=False, indent=2))
-
-
-def parse_args() -> Any:
-    parser = FlexibleArgumentParser(description="Offline Qwen3-TTS word timestamp example")
-    parser.add_argument("--model", default="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", help="Qwen3-TTS model path/name")
-    parser.add_argument(
-        "--forced-aligner",
-        default=None,
-        help="Qwen3 forced aligner model path/name",
-    )
-    parser.add_argument(
-        "--forced-aligner-config",
-        default=None,
-        help="Optional YAML file for forced aligner settings (incl. gpu_memory_utilization)",
-    )
-    parser.add_argument("--stage-configs-path", default=None, help="Qwen3-TTS deploy YAML")
-    parser.add_argument("--text", default="Hello world.", help="Text to synthesize and align")
-    parser.add_argument("--language", default="English", help="Qwen3-TTS language field")
-    parser.add_argument("--speaker", default="Vivian", help="CustomVoice speaker name")
-    parser.add_argument("--instructions", default="", help="Optional speaking style instruction")
-    parser.add_argument("--max-new-tokens", type=int, default=2048, help="TTS max_new_tokens")
-    parser.add_argument("--output-dir", default="output_audio", help="Directory for WAV and JSON sidecar")
-    parser.add_argument("--log-stats", action="store_true", default=False, help="Enable vLLM-Omni stats logging")
-    nullify_stage_engine_defaults(parser)
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    main(parse_args())

examples/offline_inference/text_to_video/text_to_video.md

@@ -9,7 +9,6 @@ A unified script for text-to-video generation. Supports multiple models with mod
 | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB |
 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB |
 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required |
-| `BestWishYsh/Helios-Base` / `Helios-Mid` / `Helios-Distilled` | 384x640 | 99 | 50 | 5.0 / 5.0 / 1.0 | — |
 
 ## Local CLI Usage
 
@@ -101,49 +100,6 @@ python text_to_video.py \
   --output quick_test.mp4
 ```
 
-### Helios (T2V)
-
-Helios ships three variants. Model-specific knobs (declared in
-`vllm_omni/model_extras/helios.py`) are passed via the generic `--extra-body`
-JSON flag rather than bespoke per-model flags.
-
-**Helios-Base** (Stage 1 only):
-
-```bash
-python text_to_video.py \
-  --model BestWishYsh/Helios-Base \
-  --prompt "A dynamic time-lapse of scenery rushing past the window of a speeding train." \
-  --guidance-scale 5.0 \
-  --output helios_t2v_base.mp4
-```
-
-**Helios-Mid** (Stage 2 pyramid + CFG-Zero*):
-
-```bash
-python text_to_video.py \
-  --model BestWishYsh/Helios-Mid \
-  --prompt "A dynamic time-lapse of scenery rushing past the window of a speeding train." \
-  --guidance-scale 5.0 \
-  --extra-body '{"is_enable_stage2": true, "pyramid_num_inference_steps_list": [20, 20, 20], "use_cfg_zero_star": true, "use_zero_init": true, "zero_steps": 1}' \
-  --output helios_t2v_mid.mp4
-```
-
-**Helios-Distilled** (Stage 2 pyramid + DMD, few-step):
-
-```bash
-python text_to_video.py \
-  --model BestWishYsh/Helios-Distilled \
-  --prompt "A dynamic time-lapse of scenery rushing past the window of a speeding train." \
-  --num-frames 240 \
-  --guidance-scale 1.0 \
-  --extra-body '{"is_enable_stage2": true, "pyramid_num_inference_steps_list": [2, 2, 2], "is_amplify_first_chunk": true}' \
-  --output helios_t2v_distilled.mp4
-```
-
-> Helios image-to-video (I2V) and video-to-video (V2V) require image/video
-> conditioning tensors that cannot be passed through the JSON `--extra-body`
-> flag; they are out of scope for this text-to-video example.
-
 ## Key Arguments
 
 ### Common
@@ -156,7 +112,6 @@ python text_to_video.py \
 - `--num-inference-steps`: sampling steps. Default depends on model.
 - `--fps`: frames per second for the saved MP4.
 - `--output`: path to save the generated video.
-- `--extra-body`: JSON dict of model-specific knobs (declared in `vllm_omni/model_extras/`), merged into sampling `extra_args`. See the Helios recipes above.
 - `--vae-use-slicing`: enable VAE slicing for memory optimization.
 - `--vae-use-tiling`: enable VAE tiling for memory optimization.
 - `--cfg-parallel-size`: set it to 2 to enable CFG Parallel. See more examples in [`user_guide`](../../../docs/user_guide/diffusion/parallelism_acceleration.md#cfg-parallel).

examples/offline_inference/text_to_video/text_to_video.py

@@ -35,15 +35,6 @@ _MODEL_PRESETS = {
         "fps": 24,
         "output": "hunyuan_video_15_output.mp4",
     },
-    "helios": {
-        "height": 384,
-        "width": 640,
-        "num_frames": 99,
-        "num_inference_steps": 50,
-        "guidance_scale": 5.0,
-        "fps": 16,
-        "output": "helios_output.mp4",
-    },
 }
 
 
@@ -51,8 +42,6 @@ def _detect_preset(model: str) -> dict:
     model_lower = model.lower()
     if "hunyuan" in model_lower:
         return _MODEL_PRESETS["hunyuan"]
-    if "helios" in model_lower:
-        return _MODEL_PRESETS["helios"]
     return _MODEL_PRESETS["wan"]
 
 
@@ -66,26 +55,10 @@ def parse_profiler_config(value: str) -> dict[str, Any]:
     return config
 
 
-def parse_extra_body(value: str) -> dict[str, Any]:
-    """Parse a JSON object of model-specific extra_body params.
-
-    Pipeline-declared knobs (see vllm_omni/model_extras/) are merged into
-    OmniDiffusionSamplingParams.extra_args, so a single generic example can
-    drive model-specific behaviour without bespoke per-model flags.
-    """
-    try:
-        body = json.loads(value)
-    except json.JSONDecodeError as e:
-        raise argparse.ArgumentTypeError(f"--extra-body must be valid JSON: {e}") from e
-    if not isinstance(body, dict):
-        raise argparse.ArgumentTypeError("--extra-body must be a JSON object")
-    return body
-
-
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(
         description="Generate a video from a text prompt. "
-        "Supports Wan2.2, HunyuanVideo-1.5, Helios, and other text-to-video models."
+        "Supports Wan2.2, HunyuanVideo-1.5, and other text-to-video models."
     )
     parser.add_argument(
         "--model",
@@ -101,14 +74,6 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument("--prompt", default="A serene lakeside sunrise with mist over the water.", help="Text prompt.")
     parser.add_argument("--negative-prompt", default="", help="Negative prompt.")
-    parser.add_argument(
-        "--extra-body",
-        type=parse_extra_body,
-        default=None,
-        help="JSON dict of model-specific extra_body params (declared in vllm_omni/model_extras/), "
-        "merged into sampling extra_args. Example (Helios-Distilled): "
-        '\'{"is_enable_stage2": true, "pyramid_num_inference_steps_list": [2, 2, 2], "is_amplify_first_chunk": true}\'.',
-    )
     parser.add_argument("--seed", type=int, default=42, help="Random seed.")
     parser.add_argument("--guidance-scale", type=float, default=None, help="CFG scale. Default: model-specific.")
     parser.add_argument(
@@ -392,9 +357,6 @@ def main():
     )
     if args.guidance_scale_high is not None:
         sampling_kwargs["guidance_scale_2"] = args.guidance_scale_high
-    if args.extra_body:
-        # Model-specific knobs (declared in vllm_omni/model_extras/) routed via extra_args.
-        sampling_kwargs["extra_args"] = dict(args.extra_body)
 
     generation_start = time.perf_counter()
     frames = omni.generate(

examples/offline_inference/x_to_video_audio/x_to_video_audio.md

@@ -87,63 +87,6 @@ python x_to_video_audio.py \
 ```
 
 
-## MagiHuman (`--model-type magi-human`)
-
-MagiHuman is a text → video+audio model with a DiT MoE backbone and a ~9B-param
-T5Gemma text encoder. A detailed text prompt is the only required input; an optional
-image and/or audio file may be supplied for conditioning. Natively supports Tensor
-Parallelism. For an 80GB node, `--tensor-parallel-size 4` is recommended to shard
-the MoE weights and the text encoder.
-
-> Install [MagiCompiler](https://github.com/SandAI-org/MagiCompiler) for correct
-> attention-kernel behaviour (the pipeline otherwise falls back to stubs).
-
-Text-only generation:
-
-```bash
-python x_to_video_audio.py \
-  --model-type magi-human \
-  --model /path/to/daVinci-MagiHuman \
-  --prompt "A young woman with long, wavy golden blonde hair... <dialogue and background sound>" \
-  --tensor-parallel-size 4 \
-  --height 256 --width 448 \
-  --num-inference-steps 8 \
-  --seed 52 \
-  --extra-body '{"seconds": 5, "sr_height": 1080, "sr_width": 1920, "sr_num_inference_steps": 5}' \
-  --output output_magihuman.mp4
-```
-
-With optional image and audio conditioning:
-
-```bash
-python x_to_video_audio.py \
-  --model-type magi-human \
-  --model /path/to/daVinci-MagiHuman \
-  --prompt "A young woman..." \
-  --tensor-parallel-size 4 \
-  --height 256 --width 448 \
-  --num-inference-steps 8 \
-  --seed 52 \
-  --extra-body '{"image_path": "/path/to/ref.jpg", "audio_path": "/path/to/ref.wav", "sr_height": 1080, "sr_width": 1920, "sr_num_inference_steps": 5}' \
-  --output output_magihuman.mp4
-```
-
-MagiHuman-specific arguments are passed as a JSON dict via `--extra-body` (declared
-in `vllm_omni/model_extras/magi_human.py`, routed via `extra_args`):
-
-- `seconds`: output duration in seconds (default 10; ignored when `audio_path` is set,
-  because the audio length then determines the number of frames).
-- `audio_path`: path to an audio file for audio-to-video conditioning. When provided,
-  the audio drives the frame count and is encoded as a latent condition into the DiT
-  (`is_a2v` mode). Omit for pure text-to-video+audio generation.
-- `image_path`: path to an image file for visual conditioning. Applied at both the
-  base-resolution (BR) and super-resolution (SR) stages when SR is enabled.
-- `sr_height` / `sr_width`: super-resolution output resolution. SR stage is skipped
-  when these are omitted.
-- `sr_num_inference_steps`: denoising steps for the SR stage.
-
-## DreamID-Omni arguments
-
 Key arguments:
 - `--prompt`: text description (string).
 - `--model`: path to the model local directory.

examples/offline_inference/x_to_video_audio/x_to_video_audio.py

@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 import argparse
-import json
 import re
 import time
 from typing import Any
@@ -21,26 +20,14 @@ AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".m4a", ".aac", ".ogg"}
 
 
 def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Offline inference for X -> video+audio models.")
-    parser.add_argument("--model", required=True, help="Model ckpt root directory.")
-    parser.add_argument(
-        "--model-type",
-        default="dreamid-omni",
-        choices=["dreamid-omni", "magi-human"],
-        help="Model type.",
-    )
+    parser = argparse.ArgumentParser(description="Offline inference for DreamID-Omni (video + audio).")
+    parser.add_argument("--model", required=True, help="DreamID ckpt root directory.")
+    parser.add_argument("--model-type", default="dreamid-omni", help="Model type.")
     parser.add_argument("--prompt", default=None, help="Text prompt.")
 
     parser.add_argument("--image-path", type=str, nargs="+", help="list of image-path")
     parser.add_argument("--audio-path", type=str, nargs="+", help="list of audio-path")
     parser.add_argument("--prompt-file", type=str, default=None, help="Text prompt in json format.")
-    parser.add_argument(
-        "--extra-body",
-        type=str,
-        default=None,
-        help="[magi-human] JSON dict of model-specific extra params (declared in vllm_omni/model_extras/), "
-        'merged into sampling extra_args. Example: \'{"image_path": "/path/to/img.jpg", "seconds": 5}\'.',
-    )
 
     parser.add_argument("--height", type=int, default=704, help="Video height.")
     parser.add_argument("--width", type=int, default=1280, help="Video width.")
@@ -155,6 +142,8 @@ def main() -> None:
 
     text_prompt = args.prompt
     if args.prompt_file:
+        import json
+
         with open(args.prompt_file) as f:
             text_prompt = json.load(f)
             text_prompt = re.sub(
@@ -166,36 +155,25 @@ def main() -> None:
             text_prompt = re.sub(r"\[[A-Z_]+\]", "", text_prompt)
             text_prompt = re.sub(r"\n\s*\n", "\n", text_prompt).strip()
 
-    if args.model_type == "magi-human":
-        prompt = text_prompt
-        extra_args = json.loads(args.extra_body) if args.extra_body else {}
-        sampling_params = OmniDiffusionSamplingParams(
-            height=args.height,
-            width=args.width,
-            num_inference_steps=args.num_inference_steps,
-            seed=args.seed,
-            extra_args=extra_args,
-        )
-    else:
-        image, audio = load_image_and_audio(args.image_path, args.audio_path)
+    image, audio = load_image_and_audio(args.image_path, args.audio_path)
 
-        prompt = {
-            "prompt": text_prompt,
-            "video_negative_prompt": args.video_negative_prompt,
-            "audio_negative_prompt": args.audio_negative_prompt,
-            "multi_modal_data": {"image": image, "audio": audio},
-        }
+    prompt = {
+        "prompt": text_prompt,
+        "video_negative_prompt": args.video_negative_prompt,
+        "audio_negative_prompt": args.audio_negative_prompt,
+        "multi_modal_data": {"image": image, "audio": audio},
+    }
 
-        sampling_params = OmniDiffusionSamplingParams(
-            height=args.height,
-            width=args.width,
-            num_inference_steps=args.num_inference_steps,
-            seed=args.seed,
-            extra_args={
-                "solver_name": args.solver_name,
-                "shift": args.shift,
-            },
-        )
+    sampling_params = OmniDiffusionSamplingParams(
+        height=args.height,
+        width=args.width,
+        num_inference_steps=args.num_inference_steps,
+        seed=args.seed,
+        extra_args={
+            "solver_name": args.solver_name,
+            "shift": args.shift,
+        },
+    )
 
     parallel_config = DiffusionParallelConfig(
         cfg_parallel_size=args.cfg_parallel_size,
@@ -237,28 +215,22 @@ def main() -> None:
     elapsed = time.perf_counter() - start
 
     if not outputs:
-        raise RuntimeError("No output returned from the model.")
+        raise RuntimeError("No output returned from DreamID-Omni.")
     result = outputs[0]
     if not result.images:
-        raise RuntimeError("No video frames found in model output.")
+        raise RuntimeError("No video frames found in DreamID-Omni output.")
     generated_video = result.images[0]
     mm = result.multimodal_output or {}
     generated_audio = mm.get("audio")
+    fps = int(mm.get("fps", 24))
+    sample_rate = int(mm.get("audio_sample_rate", 16000))
 
-    if args.model_type == "magi-human":
-        # MagiHuman returns frames already as (F, H, W, C) uint8, ready to mux.
-        fps = float(mm.get("fps", 25))
-        sample_rate = int(mm.get("audio_sample_rate", 24000))
-        frames = generated_video
-    else:
-        fps = float(mm.get("fps", 24))
-        sample_rate = int(mm.get("audio_sample_rate", 16000))
-        # DreamID-Omni returns video as (C, F, H, W) float32 in [-1, 1].
-        # mux_video_audio_bytes expects (F, H, W, C) uint8.
-        if not isinstance(generated_video, np.ndarray) or generated_video.ndim != 4:
-            raise RuntimeError(f"Unexpected video shape: {getattr(generated_video, 'shape', None)}")
-        frames = generated_video.transpose(1, 2, 3, 0)
-        frames = (np.clip((frames + 1.0) / 2.0, 0.0, 1.0) * 255.0).round().astype(np.uint8)
+    # DreamID-Omni returns video as (C, F, H, W) float32 in [-1, 1].
+    # mux_video_audio_bytes expects (F, H, W, C) uint8.
+    if not isinstance(generated_video, np.ndarray) or generated_video.ndim != 4:
+        raise RuntimeError(f"Unexpected video shape: {getattr(generated_video, 'shape', None)}")
+    frames = generated_video.transpose(1, 2, 3, 0)
+    frames = (np.clip((frames + 1.0) / 2.0, 0.0, 1.0) * 255.0).round().astype(np.uint8)
 
     audio_np = None
     if generated_audio is not None:

examples/online_serving/aura_omni/README.md

@@ -1,146 +0,0 @@
-# AURA Omni Native Pipeline
-
-`aura_omni` serves AURA as a native multi-stage vLLM-Omni pipeline:
-
-```text
-Qwen3-ASR -> AURA/Qwen3-VL -> Qwen3-TTS Talker -> Qwen3-TTS Code2Wav
-```
-
-The pipeline has three semantic modules, but four engine stages because the
-existing Qwen3-TTS implementation is natively split into Talker and Code2Wav.
-
-Start the server with the deploy profile:
-
-```bash
-vllm serve aurateam/AURA \
-  --omni \
-  --port 8091 \
-  --deploy-config vllm_omni/deploy/aura_omni.yaml \
-  --served-model-name aurateam/AURA \
-  --trust-remote-code
-```
-
-The deploy file sets per-stage model repos:
-
-- Stage 0 ASR: `Qwen/Qwen3-ASR-1.7B`
-- Stage 1 AURA: `aurateam/AURA`
-- Stage 2/3 TTS: `Qwen/Qwen3-TTS-12Hz-1.7B-Base`
-
-For local weights, edit the `model` value on each stage in
-`vllm_omni/deploy/aura_omni.yaml`. The deploy profile includes
-`pipeline: aura_omni`, so the server uses this four-stage topology even when
-the command-line model path points at one component checkpoint.
-
-Expected request shape:
-
-- Send microphone audio as the Stage 0 multimodal audio input.
-- Include video frames in the original request `multi_modal_data`; the
-  `asr2aura` processor carries them forward to AURA.
-- Optional `additional_information` keys:
-  - `aura_system_prompt`
-  - `tts_task_type`
-  - `tts_language`
-  - `tts_speaker`
-  - `tts_instruct`
-  - `tts_ref_audio`
-  - `tts_ref_text`
-  - `tts_x_vector_only_mode`
-  - `tts_pass_token_ids`
-
-If AURA emits `<|silent|>`, the `aura2tts` processor returns no TTS request, so
-the TTS stages are skipped for that turn.
-
-## GPU Utilization Recommendation
-
-Tune `gpu_memory_utilization` per stage in `vllm_omni/deploy/aura_omni.yaml`.
-Recommended baseline on one GPU for H200
-
-- Stage 0 (ASR): `0.10`
-- Stage 1 (AURA): `0.4`
-- Stage 2 (Qwen3-TTS Talker): `0.20`
-- Stage 3 (Qwen3-TTS Code2Wav): `0.20`
-
-## Python Client
-
-```bash
-python examples/online_serving/aura_omni/openai_chat_completion_client.py \
-  --host localhost \
-  --port 8091 \
-  --model aurateam/AURA \
-  --modalities text,audio
-```
-
-Use local media:
-
-```bash
-python examples/online_serving/aura_omni/openai_chat_completion_client.py \
-  --audio-path /path/to/input.wav \
-  --video-path /path/to/video.mp4 \
-  --output-dir output_aura_omni_online
-```
-
-Base voice clone mode (default, recommended as x-vector while debugging ICL):
-
-```bash
-python examples/online_serving/aura_omni/openai_chat_completion_client.py \
-  --tts-task-type Base \
-  --tts-ref-audio vllm-omni/tests/assets/qwen3_tts/clone_2.wav \
-  --tts-ref-text "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
-```
-
-Enable AURA token-id passthrough explicitly:
-
-```bash
-python examples/online_serving/aura_omni/openai_chat_completion_client.py \
-  --tts-pass-token-ids
-```
-
-CustomVoice mode requires stages 2 and 3 in `aura_omni.yaml` to point at a
-Qwen3-TTS CustomVoice checkpoint:
-
-```bash
-python examples/online_serving/aura_omni/openai_chat_completion_client.py \
-  --tts-task-type CustomVoice \
-  --tts-speaker Vivian
-```
-
-By default, AURA responses are passed to Qwen3-TTS as text. Set
-`tts_pass_token_ids=true` to pass AURA-generated assistant token ids directly
-to Qwen3-TTS instead. The processor still uses AURA token ids, when available,
-to estimate the Talker prompt length in the default text path.
-
-## Curl
-
-```bash
-cd examples/online_serving/aura_omni
-bash run_curl_multimodal_generation.sh
-```
-
-Set `PORT`, `MODEL`, or `OUTPUT_DIR` to override defaults:
-
-```bash
-PORT=8666 MODEL=aurateam/AURA bash run_curl_multimodal_generation.sh
-TTS_PASS_TOKEN_IDS=true PORT=8666 MODEL=aurateam/AURA bash run_curl_multimodal_generation.sh
-```
-
-## Gradio
-
-Launch the server and Gradio UI together:
-
-```bash
-cd examples/online_serving/aura_omni
-bash run_gradio_demo.sh
-```
-
-If the server is already running:
-
-```bash
-python examples/online_serving/aura_omni/gradio_demo.py \
-  --model aurateam/AURA \
-  --api-base http://localhost:8091/v1
-```
-
-## Offline
-
-For offline inference, see
-[`examples/offline_inference/aura_omni`](../../offline_inference/aura_omni/).

examples/online_serving/aura_omni/gradio_demo.py

@@ -1,305 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Gradio demo for AURA Omni online serving."""
-
-from __future__ import annotations
-
-import argparse
-import base64
-import io
-from pathlib import Path
-from typing import Any
-
-try:
-    import gradio as gr
-except ImportError:
-    raise ImportError("gradio is required to run this demo. Install it with: pip install 'vllm-omni[demo]'") from None
-
-import numpy as np
-import soundfile as sf
-from openai import OpenAI
-
-from vllm_omni.model_executor.stage_input_processors.aura_omni import (
-    DEFAULT_QWEN3_TTS_REF_TEXT,
-    default_qwen3_tts_ref_audio_path,
-)
-
-SEED = 42
-DEFAULT_TTS_REF_AUDIO = default_qwen3_tts_ref_audio_path()
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(description="Gradio demo for AURA Omni online inference.")
-    parser.add_argument("--model", default="aurateam/AURA", help="Model name served by vLLM.")
-    parser.add_argument("--api-base", default="http://localhost:8091/v1", help="OpenAI-compatible API base.")
-    parser.add_argument("--ip", default="127.0.0.1", help="Gradio host.")
-    parser.add_argument("--port", type=int, default=7862, help="Gradio port.")
-    parser.add_argument("--share", action="store_true", help="Share the Gradio demo publicly.")
-    return parser.parse_args()
-
-
-def _audio_to_data_url(audio_file: Any | None) -> str | None:
-    if audio_file is None:
-        return None
-
-    def _path_to_data_url(path_str: str) -> str | None:
-        if not path_str:
-            return None
-        path = Path(path_str)
-        if not path.exists():
-            return None
-        with open(path, "rb") as f:
-            return f"data:audio/wav;base64,{base64.b64encode(f.read()).decode('utf-8')}"
-
-    if isinstance(audio_file, str):
-        return _path_to_data_url(audio_file)
-
-    # Gradio >=5 may return dict payloads with local temp path.
-    if isinstance(audio_file, dict):
-        path_val = audio_file.get("path") or audio_file.get("name")
-        if isinstance(path_val, str):
-            return _path_to_data_url(path_val)
-        return None
-
-    if isinstance(audio_file, tuple):
-        # Common shape: (sample_rate, np.ndarray)
-        if len(audio_file) == 2 and isinstance(audio_file[0], (int, float)):
-            sample_rate, audio_np = audio_file
-            audio_np = np.asarray(audio_np)
-            if audio_np.ndim > 1:
-                audio_np = audio_np[:, 0]
-            if audio_np.dtype != np.int16:
-                audio_np = np.clip(audio_np.astype(np.float32), -1.0, 1.0)
-            buf = io.BytesIO()
-            sf.write(buf, audio_np, int(sample_rate), format="WAV")
-            return f"data:audio/wav;base64,{base64.b64encode(buf.getvalue()).decode('utf-8')}"
-        # Compatibility shape from some gradio builds: (filepath, (sr, np.ndarray))
-        if len(audio_file) >= 1 and isinstance(audio_file[0], str):
-            path_url = _path_to_data_url(audio_file[0])
-            if path_url is not None:
-                return path_url
-    return None
-
-
-def _video_to_data_url(video_file: str | None) -> str | None:
-    if not video_file:
-        return None
-    path = Path(video_file)
-    mime = {
-        ".mp4": "video/mp4",
-        ".webm": "video/webm",
-        ".mov": "video/quicktime",
-        ".avi": "video/x-msvideo",
-        ".mkv": "video/x-matroska",
-    }.get(path.suffix.lower(), "video/mp4")
-    with open(path, "rb") as f:
-        return f"data:{mime};base64,{base64.b64encode(f.read()).decode('utf-8')}"
-
-
-def _sampling_params_list() -> list[dict[str, Any]]:
-    return [
-        {"temperature": 0.0, "top_p": 1.0, "top_k": -1, "max_tokens": 256, "seed": SEED},
-        {
-            "temperature": 0.5,
-            "top_p": 1.0,
-            "top_k": -1,
-            "max_tokens": 256,
-            "seed": SEED,
-            "repetition_penalty": 1.0,
-        },
-        {
-            "temperature": 0.9,
-            "top_k": 50,
-            "max_tokens": 4096,
-            "seed": SEED,
-            "detokenize": False,
-            "repetition_penalty": 1.05,
-            "stop_token_ids": [2150],
-        },
-        {
-            "temperature": 0.0,
-            "top_p": 1.0,
-            "top_k": -1,
-            "max_tokens": 65536,
-            "seed": SEED,
-            "repetition_penalty": 1.0,
-        },
-    ]
-
-
-def _decode_audio_choice(audio_data: str | None) -> tuple[int, np.ndarray] | None:
-    if not audio_data:
-        return None
-    wav_bytes = base64.b64decode(audio_data)
-    audio_np, sample_rate = sf.read(io.BytesIO(wav_bytes))
-    if audio_np.ndim > 1:
-        audio_np = audio_np[:, 0]
-    return int(sample_rate), audio_np.astype(np.float32)
-
-
-def build_interface(client: OpenAI, model: str):
-    def run(
-        audio_file,
-        video_file,
-        prompt: str,
-        aura_system_prompt: str,
-        tts_task_type: str,
-        tts_language: str,
-        tts_speaker: str,
-        tts_instruct: str,
-        tts_ref_audio: str,
-        tts_ref_text: str,
-        tts_x_vector_only_mode: bool,
-        tts_pass_token_ids: bool,
-    ):
-        audio_url = _audio_to_data_url(audio_file)
-        video_url = _video_to_data_url(video_file)
-        if not audio_url:
-            return "Please provide audio input for the ASR stage.", None
-        if tts_task_type == "Base" and (not (tts_ref_audio or "").strip() or not (tts_ref_text or "").strip()):
-            return "Base TTS requires both reference audio and reference transcript.", None
-        content: list[dict[str, Any]] = [{"type": "audio_url", "audio_url": {"url": audio_url}}]
-        if video_url:
-            content.append({"type": "video_url", "video_url": {"url": video_url}})
-        content.append(
-            {"type": "text", "text": prompt or "Use the audio and video together to decide whether a reply is needed."}
-        )
-
-        additional_information: dict[str, Any] = {
-            "aura_system_prompt": aura_system_prompt,
-            "tts_task_type": tts_task_type,
-            "tts_instruct": tts_instruct,
-            "tts_pass_token_ids": bool(tts_pass_token_ids),
-        }
-        if tts_task_type == "CustomVoice":
-            additional_information.update(
-                {
-                    "tts_language": tts_language or "English",
-                    "tts_speaker": tts_speaker or "Vivian",
-                }
-            )
-        else:
-            additional_information.update(
-                {
-                    "tts_ref_audio": tts_ref_audio.strip() if tts_ref_audio else None,
-                    "tts_ref_text": tts_ref_text.strip() if tts_ref_text else None,
-                    "tts_x_vector_only_mode": bool(tts_x_vector_only_mode),
-                }
-            )
-
-        response = client.chat.completions.create(
-            model=model,
-            messages=[{"role": "user", "content": content}],
-            modalities=["text", "audio"],
-            extra_body={
-                "sampling_params_list": _sampling_params_list(),
-                "additional_information": additional_information,
-            },
-            timeout=600.0,
-        )
-
-        text_parts: list[str] = []
-        audio_output = None
-        for choice in response.choices:
-            message = choice.message
-            if message.content:
-                text_parts.append(str(message.content))
-            if getattr(message, "audio", None):
-                audio_output = _decode_audio_choice(message.audio.data)
-        return "\n\n".join(text_parts) or "No text output.", audio_output
-
-    default_system = (
-        "You are receiving a live video stream where the final frame is the present moment. "
-        "Respond only when a response is needed. Otherwise output '<|silent|>'. Respond in English."
-    )
-
-    def _toggle_tts_advanced(task_type: str):
-        is_base = task_type == "Base"
-        return (
-            gr.update(visible=not is_base),
-            gr.update(visible=is_base),
-        )
-
-    with gr.Blocks() as demo:
-        gr.Markdown("# AURA Omni")
-        gr.Markdown("ASR -> AURA/Qwen3-VL -> Qwen3-TTS Talker -> Code2Wav")
-        with gr.Row():
-            audio_input = gr.Audio(label="Audio input", sources=["upload", "microphone"], type="numpy")
-            video_input = gr.Video(label="Video input (optional)", sources=["upload"])
-        prompt = gr.Textbox(
-            label="Instruction",
-            value="Use the audio and video together to decide whether a reply is needed. If needed, respond briefly in English.",
-            lines=2,
-        )
-        with gr.Accordion("Advanced", open=False):
-            aura_system_prompt = gr.Textbox(label="AURA system prompt", value=default_system, lines=4)
-            tts_task_type = gr.Radio(choices=["Base", "CustomVoice"], value="Base", label="TTS task type")
-            tts_instruct = gr.Textbox(label="TTS instruction", value="")
-            tts_pass_token_ids = gr.Checkbox(
-                label="Pass AURA token ids directly to TTS",
-                value=False,
-            )
-
-            with gr.Group(visible=False) as customvoice_config_group:
-                tts_language = gr.Dropdown(
-                    choices=["Chinese", "English", "Japanese", "Korean", "Cantonese"],
-                    value="Chinese",
-                    label="CustomVoice language",
-                )
-                tts_speaker = gr.Textbox(label="CustomVoice speaker", value="Vivian")
-
-            with gr.Group(visible=True) as base_config_group:
-                tts_ref_audio = gr.Textbox(
-                    label="Base reference audio path/URL",
-                    value=DEFAULT_TTS_REF_AUDIO,
-                )
-                tts_ref_text = gr.Textbox(
-                    label="Base reference transcript",
-                    value=DEFAULT_QWEN3_TTS_REF_TEXT,
-                    lines=3,
-                )
-                tts_x_vector_only_mode = gr.Checkbox(
-                    label="Base x-vector only mode (disable ICL)",
-                    value=False,
-                )
-
-            tts_task_type.change(
-                _toggle_tts_advanced,
-                inputs=[tts_task_type],
-                outputs=[customvoice_config_group, base_config_group],
-            )
-        button = gr.Button("Generate", variant="primary")
-        with gr.Row():
-            text_output = gr.Textbox(label="Text output", lines=8)
-            audio_output = gr.Audio(label="Audio output", interactive=False)
-        button.click(
-            run,
-            inputs=[
-                audio_input,
-                video_input,
-                prompt,
-                aura_system_prompt,
-                tts_task_type,
-                tts_language,
-                tts_speaker,
-                tts_instruct,
-                tts_ref_audio,
-                tts_ref_text,
-                tts_x_vector_only_mode,
-                tts_pass_token_ids,
-            ],
-            outputs=[text_output, audio_output],
-        )
-        demo.queue()
-    return demo
-
-
-def main() -> None:
-    args = parse_args()
-    client = OpenAI(api_key="EMPTY", base_url=args.api_base)
-    demo = build_interface(client, args.model)
-    demo.launch(server_name=args.ip, server_port=args.port, share=args.share)
-
-
-if __name__ == "__main__":
-    main()

examples/online_serving/aura_omni/openai_chat_completion_client.py

@@ -1,199 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""OpenAI-compatible client for the AURA Omni pipeline."""
-
-from __future__ import annotations
-
-import base64
-import io
-import os
-
-import soundfile as sf
-from openai import OpenAI
-from vllm.assets.audio import AudioAsset
-from vllm.utils.argparse_utils import FlexibleArgumentParser
-
-from vllm_omni.model_executor.stage_input_processors.aura_omni import (
-    DEFAULT_QWEN3_TTS_REF_TEXT,
-    default_qwen3_tts_ref_audio_path,
-)
-
-SEED = 42
-DEFAULT_MODEL = "aurateam/AURA"
-DEFAULT_VIDEO_URL = "https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
-
-
-def _encode_file(path: str) -> str:
-    with open(path, "rb") as f:
-        return base64.b64encode(f.read()).decode("utf-8")
-
-
-def _data_url(path: str, default_mime: str) -> str:
-    suffix = os.path.splitext(path)[1].lower()
-    mime_by_suffix = {
-        ".wav": "audio/wav",
-        ".mp3": "audio/mpeg",
-        ".ogg": "audio/ogg",
-        ".flac": "audio/flac",
-        ".m4a": "audio/mp4",
-        ".mp4": "video/mp4",
-        ".webm": "video/webm",
-        ".mov": "video/quicktime",
-        ".avi": "video/x-msvideo",
-        ".mkv": "video/x-matroska",
-    }
-    return f"data:{mime_by_suffix.get(suffix, default_mime)};base64,{_encode_file(path)}"
-
-
-def media_url(path_or_url: str | None, *, kind: str) -> str:
-    if path_or_url:
-        if path_or_url.startswith(("http://", "https://", "data:")):
-            return path_or_url
-        if not os.path.exists(path_or_url):
-            raise FileNotFoundError(f"{kind} file not found: {path_or_url}")
-        return _data_url(path_or_url, "audio/wav" if kind == "audio" else "video/mp4")
-    if kind == "audio":
-        return AudioAsset("mary_had_lamb").url
-    return DEFAULT_VIDEO_URL
-
-
-def sampling_params_list() -> list[dict]:
-    return [
-        {"temperature": 0.0, "top_p": 1.0, "top_k": -1, "max_tokens": 256, "seed": SEED},
-        {
-            "temperature": 0.5,
-            "top_p": 1.0,
-            "top_k": -1,
-            "max_tokens": 256,
-            "seed": SEED,
-            "repetition_penalty": 1.0,
-        },
-        {
-            "temperature": 0.9,
-            "top_k": 50,
-            "max_tokens": 4096,
-            "seed": SEED,
-            "detokenize": False,
-            "repetition_penalty": 1.05,
-            "stop_token_ids": [2150],
-        },
-        {
-            "temperature": 0.0,
-            "top_p": 1.0,
-            "top_k": -1,
-            "max_tokens": 65536,
-            "seed": SEED,
-            "repetition_penalty": 1.0,
-        },
-    ]
-
-
-def parse_modalities(value: str | None) -> list[str] | None:
-    if not value:
-        return None
-    return [item.strip() for item in value.split(",") if item.strip()]
-
-
-def save_response(response, output_dir: str) -> None:
-    os.makedirs(output_dir, exist_ok=True)
-    for idx, choice in enumerate(response.choices):
-        message = choice.message
-        if message.content:
-            out_txt = os.path.join(output_dir, f"choice_{idx}.txt")
-            with open(out_txt, "w", encoding="utf-8") as f:
-                f.write(str(message.content).strip() + "\n")
-            print(f"Text saved to {out_txt}")
-            print(message.content)
-        if getattr(message, "audio", None):
-            audio_bytes = base64.b64decode(message.audio.data)
-            audio_np, sample_rate = sf.read(io.BytesIO(audio_bytes))
-            out_wav = os.path.join(output_dir, f"choice_{idx}.wav")
-            sf.write(out_wav, audio_np, int(sample_rate), format="WAV")
-            print(f"Audio saved to {out_wav}")
-
-
-def main(args) -> None:
-    client = OpenAI(base_url=f"http://{args.host}:{args.port}/v1", api_key="EMPTY")
-    messages = [
-        {
-            "role": "user",
-            "content": [
-                {"type": "audio_url", "audio_url": {"url": media_url(args.audio_path, kind="audio")}},
-                {"type": "video_url", "video_url": {"url": media_url(args.video_path, kind="video")}},
-                {"type": "text", "text": args.prompt},
-            ],
-        }
-    ]
-    response = client.chat.completions.create(
-        model=args.model,
-        messages=messages,
-        modalities=parse_modalities(args.modalities),
-        extra_body={
-            "sampling_params_list": sampling_params_list(),
-            "additional_information": {
-                "aura_system_prompt": args.aura_system_prompt,
-                "tts_task_type": args.tts_task_type,
-                "tts_language": args.tts_language,
-                "tts_speaker": args.tts_speaker,
-                "tts_instruct": args.tts_instruct,
-                "tts_ref_audio": args.tts_ref_audio,
-                "tts_ref_text": args.tts_ref_text,
-                "tts_x_vector_only_mode": args.tts_x_vector_only_mode,
-                "tts_pass_token_ids": args.tts_pass_token_ids,
-            },
-        },
-        timeout=args.timeout,
-    )
-    save_response(response, args.output_dir)
-
-
-def parse_args():
-    parser = FlexibleArgumentParser(description="AURA Omni online serving client")
-    parser.add_argument("--host", default="localhost")
-    parser.add_argument("--port", type=int, default=8091)
-    parser.add_argument("--model", default=DEFAULT_MODEL)
-    parser.add_argument("--audio-path", default=None, help="Audio file, URL, or data URL.")
-    parser.add_argument("--video-path", default=None, help="Video file, URL, or data URL.")
-    parser.add_argument(
-        "--prompt",
-        default="Use the audio and video together to decide whether a reply is needed. If needed, respond briefly in English.",
-    )
-    parser.add_argument("--modalities", default="text,audio")
-    parser.add_argument("--output-dir", default="output_aura_omni_online")
-    parser.add_argument(
-        "--aura-system-prompt",
-        default=(
-            "You are receiving a live video stream where the final frame is the present moment. "
-            "Respond only when a response is needed. Otherwise output '<|silent|>'. Respond in English."
-        ),
-    )
-    parser.add_argument("--tts-task-type", default="Base", choices=["Base", "CustomVoice"])
-    parser.add_argument("--tts-language", default="English")
-    parser.add_argument("--tts-speaker", default="Vivian")
-    parser.add_argument("--tts-instruct", default="")
-    parser.add_argument(
-        "--tts-ref-audio",
-        default=default_qwen3_tts_ref_audio_path(),
-        help="Base-mode reference audio path/URL visible to server.",
-    )
-    parser.add_argument(
-        "--tts-ref-text",
-        default=DEFAULT_QWEN3_TTS_REF_TEXT,
-        help="Base-mode reference audio transcript.",
-    )
-    parser.add_argument(
-        "--tts-x-vector-only-mode",
-        action="store_true",
-        help="Use speaker embedding only for Base mode (disable ICL ref_text conditioning).",
-    )
-    parser.add_argument(
-        "--tts-pass-token-ids",
-        action="store_true",
-        help="Pass AURA-generated assistant token ids directly to Qwen3-TTS. Defaults to sending text.",
-    )
-    parser.add_argument("--timeout", type=float, default=600.0)
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    main(parse_args())

examples/online_serving/aura_omni/run_curl_multimodal_generation.sh

@@ -1,60 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-PORT="${PORT:-8091}"
-MODEL="${MODEL:-aurateam/AURA}"
-OUTPUT_DIR="${OUTPUT_DIR:-output_aura_omni_online}"
-TTS_PASS_TOKEN_IDS="${TTS_PASS_TOKEN_IDS:-false}"
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-VLLM_OMNI_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
-CLONE_REF_AUDIO="${VLLM_OMNI_ROOT}/tests/assets/qwen3_tts/clone_2.wav"
-CLONE_REF_TEXT="Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
-mkdir -p "$OUTPUT_DIR"
-
-MARY_HAD_LAMB_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/mary_had_lamb.ogg"
-SAMPLE_VIDEO_URL="https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
-
-request_body=$(cat <<EOF
-{
-  "model": "$MODEL",
-  "modalities": ["text", "audio"],
-  "sampling_params_list": [
-    {"temperature": 0.0, "top_p": 1.0, "top_k": -1, "max_tokens": 256, "seed": 42},
-    {"temperature": 0.5, "top_p": 1.0, "top_k": -1, "max_tokens": 256, "seed": 42, "repetition_penalty": 1.0},
-    {"temperature": 0.9, "top_k": 50, "max_tokens": 4096, "seed": 42, "detokenize": false, "repetition_penalty": 1.05, "stop_token_ids": [2150]},
-    {"temperature": 0.0, "top_p": 1.0, "top_k": -1, "max_tokens": 65536, "seed": 42, "repetition_penalty": 1.0}
-  ],
-  "additional_information": {
-    "aura_system_prompt": "You are receiving a live video stream where the final frame is the present moment. Respond only when a response is needed. Otherwise output '<|silent|>'. Respond in English.",
-    "tts_task_type": "Base",
-    "tts_ref_audio": "file://${CLONE_REF_AUDIO}",
-    "tts_ref_text": "${CLONE_REF_TEXT}",
-    "tts_language": "English",
-    "tts_speaker": "Vivian",
-    "tts_instruct": "",
-    "tts_pass_token_ids": ${TTS_PASS_TOKEN_IDS}
-  },
-  "messages": [{
-    "role": "user",
-    "content": [
-      {"type": "audio_url", "audio_url": {"url": "$MARY_HAD_LAMB_AUDIO_URL"}},
-      {"type": "video_url", "video_url": {"url": "$SAMPLE_VIDEO_URL"}},
-      {"type": "text", "text": "Use the audio and video together to decide whether a reply is needed. If needed, respond briefly in English."}
-    ]
-  }]
-}
-EOF
-)
-
-response=$(curl -sS --retry 3 --retry-delay 3 --retry-connrefused \
-  -X POST "http://localhost:${PORT}/v1/chat/completions" \
-  -H "Content-Type: application/json" \
-  -d "$request_body")
-
-echo "$response" | jq '.choices[].message.content'
-
-audio_b64=$(echo "$response" | jq -r '.choices[]?.message.audio.data // empty' | head -n 1)
-if [[ -n "$audio_b64" ]]; then
-  echo "$audio_b64" | base64 -d > "${OUTPUT_DIR}/aura_omni_output.wav"
-  echo "Audio saved to ${OUTPUT_DIR}/aura_omni_output.wav"
-fi

examples/online_serving/aura_omni/run_gradio_demo.sh

@@ -1,79 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-MODEL="aurateam/AURA"
-SERVER_MODEL="aurateam/AURA"
-DEPLOY_CONFIG="/data/yrr/vllm-omni/vllm_omni/deploy/aura_omni.yaml"
-SERVER_PORT=8091
-GRADIO_PORT=7862
-SERVER_HOST="0.0.0.0"
-GRADIO_IP="127.0.0.1"
-GRADIO_SHARE=false
-
-
-while [[ $# -gt 0 ]]; do
-  case "$1" in
-    --model) MODEL="$2"; shift 2 ;;
-    --server-model) SERVER_MODEL="$2"; shift 2 ;;
-    --deploy-config) DEPLOY_CONFIG="$2"; shift 2 ;;
-    --server-port) SERVER_PORT="$2"; shift 2 ;;
-    --gradio-port) GRADIO_PORT="$2"; shift 2 ;;
-    --server-host) SERVER_HOST="$2"; shift 2 ;;
-    --gradio-ip) GRADIO_IP="$2"; shift 2 ;;
-    --share) GRADIO_SHARE=true; shift ;;
-    --help)
-      echo "Usage: $0 [--model SERVED_MODEL_NAME] [--server-model MODEL_PATH] [--deploy-config YAML] [--server-port PORT] [--gradio-port PORT] [--share]"
-      exit 0
-      ;;
-    *) echo "Unknown option: $1"; exit 1 ;;
-  esac
-done
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-API_BASE="http://localhost:${SERVER_PORT}/v1"
-LOG_FILE="/tmp/aura_omni_vllm_${SERVER_PORT}.log"
-
-cleanup() {
-  echo "Shutting down..."
-  [[ -n "${SERVER_PID:-}" ]] && kill "$SERVER_PID" 2>/dev/null || true
-  [[ -n "${GRADIO_PID:-}" ]] && kill "$GRADIO_PID" 2>/dev/null || true
-}
-trap cleanup SIGINT SIGTERM EXIT
-
-vllm serve "$SERVER_MODEL" \
-  --omni \
-  --host "$SERVER_HOST" \
-  --port "$SERVER_PORT" \
-  --deploy-config "$DEPLOY_CONFIG" \
-  --served-model-name "$MODEL" \
-  --trust-remote-code 2>&1 | tee "$LOG_FILE" &
-SERVER_PID=$!
-
-echo "Waiting for server startup..."
-for _ in $(seq 1 600); do
-  if grep -q "Application startup complete" "$LOG_FILE" 2>/dev/null; then
-    break
-  fi
-  if ! kill -0 "$SERVER_PID" 2>/dev/null; then
-    echo "vLLM server exited before startup completed"
-    wait "$SERVER_PID" || true
-    exit 1
-  fi
-  sleep 1
-done
-
-# cd "$SCRIPT_DIR"
-GRADIO_CMD=(python gradio_demo.py --model "$MODEL" --api-base "$API_BASE" --ip "$GRADIO_IP" --port "$GRADIO_PORT")
-if [[ "$GRADIO_SHARE" == "true" ]]; then
-  GRADIO_CMD+=(--share)
-fi
-"${GRADIO_CMD[@]}" &
-GRADIO_PID=$!
-
-echo "vLLM server: http://${SERVER_HOST}:${SERVER_PORT}"
-echo "Gradio demo: http://${GRADIO_IP}:${GRADIO_PORT}"
-if [[ -n "${SERVER_PID:-}" ]]; then
-  wait "$SERVER_PID" "$GRADIO_PID"
-else
-  wait "$GRADIO_PID"
-fi

examples/online_serving/bagel/README.md

@@ -0,0 +1,404 @@
+# BAGEL-7B-MoT
+
+## Installation
+
+Please refer to [README.md](../../../README.md)
+
+## Architecture
+
+BAGEL-7B-MoT is a Mixture-of-Transformers (MoT) model supporting both image generation and understanding. It offers two deployment topologies:
+
+| Topology | Stages | Description |
+| :------- | :----- | :---------- |
+| **Two-stage** (default) | Stage 0 (Thinker, AR) + Stage 1 (DiT, Diffusion) | Thinker handles text/understanding via vLLM AR engine; DiT handles image generation. KV cache is transferred between stages. |
+| **Single-stage** | Stage 0 (DiT, Diffusion) only | The DiT stage contains a full LLM, ViT, VAE, and tokenizer internally. All modalities are handled within a single diffusion process. |
+
+Both topologies support all four modalities: `text2img`, `img2img`, `img2text`, `text2text`.
+
+> **Note**: These examples work with the default configuration on an **NVIDIA A100 (80GB)**. We also tested on dual **NVIDIA RTX 5000 Ada (32GB each)**. For dual-GPU setups, modify the deploy YAML to distribute stages across devices.
+
+## Launch the Server
+
+### Two-Stage (Default)
+
+The default pipeline is auto-detected from the model. No extra flags needed:
+
+```bash
+vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091
+```
+
+Or use the convenience script:
+
+```bash
+cd examples/online_serving/bagel
+bash run_server.sh
+
+# Initialize each stage in a discrete isolated process terminal
+bash run_server_stage_cli.sh --stage 0
+bash run_server_stage_cli.sh --stage 1
+```
+
+To use a custom deploy YAML, pass it via `--deploy-config`:
+
+```bash
+vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 \
+    --deploy-config /path/to/deploy_config.yaml
+```
+
+See [`bagel.yaml`](../../../vllm_omni/deploy/bagel.yaml) for the default two-stage deploy configuration.
+
+### Single-Stage
+
+The DiT stage contains a full LLM, ViT, VAE, and tokenizer, so it can handle all modalities (text2img, img2img, img2text, text2text, think) without a separate Thinker stage:
+
+```bash
+vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 \
+    --deploy-config vllm_omni/deploy/bagel_single_stage.yaml
+```
+
+See [`bagel_single_stage.yaml`](../../../vllm_omni/deploy/bagel_single_stage.yaml) for configuration. The `pipeline: bagel_single_stage` field selects the single-stage topology from the pipeline registry.
+
+### Tensor Parallelism (TP)
+
+For larger models or multi-GPU environments, enable TP via CLI:
+
+```bash
+vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 --tensor-parallel-size 2
+```
+
+Or set `tensor_parallel_size` per stage in a custom deploy YAML.
+
+### VAE Patch Parallelism
+
+[VAE Patch Parallelism](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/parallelism/vae_patch_parallel.html) distributes Bagel VAE **decode/encode** across multiple GPUs by splitting latent tiles. It lowers **per-GPU peak memory during VAE decode**, which helps high-resolution `text2img` / `img2img` when VAE becomes a bottleneck.
+
+**Scope for Bagel:**
+
+| Topology | VAE patch parallel |
+| :------- | :----------------- |
+| **Single-stage** (DiT only) | Supported on stage 0 (`BagelPipeline` + `DistributedAutoEncoder`) |
+| **Two-stage** | Supported on **stage 1 (DiT)** only; stage 0 (Thinker) uses encoder-only VAE and is unrelated |
+
+**Requirements:**
+
+- `vae_patch_parallel_size > 1` and a distributed VAE (`DistributedAutoEncoder` on the DiT pipeline).
+- The DiT process group must have at least `vae_patch_parallel_size` ranks. In practice this means the diffusion stage `world_size` must be ≥ 2 (commonly `tensor_parallel_size=2` on that stage).
+- `vae_use_tiling` must be enabled. If you set `vae_patch_parallel_size > 1` and omit tiling, the registry auto-enables `vae_use_tiling` at startup.
+
+VAE patch parallel **reuses the DiT process group** (`dit_group`); it does not create a separate VAE-only worker pool. It is not a substitute for single-GPU VAE tiling (`vae_pp=1`).
+
+**Online serving (single-stage, 2 GPUs):**
+
+```bash
+CUDA_VISIBLE_DEVICES=0,1 vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 \
+    --deploy-config vllm_omni/deploy/bagel_single_stage.yaml \
+    --tensor-parallel-size 2 \
+    --vae-patch-parallel-size 2 \
+    --vae-use-tiling
+```
+
+**Online serving (two-stage, VAE PP on DiT stage 1):** use a custom deploy YAML, for example:
+
+```yaml
+stages:
+  - stage_id: 0
+    devices: "0"
+    # Thinker (AR) — no VAE patch parallel here
+
+  - stage_id: 1
+    devices: "0,1"
+    vae_use_tiling: true
+    parallel_config:
+      tensor_parallel_size: 2
+      vae_patch_parallel_size: 2
+```
+
+```bash
+vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 \
+    --deploy-config /path/to/bagel_vae_pp.yaml
+```
+
+**Verify it is active** (check server logs at startup):
+
+```text
+INFO ... vae_patch_parallel_size=2 requires vae_use_tiling; automatically enabling it.
+```
+
+| CLI flag | Default | Description |
+| :------- | :------ | :---------- |
+| `--vae-patch-parallel-size` | `1` | Number of DiT ranks used for VAE tile parallelism. Set to `2` or higher to enable. Should be ≤ DiT process group size (typically match `--tensor-parallel-size` on the diffusion stage). |
+| `--vae-use-tiling` | off | Enable VAE spatial tiling. Required for VAE patch parallel (auto-enabled when `vae_patch_parallel_size > 1`). |
+
+#### Hybrid Sharded Data Parallel (HSDP)
+
+For larger Bagel deployments on multiple GPUs, you can enable HSDP (Hybrid Sharded Data Parallel) by modifying the stage configuration (for example, [`bagel.yaml`](../../../vllm_omni/deploy/bagel.yaml)). HSDP shards transformer weights across GPUs to reduce per-GPU memory usage.
+
+1. **Enable HSDP**: Set `use_hsdp: true`.
+2. **Set shard size**: Set `hsdp_shard_size` to the number of GPUs used for sharding (for example, `4`).
+3. **Set replicate size**: Usually keep `hsdp_replicate_size: 1` unless you want replicated HSDP groups.
+4. **Set devices**: Specify the comma-separated GPU IDs used by the diffusion stage (for example, `"0,1,2,3"`).
+
+Example configuration for HSDP across 4 GPUs:
+```yaml
+  - stage_id: 1
+    devices: "0,1,2,3"
+    parallel_config:
+      use_hsdp: true
+      hsdp_shard_size: 4
+      hsdp_replicate_size: 1
+```
+
+### Multi-Node Deployment
+
+Deploy each stage on a **separate node** for better resource utilization. Replace `<ORCHESTRATOR_IP>` with the actual IP address of your orchestrator node.
+
+**1. Launch Stage 0 (Thinker / Orchestrator)** on the orchestrator node:
+
+```bash
+# API server port for client requests: 8000
+vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni \
+    --port 8000 \
+    --stage-id 0 \
+    --omni-master-address <ORCHESTRATOR_IP> \
+    --omni-master-port 8091
+```
+
+**2. Launch Stage 1 (DiT)** on the remote node in headless mode:
+
+```bash
+vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni \
+    --stage-id 1 \
+    --headless \
+    --omni-master-address <ORCHESTRATOR_IP> \
+    --omni-master-port 8091
+```
+
+Or use the convenience script:
+
+```bash
+# Terminal 1: Stage 0
+bash run_server_stage_cli.sh --stage 0
+
+# Terminal 2: Stage 1
+bash run_server_stage_cli.sh --stage 1
+
+# With extra args
+bash run_server_stage_cli.sh --stage 0 -- --tensor-parallel-size 2
+bash run_server_stage_cli.sh --stage 1 -- --gpu-memory-utilization 0.9
+```
+
+**vllm serve arguments:**
+
+| Argument | Description |
+| :------- | :---------- |
+| `--stage-id` | Which stage this process runs (0 = Thinker, 1 = DiT) |
+| `--headless` | Run without the API server (worker-only mode) |
+| `-oma` / `--omni-master-address` | Orchestrator master address |
+| `-omp` / `--omni-master-port` | Orchestrator master port |
+
+> [!IMPORTANT]
+> **Startup Order**: Stage 0 (orchestrator) must be launched **before** Stage 1 (headless).
+> Stage 0 will appear to hang on startup until Stage 1 (worker) connects — this is expected behavior.
+
+### Inter-Stage Connectors
+
+When deploying stages across nodes, configure the connector type in the deploy YAML:
+
+- **SharedMemoryConnector** (default): Used for single-node deployments. No explicit configuration needed.
+- **MooncakeTransferEngineConnector**: For multi-node setups with RDMA hardware. Defined in [`bagel.yaml`](../../../vllm_omni/deploy/bagel.yaml) under `connectors.rdma_connector`.
+
+To use Mooncake, create a custom deploy YAML that binds `output_connectors` / `input_connectors` on each stage to the `rdma_connector` defined in the `connectors` section.
+
+## Send Requests
+
+```bash
+cd examples/online_serving/bagel
+```
+
+### Text to Image (text2img)
+
+**Python client:**
+
+```bash
+python openai_chat_client.py \
+    --prompt "A beautiful sunset over mountains" \
+    --modality text2img \
+    --output sunset.png \
+    --steps 50
+```
+
+**curl:**
+
+```bash
+curl http://localhost:8091/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "messages": [{"role": "user", "content": [{"type": "text", "text": "<|im_start|>A beautiful sunset over mountains<|im_end|>"}]}],
+    "modalities": ["image"],
+    "height": 512,
+    "width": 512,
+    "num_inference_steps": 50,
+    "seed": 42
+  }'
+```
+
+### Image to Image (img2img)
+
+**Python client:**
+
+```bash
+python openai_chat_client.py \
+    --prompt "Make the cat stand up" \
+    --modality img2img \
+    --image-url /path/to/input.jpg \
+    --output transformed.png
+```
+
+**curl:**
+
+```bash
+IMAGE_BASE64=$(base64 -w 0 cat.jpg)
+
+cat <<EOF > payload.json
+{
+    "messages": [{
+      "role": "user",
+      "content": [
+        {"type": "text", "text": "<|im_start|>Make the cat stand up<|im_end|>"},
+        {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,${IMAGE_BASE64}"}}
+      ]
+    }],
+    "modalities": ["image"],
+    "height": 512,
+    "width": 512,
+    "num_inference_steps": 50,
+    "seed": 42
+}
+EOF
+
+curl http://localhost:8091/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d @payload.json
+```
+
+### Image to Text (img2text)
+
+**Python client:**
+
+```bash
+python openai_chat_client.py \
+    --prompt "Describe this image in detail" \
+    --modality img2text \
+    --image-url /path/to/image.jpg
+```
+
+**curl:**
+
+```bash
+IMAGE_BASE64=$(base64 -w 0 cat.jpg)
+
+cat <<EOF > payload.json
+{
+  "messages": [{
+    "role": "user",
+    "content": [
+      {"type": "text", "text": "<|im_start|>user\n<|image_pad|>\nDescribe this image in detail<|im_end|>\n<|im_start|>assistant\n"},
+      {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,${IMAGE_BASE64}"}}
+    ]
+  }],
+  "modalities": ["text"]
+}
+EOF
+
+curl http://localhost:8091/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d @payload.json
+```
+
+### Text to Text (text2text)
+
+**Python client:**
+
+```bash
+python openai_chat_client.py \
+    --prompt "What is the capital of France?" \
+    --modality text2text
+```
+
+**curl:**
+
+```bash
+curl http://localhost:8091/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "messages": [{"role": "user", "content": [{"type": "text", "text": "<|im_start|>user\nWhat is the capital of France?<|im_end|>\n<|im_start|>assistant\n"}]}],
+    "modalities": ["text"]
+  }'
+```
+
+### Python Client Arguments
+
+| Argument | Default | Description |
+| :------- | :------ | :---------- |
+| `--prompt` / `-p` | `A cute cat` | Text prompt |
+| `--output` / `-o` | `bagel_output.png` | Output file path |
+| `--server` / `-s` | `http://localhost:8091` | Server URL |
+| `--image-url` / `-i` | `None` | Input image URL or local path (img2img/img2text) |
+| `--modality` / `-m` | `text2img` | `text2img`, `img2img`, `img2text`, `text2text` |
+| `--height` | `512` | Image height in pixels |
+| `--width` | `512` | Image width in pixels |
+| `--steps` | `25` | Number of inference steps |
+| `--seed` | `42` | Random seed |
+| `--negative` | `None` | Negative prompt for CFG |
+
+Example with custom parameters:
+
+```bash
+python openai_chat_client.py \
+    --prompt "A futuristic city" \
+    --modality text2img \
+    --height 768 \
+    --width 768 \
+    --steps 50 \
+    --seed 42 \
+    --negative "blurry, low quality"
+```
+
+## Configuration Reference
+
+### Deploy YAML Files
+
+| File | Description |
+| :--- | :---------- |
+| [`bagel.yaml`](../../../vllm_omni/deploy/bagel.yaml) | Two-stage default (Thinker + DiT on GPU 0) |
+| [`bagel_single_stage.yaml`](../../../vllm_omni/deploy/bagel_single_stage.yaml) | Single-stage (DiT only) |
+
+### Key Deploy YAML Fields
+
+| Field | Scope | Description |
+| :---- | :---- | :---------- |
+| `pipeline` | top-level | Override auto-detected pipeline (e.g. `bagel_single_stage`) |
+| `stages[].stage_id` | per-stage | Stage identifier (0, 1, ...) |
+| `stages[].devices` | per-stage | GPU device IDs (e.g. `"0"`, `"0,1"`) |
+| `stages[].max_num_seqs` | per-stage | Maximum concurrent sequences |
+| `stages[].gpu_memory_utilization` | per-stage | Fraction of GPU memory to use |
+| `stages[].enforce_eager` | per-stage | Disable CUDA graphs |
+| `stages[].tensor_parallel_size` | per-stage | TP degree for this stage |
+| `stages[].parallel_config.vae_patch_parallel_size` | per-stage (DiT) | VAE tile parallelism degree (DiT stage only) |
+| `stages[].vae_use_tiling` | per-stage (DiT) | Enable VAE tiling (required for VAE patch parallel) |
+| `connectors` | top-level | Define available connector instances (SHM, Mooncake) |
+| `platforms` | top-level | Platform-specific overrides (e.g. `xpu`) |
+
+## FAQ
+
+- If you encounter OOM errors, try decreasing `max_model_len` or `gpu_memory_utilization` in the deploy YAML.
+
+**Two-stage VRAM usage:**
+
+| Stage | VRAM |
+| :---- | :--- |
+| Stage 0 (Thinker) | **15.04 GiB + KV Cache** |
+| Stage 1 (DiT) | **26.50 GiB** |
+| Total | **~42 GiB + KV Cache** |
+
+**Single-stage VRAM usage:** The DiT loads the full model (~42 GiB) in one process.

examples/online_serving/bagel/openai_chat_client.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+"""
+Bagel OpenAI-compatible chat client for image generation and multimodal tasks.
+
+Usage:
+    python openai_chat_client.py --prompt "A cute cat" --output output.png
+    python openai_chat_client.py --prompt "Describe this image" --image-url https://example.com/image.png
+"""
+
+import argparse
+import base64
+from pathlib import Path
+
+import requests
+
+
+def generate_image(
+    prompt: str,
+    server_url: str = "http://localhost:8091",
+    image_url: str | None = None,
+    height: int | None = None,
+    width: int | None = None,
+    steps: int | None = None,
+    seed: int | None = None,
+    negative_prompt: str | None = None,
+    modality: str = "text2img",  # "text2img" (default), "img2img", "img2text", "text2text"
+) -> bytes | str | None:
+    """Generate an image or text using the chat completions API.
+
+    Args:
+        prompt: Text description or prompt
+        server_url: Server URL
+        image_url: URL or path to input image (for img2img/img2text)
+        height: Image height in pixels
+        width: Image width in pixels
+        steps: Number of inference steps
+        seed: Random seed
+        negative_prompt: Negative prompt
+        modality: Task modality hint
+
+    Returns:
+        Image bytes (for image outputs) or Text string (for text outputs) or None if failed
+    """
+
+    # Construct Message Content
+    content = [{"type": "text", "text": f"<|im_start|>{prompt}<|im_end|>"}]
+
+    if image_url:
+        # Check if local file
+        if Path(image_url).exists():
+            with open(image_url, "rb") as f:
+                b64_data = base64.b64encode(f.read()).decode("utf-8")
+                final_image_url = f"data:image/jpeg;base64,{b64_data}"
+        else:
+            final_image_url = image_url
+
+        content.append({"type": "image_url", "image_url": {"url": final_image_url}})
+
+    messages = [{"role": "user", "content": content}]
+
+    # Build request payload with all parameters at top level
+    # Note: vLLM ignores "extra_body", so we put parameters directly in the payload
+    payload = {"messages": messages}
+
+    # Set output modalities at top level
+    if modality == "text2img" or modality == "img2img":
+        payload["modalities"] = ["image"]
+    elif modality == "img2text" or modality == "text2text":
+        payload["modalities"] = ["text"]
+
+    # Add generation parameters directly to payload
+    if height is not None:
+        payload["height"] = height
+    if width is not None:
+        payload["width"] = width
+    if steps is not None:
+        payload["num_inference_steps"] = steps
+    if seed is not None:
+        payload["seed"] = seed
+    if negative_prompt:
+        payload["negative_prompt"] = negative_prompt
+
+    # Send request
+    try:
+        print(f"Sending request to {server_url} with modality {modality}...")
+        response = requests.post(
+            f"{server_url}/v1/chat/completions",
+            headers={"Content-Type": "application/json"},
+            json=payload,
+            timeout=300,
+        )
+        response.raise_for_status()
+        data = response.json()
+
+        # Extract content - check ALL choices since server may return multiple
+        # (e.g., text in choices[0], image in choices[1])
+        choices = data.get("choices", [])
+
+        # First pass: look for image output in any choice
+        for choice in choices:
+            choice_content = choice.get("message", {}).get("content")
+
+            # Handle Image Output
+            if isinstance(choice_content, list) and len(choice_content) > 0:
+                first_item = choice_content[0]
+                if isinstance(first_item, dict) and "image_url" in first_item:
+                    img_url_str = first_item["image_url"].get("url", "")
+                    if img_url_str.startswith("data:image"):
+                        _, b64_data = img_url_str.split(",", 1)
+                        return base64.b64decode(b64_data)
+
+        # Second pass: look for text output if no image found
+        for choice in choices:
+            choice_content = choice.get("message", {}).get("content")
+            if isinstance(choice_content, str) and choice_content:
+                return choice_content
+
+        print(f"Unexpected response format: {choices}")
+        return None
+
+    except Exception as e:
+        print(f"Error: {e}")
+        return None
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Bagel multimodal chat client")
+    parser.add_argument("--prompt", "-p", default="A cute cat", help="Text prompt")
+    parser.add_argument("--output", "-o", default="bagel_output.png", help="Output file (for image results)")
+    parser.add_argument("--server", "-s", default="http://localhost:8091", help="Server URL")
+
+    # Modality Control
+    parser.add_argument("--image-url", "-i", type=str, help="Input image URL or local path")
+    parser.add_argument(
+        "--modality",
+        "-m",
+        default="text2img",
+        choices=["text2img", "img2img", "img2text", "text2text"],
+        help="Task modality",
+    )
+
+    # Generation Params
+    parser.add_argument("--height", type=int, default=512, help="Image height")
+    parser.add_argument("--width", type=int, default=512, help="Image width")
+    parser.add_argument("--steps", type=int, default=25, help="Inference steps")
+    parser.add_argument("--seed", type=int, default=42, help="Random seed")
+    parser.add_argument("--negative", help="Negative prompt")
+
+    args = parser.parse_args()
+
+    print(f"Mode: {args.modality}")
+    if args.image_url:
+        print(f"Input Image: {args.image_url}")
+
+    result = generate_image(
+        prompt=args.prompt,
+        server_url=args.server,
+        image_url=args.image_url,
+        height=args.height,
+        width=args.width,
+        steps=args.steps,
+        seed=args.seed,
+        negative_prompt=args.negative,
+        modality=args.modality,
+    )
+
+    if result:
+        if isinstance(result, bytes):
+            # It's an image
+            output_path = Path(args.output)
+            output_path.write_bytes(result)
+            print(f"Image saved to: {output_path}")
+            print(f"Size: {len(result) / 1024:.1f} KB")
+        elif isinstance(result, str):
+            # It's text
+            print("Response:")
+            print(result)
+    else:
+        print("Failed to generate response")
+        exit(1)
+
+
+if __name__ == "__main__":
+    main()

examples/online_serving/bagel/run_server.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# Bagel online serving startup script
+
+MODEL="${MODEL:-ByteDance-Seed/BAGEL-7B-MoT}"
+PORT="${PORT:-8091}"
+
+echo "Starting Bagel server..."
+echo "Model: $MODEL"
+echo "Port: $PORT"
+
+vllm serve "$MODEL" --omni \
+    --port "$PORT"

examples/online_serving/bagel/run_server_stage_cli.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+# Bagel multi-stage online serving startup script.
+#
+# Usage:
+#   ./run_server_stage_cli.sh --stage 0
+#   ./run_server_stage_cli.sh --stage 1
+#   ./run_server_stage_cli.sh --stage 0 -- --tensor-parallel-size 2
+#   ./run_server_stage_cli.sh --stage 1 -- --gpu-memory-utilization 0.9
+#
+# By default, `--stage all` keeps the old behavior and launches both stages in
+# one session. Use `--stage 0` / `--stage 1` to launch each stage separately in
+# different terminal sessions, with stage-specific extra CLI arguments passed
+# after `--`.
+
+set -euo pipefail
+
+MODEL="${MODEL:-ByteDance-Seed/BAGEL-7B-MoT}"
+PORT="${PORT:-8091}"
+MASTER_ADDRESS="${MASTER_ADDRESS:-127.0.0.1}"
+MASTER_PORT="${MASTER_PORT:-8092}"
+STAGE="all"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+DEPLOY_CONFIG="${DEPLOY_CONFIG:-$SCRIPT_DIR/../../../vllm_omni/deploy/bagel.yaml}"
+EXTRA_ARGS=()
+
+usage() {
+    cat <<EOF
+Usage: $0 [OPTIONS] [-- EXTRA_VLLM_ARGS...]
+
+Options:
+  --stage {0|1|all}          Stage to launch (default: all)
+  --model MODEL              Model name/path (default: $MODEL)
+  --port PORT                API port for stage 0 (default: $PORT)
+  --master-address ADDRESS   Master/orchestrator address (default: $MASTER_ADDRESS)
+  --master-port PORT         Master/orchestrator port (default: $MASTER_PORT)
+  --deploy-config PATH       Deploy config YAML path (default: $DEPLOY_CONFIG)
+  --help                     Show this help message
+
+Examples:
+  $0 --stage 0
+  $0 --stage 1
+  $0 --stage 0 -- --tensor-parallel-size 2
+  $0 --stage 1 -- --gpu-memory-utilization 0.9
+
+Notes:
+  - Use different terminal sessions to launch stage 0 and stage 1 separately.
+  - Extra args after '--' are forwarded only to the selected stage.
+  - When using '--stage all', the extra args are forwarded to both stages.
+EOF
+}
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --stage)
+            STAGE="$2"
+            shift 2
+            ;;
+        --model)
+            MODEL="$2"
+            shift 2
+            ;;
+        --port)
+            PORT="$2"
+            shift 2
+            ;;
+        --master-address)
+            MASTER_ADDRESS="$2"
+            shift 2
+            ;;
+        --master-port)
+            MASTER_PORT="$2"
+            shift 2
+            ;;
+        --deploy-config)
+            DEPLOY_CONFIG="$2"
+            shift 2
+            ;;
+        --help|-h)
+            usage
+            exit 0
+            ;;
+        --)
+            shift
+            EXTRA_ARGS=("$@")
+            break
+            ;;
+        *)
+            echo "Unknown option: $1" >&2
+            usage
+            exit 1
+            ;;
+    esac
+done
+
+if [[ "$STAGE" != "0" && "$STAGE" != "1" && "$STAGE" != "all" ]]; then
+    echo "Invalid --stage value: $STAGE" >&2
+    usage
+    exit 1
+fi
+
+print_config() {
+    echo "Model: $MODEL"
+    echo "API Port: $PORT"
+    echo "Master Address: $MASTER_ADDRESS"
+    echo "Master Port: $MASTER_PORT"
+    echo "Deploy Config: $DEPLOY_CONFIG"
+    echo "Selected Stage: $STAGE"
+    if [[ ${#EXTRA_ARGS[@]} -gt 0 ]]; then
+        echo "Extra Args: ${EXTRA_ARGS[*]}"
+    fi
+}
+
+run_stage_0() {
+    echo "Starting Stage 0 (Thinker) as master..."
+    vllm serve "$MODEL" --omni \
+        --port "$PORT" \
+        --deploy-config "$DEPLOY_CONFIG" \
+        --stage-id 0 \
+        --omni-master-address "$MASTER_ADDRESS" \
+        --omni-master-port "$MASTER_PORT" \
+        "${EXTRA_ARGS[@]}"
+}
+
+run_stage_1() {
+    echo "Starting Stage 1 (DiT) in headless mode..."
+    vllm serve "$MODEL" --omni \
+        --deploy-config "$DEPLOY_CONFIG" \
+        --stage-id 1 \
+        --headless \
+        --omni-master-address "$MASTER_ADDRESS" \
+        --omni-master-port "$MASTER_PORT" \
+        "${EXTRA_ARGS[@]}"
+}
+
+echo "Starting Bagel multi-stage server..."
+print_config
+
+case "$STAGE" in
+    0)
+        run_stage_0
+        ;;
+    1)
+        run_stage_1
+        ;;
+    all)
+        echo "Launching both stages in one session (legacy mode)..."
+        echo "Starting Stage 0 (Thinker) in background first..."
+        run_stage_0 &
+        STAGE_0_PID=$!
+
+        cleanup() {
+            if [[ -n "${STAGE_0_PID:-}" ]]; then
+                kill "$STAGE_0_PID" 2>/dev/null || true
+                wait "$STAGE_0_PID" 2>/dev/null || true
+            fi
+        }
+
+        trap cleanup EXIT INT TERM
+
+        echo "Waiting briefly for Stage 0 to initialize..."
+        sleep 2
+        run_stage_1
+        ;;
+esac

examples/online_serving/helios/README.md

@@ -0,0 +1,175 @@
+# Helios Video Generation via Online API
+
+This example demonstrates how to use the `/v1/videos` API with Helios models using the generic `extra_params` field.
+
+## Overview
+
+The `/v1/videos` API now supports model-specific parameters through the `extra_params` field, which accepts a JSON object containing any model-specific configuration. This allows supporting new models like Helios without modifying the API.
+
+## Helios Model Variants
+
+- **Helios-Base**: Basic T2V/I2V/V2V generation (Stage 1 only)
+- **Helios-Mid**: Advanced generation with CFG-Zero* (Stage 2)
+- **Helios-Distilled**: Fast generation with DMD (Stage 2)
+
+## API Usage Examples
+
+### 1. T2V (Text-to-Video) - Helios-Base
+
+Basic text-to-video generation:
+
+```bash
+curl -X POST http://localhost:8000/v1/videos \
+  -F "prompt=A serene lakeside sunrise with mist over the water." \
+  -F "model=BestWishYsh/Helios-Base" \
+  -F "width=640" \
+  -F "height=384" \
+  -F "num_frames=99" \
+  -F "num_inference_steps=50" \
+  -F "guidance_scale=5.0" \
+  -F "seed=42"
+```
+
+### 2. I2V (Image-to-Video) - Helios-Base
+
+Generate video from an input image:
+
+```bash
+curl -X POST http://localhost:8000/v1/videos \
+  -F "prompt=The lake water gently ripples as morning mist rises." \
+  -F "model=BestWishYsh/Helios-Base" \
+  -F "input_reference=@/path/to/image.jpg" \
+  -F "width=640" \
+  -F "height=384" \
+  -F "num_frames=99" \
+  -F "guidance_scale=5.0"
+```
+
+### 3. Helios-Mid with Stage 2 + CFG-Zero*
+
+Advanced generation with pyramid multi-stage denoising:
+
+```bash
+curl -X POST http://localhost:8000/v1/videos \
+  -F "prompt=A serene lakeside sunrise with mist over the water." \
+  -F "model=BestWishYsh/Helios-Mid" \
+  -F "width=640" \
+  -F "height=384" \
+  -F "guidance_scale=5.0" \
+  -F 'extra_params={
+    "is_enable_stage2": true,
+    "pyramid_num_stages": 3,
+    "pyramid_num_inference_steps_list": [20, 20, 20],
+    "use_cfg_zero_star": true,
+    "use_zero_init": true,
+    "zero_steps": 1
+  }'
+```
+
+### 4. Helios-Distilled with DMD
+
+Fast generation with Distribution Matching Distillation:
+
+```bash
+curl -X POST http://localhost:8000/v1/videos \
+  -F "prompt=A serene lakeside sunrise with mist over the water." \
+  -F "model=BestWishYsh/Helios-Distilled" \
+  -F "width=640" \
+  -F "height=384" \
+  -F "guidance_scale=1.0" \
+  -F 'extra_params={
+    "is_enable_stage2": true,
+    "pyramid_num_stages": 3,
+    "pyramid_num_inference_steps_list": [2, 2, 2],
+    "is_amplify_first_chunk": true
+  }'
+```
+
+## Model-Specific Parameters
+
+The `extra_params` field accepts a JSON object with model-specific parameters. For Helios models, supported parameters include:
+
+### Stage 2 Parameters
+- `is_enable_stage2` (bool): Enable pyramid multi-stage denoising
+- `pyramid_num_stages` (int): Number of pyramid stages (default: 3)
+- `pyramid_num_inference_steps_list` (array): Steps per stage, e.g., `[20, 20, 20]`
+
+### CFG Zero Star Parameters (Helios-Mid)
+- `use_cfg_zero_star` (bool): Enable CFG Zero Star guidance
+- `use_zero_init` (bool): Use zero initialization for first steps
+- `zero_steps` (int): Number of initial zero prediction steps (default: 1)
+
+### DMD Parameters (Helios-Distilled)
+- `is_amplify_first_chunk` (bool): Enable DMD amplification for first chunk
+
+### Video Input (V2V mode)
+For video-to-video generation, upload a video file via `input_reference`:
+
+```bash
+curl -X POST http://localhost:8000/v1/videos \
+  -F "prompt=Transform the video into a watercolor painting style." \
+  -F "model=BestWishYsh/Helios-Base" \
+  -F "input_reference=@/path/to/video.mp4" \
+  -F "guidance_scale=5.0"
+```
+
+## Python Example
+
+```python
+import requests
+import json
+
+url = "http://localhost:8000/v1/videos"
+
+# Helios-Mid with Stage 2
+data = {
+    "prompt": "A serene lakeside sunrise with mist over the water.",
+    "model": "BestWishYsh/Helios-Mid",
+    "width": 640,
+    "height": 384,
+    "guidance_scale": 5.0,
+    "extra_params": json.dumps({
+        "is_enable_stage2": True,
+        "pyramid_num_stages": 3,
+        "pyramid_num_inference_steps_list": [20, 20, 20],
+        "use_cfg_zero_star": True,
+        "use_zero_init": True,
+        "zero_steps": 1
+    })
+}
+
+response = requests.post(url, data=data)
+video_job = response.json()
+print(f"Video job created: {video_job['id']}")
+
+# Poll for completion
+import time
+while True:
+    status_response = requests.get(f"{url}/{video_job['id']}")
+    status = status_response.json()
+
+    if status['status'] == 'completed':
+        print(f"Video generated: {status['file_name']}")
+        break
+    elif status['status'] == 'failed':
+        print(f"Generation failed: {status.get('error')}")
+        break
+
+    print(f"Progress: {status['progress']}%")
+    time.sleep(2)
+```
+
+## Benefits of This Approach
+
+1. **No API Changes**: New models can be supported without modifying the API
+2. **Backward Compatible**: Existing clients continue to work
+3. **Flexible**: Any model-specific parameter can be passed
+4. **Type-Safe**: Parameters are validated at the model level
+5. **Future-Proof**: Supports models that don't exist yet
+
+## Notes
+
+- The `extra_params` field must be a valid JSON object
+- Parameters are passed directly to the model's `extra_args`
+- Invalid parameters will be caught by the model implementation
+- Tensor inputs (images/videos) should use `input_reference`, not `extra_params`

examples/online_serving/helios/helios_client.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""
+Helios video generation client example using the /v1/videos API.
+
+This example demonstrates how to use the generic extra_params field
+to pass Helios-specific parameters without modifying the API.
+"""
+
+import argparse
+import json
+import time
+from pathlib import Path
+
+import requests
+
+
+def create_video_job(
+    api_url: str,
+    prompt: str,
+    model: str,
+    width: int = 640,
+    height: int = 384,
+    num_frames: int = 99,
+    guidance_scale: float = 5.0,
+    seed: int = 42,
+    extra_params: dict | None = None,
+    input_image: Path | None = None,
+) -> dict:
+    """Create a video generation job."""
+    data = {
+        "prompt": prompt,
+        "model": model,
+        "width": width,
+        "height": height,
+        "num_frames": num_frames,
+        "guidance_scale": guidance_scale,
+        "seed": seed,
+    }
+
+    files = {}
+    if input_image:
+        files["input_reference"] = open(input_image, "rb")
+
+    if extra_params:
+        data["extra_params"] = json.dumps(extra_params)
+
+    response = requests.post(f"{api_url}/v1/videos", data=data, files=files)
+    response.raise_for_status()
+
+    if files:
+        files["input_reference"].close()
+
+    return response.json()
+
+
+def poll_video_status(api_url: str, video_id: str, poll_interval: int = 2) -> dict:
+    """Poll video generation status until completion."""
+    print(f"Polling video job: {video_id}")
+
+    while True:
+        response = requests.get(f"{api_url}/v1/videos/{video_id}")
+        response.raise_for_status()
+        status_data = response.json()
+
+        status = status_data["status"]
+        progress = status_data.get("progress", 0)
+
+        print(f"Status: {status}, Progress: {progress}%")
+
+        if status == "completed":
+            print("Video generation completed!")
+            return status_data
+        elif status == "failed":
+            error = status_data.get("error", {})
+            raise RuntimeError(f"Video generation failed: {error}")
+
+        time.sleep(poll_interval)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Helios video generation client")
+    parser.add_argument(
+        "--api-url",
+        default="http://localhost:8000",
+        help="API server URL",
+    )
+    parser.add_argument(
+        "--model",
+        default="BestWishYsh/Helios-Base",
+        help="Model name (Helios-Base, Helios-Mid, Helios-Distilled)",
+    )
+    parser.add_argument(
+        "--prompt",
+        default="A serene lakeside sunrise with mist over the water.",
+        help="Text prompt",
+    )
+    parser.add_argument(
+        "--width",
+        type=int,
+        default=640,
+        help="Video width",
+    )
+    parser.add_argument(
+        "--height",
+        type=int,
+        default=384,
+        help="Video height",
+    )
+    parser.add_argument(
+        "--num-frames",
+        type=int,
+        default=99,
+        help="Number of frames",
+    )
+    parser.add_argument(
+        "--guidance-scale",
+        type=float,
+        default=5.0,
+        help="Guidance scale",
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=42,
+        help="Random seed",
+    )
+    parser.add_argument(
+        "--input-image",
+        type=Path,
+        help="Input image for I2V mode",
+    )
+    parser.add_argument(
+        "--preset",
+        choices=["base", "mid-stage2", "distilled"],
+        default="base",
+        help="Helios preset configuration",
+    )
+
+    args = parser.parse_args()
+
+    # Define preset configurations
+    presets = {
+        "base": None,  # No extra params for base model
+        "mid-stage2": {
+            "is_enable_stage2": True,
+            "pyramid_num_stages": 3,
+            "pyramid_num_inference_steps_list": [20, 20, 20],
+            "use_cfg_zero_star": True,
+            "use_zero_init": True,
+            "zero_steps": 1,
+        },
+        "distilled": {
+            "is_enable_stage2": True,
+            "pyramid_num_stages": 3,
+            "pyramid_num_inference_steps_list": [2, 2, 2],
+            "is_amplify_first_chunk": True,
+        },
+    }
+
+    extra_params = presets[args.preset]
+
+    print("=" * 50)
+    print("Helios Video Generation")
+    print("=" * 50)
+    print(f"API URL: {args.api_url}")
+    print(f"Model: {args.model}")
+    print(f"Preset: {args.preset}")
+    print(f"Prompt: {args.prompt}")
+    print(f"Size: {args.width}x{args.height}")
+    print(f"Frames: {args.num_frames}")
+    print(f"Guidance Scale: {args.guidance_scale}")
+    if extra_params:
+        print(f"Extra Params: {json.dumps(extra_params, indent=2)}")
+    print()
+
+    # Create video job
+    print("Creating video generation job...")
+    job = create_video_job(
+        api_url=args.api_url,
+        prompt=args.prompt,
+        model=args.model,
+        width=args.width,
+        height=args.height,
+        num_frames=args.num_frames,
+        guidance_scale=args.guidance_scale,
+        seed=args.seed,
+        extra_params=extra_params,
+        input_image=args.input_image,
+    )
+
+    video_id = job["id"]
+    print(f"Video job created: {video_id}")
+    print()
+
+    # Poll for completion
+    result = poll_video_status(args.api_url, video_id)
+
+    print()
+    print("=" * 50)
+    print("Generation Complete")
+    print("=" * 50)
+    print(f"Video ID: {result['id']}")
+    print(f"File: {result.get('file_name', 'N/A')}")
+    print(f"Inference Time: {result.get('inference_time_s', 'N/A')}s")
+    print()
+
+
+if __name__ == "__main__":
+    main()

examples/online_serving/helios/run_helios_distilled.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+# Helios-Distilled with DMD Example
+# Fast generation with Distribution Matching Distillation
+
+API_URL="${API_URL:-http://localhost:8000}"
+MODEL="${MODEL:-BestWishYsh/Helios-Distilled}"
+PROMPT="${PROMPT:-A serene lakeside sunrise with mist over the water.}"
+
+echo "==================================="
+echo "Helios-Distilled with DMD"
+echo "==================================="
+echo "API URL: $API_URL"
+echo "Model: $MODEL"
+echo "Prompt: $PROMPT"
+echo ""
+
+# Model-specific parameters for Helios-Distilled
+extra_params='{
+  "is_enable_stage2": true,
+  "pyramid_num_stages": 3,
+  "pyramid_num_inference_steps_list": [2, 2, 2],
+  "is_amplify_first_chunk": true
+}'
+
+echo "Model extra params: $extra_params"
+echo ""
+
+# Create video generation job
+echo "Creating video generation job..."
+RESPONSE=$(curl -s -X POST "$API_URL/v1/videos" \
+  -F "prompt=$PROMPT" \
+  -F "model=$MODEL" \
+  -F "width=640" \
+  -F "height=384" \
+  -F "guidance_scale=1.0" \
+  -F "seed=42" \
+  -F "extra_params=$extra_params")
+
+echo "Response: $RESPONSE"
+echo ""
+
+# Extract video ID
+VIDEO_ID=$(echo "$RESPONSE" | grep -o '"id":"[^"]*"' | cut -d'"' -f4)
+
+if [ -z "$VIDEO_ID" ]; then
+  echo "Error: Failed to create video job"
+  exit 1
+fi
+
+echo "Video job created: $VIDEO_ID"
+echo ""
+
+# Poll for completion
+echo "Polling for completion..."
+while true; do
+  STATUS_RESPONSE=$(curl -s "$API_URL/v1/videos/$VIDEO_ID")
+  STATUS=$(echo "$STATUS_RESPONSE" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
+  PROGRESS=$(echo "$STATUS_RESPONSE" | grep -o '"progress":[0-9]*' | cut -d':' -f2)
+
+  echo "Status: $STATUS, Progress: $PROGRESS%"
+
+  if [ "$STATUS" = "completed" ]; then
+    echo ""
+    echo "Video generation completed!"
+    echo "Full response:"
+    echo "$STATUS_RESPONSE" | jq '.'
+    break
+  elif [ "$STATUS" = "failed" ]; then
+    echo ""
+    echo "Video generation failed!"
+    echo "$STATUS_RESPONSE" | jq '.'
+    exit 1
+  fi
+
+  sleep 2
+done

examples/online_serving/helios/run_helios_mid_stage2.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+# Helios-Mid with Stage 2 + CFG-Zero* Example
+# This demonstrates advanced generation with pyramid multi-stage denoising
+
+API_URL="${API_URL:-http://localhost:8000}"
+MODEL="${MODEL:-BestWishYsh/Helios-Mid}"
+PROMPT="${PROMPT:-A serene lakeside sunrise with mist over the water.}"
+
+echo "==================================="
+echo "Helios-Mid Stage 2 + CFG-Zero*"
+echo "==================================="
+echo "API URL: $API_URL"
+echo "Model: $MODEL"
+echo "Prompt: $PROMPT"
+echo ""
+
+# Model-specific parameters for Helios-Mid
+extra_params='{
+  "is_enable_stage2": true,
+  "pyramid_num_stages": 3,
+  "pyramid_num_inference_steps_list": [20, 20, 20],
+  "use_cfg_zero_star": true,
+  "use_zero_init": true,
+  "zero_steps": 1
+}'
+
+echo "Model extra params: $extra_params"
+echo ""
+
+# Create video generation job
+echo "Creating video generation job..."
+RESPONSE=$(curl -s -X POST "$API_URL/v1/videos" \
+  -F "prompt=$PROMPT" \
+  -F "model=$MODEL" \
+  -F "width=640" \
+  -F "height=384" \
+  -F "guidance_scale=5.0" \
+  -F "seed=42" \
+  -F "extra_params=$extra_params")
+
+echo "Response: $RESPONSE"
+echo ""
+
+# Extract video ID
+VIDEO_ID=$(echo "$RESPONSE" | grep -o '"id":"[^"]*"' | cut -d'"' -f4)
+
+if [ -z "$VIDEO_ID" ]; then
+  echo "Error: Failed to create video job"
+  exit 1
+fi
+
+echo "Video job created: $VIDEO_ID"
+echo ""
+
+# Poll for completion
+echo "Polling for completion..."
+while true; do
+  STATUS_RESPONSE=$(curl -s "$API_URL/v1/videos/$VIDEO_ID")
+  STATUS=$(echo "$STATUS_RESPONSE" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
+  PROGRESS=$(echo "$STATUS_RESPONSE" | grep -o '"progress":[0-9]*' | cut -d':' -f2)
+
+  echo "Status: $STATUS, Progress: $PROGRESS%"
+
+  if [ "$STATUS" = "completed" ]; then
+    echo ""
+    echo "Video generation completed!"
+    echo "Full response:"
+    echo "$STATUS_RESPONSE" | jq '.'
+    break
+  elif [ "$STATUS" = "failed" ]; then
+    echo ""
+    echo "Video generation failed!"
+    echo "$STATUS_RESPONSE" | jq '.'
+    exit 1
+  fi
+
+  sleep 2
+done

examples/online_serving/helios/run_helios_t2v.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+# Helios Text-to-Video Generation Example
+# This script demonstrates how to use the /v1/videos API with Helios models
+
+API_URL="${API_URL:-http://localhost:8000}"
+MODEL="${MODEL:-BestWishYsh/Helios-Base}"
+PROMPT="${PROMPT:-A serene lakeside sunrise with mist over the water.}"
+
+echo "==================================="
+echo "Helios T2V Generation"
+echo "==================================="
+echo "API URL: $API_URL"
+echo "Model: $MODEL"
+echo "Prompt: $PROMPT"
+echo ""
+
+# Create video generation job
+echo "Creating video generation job..."
+RESPONSE=$(curl -s -X POST "$API_URL/v1/videos" \
+  -F "prompt=$PROMPT" \
+  -F "model=$MODEL" \
+  -F "width=640" \
+  -F "height=384" \
+  -F "num_frames=99" \
+  -F "num_inference_steps=50" \
+  -F "guidance_scale=5.0" \
+  -F "seed=42")
+
+echo "Response: $RESPONSE"
+echo ""
+
+# Extract video ID
+VIDEO_ID=$(echo "$RESPONSE" | grep -o '"id":"[^"]*"' | cut -d'"' -f4)
+
+if [ -z "$VIDEO_ID" ]; then
+  echo "Error: Failed to create video job"
+  exit 1
+fi
+
+echo "Video job created: $VIDEO_ID"
+echo ""
+
+# Poll for completion
+echo "Polling for completion..."
+while true; do
+  STATUS_RESPONSE=$(curl -s "$API_URL/v1/videos/$VIDEO_ID")
+  STATUS=$(echo "$STATUS_RESPONSE" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
+  PROGRESS=$(echo "$STATUS_RESPONSE" | grep -o '"progress":[0-9]*' | cut -d':' -f2)
+
+  echo "Status: $STATUS, Progress: $PROGRESS%"
+
+  if [ "$STATUS" = "completed" ]; then
+    echo ""
+    echo "Video generation completed!"
+    echo "Full response:"
+    echo "$STATUS_RESPONSE" | jq '.'
+    break
+  elif [ "$STATUS" = "failed" ]; then
+    echo ""
+    echo "Video generation failed!"
+    echo "$STATUS_RESPONSE" | jq '.'
+    exit 1
+  fi
+
+  sleep 2
+done

examples/online_serving/image_to_image/README.md

@@ -1,8 +1,6 @@
 # Image-To-Image
 
-This example demonstrates how to deploy image-to-image models for online image editing service using vLLM-Omni.
-
-Supported models include Qwen-Image-Edit, BAGEL, and other image-to-image pipelines.
+This example demonstrates how to deploy Qwen-Image-Edit model for online image editing service using vLLM-Omni.
 
 For **multi-image** input editing, use **Qwen-Image-Edit-2509** (QwenImageEditPlusPipeline) and send multiple images in the user message content.
 
@@ -23,12 +21,6 @@ vllm serve Qwen/Qwen-Image-Edit --omni --port 8092
 vllm serve Qwen/Qwen-Image-Edit-2509 --omni --port 8092
 ```
 
-### BAGEL
-
-```bash
-vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091
-```
-
 ### Start with Parameters
 
 
@@ -127,16 +119,6 @@ python openai_chat_client.py --input input.png --prompt "Convert to oil painting
 python openai_chat_client.py --input input1.png input2.png --prompt "Combine these images into a single scene" --output output.png
 ```
 
-Pass model-specific parameters through `--extra-body` (e.g. for BAGEL):
-
-```bash
-python openai_chat_client.py \
-  --input input.png \
-  --prompt "Make the scene look like a watercolor painting" \
-  --server http://localhost:8091 \
-  --extra-body '{"cfg_text_scale": 4.0, "cfg_img_scale": 1.5}'
-```
-
 ### Method 4: Using Gradio Demo
 
 ```bash
@@ -336,9 +318,6 @@ count, use `size` and `n` rather than `height`, `width`, or
 | `layers`                 | int   | 4       | Number of layers (Qwen-Image-Layered) |
 | `resolution`             | int   | 640     | Resolution, 640 or 1024 (Qwen-Image-Layered) |
 
-Models like BAGEL accept additional parameters via `extra_body` (e.g.
-`cfg_text_scale`, `cfg_img_scale`). See the [BAGEL recipe](../../../recipes/Bagel/BAGEL-7B-MoT.md) for the full list.
-
 ## Response Format
 
 ```json

examples/online_serving/image_to_image/openai_chat_client.py

@@ -38,7 +38,6 @@ def edit_image(
     guidance_scale: float | None = None,
     seed: int | None = None,
     negative_prompt: str | None = None,
-    extra_body: dict | None = None,
 ) -> bytes | None:
     """Edit an image using the chat completions API.
 
@@ -52,7 +51,6 @@ def edit_image(
         guidance_scale: CFG guidance scale
         seed: Random seed
         negative_prompt: Negative prompt
-        extra_body: Additional model-specific params (e.g. cfg_text_scale for BAGEL)
 
     Returns:
         Edited image bytes or None if failed
@@ -77,26 +75,24 @@ def edit_image(
     ]
 
     # Build extra_body with generation parameters
-    merged_extra_body: dict[str, object] = {}
+    extra_body = {}
     if height is not None:
-        merged_extra_body["height"] = height
+        extra_body["height"] = height
     if width is not None:
-        merged_extra_body["width"] = width
+        extra_body["width"] = width
     if steps is not None:
-        merged_extra_body["num_inference_steps"] = steps
+        extra_body["num_inference_steps"] = steps
     if guidance_scale is not None:
-        merged_extra_body["guidance_scale"] = guidance_scale
+        extra_body["guidance_scale"] = guidance_scale
     if seed is not None:
-        merged_extra_body["seed"] = seed
+        extra_body["seed"] = seed
     if negative_prompt:
-        merged_extra_body["negative_prompt"] = negative_prompt
-    if extra_body:
-        merged_extra_body.update(extra_body)
+        extra_body["negative_prompt"] = negative_prompt
 
     # Build request payload
-    payload: dict[str, object] = {"messages": messages}
-    if merged_extra_body:
-        payload["extra_body"] = merged_extra_body
+    payload = {"messages": messages}
+    if extra_body:
+        payload["extra_body"] = extra_body
 
     # Send request
     try:
@@ -125,19 +121,6 @@ def edit_image(
         return None
 
 
-def parse_extra_body(value: str) -> dict:
-    """Parse a JSON string into a dict for --extra-body."""
-    import json
-
-    try:
-        obj = json.loads(value)
-    except json.JSONDecodeError as e:
-        raise argparse.ArgumentTypeError(f"--extra-body must be valid JSON: {e}") from e
-    if not isinstance(obj, dict):
-        raise argparse.ArgumentTypeError("--extra-body must be a JSON object")
-    return obj
-
-
 def main():
     parser = argparse.ArgumentParser(description="Qwen-Image-Edit chat client")
     parser.add_argument("--input", "-i", required=True, nargs="+", help="Input image path(s)")
@@ -150,12 +133,6 @@ def main():
     parser.add_argument("--guidance", type=float, default=7.5, help="Guidance scale")
     parser.add_argument("--seed", type=int, default=0, help="Random seed")
     parser.add_argument("--negative", help="Negative prompt")
-    parser.add_argument(
-        "--extra-body",
-        type=parse_extra_body,
-        default=None,
-        help="JSON object merged into request extra_body, e.g. '{\"cfg_text_scale\": 4.0}'.",
-    )
 
     args = parser.parse_args()
 
@@ -175,7 +152,6 @@ def main():
         guidance_scale=args.guidance,
         seed=args.seed,
         negative_prompt=args.negative,
-        extra_body=args.extra_body,
     )
 
     if image_bytes:

examples/online_serving/image_to_video/README.md

@@ -120,17 +120,14 @@ Use `/v1/videos/sync` if you want to write the MP4 directly to a file. `POST /v1
 Generated video files are stored on local disk by the async video API.
 Local file storage behavior can be controlled via the following environment variables:
 
-- `VLLM_OMNI_SERVER_STORAGE__PATH`: directory used for generated files (default: `/tmp/storage`)
-- `VLLM_OMNI_SERVER_STORAGE__FILE_CONCURRENCY`: max concurrent save/delete operations (default: `4`)
-
-`VLLM_OMNI_STORAGE_PATH` and `VLLM_OMNI_STORAGE_MAX_CONCURRENCY` are deprecated and will be
-removed in a future release; use the names above instead.
+- `VLLM_OMNI_STORAGE_PATH`: directory used for generated files (default: `/tmp/storage`)
+- `VLLM_OMNI_STORAGE_MAX_CONCURRENCY`: max concurrent save/delete operations (default: `4`)
 
 Example:
 
 ```bash
-export VLLM_OMNI_SERVER_STORAGE__PATH=/var/tmp/vllm-omni-videos
-export VLLM_OMNI_SERVER_STORAGE__FILE_CONCURRENCY=8
+export VLLM_OMNI_STORAGE_PATH=/var/tmp/vllm-omni-videos
+export VLLM_OMNI_STORAGE_MAX_CONCURRENCY=8
 ```
 
 ## API Calls

examples/online_serving/joyvl_interaction/bridges/asr_bridge.py

@@ -1,131 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-from __future__ import annotations
-
-import argparse
-import base64
-import io
-import json
-import struct
-import wave
-
-import aiohttp
-from aiohttp import web
-
-_HEADER = struct.Struct(">iii")
-_SAMPLE_RATE = 16000
-_DEFAULT_PROMPT = "Transcribe the audio."
-_ASR_TEXT_TAG = "<asr_text>"
-
-
-def _clean_transcript(raw: str) -> str:
-    """Qwen3-ASR emits ``language <Lang><asr_text><transcript>``; keep the transcript."""
-    text = raw or ""
-    marker = text.rfind(_ASR_TEXT_TAG)
-    if marker != -1:
-        text = text[marker + len(_ASR_TEXT_TAG) :]
-    return text.replace("</asr_text>", "").strip()
-
-
-def _pcm_to_wav_b64(pcm: bytes, sample_rate: int) -> str:
-    buf = io.BytesIO()
-    with wave.open(buf, "wb") as wav:
-        wav.setnchannels(1)
-        wav.setsampwidth(2)
-        wav.setframerate(sample_rate)
-        wav.writeframes(pcm)
-    return base64.b64encode(buf.getvalue()).decode()
-
-
-def _result(text: str, *, final: bool) -> dict:
-    return {
-        "asr_response": {
-            "event_type": "IS_FINAL" if final else "IS_PARTIAL",
-            "recognition_result": {"hypothesis": [{"text": text, "confidence": None}]},
-        },
-        "mid": "",
-        "code": 0,
-        "msg": "ok",
-    }
-
-
-async def _transcribe(cfg: dict, pcm: bytes) -> str:
-    if not pcm:
-        return ""
-    wav_b64 = _pcm_to_wav_b64(pcm, cfg["sample_rate"])
-    body = {
-        "model": cfg["model"],
-        "messages": [
-            {
-                "role": "user",
-                "content": [
-                    {"type": "input_audio", "input_audio": {"data": wav_b64, "format": "wav"}},
-                    {"type": "text", "text": cfg["prompt"]},
-                ],
-            }
-        ],
-        "temperature": 0.0,
-        "max_tokens": cfg["max_tokens"],
-    }
-    url = cfg["backend_url"].rstrip("/") + "/v1/chat/completions"
-    async with aiohttp.ClientSession() as session:
-        async with session.post(url, json=body) as resp:
-            payload = await resp.json()
-    choices = payload.get("choices") or []
-    if not choices:
-        return ""
-    return _clean_transcript(choices[0].get("message", {}).get("content") or "")
-
-
-async def _handle(request: web.Request) -> web.WebSocketResponse:
-    ws = web.WebSocketResponse(max_msg_size=0)
-    await ws.prepare(request)
-    cfg = request.app["cfg"]
-    pcm = bytearray()
-    try:
-        async for msg in ws:
-            if msg.type == aiohttp.WSMsgType.BINARY and len(msg.data) >= _HEADER.size:
-                seqid, _, _ = _HEADER.unpack(msg.data[: _HEADER.size])
-                pcm.extend(msg.data[_HEADER.size :])
-                if seqid < 0:
-                    text = await _transcribe(cfg, bytes(pcm))
-                    await ws.send_str(json.dumps(_result(text, final=True), ensure_ascii=False))
-                    pcm.clear()
-            elif msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.CLOSED, aiohttp.WSMsgType.ERROR):
-                break
-    finally:
-        if not ws.closed:
-            await ws.close()
-    return ws
-
-
-def create_app(backend_url: str, model: str, sample_rate: int, prompt: str, max_tokens: int) -> web.Application:
-    app = web.Application()
-    app["cfg"] = {
-        "backend_url": backend_url,
-        "model": model,
-        "sample_rate": sample_rate,
-        "prompt": prompt,
-        "max_tokens": max_tokens,
-    }
-    app.router.add_get("/v1/asr", _handle)
-    return app
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description="JoyVL webui <-> Qwen3-ASR (chat input_audio) bridge")
-    parser.add_argument("--host", default="0.0.0.0")
-    parser.add_argument("--port", type=int, default=8093)
-    parser.add_argument("--backend-url", default="http://127.0.0.1:8094")
-    parser.add_argument("--model", default="Qwen/Qwen3-ASR-1.7B")
-    parser.add_argument("--sample-rate", type=int, default=_SAMPLE_RATE)
-    parser.add_argument("--prompt", default=_DEFAULT_PROMPT)
-    parser.add_argument("--max-tokens", type=int, default=256)
-    args = parser.parse_args()
-    app = create_app(args.backend_url, args.model, args.sample_rate, args.prompt, args.max_tokens)
-    web.run_app(app, host=args.host, port=args.port)
-
-
-if __name__ == "__main__":
-    main()

examples/online_serving/joyvl_interaction/bridges/tts_bridge.py

@@ -1,117 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-from __future__ import annotations
-
-import argparse
-import asyncio
-import json
-import logging
-
-import aiohttp
-from aiohttp import web
-
-try:
-    import cn2an  # optional: Arabic-digit -> Chinese number normalization
-except ImportError:
-    cn2an = None
-
-logger = logging.getLogger("tts_bridge")
-
-
-def _normalize(text: str) -> str:
-    """Spell out Arabic digits in Chinese so short counting outputs synthesize
-    intelligibly: bare "1次" garbles on Qwen3-TTS (and others), while "一次"
-    is clean. ``cn2an.transform`` rewrites digits in place ("第3次"->"第三次")
-    and is pure-Python (~microseconds). No-op if cn2an is unavailable."""
-    if not cn2an or not text:
-        return text
-    try:
-        return cn2an.transform(text, "an2cn")
-    except Exception as err:  # never let TN break synthesis
-        logger.debug("cn2an normalize skipped: %s", err)
-        return text
-
-
-async def _pump_backend_to_front(back: aiohttp.ClientWebSocketResponse, front: web.WebSocketResponse) -> None:
-    async for msg in back:
-        if msg.type == aiohttp.WSMsgType.BINARY:
-            await front.send_bytes(msg.data)
-        elif msg.type == aiohttp.WSMsgType.TEXT:
-            event = json.loads(msg.data).get("type")
-            if event == "session.done":
-                await front.send_str(json.dumps({"type": "response.done"}))
-                return
-            if event == "error":
-                await front.send_str(json.dumps({"type": "error", "error": "tts backend error"}))
-                return
-        elif msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.CLOSED, aiohttp.WSMsgType.ERROR):
-            return
-
-
-async def _handle(request: web.Request) -> web.WebSocketResponse:
-    front = web.WebSocketResponse(heartbeat=20, max_msg_size=0)
-    await front.prepare(request)
-    cfg = request.app["cfg"]
-
-    session = aiohttp.ClientSession()
-    back: aiohttp.ClientWebSocketResponse | None = None
-    pump: asyncio.Task | None = None
-    parts: list[str] = []
-    try:
-        async for msg in front:
-            if msg.type != web.WSMsgType.TEXT:
-                continue
-            data = json.loads(msg.data)
-
-            if "config" in data and back is None:
-                voice = (data["config"] or {}).get("voice") or cfg["voice"]
-                back = await session.ws_connect(cfg["backend_url"], max_msg_size=0)
-                await back.send_str(
-                    json.dumps({"type": "session.config", "response_format": cfg["response_format"], "voice": voice})
-                )
-                pump = asyncio.create_task(_pump_backend_to_front(back, front))
-            elif data.get("type") == "input_text.append" and back is not None:
-                parts.append(data.get("text", ""))
-            elif data.get("type") == "input_text.commit" and back is not None:
-                # Normalize the full utterance once (digits may span chunks), then send.
-                await back.send_str(json.dumps({"type": "input.text", "text": _normalize("".join(parts))}))
-                parts = []
-                await back.send_str(json.dumps({"type": "input.done"}))
-                if pump is not None:
-                    await pump
-                break
-    except Exception as err:
-        logger.warning("tts bridge error: %s", err)
-    finally:
-        if pump is not None and not pump.done():
-            pump.cancel()
-        if back is not None and not back.closed:
-            await back.close()
-        await session.close()
-        if not front.closed:
-            await front.close()
-    return front
-
-
-def create_app(backend_url: str, voice: str, response_format: str) -> web.Application:
-    app = web.Application()
-    app["cfg"] = {"backend_url": backend_url, "voice": voice, "response_format": response_format}
-    app.router.add_get("/v1/tts", _handle)
-    return app
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description="JoyVL webui <-> Qwen3-TTS websocket bridge")
-    parser.add_argument("--host", default="0.0.0.0")
-    parser.add_argument("--port", type=int, default=8092)
-    parser.add_argument("--backend-url", default="ws://127.0.0.1:8091/v1/audio/speech/stream")
-    parser.add_argument("--voice", default="vivian")
-    parser.add_argument("--response-format", default="pcm")
-    args = parser.parse_args()
-    logging.basicConfig(level=logging.INFO)
-    web.run_app(create_app(args.backend_url, args.voice, args.response_format), host=args.host, port=args.port)
-
-
-if __name__ == "__main__":
-    main()

examples/online_serving/joyvl_interaction/cli/run_cli_demo.py

@@ -1,32 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-from __future__ import annotations
-
-import argparse
-
-from stream_client import stream
-
-ICON = {"response": "SPEAK   ", "silence": "silence ", "delegate": "DELEGATE"}
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser()
-    parser.add_argument("video")
-    parser.add_argument("--server", default="http://127.0.0.1:8070")
-    parser.add_argument("--query", default=None)
-    parser.add_argument("--fps", type=float, default=1.0)
-    parser.add_argument("--session-id", default="cli")
-    args = parser.parse_args()
-
-    for tick in stream(args.video, args.server, args.session_id, args.query, args.fps):
-        if tick.action == "silence":
-            continue
-        line = f"[{tick.t:6.1f}s] {ICON.get(tick.action)} {tick.text}"
-        if tick.delegation:
-            line += f"  (-> {tick.delegation.get('question', '')})"
-        print(line, flush=True)
-
-
-if __name__ == "__main__":
-    main()

examples/online_serving/joyvl_interaction/cli/stream_client.py

@@ -1,79 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-from __future__ import annotations
-
-import base64
-import time
-from collections.abc import Iterator
-from dataclasses import dataclass
-
-import cv2
-import requests
-
-
-@dataclass
-class Tick:
-    index: int
-    t: float
-    action: str
-    text: str
-    skipped: bool
-    latency_ms: float
-    delegation: dict | None
-
-
-def iter_frames(video_path: str, fps: float) -> Iterator[tuple[float, bytes]]:
-    cap = cv2.VideoCapture(video_path)
-    src_fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
-    step = max(1, round(src_fps / fps))
-    idx = 0
-    while True:
-        ok, frame = cap.read()
-        if not ok:
-            break
-        if idx % step == 0:
-            ok, buf = cv2.imencode(".jpg", frame)
-            if ok:
-                yield idx / src_fps, buf.tobytes()
-        idx += 1
-    cap.release()
-
-
-def stream(
-    video_path: str,
-    server: str,
-    session_id: str,
-    query: str | None = None,
-    fps: float = 1.0,
-    realtime: bool = False,
-) -> Iterator[Tick]:
-    """Send each sampled frame as one chat completion; yield the parsed action.
-
-    ``query`` is attached to the first tick to arm the session; inject later
-    queries by posting another frame turn with a new text part.
-    """
-    url = server.rstrip("/") + "/v1/chat/completions"
-    requests.post(server.rstrip("/") + "/reset", json={"session_id": session_id}, timeout=10)
-    for i, (t, jpeg) in enumerate(iter_frames(video_path, fps)):
-        data_url = "data:image/jpeg;base64," + base64.b64encode(jpeg).decode()
-        content = [{"type": "image_url", "image_url": {"url": data_url}}]
-        if i == 0 and query:
-            content.insert(0, {"type": "text", "text": query})
-        resp = requests.post(
-            url,
-            json={"session_id": session_id, "messages": [{"role": "user", "content": content}]},
-            timeout=120,
-        ).json()
-        info = resp.get("interaction", {})
-        yield Tick(
-            index=info.get("frame_index", i + 1),
-            t=t,
-            action=info.get("action", "?"),
-            text=resp["choices"][0]["message"]["content"],
-            skipped=info.get("inference_skipped", False),
-            latency_ms=info.get("latency_ms", 0.0),
-            delegation=info.get("delegation"),
-        )
-        if realtime:
-            time.sleep(1.0 / fps)

examples/online_serving/joyvl_interaction/rtsp/README.md

@@ -1,46 +0,0 @@
-# RTSP local streaming
-
-Simulate an RTSP camera from a local video file so the WebUI (or any RTSP client) can
-test the RTSP input path without a physical IP camera. RTSP is a **webui-side input** —
-the browser pulls the stream and feeds frames to the orchestrator over the normal API;
-no serving-layer code is involved.
-
-Needs [`ffmpeg`](https://ffmpeg.org/) and [MediaMTX](https://github.com/bluenviron/mediamtx/releases)
-(download the binary + `mediamtx.yml` and keep them next to these scripts, or point
-`MEDIAMTX_BIN` / `MEDIAMTX_CONFIG` at them).
-
-## 1. Start the RTSP server
-
-```bash
-bash ./mediamtx.sh        # MediaMTX, listens on :8554
-```
-
-## 2. Push a local video as an RTSP stream
-
-```bash
-bash ./rtsp.sh ./videos/example.mp4 rtsp://127.0.0.1:8554/fire1
-```
-
-`rtsp.sh` takes `[video-path] [rtsp-url]` (or set `VIDEO_PATH` / `RTSP_URL`). To stream a
-whole directory of clips, one stream per file:
-
-```bash
-bash ./rtsp_all.sh ./videos rtsp://127.0.0.1:8554
-```
-
-## 3. Use it in the WebUI
-
-Enter the URL in the WebUI RTSP box:
-
-```
-rtsp://127.0.0.1:8554/fire1
-```
-
-If the WebUI runs on another machine, replace `127.0.0.1` with the MediaMTX host's IP.
-
-## Notes
-
-- `rtsp.sh` transcodes audio to AAC; replace `-c:a aac -b:a 128k -ar 44100` with `-an`
-  if the source clip has no audio track and your `ffmpeg` errors on it.
-- If a client cannot connect, check that port `8554` is reachable and not blocked by a
-  firewall, container network, or security group.

examples/online_serving/joyvl_interaction/rtsp/mediamtx.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# Start a local MediaMTX RTSP server (listens on :8554 by default).
-# Download the binary + mediamtx.yml from https://github.com/bluenviron/mediamtx/releases
-# and either keep them next to this script or point the vars below at them.
-
-MEDIAMTX_BIN="${MEDIAMTX_BIN:-./mediamtx}"
-MEDIAMTX_CONFIG="${MEDIAMTX_CONFIG:-./mediamtx.yml}"
-
-exec "$MEDIAMTX_BIN" "$MEDIAMTX_CONFIG"

examples/online_serving/joyvl_interaction/rtsp/rtsp.sh

@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# Loop a local video file and push it to an RTSP URL, simulating an RTSP camera.
-#   bash ./rtsp.sh [video-path] [rtsp-output-url]
-# Defaults stream ./videos/example.mp4 to rtsp://127.0.0.1:8554/fire1
-
-VIDEO_PATH="${1:-${VIDEO_PATH:-./videos/example.mp4}}"
-RTSP_URL="${2:-${RTSP_URL:-rtsp://127.0.0.1:8554/fire1}}"
-
-if [[ ! -f "$VIDEO_PATH" ]]; then
-  echo "Video file not found: $VIDEO_PATH" >&2
-  exit 1
-fi
-
-echo "Input video: $VIDEO_PATH"
-echo "RTSP output: $RTSP_URL"
-
-# Drop "-c:a aac -b:a 128k -ar 44100" for "-an" if the source has no audio track.
-exec ffmpeg \
-  -hide_banner \
-  -loglevel info \
-  -re \
-  -stream_loop -1 \
-  -i "$VIDEO_PATH" \
-  -vf "scale='min(1280,iw)':-2" \
-  -c:v libx264 \
-  -preset veryfast \
-  -tune zerolatency \
-  -b:v 2500k \
-  -maxrate 2500k \
-  -bufsize 5000k \
-  -c:a aac \
-  -b:a 128k \
-  -ar 44100 \
-  -f rtsp \
-  -rtsp_transport udp \
-  "$RTSP_URL"

examples/online_serving/joyvl_interaction/rtsp/rtsp_all.sh

@@ -1,64 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# Push every .mp4 in a directory as its own RTSP stream (one ffmpeg per file),
-# named rtsp://<base>/<filename-without-extension>.
-#   bash ./rtsp_all.sh [video-dir] [rtsp-base-url]
-
-VIDEO_DIR="${1:-./videos}"
-RTSP_BASE="${2:-rtsp://127.0.0.1:8554}"
-
-if [[ ! -d "$VIDEO_DIR" ]]; then
-  echo "Video directory not found: $VIDEO_DIR" >&2
-  exit 1
-fi
-
-shopt -s nullglob
-VIDEO_FILES=("$VIDEO_DIR"/*.mp4)
-shopt -u nullglob
-
-if [[ ${#VIDEO_FILES[@]} -eq 0 ]]; then
-  echo "No video files found in: $VIDEO_DIR" >&2
-  exit 1
-fi
-
-echo "Found ${#VIDEO_FILES[@]} video(s) in $VIDEO_DIR"
-
-PIDS=()
-cleanup() {
-  echo "Stopping all streams..."
-  for pid in "${PIDS[@]}"; do
-    kill "$pid" 2>/dev/null || true
-  done
-  wait 2>/dev/null
-}
-trap cleanup EXIT INT TERM
-
-for video in "${VIDEO_FILES[@]}"; do
-  stream_name="$(basename "${video%.*}")"
-  rtsp_url="${RTSP_BASE}/${stream_name}"
-  echo "Starting stream: $rtsp_url  <-  $(basename "$video")"
-  ffmpeg \
-    -hide_banner \
-    -loglevel error \
-    -re \
-    -stream_loop -1 \
-    -i "$video" \
-    -vf "format=yuv420p,scale='min(1280,iw)':'min(720,ih)':force_original_aspect_ratio=decrease:force_divisible_by=2" \
-    -c:v libx264 \
-    -preset veryfast \
-    -tune zerolatency \
-    -b:v 4000k \
-    -maxrate 4500k \
-    -bufsize 9000k \
-    -c:a aac \
-    -b:a 128k \
-    -ar 44100 \
-    -f rtsp \
-    -rtsp_transport tcp \
-    "$rtsp_url" &
-  PIDS+=($!)
-done
-
-echo "All ${#VIDEO_FILES[@]} streams started. Press Ctrl+C to stop."
-wait

examples/online_serving/joyvl_interaction/scripts/start_all.sh

@@ -1,108 +0,0 @@
-#!/bin/bash
-# One-shot JoyVL demo: model -> interaction orchestrator -> JD webui.
-#
-# The model is the only fixed component; every external module is pluggable and
-# configured here via env (unset = disabled), matching JoyVL's design:
-#
-#   TTS_URL                       voice out  (point at our tts_bridge, or any OpenAI-Realtime-style TTS WS)
-#   ASR_URL                       voice in   (point at our asr_bridge, or any compatible ASR WS)
-#   DELEGATION_BACKEND_URL        orchestrator background brain for </delegation> (OpenAI-compatible; DELEGATION_MODEL/DELEGATION_API_KEY pick model+key)
-#                                 DELEGATION_KIND=chat (default) text/VL brain answers; =image a text-to-image model generates a picture
-#                                 chat+Claude: DELEGATION_BACKEND_URL=https://api.anthropic.com/v1/ DELEGATION_MODEL=claude-... DELEGATION_API_KEY=sk-ant-...
-#                                 image+Qwen-Image: DELEGATION_KIND=image DELEGATION_BACKEND_URL=http://127.0.0.1:8091/v1
-#   BACKGROUND_CODEX_API_URL      webui-side background agent
-#   ENABLE_MEMORY (default 1)     3-tier memory; SUMMARIZER_BACKEND_URL/MODEL pick the summarizer (default: reuse main)
-#
-# Usage:  bash start_all.sh
-#         TTS_URL=ws://host/v1/tts ASR_URL=ws://host/v1/asr GPU=0 bash start_all.sh
-set -o pipefail
-
-CODE="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
-VENV="${VENV:-$CODE/.venv}"
-WEBUI_DIR="${WEBUI_DIR:-$CODE/joyvl-interaction/joyvl-interaction-webui}"
-MODEL="${MODEL:-jdopensource/JoyAI-VL-Interaction-Preview}"
-SERVED_NAME="${SERVED_NAME:-JoyAI-VL-Interaction-Preview}"
-GPU="${GPU:-0}"
-MODEL_PORT="${MODEL_PORT:-8061}"
-ORCH_PORT="${ORCH_PORT:-8070}"
-WEBUI_PORT="${WEBUI_PORT:-8999}"
-WEBUI_USERNAME="${WEBUI_USERNAME:-admin}"
-WEBUI_PASSWORD="${WEBUI_PASSWORD:-changeme}"   # override via env for any exposed deployment
-ENABLE_MEMORY="${ENABLE_MEMORY:-1}"
-RESPONSE_DEDUP="${RESPONSE_DEDUP:-0.85}"   # demo drops near-duplicate narration; set 1.0 for reference (exact-only)
-TTS_URL="${TTS_URL:-}"
-ASR_URL="${ASR_URL:-}"
-BACKGROUND_CODEX_API_URL="${BACKGROUND_CODEX_API_URL:-}"
-
-LOG=/tmp/joyvl_demo
-mkdir -p "$LOG"
-
-[ -d "$WEBUI_DIR" ] || { echo "! WEBUI_DIR not found: $WEBUI_DIR (set WEBUI_DIR=...)"; exit 1; }
-
-kill_port() {
-  local p
-  p=$(ss -tlnp 2>/dev/null | grep ":$1 " | grep -oE 'pid=[0-9]+' | head -1 | cut -d= -f2)
-  [ -n "${p:-}" ] && kill "$p" 2>/dev/null
-  return 0
-}
-wait_http() {
-  for _ in $(seq 1 150); do
-    curl -sk -m2 "$1" -o /dev/null 2>/dev/null && return 0
-    sleep 2
-  done
-  echo "  ! timeout waiting for $1"
-  return 1
-}
-
-echo "Resetting ports ${MODEL_PORT} ${ORCH_PORT} ${WEBUI_PORT}…"
-for p in "$MODEL_PORT" "$ORCH_PORT" "$WEBUI_PORT"; do kill_port "$p"; done
-sleep 3
-
-[ -f "$VENV/bin/activate" ] && source "$VENV/bin/activate"
-
-# detach into a new session so servers outlive this launcher
-spawn() { local log="$1"; shift; setsid "$@" > "$log" 2>&1 < /dev/null & }
-
-echo "[1/3] model on GPU ${GPU}…"
-spawn "$LOG/model.log" env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 CUDA_VISIBLE_DEVICES="$GPU" \
-  vllm serve "$MODEL" --served-model-name "$SERVED_NAME" --port "$MODEL_PORT" \
-  --gpu-memory-utilization 0.85 --max-model-len 131072 --enable-prefix-caching \
-  --limit-mm-per-prompt '{"image":256,"video":1}'
-wait_http "http://127.0.0.1:${MODEL_PORT}/health" && echo "  up :${MODEL_PORT}"
-
-echo "[2/3] orchestrator (memory=${ENABLE_MEMORY})…"
-ORCH_ARGS=(--port "$ORCH_PORT" --main-backend-url "http://127.0.0.1:${MODEL_PORT}/v1"
-           --main-model "$SERVED_NAME"
-           --response-dedup-threshold "$RESPONSE_DEDUP")
-# delegation: point at a stronger background brain to enable it; otherwise keep it off
-if [ -n "${DELEGATION_BACKEND_URL:-}" ]; then
-  ORCH_ARGS+=(--delegation-backend-url "$DELEGATION_BACKEND_URL")
-  ORCH_ARGS+=(--delegation-kind "${DELEGATION_KIND:-chat}")
-  [ -n "${DELEGATION_MODEL:-}" ] && ORCH_ARGS+=(--delegation-model "$DELEGATION_MODEL")
-  [ -n "${DELEGATION_API_KEY:-}" ] && ORCH_ARGS+=(--delegation-api-key "$DELEGATION_API_KEY")
-else
-  ORCH_ARGS+=(--no-delegation)
-fi
-[ "$ENABLE_MEMORY" = "0" ] && ORCH_ARGS+=(--no-memory)
-[ -n "${SUMMARIZER_BACKEND_URL:-}" ] && ORCH_ARGS+=(--summarizer-backend-url "$SUMMARIZER_BACKEND_URL")
-[ -n "${SUMMARIZER_MODEL:-}" ] && ORCH_ARGS+=(--summarizer-model "$SUMMARIZER_MODEL")
-spawn "$LOG/orch.log" env PYTHONPATH="$CODE" python -m vllm_omni.experimental.fullduplex.joyvl.serving.server "${ORCH_ARGS[@]}"
-wait_http "http://127.0.0.1:${ORCH_PORT}/health" && echo "  up :${ORCH_PORT}"
-
-echo "[3/3] JD webui…"
-spawn "$LOG/webui.log" env WEBUI_USERNAME="$WEBUI_USERNAME" WEBUI_PASSWORD="$WEBUI_PASSWORD" \
-  TTS_URL="$TTS_URL" ASR_URL="$ASR_URL" BACKGROUND_CODEX_API_URL="$BACKGROUND_CODEX_API_URL" \
-  bash -c "cd '$WEBUI_DIR' && exec bash scripts/start_server.sh --api-base 'http://127.0.0.1:${ORCH_PORT}/v1'"
-webui_up=0
-for _ in $(seq 1 60); do grep -q "Server startup complete" "$LOG/webui.log" 2>/dev/null && { webui_up=1; break; }; sleep 1; done
-[ "$webui_up" = 1 ] || { echo "! webui did not start — see $LOG/webui.log"; exit 1; }
-
-IP=$(hostname -I 2>/dev/null | awk '{print $1}')
-echo
-echo "=== JoyVL demo ready ==="
-echo "  webui  : https://${IP}:${WEBUI_PORT}   (user ${WEBUI_USERNAME} / pass ${WEBUI_PASSWORD})"
-echo "  memory : $([ "$ENABLE_MEMORY" = 0 ] && echo off || echo on)"
-echo "  TTS    : ${TTS_URL:-<unset>}"
-echo "  ASR    : ${ASR_URL:-<unset>}"
-echo "  agent  : ${BACKGROUND_CODEX_API_URL:-<unset>}"
-echo "  logs   : ${LOG}/{model,orch,webui}.log"

examples/online_serving/joyvl_interaction/scripts/start_asr.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-# Serve Qwen3-ASR (the same audio-LLM AURA uses as its ASR stage) and run the
-# webui<->ASR bridge. Then start the webui with:
-#   ASR_URL=ws://127.0.0.1:8093/v1/asr bash scripts/start_server.sh
-set -euo pipefail
-
-ASR_MODEL="${ASR_MODEL:-Qwen/Qwen3-ASR-1.7B}"
-ASR_PORT="${ASR_PORT:-8094}"
-BRIDGE_PORT="${BRIDGE_PORT:-8093}"
-GPU="${GPU:-2}"
-SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
-
-# Qwen3-ASR is audio-in / text-out; served as a normal multimodal chat model.
-CUDA_VISIBLE_DEVICES="${GPU}" vllm serve "${ASR_MODEL}" \
-    --host 0.0.0.0 --port "${ASR_PORT}" \
-    --gpu-memory-utilization 0.3 --trust-remote-code &
-ASR_PID=$!
-trap 'kill ${ASR_PID} 2>/dev/null || true' EXIT
-
-until curl -s "http://127.0.0.1:${ASR_PORT}/health" >/dev/null 2>&1; do sleep 2; done
-python "${SCRIPT_DIR}/../bridges/asr_bridge.py" \
-    --port "${BRIDGE_PORT}" \
-    --backend-url "http://127.0.0.1:${ASR_PORT}" \
-    --model "${ASR_MODEL}"

examples/online_serving/joyvl_interaction/scripts/start_interaction.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-# Launch the interaction orchestrator in front of a running model server.
-set -euo pipefail
-
-PORT="${PORT:-8070}"
-MAIN_BACKEND_URL="${MAIN_BACKEND_URL:-http://127.0.0.1:8061/v1}"
-MAIN_MODEL="${MAIN_MODEL:-JoyAI-VL-Interaction-Preview}"
-PERSONA="${PERSONA:-default}"
-
-# By default memory reuses the main model as its own summarizer. Set
-# SUMMARIZER_BACKEND_URL/SUMMARIZER_MODEL to a dedicated Qwen3-VL-4B server,
-# or pass --no-memory for the lightest setup.
-EXTRA=()
-[[ -n "${SUMMARIZER_BACKEND_URL:-}" ]] && EXTRA+=(--summarizer-backend-url "${SUMMARIZER_BACKEND_URL}")
-[[ -n "${SUMMARIZER_MODEL:-}" ]] && EXTRA+=(--summarizer-model "${SUMMARIZER_MODEL}")
-[[ "${NO_MEMORY:-0}" == "1" ]] && EXTRA+=(--no-memory)
-
-python -m vllm_omni.experimental.fullduplex.joyvl.serving.server \
-    --port "${PORT}" \
-    --main-backend-url "${MAIN_BACKEND_URL}" \
-    --main-model "${MAIN_MODEL}" \
-    --persona "${PERSONA}" \
-    "${EXTRA[@]}"

examples/online_serving/joyvl_interaction/scripts/start_model.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-# Serve the JoyVL interaction model as a plain VLM (NOT --omni: it is a
-# standard Qwen3-VL autoregressive model, not an omni/diffusion model).
-set -euo pipefail
-
-MODEL="${MODEL:-jdopensource/JoyAI-VL-Interaction-Preview}"
-SERVED_NAME="${SERVED_NAME:-JoyAI-VL-Interaction-Preview}"
-PORT="${PORT:-8061}"
-GPU="${GPU:-0}"
-# Covers the default short-term window (chunk_frames=100) + key_frames=0 (all frames
-# summarized). Lower MAX_MODEL_LEN / IMAGE_LIMIT (and --chunk-frames) together for ~24GB.
-MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}"
-IMAGE_LIMIT="${IMAGE_LIMIT:-256}"
-
-CUDA_VISIBLE_DEVICES="${GPU}" vllm serve "${MODEL}" \
-    --served-model-name "${SERVED_NAME}" \
-    --port "${PORT}" \
-    --max-model-len "${MAX_MODEL_LEN}" \
-    --gpu-memory-utilization 0.85 \
-    --enable-prefix-caching \
-    --enable-chunked-prefill \
-    --limit-mm-per-prompt "{\"image\":${IMAGE_LIMIT},\"video\":1}"

examples/online_serving/joyvl_interaction/scripts/start_tts.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-# Serve Qwen3-TTS (vLLM-Omni) and run the webui<->TTS bridge.
-# Then start the webui with:  TTS_URL=ws://127.0.0.1:8092/v1/tts bash scripts/start_server.sh
-set -euo pipefail
-
-TTS_MODEL="${TTS_MODEL:-Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice}"
-TTS_PORT="${TTS_PORT:-8091}"
-BRIDGE_PORT="${BRIDGE_PORT:-8092}"
-GPU="${GPU:-1}"
-VOICE="${VOICE:-vivian}"
-SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
-
-CUDA_VISIBLE_DEVICES="${GPU}" vllm-omni serve "${TTS_MODEL}" \
-    --deploy-config vllm_omni/deploy/qwen3_tts.yaml \
-    --host 0.0.0.0 --port "${TTS_PORT}" \
-    --gpu-memory-utilization 0.9 --trust-remote-code --omni &
-TTS_PID=$!
-trap 'kill ${TTS_PID} 2>/dev/null || true' EXIT
-
-# wait for the TTS server, then run the bridge in the foreground
-until curl -s "http://127.0.0.1:${TTS_PORT}/health" >/dev/null 2>&1; do sleep 2; done
-python "${SCRIPT_DIR}/../bridges/tts_bridge.py" \
-    --port "${BRIDGE_PORT}" \
-    --backend-url "ws://127.0.0.1:${TTS_PORT}/v1/audio/speech/stream" \
-    --voice "${VOICE}"

examples/online_serving/qwen3_omni/README.md

@@ -24,15 +24,12 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
     --deploy-config /path/to/your_deploy_config.yaml
 ```
 
-For a 3x-GPU multi-replica layout (talker/code2wav scale-out on cuda:1,2),
-use `--stage-overrides` on top of the default config:
+For the bundled 3x-GPU multi-replica layout (talker/code2wav scale-out),
+use:
 
 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
-    --stage-overrides '{
-        "1": {"num_replicas": 2, "devices": "1,2"},
-        "2": {"num_replicas": 2, "devices": "1,2"}
-    }'
+    --deploy-config vllm_omni/deploy/qwen3_omni_moe_multi_replicas.yaml
 ```
 
 ### Launch individual stages (stage-based CLI)

examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml

@@ -20,7 +20,7 @@ stage_args:
       trust_remote_code: true
       engine_output_type: text
       distributed_executor_backend: "mp"
-      enable_prefix_caching: true
+      enable_prefix_caching: false
       hf_config_name: thinker_config
       tensor_parallel_size: 2
     final_output: true