[Bugfix] Skip HWR store-size scans when no limit is configured

Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>

.buildkite/amd/test-amd-merge.yml

@@ -72,7 +72,22 @@ steps:
         - export VLLM_LOGGING_LEVEL=DEBUG
         - export VLLM_WORKER_MULTIPROC_METHOD=spawn
         - pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
-        - pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
+        - >-
+          pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
+          --deselect='tests/diffusion/offloader/test_diffusion_layerwise_offload.py::test_layerwise_offload_diffusion_model[stabilityai/stable-audio-open-1.0]'
+
+    # TODO: Diagnose unstable Stable Audio layerwise-offload memory savings on ROCm.
+    - label: "Diffusion Stable Audio Layerwise Offload Test (quarantined)"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: NonBlocking
+      commands:
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - >-
+          pytest -s -v
+          'tests/diffusion/offloader/test_diffusion_layerwise_offload.py::test_layerwise_offload_diffusion_model[stabilityai/stable-audio-open-1.0]'
 
     - label: "Diffusion Sequence Parallelism Test (Need 4 GPUs) · Shard 1/2"
       agent_pool: mi300_4
@@ -223,13 +238,10 @@ steps:
       grade: Blocking
       timeout_in_minutes: 50
       commands:
-        - |
-          timeout 40m bash -c '
-            export VLLM_LOGGING_LEVEL=DEBUG
-            export VLLM_WORKER_MULTIPROC_METHOD=spawn
-            export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-            pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-          '
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+        - timeout 40m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model and cuda" --run-level "advanced_model"
 
 # TODO: Bagel test on ROCm is very unstable. @tjtanaa
 # Need to debug before reneable numerical changes across large PRs

.buildkite/common/ci_mirror_hardwares.yml

@@ -1,19 +1,31 @@
 # Uploader-only hardware presets for CUDA / NPU test YAML.
 # Referenced as ``mirror_hardwares: l4_1`` on a step; ``upload_pipeline.py`` expands to
-# ``agents`` (+ optional top-level ``image`` for NPU) + ``plugins``, then strips
-# ``mirror_hardwares`` before Buildkite upload.
+# ``agents`` (+ optional top-level ``image`` for NPU) + ``plugins`` (+ ``retry``
+# for ``l4_*``), then strips ``mirror_hardwares`` before Buildkite upload.
+#
+# Omit the key: compose {chip}_{n} from pytest -m SKU + cards_n (max of several
+# positives; not cards_1 with no positive cards_* → that chip's highest existing preset).
+# Unset MIRROR_HW matches H100 or L4 in -m (both → H100); no match skips the step.
+# MIRROR_HW=b200 matches B200 in -m (else the step is skipped).
+# MIRROR_HW must be empty or b200 (case-insensitive); unknown values fail the upload.
+# CUDA preset strings (h100_4, l4_1) are omitted when MIRROR_HW=b200.
+# Unset MIRROR_HW keeps string presets. NPU presets ignore MIRROR_HW.
 #
 # CUDA naming: {chip}_{count}
-#   l4_1, l4_4, h100_1 … h100_4
+#   l4_1, l4_2, l4_3, l4_4, h100_1 … h100_4, b200_1 … b200_4
 # NPU naming: {queue}_{resource_class}
 #   a2b3_npu_1  — ascend-a2b3, resource_class npu-1 (A2/B3 CI image)
 #   a2b3_npu_4  — ascend-a2b3, resource_class npu-4
-#   a3_npu_2    — ascend-a3, resource_class npu-2
+#   a3_npu_2    — ascend-a3, resource_class npu-2 (A3 CI image tag …:$BUILDKITE_COMMIT-a3)
 #
-# The ``l4_*`` presets run under the ``docker`` plugin, which does not put
-# ``buildkite-agent`` on PATH inside the container the way the ``kubernetes``
-# presets do, so they set ``mount-buildkite-agent: true``; without it any step
-# calling ``buildkite-agent`` (artifact upload, annotate, meta-data) fails there.
+# B200 (b200-k8s): matches vLLM device ``b200-k8s`` from ci-infra
+# (AgentQueue.B200_K8S + b200_plugin_template). Queue routes the K8s pool;
+# podSpec uses hostNetwork + /raid|/mnt/shared (no H100-style nodeSelector).
+#
+# The ``l4_*`` presets run on the EKS ``l4-k8s`` queue (agent-stack-k8s),
+# packed onto g6.12xlarge L4 nodes (``vllm.ci/gpu-pool=l4x4``). GPU / CPU /
+# memory / shm requests scale with card count (1 GPU = 10 CPU / 40Gi / 24Gi shm).
+# Retry covers lost agents / killed pods, not pytest or OOM (exit 1 / 137).
 
 x-npu-k8s-plugins: &npu_k8s_plugins
   - kubernetes:
@@ -21,39 +33,224 @@ x-npu-k8s-plugins: &npu_k8s_plugins
         imagePullSecrets:
           - name: swr-secret
 
+# Infra-only retry. Omit exit 1 (pytest failed) and 137 (cgroup OOM / SIGKILL
+# of the test process). ci-infra ``K8S_RETRY`` still retries exit 1.
+x-l4-k8s-retry: &l4_k8s_retry
+  automatic:
+    - exit_status: -1
+      limit: 1
+    - exit_status: 128
+      limit: 1
+    - signal_reason: agent_stop
+      limit: 1
+    - signal_reason: agent_refused
+      limit: 1
+
 
 mirror_hardwares:
   l4_1:
     agents:
-      queue: gpu_1_queue
+      queue: l4-k8s
+    retry: *l4_k8s_retry
     plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          mount-buildkite-agent: true
-          shm-size: "8gb"
-          environment:
-            - HF_HOME=/fsx/hf_cache
-            - HF_TOKEN
-          volumes:
-            - /fsx/hf_cache:/fsx/hf_cache
+      - kubernetes:
+          metadata:
+            annotations:
+              cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+              karpenter.sh/do-not-disrupt: "true"
+          podSpec:
+            priorityClassName: ci
+            serviceAccountName: buildkite-gpu-jobs
+            tolerations:
+              - key: nvidia.com/gpu
+                operator: Exists
+                effect: NoSchedule
+            nodeSelector:
+              vllm.ci/gpu-pool: l4x4
+            containers:
+              - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 1
+                  requests:
+                    nvidia.com/gpu: 1
+                    cpu: "10"
+                    memory: 40Gi
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /fsx/hf_cache
+                env:
+                  - name: HF_HOME
+                    value: /fsx/hf_cache
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+                  sizeLimit: 24Gi
+              - name: hf-cache
+                hostPath:
+                  path: /fsx/hf_cache
+                  type: DirectoryOrCreate
+
+  l4_2:
+    agents:
+      queue: l4-k8s
+    retry: *l4_k8s_retry
+    plugins:
+      - kubernetes:
+          metadata:
+            annotations:
+              cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+              karpenter.sh/do-not-disrupt: "true"
+          podSpec:
+            priorityClassName: ci
+            serviceAccountName: buildkite-gpu-jobs
+            tolerations:
+              - key: nvidia.com/gpu
+                operator: Exists
+                effect: NoSchedule
+            nodeSelector:
+              vllm.ci/gpu-pool: l4x4
+            containers:
+              - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 2
+                  requests:
+                    nvidia.com/gpu: 2
+                    cpu: "20"
+                    memory: 80Gi
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /fsx/hf_cache
+                env:
+                  - name: HF_HOME
+                    value: /fsx/hf_cache
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+                  sizeLimit: 48Gi
+              - name: hf-cache
+                hostPath:
+                  path: /fsx/hf_cache
+                  type: DirectoryOrCreate
+
+  l4_3:
+    agents:
+      queue: l4-k8s
+    retry: *l4_k8s_retry
+    plugins:
+      - kubernetes:
+          metadata:
+            annotations:
+              cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+              karpenter.sh/do-not-disrupt: "true"
+          podSpec:
+            priorityClassName: ci
+            serviceAccountName: buildkite-gpu-jobs
+            tolerations:
+              - key: nvidia.com/gpu
+                operator: Exists
+                effect: NoSchedule
+            nodeSelector:
+              vllm.ci/gpu-pool: l4x4
+            containers:
+              - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 3
+                  requests:
+                    nvidia.com/gpu: 3
+                    cpu: "30"
+                    memory: 120Gi
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /fsx/hf_cache
+                env:
+                  - name: HF_HOME
+                    value: /fsx/hf_cache
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+                  sizeLimit: 72Gi
+              - name: hf-cache
+                hostPath:
+                  path: /fsx/hf_cache
+                  type: DirectoryOrCreate
 
   l4_4:
     agents:
-      queue: gpu_4_queue
+      queue: l4-k8s
+    retry: *l4_k8s_retry
     plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          mount-buildkite-agent: true
-          shm-size: "8gb"
-          environment:
-            - HF_HOME=/fsx/hf_cache
-            - HF_TOKEN
-          volumes:
-            - /fsx/hf_cache:/fsx/hf_cache
+      - kubernetes:
+          metadata:
+            annotations:
+              cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+              karpenter.sh/do-not-disrupt: "true"
+          podSpec:
+            priorityClassName: ci
+            serviceAccountName: buildkite-gpu-jobs
+            tolerations:
+              - key: nvidia.com/gpu
+                operator: Exists
+                effect: NoSchedule
+            nodeSelector:
+              vllm.ci/gpu-pool: l4x4
+            containers:
+              - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 4
+                  requests:
+                    nvidia.com/gpu: 4
+                    cpu: "40"
+                    memory: 160Gi
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /fsx/hf_cache
+                env:
+                  - name: HF_HOME
+                    value: /fsx/hf_cache
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+                  sizeLimit: 96Gi
+              - name: hf-cache
+                hostPath:
+                  path: /fsx/hf_cache
+                  type: DirectoryOrCreate
 
   h100_1:
     agents:
@@ -195,6 +392,211 @@ mirror_hardwares:
                   path: /mnt/hf-cache
                   type: DirectoryOrCreate
 
+  # vLLM device: b200-k8s → agents.queue b200-k8s + Buildkite K8s agent-stack podSpec
+  b200_1:
+    agents:
+      queue: b200-k8s
+    plugins:
+      - kubernetes:
+          podSpec:
+            runtimeClassName: nvidia
+            hostNetwork: true
+            dnsPolicy: ClusterFirstWithHostNet
+            imagePullSecrets:
+              - name: k8s-ecr-login-renew-docker-secret
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 1
+                securityContext:
+                  capabilities:
+                    add:
+                      - IPC_LOCK
+                      - SYS_RESOURCE
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: raid
+                    mountPath: /raid
+                  - name: shared
+                    mountPath: /mnt/shared
+                env:
+                  - name: NCCL_CUMEM_HOST_ENABLE
+                    value: "0"
+                  - name: HF_HOME
+                    value: /raid/hf_cache
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: raid
+                hostPath:
+                  path: /raid
+                  type: DirectoryOrCreate
+              - name: shared
+                hostPath:
+                  path: /mnt/shared
+                  type: DirectoryOrCreate
+
+  b200_2:
+    agents:
+      queue: b200-k8s
+    plugins:
+      - kubernetes:
+          podSpec:
+            runtimeClassName: nvidia
+            hostNetwork: true
+            dnsPolicy: ClusterFirstWithHostNet
+            imagePullSecrets:
+              - name: k8s-ecr-login-renew-docker-secret
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 2
+                securityContext:
+                  capabilities:
+                    add:
+                      - IPC_LOCK
+                      - SYS_RESOURCE
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: raid
+                    mountPath: /raid
+                  - name: shared
+                    mountPath: /mnt/shared
+                env:
+                  - name: NCCL_CUMEM_HOST_ENABLE
+                    value: "0"
+                  - name: HF_HOME
+                    value: /raid/hf_cache
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: raid
+                hostPath:
+                  path: /raid
+                  type: DirectoryOrCreate
+              - name: shared
+                hostPath:
+                  path: /mnt/shared
+                  type: DirectoryOrCreate
+
+  b200_3:
+    agents:
+      queue: b200-k8s
+    plugins:
+      - kubernetes:
+          podSpec:
+            runtimeClassName: nvidia
+            hostNetwork: true
+            dnsPolicy: ClusterFirstWithHostNet
+            imagePullSecrets:
+              - name: k8s-ecr-login-renew-docker-secret
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 3
+                securityContext:
+                  capabilities:
+                    add:
+                      - IPC_LOCK
+                      - SYS_RESOURCE
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: raid
+                    mountPath: /raid
+                  - name: shared
+                    mountPath: /mnt/shared
+                env:
+                  - name: NCCL_CUMEM_HOST_ENABLE
+                    value: "0"
+                  - name: HF_HOME
+                    value: /raid/hf_cache
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: raid
+                hostPath:
+                  path: /raid
+                  type: DirectoryOrCreate
+              - name: shared
+                hostPath:
+                  path: /mnt/shared
+                  type: DirectoryOrCreate
+
+  b200_4:
+    agents:
+      queue: b200-k8s
+    plugins:
+      - kubernetes:
+          podSpec:
+            runtimeClassName: nvidia
+            hostNetwork: true
+            dnsPolicy: ClusterFirstWithHostNet
+            imagePullSecrets:
+              - name: k8s-ecr-login-renew-docker-secret
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 4
+                securityContext:
+                  capabilities:
+                    add:
+                      - IPC_LOCK
+                      - SYS_RESOURCE
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: raid
+                    mountPath: /raid
+                  - name: shared
+                    mountPath: /mnt/shared
+                env:
+                  - name: NCCL_CUMEM_HOST_ENABLE
+                    value: "0"
+                  - name: HF_HOME
+                    value: /raid/hf_cache
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: raid
+                hostPath:
+                  path: /raid
+                  type: DirectoryOrCreate
+              - name: shared
+                hostPath:
+                  path: /mnt/shared
+                  type: DirectoryOrCreate
+
   a2b3_npu_1:
     agents:
       queue: "ascend-a2b3"

.buildkite/common/scripts/skip_ci.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 # SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """Shared Buildkite skip-ci and CI-YAML level targeting logic.
 
 ``resolve_ci_decision(changed_files, ...)`` buckets explicit file lists (tests, dry-run).
@@ -10,12 +11,6 @@ CLI (for bootstrap scripts; exit code is the signal):
   python3 skip_ci.py gate <platform> <level>
       Exit 0 when bootstrap should stop (skip-all, or that L2/L3 target is off).
       Prints ``skip-all`` or ``skip-l23`` on stdout; logs the decision once.
-  python3 skip_ci.py check-skip-all
-      Exit 0 when only docs/skip-mark changes → bootstrap skips entire CI upload.
-  python3 skip_ci.py check-skip-l2-l3 <platform> <level>
-      Exit 0 when that L2/L3 pipeline should be skipped (CI config YAML-only diffs).
-  python3 skip_ci.py print-annotate
-      Print decision message for Buildkite annotation.
 """
 
 from __future__ import annotations
@@ -258,11 +253,10 @@ class CiDecision:
 
 @dataclass(frozen=True)
 class CiContext:
-    """Skip-ci decision plus git diff inputs used to compute it."""
+    """Skip-ci decision plus changed files used for pipeline filtering."""
 
     decision: CiDecision
     changed_files: list[str] | None
-    diff_range: str | None
 
 
 def _log(message: str) -> None:
@@ -332,7 +326,7 @@ def resolve_ci_context_from_git() -> CiContext:
     diff_range = _resolve_diff_range()
     changed_files = _changed_files_for_diff_range(diff_range)
     decision = resolve_ci_decision(changed_files, diff_range=diff_range)
-    return CiContext(decision=decision, changed_files=changed_files, diff_range=diff_range)
+    return CiContext(decision=decision, changed_files=changed_files)
 
 
 # --- Doc / skip-mark helpers ---
@@ -571,23 +565,6 @@ def main() -> int:
     gate_parser.add_argument("platform", help="cuda, amd, intel, or npu")
     gate_parser.add_argument("level", choices=("l2", "l3"), help="test level")
 
-    subparsers.add_parser(
-        "check-skip-all",
-        help="exit 0 when docs/skip-mark-only diff should skip entire CI upload",
-    )
-
-    skip_parser = subparsers.add_parser(
-        "check-skip-l2-l3",
-        help="exit 0 when PLATFORM l2/l3 pipeline should be skipped (CI config YAML-only diffs)",
-    )
-    skip_parser.add_argument("platform", help="cuda, amd, intel, or npu")
-    skip_parser.add_argument("level", choices=("l2", "l3"), help="test level")
-
-    subparsers.add_parser(
-        "print-annotate",
-        help="print human-readable skip reason for Buildkite annotation",
-    )
-
     args = parser.parse_args()
     decision = resolve_ci_context_from_git().decision
 
@@ -600,16 +577,6 @@ def main() -> int:
             return 0
         return 1
 
-    if args.command == "check-skip-all":
-        return 0 if decision.skip_all else 1
-
-    if args.command == "check-skip-l2-l3":
-        return 0 if not decision.is_run(args.platform, args.level) else 1
-
-    if args.command == "print-annotate":
-        print(decision.message or "CI will run normally")
-        return 0
-
     return 1
 
 

.buildkite/common/scripts/upload_pipeline.py

@@ -12,8 +12,19 @@ Bootstrap mode (``bootstrap-upload-steps.yml``):
 
 Test pipeline mode (e.g. test-merge.yml):
   - Drop steps whose ``source_file_dependencies`` do not match changed files.
-  - Expand uploader-only ``mirror_hardwares: l4_1`` into ``agents`` (+ optional ``image``
+  - Expand uploader-only ``mirror_hardwares`` into ``agents`` (+ optional ``image``
     for NPU) + ``plugins`` (see ci_mirror_hardwares.yml).
+  - Omit ``mirror_hardwares`` to compose ``{chip}_{n}`` from pytest ``-m`` SKU
+    markers plus ``cards_n`` (max of several positives; ``not cards_1`` with no
+    positive ``cards_*`` uses that chip's highest existing preset). An explicit
+    preset string (``h100_2``) is used as-is (marks are ignored). Names that
+    are not keys in ``ci_mirror_hardwares.yml`` fail the upload.
+
+    Inferred chip: unset ``MIRROR_HW`` matches ``H100`` or ``L4`` in ``-m``
+    (both → H100); no match skips the step. ``MIRROR_HW=b200`` matches
+    ``B200`` in ``-m`` (otherwise skipped). ``-m`` is not rewritten.
+    ``MIRROR_HW`` must be empty or ``b200``; unknown values fail the upload.
+    A CUDA preset string such as ``h100_4`` is omitted when ``MIRROR_HW=b200``.
 
 Usage:
   python3 upload_pipeline.py [--upload] [--all | --e2e] <pipeline.yml>
@@ -25,11 +36,13 @@ from __future__ import annotations
 
 import argparse
 import copy
+import os
+import re
 import subprocess
 import sys
 from functools import lru_cache
 from pathlib import Path
-from typing import Any
+from typing import Any, Literal
 
 try:
     import yaml
@@ -45,6 +58,14 @@ from skip_ci import (
     resolve_ci_context_from_git,
 )
 
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from tests.helpers.mark import (  # noqa: E402
+    get_skus_for_platform,
+    get_supported_card_counts,
+)
+
 # --- Constants ---
 
 LOG = "upload_pipeline"
@@ -233,32 +254,257 @@ def _load_mirror_hardwares() -> dict[str, dict[str, Any]]:
     return presets
 
 
-def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any]:
-    """Replace uploader-only ``mirror_hardwares`` with preset fields from ci_mirror_hardwares.yml."""
-    hardware = step.get("mirror_hardwares")
-    if hardware is None:
-        return step
+@lru_cache(maxsize=1)
+def _cuda_mirror_chips() -> tuple[str, ...]:
+    """Lowercase CUDA SKUs that have at least one ``{chip}_{n}`` preset.
 
-    if not isinstance(hardware, str) or not hardware.strip():
+    Longer tokens first so ``h100`` is not matched as a prefix of ``h200``.
+    """
+    skus = get_skus_for_platform("cuda")
+    known = _load_mirror_hardwares()
+    chips = {
+        sku.lower() for sku in skus if any(name == sku.lower() or name.startswith(f"{sku.lower()}_") for name in known)
+    }
+    return tuple(sorted(chips, key=lambda chip: (-len(chip), chip)))
+
+
+def _read_cards_marks(expr: str) -> tuple[set[int], bool]:
+    """Parse registered ``cards_n`` / ``not cards_n`` from a pytest ``-m`` expr.
+
+    Returns positive card counts and whether any ``not cards_*`` is present.
+    """
+    counts = get_supported_card_counts()
+    alt = "|".join(str(n) for n in sorted(counts, key=lambda n: (-len(str(n)), -n))) if counts else "(?!)"
+    positives: set[int] = set()
+    has_not_cards = False
+    for match in re.finditer(rf"\b(?:(not)\s+)?cards_({alt})\b", expr):
+        if match.group(1):
+            has_not_cards = True
+        else:
+            positives.add(int(match.group(2)))
+    return positives, has_not_cards
+
+
+def _read_hardware_marks(expr: str) -> set[str]:
+    """Return lowercase positive CUDA SKUs that have a mirror preset.
+
+    ``not H100`` is ignored. SKUs with no ``{chip}_*`` preset (e.g. H200) are dropped.
+    """
+    skus = sorted(get_skus_for_platform("cuda"), key=lambda s: (-len(s), s))
+    if not skus:
+        return set()
+    alt = "|".join(re.escape(sku) for sku in skus)
+    mirror = set(_cuda_mirror_chips())
+    chips: set[str] = set()
+    for match in re.finditer(rf"\b(?:(not)\s+)?({alt})\b", expr):
+        if match.group(1):
+            continue
+        chip = match.group(2).lower()
+        if chip in mirror:
+            chips.add(chip)
+    return chips
+
+
+# CI policy: not pytest-mark registry.
+_SUPPORTED_MIRROR_HW_SELECTORS = frozenset({"b200"})
+# Unset MIRROR_HW: match H100 then L4 in -m (skip if neither).
+_DEFAULT_INFER_CHIPS = ("h100", "l4")
+_PYTEST_MARKER_ARG = re.compile(r"-m\s+(?:\"([^\"]*)\"|'([^']*)'|(\S+))")
+_CARDS_CHIP_MAX: Literal["max"] = "max"
+
+
+def _get_mirror_hw_selector() -> str:
+    """Return lowercase ``MIRROR_HW``, or empty to keep string presets / marker chips.
+
+    A set value must be ``b200`` (case-insensitive). Unknown values fail closed
+    so a typo cannot silently drop the CUDA pipeline.
+    """
+    selector = os.environ.get("MIRROR_HW", "").strip().lower()
+    if not selector:
+        return ""
+    if selector not in _SUPPORTED_MIRROR_HW_SELECTORS:
+        allowed = " or ".join(repr(s) for s in sorted(_SUPPORTED_MIRROR_HW_SELECTORS))
+        raise ValueError(f"unsupported MIRROR_HW={selector!r}; expected empty or {allowed}")
+    return selector
+
+
+def _read_pytest_marks(commands: Any) -> tuple[set[str], int | Literal["max"] | None]:
+    """Read pytest ``-m`` from step commands: CUDA SKU chips and cards count.
+
+    *chips* are lowercase SKUs from positive markers (``not H100`` is ignored).
+    *cards* is the max positive ``cards_n``, ``"max"`` when only ``not cards_*``
+    is present, or ``None``.
+    """
+    chunks: list[str] = []
+
+    def _collect(value: Any) -> None:
+        if value is None:
+            return
+        if isinstance(value, str):
+            chunks.append(value)
+        elif isinstance(value, list):
+            for part in value:
+                _collect(part)
+        else:
+            chunks.append(str(value))
+
+    _collect(commands)
+    text = "\n".join(chunks)
+
+    chips: set[str] = set()
+    positives: set[int] = set()
+    has_not_cards = False
+    for match in _PYTEST_MARKER_ARG.finditer(text):
+        expr = match.group(1) or match.group(2) or match.group(3) or ""
+        chips |= _read_hardware_marks(expr)
+        card_counts, not_cards = _read_cards_marks(expr)
+        positives.update(card_counts)
+        has_not_cards = has_not_cards or not_cards
+
+    if positives:
+        cards: int | Literal["max"] | None = max(positives)
+    elif has_not_cards:
+        cards = _CARDS_CHIP_MAX
+    else:
+        cards = None
+    return chips, cards
+
+
+def _compose_mirror_hardware_name(
+    chips: set[str],
+    cards: int | Literal["max"] | None,
+    *,
+    step_label: str,
+) -> str | None:
+    """Assemble ``{chip}_{n}`` from pytest ``-m`` chips/cards, or None to skip.
+
+    Unset ``MIRROR_HW``: ``H100`` if in ``-m``, else ``L4``, else skip.
+    Set ``MIRROR_HW``: that chip if in ``-m``, else skip.
+    ``cards`` is a registered ``cards_n``, ``"max"`` for that chip's highest
+    existing preset, or None (error after a chip matched — no ``cards_*``).
+    """
+    selector = _get_mirror_hw_selector()
+    if selector:
+        chip = selector if selector in chips else None
+        skip_reason = f"MIRROR_HW={selector!r} not in -m SKUs {sorted(chips)}"
+    else:
+        chip = next((c for c in _DEFAULT_INFER_CHIPS if c in chips), None)
+        skip_reason = f"no H100 or L4 in -m SKUs {sorted(chips)}"
+    if chip is None:
+        _log(f"skip {step_label}: {skip_reason}")
+        return None
+    if cards is None:
         raise ValueError(
-            f"mirror_hardwares must be a non-empty string in step {_get_step_label(step)!r}",
+            f"step {step_label!r} has a pytest -m SKU marker but no cards_* "
+            f"(or not cards_*); add cards_n or set mirror_hardwares to force a preset",
         )
 
-    preset = _load_mirror_hardwares().get(hardware)
+    count_source = "not cards_*" if cards == _CARDS_CHIP_MAX else f"cards_{cards}"
+    count = None if cards == _CARDS_CHIP_MAX else cards
+    registered = get_supported_card_counts()
+    if count is not None and count not in registered:
+        supported = ", ".join(f"cards_{n}" for n in sorted(registered)) or "(none)"
+        raise ValueError(
+            f"{count_source} in step {step_label!r} is not a registered cards_* mark; supported: {supported}",
+        )
+    known = _load_mirror_hardwares()
+    if count is None:
+        available = [n for n in sorted(registered) if f"{chip}_{n}" in known]
+        if not available:
+            raise ValueError(
+                f"{count_source} in step {step_label!r} has no CUDA count presets for {chip}",
+            )
+        count = max(available)
+    chosen = f"{chip}_{count}"
+    if chosen not in known:
+        raise ValueError(
+            f"{count_source} in step {step_label!r} has no preset {chosen!r}",
+        )
+    _log(f"{step_label}: {count_source} → {chosen!r}")
+    return chosen
+
+
+def _resolve_mirror_hardware_name(hardware: Any, *, step_label: str) -> str | None:
+    """Map ``mirror_hardwares`` to a ``ci_mirror_hardwares.yml`` preset, or None to skip.
+
+    The value is a preset name (``h100_2``). Unknown names fail later when the
+    registry is loaded. CUDA names are omitted when ``MIRROR_HW`` names a
+    different chip; NPU strings ignore ``MIRROR_HW``.
+    """
+    if isinstance(hardware, bool):
+        raise ValueError(f"mirror_hardwares must not be a boolean in step {step_label!r}")
+    if isinstance(hardware, str):
+        name = hardware.strip()
+    elif isinstance(hardware, int):
+        name = str(hardware)
+    else:
+        raise ValueError(
+            f"mirror_hardwares must be a preset string in step {step_label!r}",
+        )
+    if not name:
+        raise ValueError(f"mirror_hardwares must be a non-empty string in step {step_label!r}")
+
+    selector = _get_mirror_hw_selector()
+    token = name.lower()
+    chip = next(
+        (c for c in _cuda_mirror_chips() if token == c or token.startswith(f"{c}_")),
+        None,
+    )
+    if selector and chip is not None and chip != selector:
+        _log(
+            f"skip {step_label}: preset {name!r} is {chip} hardware; MIRROR_HW={selector!r}",
+        )
+        return None
+    return name
+
+
+def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any] | None:
+    """Replace ``mirror_hardwares`` (or infer it from ``-m``) with preset agents/plugins.
+
+    Explicit preset string: used as-is; pytest marks are ignored.
+    Omitted key: match H100/L4 (or ``MIRROR_HW``) in ``-m``, then compose
+    ``{chip}_{n}`` from ``cards_n``. No match skips the step. Steps with no
+    SKU/cards and no ``mirror_hardwares`` are left unchanged (CPU jobs).
+
+    Returns None when the current ``MIRROR_HW`` selector does not apply.
+    """
+    step_label = _get_step_label(step)
+    has_pool = step.get("agents") is not None or step.get("plugins") is not None or step.get("image") is not None
+
+    if "mirror_hardwares" not in step:
+        if has_pool:
+            return step
+        chips, cards = _read_pytest_marks(step.get("commands"))
+        if not chips and cards is None:
+            return step
+        preset_name = _compose_mirror_hardware_name(chips, cards, step_label=step_label)
+    else:
+        if has_pool:
+            raise ValueError(
+                f"step {step_label!r} sets mirror_hardwares together with agents/plugins/image; "
+                f"use mirror_hardwares only",
+            )
+        preset_name = _resolve_mirror_hardware_name(
+            step.get("mirror_hardwares"),
+            step_label=step_label,
+        )
+
+    if preset_name is None:
+        return None
+
+    preset = _load_mirror_hardwares().get(preset_name)
     if preset is None:
         known = ", ".join(sorted(_load_mirror_hardwares()))
         raise ValueError(
-            f"unknown mirror_hardwares {hardware!r} in step {_get_step_label(step)!r}; known: {known}",
-        )
-
-    if step.get("agents") is not None or step.get("plugins") is not None or step.get("image") is not None:
-        raise ValueError(
-            f"step {_get_step_label(step)!r} sets mirror_hardwares together with agents/plugins/image; "
-            "use mirror_hardwares only",
+            f"unknown mirror_hardwares {preset_name!r} in step {step_label!r}; known: {known}",
         )
 
     expanded = copy.deepcopy(preset)
-    return {key: value for key, value in step.items() if key != "mirror_hardwares"} | expanded
+    merged = {key: value for key, value in step.items() if key != "mirror_hardwares"} | expanded
+    # Preset retry (K8S_RETRY on l4_*) must not clobber a step that opted out.
+    if "retry" in step:
+        merged["retry"] = step["retry"]
+    return merged
 
 
 def _match_source_file(changed_files: list[str], prefixes: list[str]) -> bool:
@@ -297,7 +543,7 @@ def _process_test_steps(
         nested = step.get("steps")
         if nested is not None:
             kept_nested = _process_test_steps(nested, changed_files)
-            if changed_files is not None and not kept_nested:
+            if not kept_nested:
                 _log(f"omit empty group {_get_step_label(step)!r}")
                 continue
             new_step = {key: value for key, value in step.items() if key != "source_file_dependencies"}
@@ -305,14 +551,11 @@ def _process_test_steps(
             processed.append(new_step)
             continue
 
-        if deps is not None:
-            processed.append(
-                _expand_mirror_hardwares(
-                    {key: value for key, value in step.items() if key != "source_file_dependencies"},
-                ),
-            )
-        else:
-            processed.append(_expand_mirror_hardwares(step))
+        leaf = {key: value for key, value in step.items() if key != "source_file_dependencies"}
+        expanded = _expand_mirror_hardwares(leaf)
+        if expanded is None:
+            continue
+        processed.append(expanded)
 
     return processed
 
@@ -338,6 +581,10 @@ def _render_test_pipeline(
     e2e_only: bool = False,
 ) -> dict[str, Any]:
     """Filter steps by PR diff and strip uploader-only ``source_file_dependencies`` metadata."""
+    # Validate MIRROR_HW once up front. Per-step skip must not run for typos
+    # (e.g. b20o), or a pipeline can shrink to leftover CPU-only steps and
+    # still upload successfully.
+    _get_mirror_hw_selector()
     steps = doc.get("steps")
     if not isinstance(steps, list):
         return doc
@@ -367,10 +614,7 @@ def _render_pipeline(
 
     text = path.read_text(encoding="utf-8")
     ctx = resolve_ci_context_from_git()
-    if force_all or e2e_only:
-        changed_files = None
-    else:
-        changed_files = ctx.changed_files
+    changed_files = None if force_all or e2e_only else ctx.changed_files
 
     doc = yaml.safe_load(text)
     if not isinstance(doc, dict):

.buildkite/cuda/test-merge.yml

@@ -1,6 +1,8 @@
+# CUDA CI L3 (`advanced_model` e2e) plus L1 CPU Simple Test (`core_model and cpu`).
+# Uploaded by Upload Merge Pipeline: PR label `merge-test`, or post-merge on main.
+
 env:
   VLLM_WORKER_MULTIPROC_METHOD: spawn
-  VLLM_USE_FLASHINFER_MOE_FP16: 0
   HF_HUB_DOWNLOAD_TIMEOUT: 300
   HF_HUB_ETAG_TIMEOUT: 60
 
@@ -77,6 +79,7 @@ steps:
     steps:
       - label: "TTS · Qwen3-TTS CustomVoice Test"
         source_file_dependencies:
+          - tests/helpers/media.py
           - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
           - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
           - vllm_omni/model_executor/models/qwen3_tts/
@@ -85,6 +88,9 @@ steps:
           - vllm_omni/model_executor/models/common/alias_free_activation.py
           - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
           - vllm_omni/deploy/qwen3_tts.yaml
+          - vllm_omni/entrypoints/openai/serving_speech.py
+          - vllm_omni/entrypoints/openai/tts_adapters/
+          - vllm_omni/engine/stage_pool.py
         commands:
           - |
             timeout 40m bash -c "
@@ -96,6 +102,7 @@ steps:
 
       - label: "TTS · Qwen3-TTS Base Test"
         source_file_dependencies:
+          - tests/helpers/media.py
           - tests/e2e/online_serving/test_qwen3_tts_base.py
           - tests/e2e/offline_inference/test_qwen3_tts_base.py
           - vllm_omni/model_executor/models/qwen3_tts/
@@ -104,6 +111,9 @@ steps:
           - vllm_omni/model_executor/models/common/alias_free_activation.py
           - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
           - vllm_omni/deploy/qwen3_tts.yaml
+          - vllm_omni/entrypoints/openai/serving_speech.py
+          - vllm_omni/entrypoints/openai/tts_adapters/
+          - vllm_omni/engine/stage_pool.py
           - .buildkite/common/scripts/run_cov_split.sh
           - pyproject.toml
         # 40 -> 60: the coverage split runs the model twice.
@@ -153,10 +163,11 @@ steps:
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'
         # h100_2, not h100_1: advanced_model asserts transcribe every audio
-        # response with Whisper, and tests/helpers/media.py only uses an
-        # accelerator for Whisper when a spare GPU exists. On a single card it
-        # grades on CPU: ~8-10 min per clip vs ~30 s, which alone overruns the
-        # 50-minute job timeout (build 2956).
+        # response with Whisper, and tests/helpers/media.py requires at least
+        # 16 GiB free VRAM on a device to run Whisper on GPU. When the model
+        # server leaves insufficient memory, Whisper falls back to CPU:
+        # ~8-10 min per clip vs ~30 s, which overruns the 50-minute job timeout
+        # (build 2956). h100_2 guarantees a second GPU with ample free VRAM.
         mirror_hardwares: h100_2
 
       - label: "Omni · MiniCPM-o 4.5 Duplex Test"
@@ -172,6 +183,21 @@ steps:
           - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
           - vllm_omni/deploy/minicpmo_4_5.yaml
           - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
+          - vllm_omni/engine/duplex/
+          - vllm_omni/entrypoints/duplex/
+          # Public duplex client library + its unit suites: client-side
+          # changes must re-run the duplex e2e group. The live client probe
+          # (test_duplex_client_live.py, single GPU, core tier) runs in the
+          # ready pipeline's duplex group.
+          - vllm_omni/clients/
+          - tests/clients/
+          - vllm_omni/entrypoints/duplex_request_client.py
+          - vllm_omni/outputs/duplex.py
+          - vllm_omni/model_executor/duplex_sampling.py
+          - tests/engine/duplex/
+          - tests/entrypoints/duplex/
+          - tests/model_executor/models/minicpmo_4_5/duplex/
+          # Legacy Realtime probe client + video stacking used by the e2e drivers.
           - vllm_omni/experimental/fullduplex/
           - vllm_omni/model_executor/models/minicpmo_4_5/
           - vllm_omni/model_executor/models/step_audio2/
@@ -231,3 +257,38 @@ steps:
               pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
             "
         mirror_hardwares: h100_1
+
+      - label: "Diffusion · MiniMax-H3 L3 DLO DP2 and FastH3"
+        source_file_dependencies:
+          - tests/e2e/online_serving/minimax_h3/
+          - tests/helpers/assertions.py
+          - tests/helpers/mark.py
+          - tests/helpers/media.py
+          - tests/helpers/runtime.py
+          - vllm_omni/diffusion/models/minimax_h3/
+          - vllm_omni/diffusion/lora/
+          - vllm_omni/diffusion/offloader/
+          - vllm_omni/diffusion/distributed/
+          - vllm_omni/entrypoints/openai/
+        # Keep four server lifecycles in one Buildkite job, but never in one
+        # pytest process. DLO uses multiprocessing shared-memory queues; the
+        # parent's resource_tracker can retain a stale /psm_* entry during
+        # teardown and strand the next Ref2VA collective.
+        # A per-file process plus a hard timeout also makes a regression fail
+        # instead of leaving the merge queue stuck indefinitely.
+        timeout_in_minutes: 150
+        commands:
+          - |
+            set -eu
+            export VLLM_IMAGE_FETCH_TIMEOUT=60
+            if [ -f vllm_omni/diffusion/models/minimax_h3/lora.py ]; then
+              export VLLM_TEST_MINIMAX_H3_TURBO_LORA=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id="lightx2v/Minimax-h3-Turbo", filename="minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors"))')
+            fi
+            if [ -f vllm_omni/diffusion/models/minimax_h3/fasth3.py ]; then
+              export VLLM_TEST_MINIMAX_H3_FASTH3_LORA=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id="FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA", filename="dense-datafree/adapter_model.safetensors"))')
+            fi
+            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_dlo_dp2_fl2va.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_dlo_dp2_ref2va.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_turbo_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_fasth3_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+        mirror_hardwares: h100_2

.buildkite/cuda/test-nightly.yml

@@ -1,6 +1,9 @@
+# CUDA CI L4 (`full_model`): nightly and release share this file.
+# Uploaded by Upload Nightly Pipeline: NIGHTLY=1, or PR labels (nightly-test / omni-test / ...).
+# Hardware: omit mirror_hardwares to compose from -m SKU + cards_n; or set a preset string to force.
+
 env:
   VLLM_WORKER_MULTIPROC_METHOD: spawn
-  VLLM_USE_FLASHINFER_MOE_FP16: 0
   HF_HUB_DOWNLOAD_TIMEOUT: 300
   HF_HUB_ETAG_TIMEOUT: 60
 
@@ -13,16 +16,14 @@ steps:
     steps:
 
       - label: ":full_moon: Omni · Function Test with H100 · Single-GPU"
-        timeout_in_minutes: 90
+        timeout_in_minutes: 120
         commands:
-          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
-        mirror_hardwares: h100_1
+          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
 
       - label: ":full_moon: Omni · Function Test with H100 · 2-GPU"
         timeout_in_minutes: 90
         commands:
-          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni and cards_2" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
-        mirror_hardwares: h100_2
+          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_2" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
 
       - label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
         timeout_in_minutes: 50
@@ -30,11 +31,11 @@ steps:
           - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni and cards_1" --run-level "full_model"
         mirror_hardwares: h100_1
 
-      - label: ":full_moon: Omni · Doc Test with L4"
+      - label: ":full_moon: Omni · Doc Test with L4 · 4-GPU"
         timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          - pytest -s -v tests/examples/ -m "full_model and omni and L4" --run-level "full_model"
+          - pytest -s -v tests/examples/ -m "full_model and omni and L4 and cards_4" --run-level "full_model"
         mirror_hardwares: l4_4
 
       - label: ":full_moon: Omni · Doc Test with H100 · 2-GPU"
@@ -178,7 +179,7 @@ steps:
       build.pull_request.labels includes "nightly-test"
     steps:
 
-      - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations)"
+      - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 2-GPU"
         timeout_in_minutes: 30
         # NOTE: Single-GPU (core_model) tiny model tests run on the ready label,
         # so nightly only adds the multi-GPU parallelism configurations.
@@ -188,7 +189,14 @@ steps:
         # for the CI, since these tests should pass irrespective of whether or not we use
         # the full weights or not.
         commands:
-          - timeout 25m pytest -sv tests/model_tests/diffusion -m 'full_model and cuda' --run-level "core_model"
+          - pytest -sv tests/model_tests/diffusion -m 'full_model and cuda and cards_2' --run-level "core_model"
+        mirror_hardwares: l4_2
+
+      - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 4-GPU"
+        timeout_in_minutes: 30
+        # CFG+TP extra_test_groups request 2×2=4 devices (see get_required_device_count).
+        commands:
+          - pytest -sv tests/model_tests/diffusion -m 'full_model and cuda and cards_4' --run-level "core_model"
         mirror_hardwares: l4_4
 
   - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
@@ -220,10 +228,21 @@ steps:
             --run-level "full_model"
         mirror_hardwares: h100_2
 
-      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · Multi-GPU"
+      - label: ":full_moon: Diffusion X2I(&A&T) · Boogu Edit CFG · 3-GPU"
         timeout_in_minutes: 120
         commands:
-          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4 and not cards_1" --run-level "full_model" --ignore=tests/e2e/accuracy
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_boogu_image_edit_expansion.py
+            -k "test_double_guidance_cfg_parallel and cfg_parallel_3_full"
+            -m "slow and diffusion and cuda and H100 and cards_3"
+            --run-level "full_model"
+        mirror_hardwares: h100_3
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · 4-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4 and cards_4" --run-level "full_model" --ignore=tests/e2e/accuracy
         mirror_hardwares: l4_4
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
@@ -259,7 +278,6 @@ steps:
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
           - export CACHE_DIT_VERSION=1.5.0
           - |
             set +e
@@ -325,6 +343,12 @@ steps:
           - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and cuda" --run-level "full_model"
         mirror_hardwares: h100_2
 
+      - label: ":full_moon: Diffusion X2V · MAGI-2 Preview Serving Smoke"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_magi2.py -m "advanced_model and diffusion and cuda and H100 and cards_4" --run-level "advanced_model"
+        mirror_hardwares: h100_4
+
       - label: ":full_moon: Diffusion X2V · Doc Test"
         timeout_in_minutes: 60
         commands:
@@ -362,7 +386,6 @@ steps:
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
           - |
             set +e
             pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_1"
@@ -412,7 +435,6 @@ steps:
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
           - |
             set +e
             pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json
@@ -427,10 +449,22 @@ steps:
     depends_on: upload-nightly-pipeline
     if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     steps:
-      - label: ":full_moon: Diffusion · Distributed Test with L4"
+      - label: ":full_moon: Diffusion · Distributed Test with L4 · 2-GPU"
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4' --run-level "full_model"
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_2' --run-level "full_model"
+        mirror_hardwares: l4_2
+
+      - label: ":full_moon: Diffusion · Distributed Test with L4 · 3-GPU"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_3' --run-level "full_model"
+        mirror_hardwares: l4_3
+
+      - label: ":full_moon: Diffusion · Distributed Test with L4 · 4-GPU"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_4' --run-level "full_model"
         mirror_hardwares: l4_4
 
       - label: ":full_moon: Diffusion · Distributed Test with H100 · 2-GPU"

.buildkite/cuda/test-ready.yml

@@ -1,6 +1,8 @@
+# CUDA CI L2 (`core_model` e2e) plus L1 CPU Simple Test (`core_model and cpu`).
+# Uploaded by Upload Ready Pipeline: PR label `ready`, or main+WEEKLY=1 with --e2e.
+
 env:
   VLLM_WORKER_MULTIPROC_METHOD: spawn
-  VLLM_USE_FLASHINFER_MOE_FP16: 0
   HF_HUB_DOWNLOAD_TIMEOUT: 300
   HF_HUB_ETAG_TIMEOUT: 60
 
@@ -51,17 +53,20 @@ steps:
           - timeout 15m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
         mirror_hardwares: l4_1
 
-      - label: "Diffusion · Distributed & Attention Test"
-        source_file_dependencies:
-          - tests/diffusion/distributed/
-          - tests/diffusion/attention/test_attention_sp.py
-          - vllm_omni/diffusion/distributed/
-          - vllm_omni/diffusion/attention/
-          - vllm_omni/diffusion/data.py
-          - vllm_omni/diffusion/models/
+      - label: "Diffusion · Distributed Test · Single-GPU"
         commands:
-          - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda' --run-level "core_model"
-          - timeout 20m pytest -sv tests/diffusion/attention/test_attention_sp.py -m 'core_model and cuda' --run-level "core_model"
+          - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_1' --run-level "core_model"
+        mirror_hardwares: l4_1
+
+      - label: "Diffusion · Distributed Test · 2-GPU"
+        commands:
+          - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_2' --run-level "core_model"
+        mirror_hardwares: l4_2
+
+      - label: "Diffusion · Distributed & Attention Test · 4-GPU"
+        commands:
+          - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_4' --run-level "core_model"
+          - timeout 20m pytest -sv tests/diffusion/attention/test_attention_sp.py -m 'core_model and cuda and L4 and cards_4' --run-level "core_model"
         mirror_hardwares: l4_4
 
       - label: "Diffusion · Model Test"
@@ -209,6 +214,21 @@ steps:
           - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
           - vllm_omni/deploy/minicpmo_4_5.yaml
           - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
+          - vllm_omni/engine/duplex/
+          - vllm_omni/entrypoints/duplex/
+          # Public duplex client library + its unit suites: client-side
+          # changes must re-run the duplex e2e group, which includes the
+          # live client probe (test_duplex_client_live.py, single GPU).
+          - vllm_omni/clients/
+          - tests/clients/
+          - tests/e2e/online_serving/test_duplex_client_live.py
+          - vllm_omni/entrypoints/duplex_request_client.py
+          - vllm_omni/outputs/duplex.py
+          - vllm_omni/model_executor/duplex_sampling.py
+          - tests/engine/duplex/
+          - tests/entrypoints/duplex/
+          - tests/model_executor/models/minicpmo_4_5/duplex/
+          # Legacy Realtime probe client + video stacking used by the e2e drivers.
           - vllm_omni/experimental/fullduplex/
           - vllm_omni/model_executor/models/minicpmo_4_5/
           - vllm_omni/model_executor/models/step_audio2/
@@ -226,8 +246,8 @@ steps:
           - vllm_omni/config/stage_config.py
         commands:
           - |
-            timeout 20m bash -c "
-              pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and cuda' --run-level 'core_model'
+            timeout 35m bash -c "
+              pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py tests/e2e/online_serving/test_duplex_client_live.py -m 'core_model and cuda' --run-level 'core_model'
             "
         mirror_hardwares: h100_1
 
@@ -245,6 +265,7 @@ steps:
 
       - label: "TTS · Qwen3-TTS CustomVoice Test"
         source_file_dependencies:
+          - tests/helpers/media.py
           - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
           - vllm_omni/model_executor/models/qwen3_tts/
           - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
@@ -252,6 +273,9 @@ steps:
           - vllm_omni/model_executor/models/common/alias_free_activation.py
           - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
           - vllm_omni/deploy/qwen3_tts.yaml
+          - vllm_omni/entrypoints/openai/serving_speech.py
+          - vllm_omni/entrypoints/openai/tts_adapters/
+          - vllm_omni/engine/stage_pool.py
         commands:
           - |
             timeout 20m bash -c "
@@ -260,25 +284,6 @@ steps:
             "
         mirror_hardwares: l4_1
 
-      - label: "TTS · Qwen3-TTS Base Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen3_tts_base.py
-          - vllm_omni/model_executor/models/qwen3_tts/
-          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-          - vllm_omni/model_executor/models/common/snake_activation.py
-          - vllm_omni/model_executor/models/common/alias_free_activation.py
-          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
-          - vllm_omni/deploy/qwen3_tts.yaml
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_WORKER_MULTIPROC_METHOD=spawn
-              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
-              export VLLM_OMNI_USE_V2_RUNNER=1
-              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m 'core_model' --run-level 'core_model'
-            "
-        mirror_hardwares: l4_1
-
       - label: "Diffusion · Wan22 Test"
         source_file_dependencies:
           - tests/e2e/online_serving/test_wan22_t2v.py

.buildkite/cuda/test-weekly.yml

@@ -1,6 +1,8 @@
+# CUDA CI L5 (`slow` / reliability / extra coverage).
+# Uploaded by Upload Weekly Pipeline: WEEKLY=1 / NON_CRITICAL=1, or PR label `weekly-test`.
+
 env:
   VLLM_WORKER_MULTIPROC_METHOD: spawn
-  VLLM_USE_FLASHINFER_MOE_FP16: 0
   HF_HUB_DOWNLOAD_TIMEOUT: 300
   HF_HUB_ETAG_TIMEOUT: 60
 
@@ -111,17 +113,24 @@ steps:
     if: build.env("NON_CRITICAL") == "1"
     steps:
 
-      # Qwen2.5 expansion (CUDA 4-card / online 2-card) stays on this l4_4 job.
-      - label: "Omni · L4 · Multi-GPU"
+      # Online Qwen2.5 expansion is 2-card; offline expansion + AutoRound are 4-card.
+      - label: "Omni · L4 · 2-GPU"
         timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and not cards_1" --run-level "core_model"
+          - pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and cards_2" --run-level "core_model"
+        mirror_hardwares: l4_2
+
+      - label: "Omni · L4 · 4-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and cards_4" --run-level "core_model"
           - >-
             pytest -s -v tests/e2e/
             --ignore=tests/e2e/online_serving/test_qwen2_5_omni_expansion.py
             --ignore=tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
-            -m "slow and L4 and omni and not cards_1"
+            -m "slow and L4 and omni and cards_4"
             --run-level "full_model"
         mirror_hardwares: l4_4
 
@@ -194,12 +203,21 @@ steps:
             --run-level "full_model"
         mirror_hardwares: l4_1
 
-      - label: "Diffusion · L4 · Multi-GPU"
+      - label: "Diffusion · L4 · 2-GPU"
         timeout_in_minutes: 60
         commands:
           - >-
             pytest -sv
-            tests/e2e/ -m "slow and diffusion and L4 and not cards_1"
+            tests/e2e/ -m "slow and diffusion and L4 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: l4_2
+
+      - label: "Diffusion · L4 · 4-GPU"
+        timeout_in_minutes: 60
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/ -m "slow and diffusion and L4 and cards_4"
             --run-level "full_model"
         mirror_hardwares: l4_4
 

.claude/skills/add-diffusion-model/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: add-diffusion-model
-description: Add a new diffusion model (text-to-image, text-to-video, image-to-video, text-to-audio, image editing) to vLLM-Omni, including Cache-DiT acceleration and parallelism support (TP, SP/USP, CFG-Parallel, HSDP). Use when integrating a new diffusion model, porting a diffusers pipeline or a custom model repo to vllm-omni, creating a new DiT transformer adapter, adding diffusion model support, or enabling multi-GPU parallelism and cache acceleration for an existing model.
+description: Add a new diffusion model (text-to-image, text-to-video, image-to-video, text-to-audio, image editing) to vLLM-Omni, including native non-Diffusers ports, reference-parity validation, Cache-DiT, offload, and parallelism support (TP, SP/USP, CFG-Parallel, HSDP). Use when integrating or reviewing a new diffusion model, porting a Diffusers pipeline or custom model repository, creating a DiT adapter, reusing shared examples, or qualifying multi-GPU and memory optimizations.
 ---
 
 # Adding a Diffusion Model to vLLM-Omni
@@ -26,9 +26,16 @@ Check the model's HF repo for `model_index.json`. This determines your path:
 |----------|----------------|----------------|
 | **Already supported** | `_class_name` in `model_index.json` matches a key in `_DIFFUSION_MODELS` in `registry.py` | Skip implementation, then validate model-specific examples, tests, and docs as needed |
 | **Diffusers-based** | Has standard `model_index.json` with `_diffusers_version`, subfolders for `transformer/`, `vae/`, etc. | Follow **Path A** below |
-| **Custom/private repo** | No diffusers `model_index.json`, weights in non-standard format, custom model code in a separate git repo | Follow **Path B** below |
+| **Native non-Diffusers model** | No Diffusers index, non-standard checkpoint hierarchy, or custom architecture in a separate repo | Follow **Path B** below; port the runtime natively unless an external adapter was explicitly requested |
 | **Hybrid** | Has some diffusers components (VAE) but custom transformer/fusion | Mix of Path A and Path B |
 
+Before coding, write a short integration contract covering runtime ownership,
+checkpoint discovery, reference revision, I/O geometry, attention semantics,
+CFG behavior, auxiliary components, target hardware, and the default deployment.
+For Path B, hybrid models, or any optimization work, read
+[references/native-model-integration-checklist.md](references/native-model-integration-checklist.md)
+and use its phase gates.
+
 ## Path A: Diffusers-Based Model
 
 For models with a standard diffusers layout. See [references/transformer-adaptation.md](references/transformer-adaptation.md) for detailed code patterns.
@@ -103,9 +110,15 @@ For custom loop structures, follow `vllm_omni/diffusion/models/progress_bar.py`
 
 ---
 
-## Path B: Custom/Private Repo Model
+## Path B: Native / Non-Diffusers Model
 
-For models without a diffusers pipeline — weights in custom format, model code in a private repo. Real examples: BAGEL, HunyuanImage3. See [references/custom-model-patterns.md](references/custom-model-patterns.md) for concrete custom-model integration patterns.
+For models without a Diffusers pipeline—weights in custom formats and model
+code in another public or private repository. Treat that repository as a pinned
+correctness oracle. A request for native support means the vLLM-Omni runtime
+must not import the reference implementation; an external adapter is appropriate
+only when the requested scope explicitly permits that dependency. See
+[references/custom-model-patterns.md](references/custom-model-patterns.md) for
+concrete integration patterns.
 
 ### B1. Understand the reference repo
 
@@ -126,16 +139,25 @@ This is the key design decision for custom models. Follow these placement rules:
 | **Pipeline orchestration** (init, forward, denoise loop) | `vllm_omni/diffusion/models/<name>/pipeline_<name>.py` | Always required |
 | **Custom transformer/backbone** (ported and adapted to vllm-omni) | `vllm_omni/diffusion/models/<name>/<name>_transformer.py` or similar | `wan2_2.py`, `fusion.py`, `bagel_transformer.py` |
 | **Custom sub-models** (VAE, fusion, autoencoder) | `vllm_omni/diffusion/models/<name>/` as separate files | `autoencoder.py`, `fusion.py` |
-| **External dependency code** (original repo utilities) | **External repo**, installed via download script or pip | original-repo package via git clone |
+| **Reference-only code** | Keep outside the runtime; use a pinned revision for golden outputs and architecture analysis | Reference inference script |
+| **Explicit external adapter dependency** | External package, only when the requested scope and maintainer direction allow it | Compatibility adapter, not native support |
 | **Hardcoded model configs** | Module-level dicts in pipeline file | `VIDEO_CONFIG`, `AUDIO_CONFIG` dicts |
 | **Download/setup script** | `examples/offline_inference/<name>/download_<name>.py` | `download_<name>.py` |
 | **Custom `model_index.json`** | Generated by download script, placed at model root | Minimal: `{"_class_name": "YourPipeline", ...}` |
 
 ### B3. Handle external dependencies
 
-If the model's code lives in a separate git repo:
+If the model's code lives in a separate git repo, first decide whether the
+requested deliverable is native support or an external adapter. Do not silently
+choose the adapter path.
 
-**Option 1: Import with graceful fallback** (recommended for models with external utils)
+**Option 1: Port the code directly** (default for native support)
+
+Copy the essential model files into `vllm_omni/diffusion/models/<name>/` and
+adapt them to shared vLLM-Omni contracts. Keep checkpoint loading strict and
+use the pinned reference only to generate parity evidence.
+
+**Option 2: Import with graceful fallback** (adapter scope only)
 
 ```python
 try:
@@ -147,11 +169,9 @@ except ImportError:
     )
 ```
 
-**Option 2: Port the code directly** (preferred when feasible)
-
-Copy the essential model files into `vllm_omni/diffusion/models/<name>/` and adapt them. This avoids external dependencies. BAGEL does this — `autoencoder.py` and `bagel_transformer.py` are ported directly.
-
-**Decision criteria**: Port if the code is self-contained and won't diverge. Use external deps if the model repo is actively maintained and the code is complex.
+Use an external runtime dependency only when the user explicitly requested an
+adapter or a maintainer approved the exception. Document the dependency,
+pinned revision, installation path, and unsupported native features.
 
 ### B4. Handle custom weight loading
 
@@ -221,7 +241,8 @@ class CustomPipeline(nn.Module):
 
 ### B5. Create the `model_index.json`
 
-Custom models need a `model_index.json` at the model root for vllm-omni to discover them. For custom models, this is minimal:
+Prefer a `model_index.json` at the model root when vLLM-Omni owns an assembled
+checkpoint directory. For custom models, this is minimal:
 
 ```json
 {
@@ -230,7 +251,14 @@ Custom models need a `model_index.json` at the model root for vllm-omni to disco
 }
 ```
 
-The `_class_name` must match a key in `_DIFFUSION_MODELS` in `registry.py`. Additional keys are model-specific (accessed via `od_config.model_config`).
+The `_class_name` must match a key in `_DIFFUSION_MODELS` in `registry.py`.
+Additional keys are model-specific (accessed via `od_config.model_config`).
+
+If the released repository is immutable and has neither a root `config.json`
+nor a Diffusers index, add it to a generic native-checkpoint signature resolver:
+match the exact Hub ID or a distinctive set of local files and return the
+pipeline class. Do not use model-name substrings or add parallel one-off
+predicates in CLI, config, and serving consumers.
 
 If the model's weights come from multiple HF repos, write a **download script** that:
 1. Downloads from each repo
@@ -292,7 +320,9 @@ Use the appropriate existing example script:
 | Image-to-Image | `examples/offline_inference/image_to_image/image_edit.py` |
 | Text-to-Audio | `examples/offline_inference/text_to_audio/text_to_audio.py` |
 
-For custom/Omni models that don't fit these categories, create a dedicated example script.
+Reuse these shared scripts even for custom models when their request and output
+contracts fit. Create a dedicated model script only when the shared category
+cannot represent the protocol, and document that gap in the PR.
 
 **Validation**: No errors, output is meaningful, quality matches reference implementation.
 
@@ -300,13 +330,17 @@ See [references/troubleshooting.md](references/troubleshooting.md) for common er
 
 ### Step 6: Add Example Scripts
 
-For Omni or custom models, create:
+Only when the shared category scripts cannot represent the model, create:
 - `examples/offline_inference/your_model_name/` — offline script + README
 - `examples/online_serving/your_model_name/` — server script + client
 - Download script if weights require assembly from multiple sources
 
 ### Step 7: Update Documentation
 
+Follow the [`add-recipe` skill](../add-recipe/SKILL.md) to add or update the
+model-family recipe and its `recipes/README.md` row with verified specifications,
+hardware, commands, feature links, and qualification evidence.
+
 Required updates:
 1. `docs/user_guide/diffusion/parallelism/overview.md` — parallelism support overview/table
 2. `docs/user_guide/diffusion/cpu_offload.md` — if CPU offload supported (add to supported models table)
@@ -346,107 +380,20 @@ Classify the model's **CI priority** first:
 
 **Keep model-specific code inside test modules — not `tests/helpers/{slug}.py`:** deploy constants, prompts, sampling dicts, and inline `request_config` / `form_data` belong in each `test_{slug}.py` and `test_{slug}_expansion.py`. Do not add per-model files under `tests/helpers/`; reuse only repo-wide harness (`mark`, `media`, `runtime`, `stage_config`, `assertions`). **L2+ online/offline e2e:** reuse or add `send_*_request` in `tests/helpers/runtime.py` — tests call the handler, not raw `omni.generate` / HTTP. See [vllm-omni-test skill](../vllm-omni-test/SKILL.md) § **Runtime send helpers**.
 
+Keep the model suite proportional using the six distinct failure owners in the
+native integration checklist. Combine supported features into a few
+parametrized E2E rows instead of creating one test per optimization.
+
 ### Step 9: Add Cache-DiT Acceleration
 
-Cache-DiT accelerates inference by caching intermediate computation results across denoising steps. After your model is working correctly on a single GPU, add cache-dit support.
+Add caching only after uncached single-device correctness. Read
+[references/cache-dit-patterns.md](references/cache-dit-patterns.md), use the
+automatic single-block-list path when possible, and add a registered
+`BlockAdapter` only for genuinely custom block topology.
 
-See [references/cache-dit-patterns.md](references/cache-dit-patterns.md) for detailed code patterns.
-
-#### 9a. Determine your model type
-
-| Model Type | Description | Action |
-|------------|-------------|--------|
-| **Standard single-transformer** | One transformer with one `ModuleList` of blocks | No code needed — `CacheDiTBackend` auto-detects via `enable_cache_for_dit()` |
-| **Multi-block-list** | One transformer with multiple block lists (e.g., `transformer_blocks` + `single_transformer_blocks`) | Write custom enabler with `BlockAdapter` |
-| **Dual-transformer** | Two transformers (e.g., high-noise + low-noise) | Write custom enabler with `BlockAdapter` wrapping both |
-
-#### 9b. Standard models — verify automatic support
-
-For standard single-transformer models, test directly:
-
-```python
-omni = Omni(
-    model="your-model-name",
-    cache_backend="cache_dit",
-    cache_config={
-        "Fn_compute_blocks": 1,
-        "Bn_compute_blocks": 0,
-        "max_warmup_steps": 4,
-    }
-)
-```
-
-Check logs for "Cache-dit enabled successfully on xxx". If it works, skip to Step 9e.
-
-#### 9c. Custom architectures — write a custom enabler
-
-For multi-block-list or dual-transformer models, write a custom enabler function:
-
-```python
-from cache_dit import BlockAdapter, ForwardPattern, ParamsModifier, DBCacheConfig
-
-def enable_cache_for_your_model(pipeline, cache_config):
-    db_cache_config = DBCacheConfig(
-        num_inference_steps=None,
-        Fn_compute_blocks=cache_config.Fn_compute_blocks,
-        Bn_compute_blocks=cache_config.Bn_compute_blocks,
-        max_warmup_steps=cache_config.max_warmup_steps,
-        max_cached_steps=cache_config.max_cached_steps,
-        max_continuous_cached_steps=cache_config.max_continuous_cached_steps,
-        residual_diff_threshold=cache_config.residual_diff_threshold,
-    )
-
-    cache_dit.enable_cache(
-        BlockAdapter(
-            transformer=pipeline.transformer,
-            blocks=[
-                pipeline.transformer.transformer_blocks,
-                pipeline.transformer.single_transformer_blocks,
-            ],
-            forward_pattern=[ForwardPattern.Pattern_1, ForwardPattern.Pattern_1],
-            params_modifiers=[ParamsModifier(...)],
-        ),
-        cache_config=db_cache_config,
-    )
-
-    def refresh_cache_context(pipeline, num_inference_steps, verbose=True):
-        cache_dit.refresh_context(
-            pipeline.transformer, num_inference_steps=num_inference_steps, verbose=verbose
-        )
-    return refresh_cache_context
-```
-
-#### 9d. Register the custom enabler
-
-Add your enabler to `CUSTOM_DIT_ENABLERS` in
-`vllm_omni/diffusion/cache/cachedit/model_specific.py`:
-
-```python
-CUSTOM_DIT_ENABLERS = {
-    "Wan22Pipeline": enable_cache_for_wan22,
-    "LongCatImagePipeline": enable_cache_for_longcat_image,
-    "YourModelPipeline": enable_cache_for_your_model,  # Add here
-}
-```
-
-#### 9e. Test Cache-DiT
-
-```python
-omni = Omni(
-    model="your-model-name",
-    cache_backend="cache_dit",
-    cache_config={
-        "Fn_compute_blocks": 1, "Bn_compute_blocks": 0,
-        "max_warmup_steps": 4, "residual_diff_threshold": 0.24,
-    }
-)
-images = omni.generate("a beautiful landscape",
-    OmniDiffusionSamplingParams(num_inference_steps=50))
-```
-
-**Verify**: 1) logs show cache enabled, 2) 1.5-2x speedup, 3) output quality acceptable vs baseline.
-
-If quality degrades, lower `residual_diff_threshold` (try 0.12-0.18) or increase `max_warmup_steps` (try 6-8).
+Verify a real cache hit and compare quality with the uncached baseline. Make a
+speed claim only at a realistic step count where warmup permits hits; an
+all-warmup smoke proves integration, not acceleration.
 
 ---
 
@@ -460,107 +407,30 @@ See [references/parallelism-patterns.md](references/parallelism-patterns.md) for
 
 #### 10a. Tensor Parallelism (TP)
 
-Shards DiT linear layers across GPUs. Requires code changes in the transformer.
-
-**What to change in the transformer**:
-1. Replace `nn.Linear` with `ColumnParallelLinear` / `RowParallelLinear` / `QKVParallelLinear`
-2. Update `load_weights()` to handle QKV fusion with `stacked_params_mapping`
-3. Use `self.to_qkv.num_heads` (local heads) instead of total heads for split sizes
-
-```python
-from vllm.model_executor.layers.linear import (
-    QKVParallelLinear, RowParallelLinear, ColumnParallelLinear,
-)
-
-# Attention: QKV → RowParallel output
-self.to_qkv = QKVParallelLinear(dim, head_dim, num_heads, num_kv_heads)
-self.to_out = RowParallelLinear(dim, dim, input_is_parallel=True)
-
-# FFN: ColumnParallel → RowParallel
-self.w1 = ColumnParallelLinear(dim, ffn_dim)
-self.w2 = RowParallelLinear(ffn_dim, dim, input_is_parallel=True)
-```
-
-**Constraints**: `num_heads % tp_size == 0` and `num_kv_heads % tp_size == 0`.
-
-**Test**: `--tensor-parallel-size 2`
+Replace compatible projections with vLLM parallel linears, preserve checkpoint
+fusion/loading, and use local head counts. Require query/KV head divisibility
+and compare a multi-rank forward with the one-rank oracle.
 
 #### 10b. Sequence Parallelism (SP / USP)
 
-Splits sequence tokens across GPUs. Non-intrusive via `_sp_plan` on the transformer class — no changes to `forward()`.
-
-**What to change in the transformer**:
-
-Add `_sp_plan` class attribute:
-
-```python
-from vllm_omni.diffusion.distributed.sp_plan import (
-    SequenceParallelInput, SequenceParallelOutput,
-)
-
-class YourTransformer(nn.Module):
-    _sp_plan = {
-        "blocks.0": {
-            "hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3),
-        },
-        "proj_out": SequenceParallelOutput(gather_dim=1, expected_dims=3),
-    }
-```
-
-If inline tensor ops (e.g., `torch.cat`) exist between shard/gather points, extract them into `nn.Module` submodules so hooks can intercept them.
-
-For RoPE that needs splitting, add an entry for the RoPE module with `split_output=True`.
-
-**Test**: `--ulysses-degree 2` (offline) or `--usp 2` (online serving)
+Prefer the declarative `_sp_plan`. For packed variable-length attention or
+learned-sink LSE correction, keep model math explicit and reuse shared exchange
+utilities. Validate uneven sequence splits, RoPE coordinates, and outputs
+against the one-rank oracle.
 
 #### 10c. CFG Parallel
 
-Distributes positive/negative CFG branches across 2 GPUs. Requires the pipeline to inherit `CFGParallelMixin`.
-
-**What to change in the pipeline**:
-
-```python
-from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
-
-class YourPipeline(nn.Module, CFGParallelMixin):
-    def diffuse(self, ...) -> torch.Tensor:
-        for i, t in enumerate(timesteps):
-            positive_kwargs = {...}
-            negative_kwargs = {...} if do_true_cfg else None
-            noise_pred = self.predict_noise_maybe_with_cfg(
-                do_true_cfg=do_true_cfg, true_cfg_scale=cfg_scale,
-                positive_kwargs=positive_kwargs, negative_kwargs=negative_kwargs,
-            )
-            latents = self.scheduler_step_maybe_with_cfg(
-                noise_pred, t, latents, do_true_cfg
-            )
-        return latents
-```
-
-Override `predict_noise()` if your transformer call is non-standard. Override `combine_cfg_noise()` for multi-output models (e.g., video + audio).
-
-**Constraint**: Exactly 2 GPUs. Only for models using classifier-free guidance.
-
-**Test**: `--cfg-parallel-size 2`
+Confirm the model uses CFG. Then reuse `CFGParallelMixin`, overriding prediction
+or recombination only for non-standard or multi-output pipelines. Distinguish a
+packed positive/negative implementation from two independent branches and
+validate the two-rank result against the packed one-rank oracle.
 
 #### 10d. HSDP (Hybrid Sharded Data Parallel)
 
-Shards transformer weights via PyTorch FSDP2 to reduce per-GPU VRAM. No code changes to the forward pass — just add a class attribute.
-
-**What to change in the transformer**:
-
-```python
-class YourTransformer(nn.Module):
-    @staticmethod
-    def _is_transformer_block(name: str, module) -> bool:
-        return "blocks" in name and name.split(".")[-1].isdigit()
-
-    _hsdp_shard_conditions = [_is_transformer_block]
-```
-
-**Constraint**: Cannot combine with TP. For standalone HSDP, set `hsdp_shard_size` explicitly.
-
-**Test**: `--use-hsdp` or `DiffusionParallelConfig(use_hsdp=True)`
+Declare layer shard conditions and ignored rank-local modules; preserve mixed
+checkpoint dtypes. HSDP cannot combine with TP. Measure parameter loading,
+FSDP materialization, warm HBM, and host PSS rather than assuming sharding saves
+peak memory. Keep the resident layout as default if HSDP is worse.
 
 #### 10e. Update parallelism documentation
 
@@ -599,6 +469,12 @@ Pipelines without `SupportsComponentDiscovery` fall back to scanning
 well-known attribute names (`transformer`, `text_encoder`, `vae`,
 etc.), which fails for non-standard names.
 
+Keep model-specific checkpoint paths, nested block aliases, layout transforms,
+and component lifecycles in the model package. Change a shared offloader only
+for a general contract, demonstrate another consumer or a framework-level bug,
+and add one focused shared regression. Avoid `if ModelName` branches in shared
+backends.
+
 ### Step 12: Performance Profiling
 
 After verifying correctness and implementing parallelism/caching, profile the model's performance to identify bottlenecks and ensure optimal execution.
@@ -608,6 +484,12 @@ See the [Profiling Single-Stage Diffusion](../../../docs/contributing/profiling.
 2. Using Nsight Systems (`nsys`) with `profiler: "cuda"` for low-overhead CUDA traces.
 3. Controlling profiling via `omni.start_profile()` and `omni.stop_profile()`.
 
+Report cold E2E, warm user latency, steady-state wave time, peak allocated and
+reserved HBM, host PSS for the full process tree, and stage boundaries. State
+whether prompt encoding and VAE/audio decoding are included. For multi-device
+layouts, plot user latency against throughput per device and keep different
+denoising-step counts on separate Pareto frontiers.
+
 ---
 
 ## Pre-commit conventions
@@ -635,7 +517,7 @@ shellcheck). In particular:
 3. **Use small models**: Test with smaller variants first
 4. **Check tensor shapes**: Most errors are reshape mismatches in attention
 5. **Add features incrementally**: Single GPU → TP → SP → CFG → HSDP → Cache-DiT
-6. **For custom models**: Get the model running with the original code first, then progressively replace components with vllm-omni equivalents
+6. **For custom models**: Run the pinned reference separately, then port the runtime natively; do not ship temporary imports from the reference implementation
 7. **Cache-DiT before parallelism tuning**: Cache-DiT is lossy — verify quality at baseline before combining with parallelism
 8. **Combine lossless + lossy**: e.g., TP + SP + Cache-DiT for maximum throughput
 
@@ -644,6 +526,7 @@ shellcheck). In particular:
 - [vllm-omni-test skill](../vllm-omni-test/SKILL.md) — L1–L4 markers, naming, Buildkite wiring, run commands
 - [Transformer Adaptation](references/transformer-adaptation.md) — porting transformers from diffusers
 - [Custom Model Patterns](references/custom-model-patterns.md) — patterns for non-diffusers models
+- [Native Model Integration Checklist](references/native-model-integration-checklist.md) — ownership boundaries, phase gates, qualification matrix, and review evidence
 - [Parallelism Patterns](references/parallelism-patterns.md) — TP, SP/USP, CFG parallel, HSDP implementation details
 - [Cache-DiT Patterns](references/cache-dit-patterns.md) — cache-dit acceleration for standard and custom architectures
 - [Troubleshooting](references/troubleshooting.md) — common errors and fixes

.claude/skills/add-diffusion-model/references/cache-dit-patterns.md

@@ -9,7 +9,9 @@ Three caching strategies:
 - **TaylorSeer**: Calibration-based prediction using Taylor expansion to estimate block outputs
 - **SCM** (Step Computation Masking): Dynamic step skipping based on configurable policies
 
-**Typical speedup**: 1.5-2.5x depending on model and configuration.
+Speedup is model-, step-, and policy-dependent. Treat a cache-enabled run as
+integration evidence until logs or a focused test show real hits and repeated
+timings show an improvement over the uncached baseline.
 
 **Official docs**: https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/cache_dit
 
@@ -243,8 +245,10 @@ vllm serve your-model --omni --port 8098 \
 
 **Verification checklist**:
 1. Logs show "Cache-dit enabled successfully on xxx"
-2. Performance: 1.5-2x speedup vs no cache
-3. Quality: compare output with `cache_backend=None`
+2. A focused test or summary confirms at least one real cache hit
+3. Quality is compared with `cache_backend=None`
+4. Performance uses a realistic step count, repeated timings, and a policy
+   whose warmup does not consume the whole run
 
 ## Excluded Models
 

.claude/skills/add-diffusion-model/references/custom-model-patterns.md

@@ -16,7 +16,7 @@ vllm_omni/diffusion/models/wan2_2/
 
 The transformer is loaded separately via `weights_sources` + `load_weights()`. Non-transformer components (VAE, text encoder) are loaded in `__init__` via `from_pretrained()`.
 
-### Custom model with external deps
+### External adapter (not native support)
 
 ```
 vllm_omni/diffusion/models/<name>/
@@ -28,7 +28,10 @@ examples/offline_inference/<task>/
 └── download_<name>.py             # Downloads weights and/or clones code repo
 ```
 
-All weights loaded eagerly in `__init__`. `load_weights()` is a no-op. External dependency imported with try/except.
+All weights are loaded eagerly in `__init__`. `load_weights()` is a no-op. The
+external dependency is imported with try/except. Use this pattern only when the
+requested scope explicitly allows an adapter; it does not satisfy a request for
+a native vLLM-Omni implementation.
 
 ### Custom model with ported code (e.g., BAGEL)
 
@@ -86,7 +89,7 @@ init → load ALL weights eagerly via custom helpers → load_weights() = no-op
 - `load_weights()` is `pass`
 - Weights may come from multiple HF repos in different formats (`.pth`, `.safetensors`)
 
-Use this when:
+Use this only for an explicitly approved external adapter when:
 - The original model has complex, well-tested loading code you don't want to rewrite
 - Weights span multiple HF repos
 - Weight format is non-standard (e.g., a single `.pth` file, not sharded safetensors)
@@ -112,7 +115,14 @@ Custom model `model_index.json` (minimal):
 }
 ```
 
-The only **required** field is `_class_name` — it must match a key in `_DIFFUSION_MODELS` in `registry.py`. Other fields are model-specific and accessible via `od_config.model_config` dict.
+The only **required** field is `_class_name` — it must match a key in
+`_DIFFUSION_MODELS` in `registry.py`. Other fields are model-specific and
+accessible via `od_config.model_config`.
+
+When the released repository cannot be modified and contains neither this file
+nor a root `config.json`, use the generic native-checkpoint signature resolver.
+Match the exact Hub ID or a distinctive local file set; never infer support from
+a model-name substring.
 
 ## External Dependency Management
 
@@ -141,6 +151,10 @@ def download_dependency():
 
 Copy essential files from the original repo into `vllm_omni/diffusion/models/<name>/`. Adapt imports to use vllm-omni utilities. Benefits: no external dependency, no git clone step. Drawback: must maintain the ported code.
 
+This is the default for native model support. Keep the reference repository at
+a pinned revision for architecture analysis and golden outputs, but do not
+import it from production code.
+
 ## Multi-Modal Input/Output Protocols
 
 Custom models that handle images, audio, or video I/O should implement protocol classes:

.claude/skills/add-diffusion-model/references/native-model-integration-checklist.md

@@ -0,0 +1,131 @@
+# Native Diffusion Model Integration Checklist
+
+Use this checklist for native non-Diffusers models, hybrid pipelines, and
+model-specific optimization work.
+
+## 1. Freeze the contract before coding
+
+Record:
+
+- Reference repository and immutable revision.
+- Official checkpoint ID, local layout, license, and required auxiliary files.
+- Pipeline stages: prompt/image encoding, denoising, video/image/audio decoding,
+  and muxing.
+- Token packing, RoPE coordinates, attention sinks/LSE behavior, MoE layout,
+  scheduler equations, CFG formulation, dtypes, and checkpoint transforms.
+- Output geometry, frame count/FPS, audio rate/channels, supported tasks, and
+  default generation steps.
+- Available local hardware versus the official qualification hardware.
+- Intended default deployment and optional optimization paths.
+
+State whether the deliverable is a native implementation or an external
+adapter. For native support, use the reference as an oracle only.
+
+## 2. Use phase gates
+
+| Phase | Required exit evidence |
+|---|---|
+| Architecture | Component/weight inventory; unsupported reference dependencies identified; ownership decisions documented |
+| Native correctness | Strict checkpoint load; tiny forward or one-step golden vs pinned reference; output contract validated |
+| Framework integration | Registry/discovery, shared request types, existing category example, serving route, docs, and recipe work without model-name branching in shared scripts |
+| Lossless distribution | One-rank oracle vs each supported TP/SP/CFG/HSDP/VAE layout; released-checkpoint smoke for the recommended layout |
+| Memory optimization | Ordinary/layerwise/DLO behavior, peak HBM, full-tree host PSS, and stage timeline; shared backend unchanged unless a general contract is proven |
+| Lossy optimization | Cache hit demonstrated; quality delta measured; speed claim made only when warmup permits hits |
+| Full qualification | Default-resolution E2E at official step count for every supported task; media metadata and hashes recorded |
+| Review readiness | Focused tests, Ruff/format, strict docs, DCO, current-main rebase, issue/RFC for large changes, concise PR evidence and limitations |
+
+Do not start performance tuning before native one-step parity. Do not claim a
+feature from topology validation alone when released weights can exercise it.
+
+## 3. Keep ownership narrow
+
+Default placement:
+
+| Concern | Owner |
+|---|---|
+| Architecture, checkpoint remap, packed attention math, nested block alias, auxiliary lifecycle | Model package |
+| Kernel/version dispatch already useful to multiple models | Shared attention utility |
+| Request-dependent example defaults | Typed model-extra entry consumed through a registry |
+| Exact-ID/local-file detection for repositories without standard metadata | Generic native-checkpoint signature table |
+| General offload/cache/HSDP bug | Shared layer plus one focused shared regression |
+
+Before changing shared code, answer:
+
+1. Is this a framework contract or only this checkpoint's layout?
+2. Can the model expose an existing declarative hook instead?
+3. Is there a second consumer or a framework-level failing test?
+4. Can the shared change avoid importing or naming the model?
+
+If the answers do not justify a shared change, keep it model-local.
+
+## 4. Reuse examples without leaking model logic
+
+- Reuse the existing `x_to_y.py` category script.
+- Resolve the pipeline class through standard metadata or native signature
+  discovery, not path substrings.
+- Put dynamic model defaults and validation in `model_extras`; let shared
+  examples consume only the generic typed API.
+- Do not create image/speech/default abstractions without a real consumer.
+- Add a dedicated example only when the shared request/output protocol cannot
+  express the model.
+
+## 5. Build a bounded evidence matrix
+
+Start with the recommended resident layout. Add only supported, useful points:
+
+- Single request: TP, SP, or hybrid TP/SP.
+- Throughput: DLO/DP layouts when concurrent requests are valid.
+- CFG: compare CFG-parallel output with the packed-CFG oracle only if the model
+  actually uses CFG.
+- HSDP: record loading and materialization peaks; it can use more HBM than a
+  resident SP layout.
+- VAE patch parallelism: compare decoded tensors or media exactly where the
+  algorithm should be lossless.
+- Cache: separate cache integration from acceleration evidence.
+
+With four devices, qualify four-device layouts such as DP4, DP2SP2, SP4, or
+CFG2xSP2. Do not present CFG2xSP4 or an official eight-device runtime as locally
+validated.
+
+For each measured layout record:
+
+- Cold E2E and warm user latency.
+- Steady-state wave time and throughput per device.
+- Prompt, denoise, video decode, audio decode, and mux times.
+- Peak allocated/reserved HBM and full-process-tree host PSS.
+- Step count, seed, output geometry, frame/FPS/audio metadata, and parity metric.
+
+Plot latency versus throughput/device. Keep one-step, four-step, and full-step
+results separate because they are not quality-equivalent.
+
+## 6. Keep tests proportional
+
+Prefer a small set with distinct failure ownership:
+
+1. Strict weight-key/dtype/load contract.
+2. Tiny native golden or one-step reference parity.
+3. One parametrized distributed-parity test covering supported layouts.
+4. Pipeline/request/output geometry and serving smoke.
+5. Shared-example/default test when the category script changes.
+6. One shared regression only for a genuine shared-layer fix.
+
+Avoid duplicating the same shape or configuration assertion across
+model-specific files. Keep expensive released-checkpoint evidence in the PR or
+recipe unless the repository's CI tier explicitly requires it.
+
+## 7. Write evidence, not marketing
+
+In the PR description include:
+
+- A compact architecture/data-flow explanation.
+- Default deployment and why it is recommended.
+- Code-change counts by component.
+- Accuracy and distributed-parity evidence tied to commands/scripts.
+- Separate Pareto figures for different step counts.
+- Supported versus validated versus planned optimizations.
+- Hardware gaps and untested official-runtime claims.
+
+Call a result a bounded local qualification unless it includes repeated timing
+and variance. State whether latency includes text/image encoding and all
+decoders. Preserve the reference revision and exact commands needed to
+reproduce the evidence.

.claude/skills/add-recipe/SKILL.md

@@ -0,0 +1,126 @@
+---
+name: add-recipe
+description: Add or update an in-repository vLLM-Omni model recipe with verified task, input, output, hardware, command, feature, and validation contracts. Use when creating files under recipes/, restructuring a recipe after review, documenting a newly supported model, or synchronizing recipe claims with support tables and shared feature guides.
+---
+
+# Add a vLLM-Omni Recipe
+
+## Source the contract
+
+Read [`recipes/TEMPLATE.md`](../../../recipes/TEMPLATE.md) and
+[`recipes/README.md`](../../../recipes/README.md) before editing. Inspect the
+canonical model card/repository, implementation, shared examples, tests, and
+available local qualification evidence.
+
+Do not infer support from a model name, sibling recipe, registry entry, or
+topology validator. Mark unexecuted configurations as configuration-only.
+
+## Route by model family
+
+Read only the contributor and user documentation relevant to the model:
+
+| Family | Contributor guide | User-facing documentation to synchronize |
+|---|---|---|
+| Diffusion | [`adding a diffusion model`](../add-diffusion-model/SKILL.md) | `docs/user_guide/diffusion_features.md`, applicable diffusion feature guides, and shared image/video/audio examples |
+| TTS | [`adding a TTS model`](../add-tts-model/SKILL.md) | `examples/offline_inference/text_to_speech/README.md`, `examples/online_serving/text_to_speech/README.md`, and `docs/serving/speech_api.md` |
+| Omni | [`adding an omni model`](../../../docs/contributing/model/adding_omni_model.md) | model-family offline/online example docs, `docs/serving/chat_completions_api.md`, and `docs/user_guide/feature_compatibility.md` |
+
+All families must update `docs/models/supported_models.md` and the matching row
+in `recipes/README.md`. Do not add a modality-specific support document when
+the repository has no such table; update the closest shared user contract.
+
+## Structure the recipe
+
+Keep the recipe task-oriented and use one hardware-specific file named
+`recipes/<vendor>/<model>-<hardware>.md`. A recipe may contain multiple
+deployment topologies on that hardware, but each additional accelerator model
+or platform requires its own suffixed file.
+
+1. **Summary:** vendor, exact model ID, runtime, modes, named hardware,
+   recommended deployment, and maintainer.
+2. **Supported model contract:** put task, input, output, and provided-profile
+   tables before setup or commands.
+3. **References:** link the canonical model source, shared examples, supported
+   model table, and feature matrix.
+4. **Checkpoint/setup:** pin revisions when possible and state required assets.
+5. **Hardware:** document the single accelerator named by the file suffix,
+   including every validated device-count/topology profile on it.
+6. **Commands:** reuse shared `examples/` entrypoints unless the model contract
+   truly requires a dedicated script. Include offline and online paths that
+   were validated.
+7. **Supported features:** use a compact model-specific topology/status table
+   with links to shared feature guides.
+8. **Verification:** provide a quick command and exact expected output
+   contract.
+9. **Qualification evidence:** report correctness, memory, and timing evidence
+   with its measurement scope and caveats.
+
+## Separate hardware from software
+
+For every locally validated hardware profile, record:
+
+- accelerator vendor/model and per-device memory;
+- number of devices;
+- interconnect (`NVLink`, `PCIe`, or the platform equivalent);
+- host memory when CPU staging/offload is material;
+- whether the profile is runtime-qualified or configuration-only.
+
+Record OS, Python, driver/runtime, framework versions, and the vLLM-Omni
+revision in a separate software-environment table. Keep precision, worker
+count, and DP/TP/SP/PP sizes with the deployment profile or exact command.
+
+Never generalize a result from one accelerator family to another or combine
+NVIDIA, AMD, NPU, or distinct accelerator models in one recipe. Create a
+separate hardware-suffixed recipe and `recipes/README.md` row. Distinguish
+upstream requirements from hardware exercised by the PR.
+
+## Keep shared documentation concise
+
+Link the applicable diffusion, TTS, omni, serving, or design guide for generic
+feature semantics and launch instructions. The recipe should contain only the
+model-specific status, valid topology, required flag difference, and evidence
+boundary.
+
+Put unsupported combinations in the same feature table. Avoid repeating the
+same model-specific prose below the global feature matrix; the recipe is the
+detailed source of truth.
+
+## Report evidence precisely
+
+For every performance or memory value, state:
+
+- checkpoint/revision and workload;
+- accelerator and device count;
+- precision and topology;
+- cold/warm scope, step count, and concurrency;
+- peak HBM and host-memory metric when measured;
+- output/quality guard and sample count;
+- monitoring overhead or other known variance.
+
+Call a single bounded run qualification evidence, not a benchmark. Do not add
+a benchmark script unless the contribution explicitly requires one.
+
+## Synchronize repository documentation
+
+Update all applicable locations:
+
+- the row in `recipes/README.md`;
+- `docs/models/supported_models.md`;
+- the family-specific documentation selected in **Route by model family**;
+- shared example documentation when commands or flags change.
+
+Keep global support tables compact. Link to the recipe instead of duplicating
+its deployment explanation.
+
+## Validate
+
+Run, at minimum:
+
+```bash
+pre-commit run --files <changed recipe/docs/skill files>
+mkdocs build --strict
+git diff --check
+```
+
+Remove generated documentation artifacts after validation. Confirm every
+recipe link resolves and every claimed profile has matching code or evidence.

.claude/skills/add-tts-model/SKILL.md

@@ -499,6 +499,13 @@ Common pre-commit failures, recovery commands for missing sign-off, and the
 full `pre-commit run` invocation for a TTS model:
 [references/precommit-dco.md](references/precommit-dco.md).
 
+## Documentation and recipe
+
+Follow the [`add-recipe` skill](../add-recipe/SKILL.md) to add or update the
+in-repository model-family recipe, the `recipes/README.md` index, the supported
+models table, TTS example documentation, and the speech API contract using
+validated evidence.
+
 ## Integration Checklist
 
 Use this checklist when integrating a new TTS model:
@@ -543,6 +550,7 @@ Use this checklist when integrating a new TTS model:
 - [ ] Buildkite entries match level: `test-ready.yml` / `test-merge.yml` or nightly TTS job / `test-nightly.yml`
 - [ ] Gradio demo working
 - [ ] Documentation added (offline + online docs, nav, supported models)
+- [ ] In-repository recipe and `recipes/README.md` row added via the `add-recipe` skill
 
 ### Phase 4: Async Chunk
 - [ ] Pipeline declares the async handoff processor

.claude/skills/production-add-diffusion-model/SKILL.md

@@ -0,0 +1,458 @@
+---
+name: production-add-diffusion-model
+description: Productionize a vLLM-Omni diffusion model after its Day-0 vertical slice works. Use when work requires official API parity and input limits, feature-combination evidence, online FP8, distributed layerwise offload, per-request Cache-DiT, CUDA/ROCm/NPU/XPU recipes, faster or sparse attention, fused operators, USP or disaggregation analysis, continuous batching and abort handling, long-running RPS stability, or accuracy/performance/reliability CI. For initial architecture porting, registry wiring, and basic weight loading, use add-diffusion-model first.
+---
+
+# Productionizing a Diffusion Model
+
+## Scope
+
+Use this skill to move a working model integration from Day-0 support to a
+measured, fail-closed production deployment. It is deliberately separate from
+[`add-diffusion-model`](../add-diffusion-model/SKILL.md); do not modify or copy
+that skill when productionizing a model. If registry, pipeline, or basic loader
+work is incomplete, read that skill first and finish the Day-0 vertical slice.
+
+MiniMax-H3 [PR #5691](https://github.com/vllm-project/vllm-omni/pull/5691)
+is the merged Day-0 case study. Its open optimization
+[issue #5700](https://github.com/vllm-project/vllm-omni/issues/5700) is an
+actively maintained roadmap and evidence map, not a list of already supported
+features. It can lag merged implementation: read its current revision, cited
+PRs, target source, and maintained recipes, then leave every unverified row
+`not tested` until the target revision has scoped evidence.
+
+Read these references before the corresponding gate:
+
+- [API and hardware recipes](references/api-and-recipes.md)
+- [Production feature patterns](references/feature-patterns.md)
+- [Performance patterns and code examples](references/performance-patterns.md)
+- [Serving, soak, and CI validation](references/production-validation.md)
+
+## Non-negotiable evidence contract
+
+Create a matrix whose row key is at least:
+
+```text
+task x API mode x execution mode/capacity x shape/schedule x attention backend x
+packed layout x cache policy x quantization x offload mode x topology x
+hardware x dtype x output representation/transport
+```
+
+Use exactly these states:
+
+| State | Meaning |
+|---|---|
+| `validated` | The exact row passed correctness and measured deployment tests, with reproducible artifacts. |
+| `limited` | A bounded subset passed; the limitation and rejection/fallback behavior are explicit. |
+| `unsupported` | A stable architectural/platform restriction is proven and fails early with an actionable error. |
+| `not tested` | No adequate evidence exists. This is the default. |
+
+Never infer compatibility from import success, server startup, another task,
+another card, or another topology. Record the model/checkpoint revision,
+vLLM-Omni commit, commands, request assets and hashes, seed/schedule/shape,
+raw output artifacts, environment, and result for every `validated` row.
+
+## Workflow
+
+Follow the gates in order. Keep a dense BF16 single-device path as the
+correctness oracle until every advertised fast path has parity evidence.
+
+### Gate 0: Freeze the official contract
+
+Pin the official implementation and checkpoint revision. Build a matrix for:
+
+- constructor/call arguments, defaults, scheduler and sigma schedule;
+- task-to-source modality/count/order rules;
+- prompt, MIME, bytes, pixels, frames, duration, FPS, steps, seed, guidance,
+  output count, and output format limits;
+- offline API and every public sync/async serving endpoint;
+- checkpoint partitions and task selection.
+
+Normalize offline, sync, and async requests through one validation contract.
+Reject an invalid task/source combination before downloads, upload persistence,
+temporary files, async job creation, or engine submission. Use 400 for invalid
+semantics and 413 for payload limits; clean partial resources on every failure.
+See [API and hardware recipes](references/api-and-recipes.md).
+
+### Gate 1: Re-prove correctness under production shapes
+
+1. Make loading strict: track expected, loaded, unexpected, and intentionally
+   ignored tensors. A fused destination is complete only after every source
+   shard arrives. Missing retained parameters abort startup.
+2. Fix reference revision, asset hashes, seed, schedule, and shape. Compare
+   component outputs, intermediate denoise latents, and final artifacts.
+3. Test unequal packed samples. Verify `cu_seqlens`, padding exclusion,
+   modality/CFG ownership, split/gather boundaries, and one output per request.
+4. Test partial process groups with non-member ranks and group size smaller
+   than world size. Do not construct a private group per component or request.
+5. Run negative API cases before positive E2E tests to prove side-effect-free
+   rejection and cleanup.
+
+### Gate 2: Wire shared fast primitives
+
+Prefer explicit, testable vLLM-Omni/vLLM operators with native fallbacks.
+`torch.compile` remains a separately validated regional optimization; it is
+not a substitute for correct operator selection or lifecycle-safe boundaries.
+
+#### RMSNorm + RoPE baseline
+
+Use the shared cross-platform operators, preserving the official Q/K norm and
+RoPE order:
+
+```python
+from vllm_omni.diffusion.layers.norm import RMSNorm
+from vllm_omni.diffusion.layers.rope import (
+    RotaryEmbedding,
+    apply_rope_to_qk,
+)
+
+self.norm_q = RMSNorm(head_dim, eps=eps)
+self.norm_k = RMSNorm(head_dim, eps=eps)
+self.rope = RotaryEmbedding(is_neox_style=False)
+
+query = self.norm_q(query)  # [B, S, Hq, D] or packed [T, Hq, D]
+key = self.norm_k(key)      # [B, S, Hkv, D] or packed [T, Hkv, D]
+query, key = apply_rope_to_qk(
+    self.rope,
+    query,
+    key,
+    (cos, sin),
+)
+```
+
+Verify `is_neox_style`, `half_head_dim`, partial rotary dimensions, cos/sin
+layout, and packed row ownership against the official model. `False` means
+interleaved/GPT-J style; do not assume the default is NeoX. Shared RMSNorm and
+RoPE are two dispatched operators, not automatically one fused kernel.
+
+For packed non-interleaved RoPE, use the shared fused boundary when its contract
+matches the model:
+
+```python
+from vllm_omni.diffusion.layers.fused_qk_norm_rope import fused_qk_norm_rope
+
+query, key = fused_qk_norm_rope(
+    query,                    # [T, Hq, D]
+    key,                      # [T, Hkv, D]
+    self.norm_q.weight,       # [D]
+    self.norm_k.weight,       # [D]
+    rope_table,               # [T, rotary_dim] = [cos | sin]
+    self.norm_q.variance_epsilon,
+)
+```
+
+The current CUDA fast path specializes BF16 `head_dim=128`, `rotary_dim=96`;
+the public function keeps an eager fallback for unsupported inputs. Verify the
+packed row/frequency contract and trace the actual fast path. Do not reshape an
+incompatible official RoPE layout merely to enter this kernel.
+
+MiniMax-H3-style partial NeoX RoPE rotates a prefix and passes the suffix
+through:
+
+```python
+self.q_norm = RMSNorm(head_dim, eps=eps, dtype=torch.bfloat16)
+self.k_norm = RMSNorm(head_dim, eps=eps, dtype=torch.bfloat16)
+self.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)
+
+def apply_partial_rope(self, x, freqs, rot_dim):
+    x_rot, x_pass = x[..., :rot_dim], x[..., rot_dim:]
+    cos, sin = torch.cos(freqs).to(x.dtype), torch.sin(freqs).to(x.dtype)
+    return torch.cat((self.rope(x_rot, cos, sin), x_pass), dim=-1)
+```
+
+The partial-RoPE snippet assumes `import torch` and lives on the owning
+attention module; adapt names and validated frequency shapes rather than
+copying it at module scope.
+
+Call the modules normally so `CustomOp` selects CUDA/HIP/NPU/XPU/native
+implementations. Do not call `forward_cuda()` directly. Compare the shared
+path with native BF16 at operator, block, denoise-trajectory, and artifact
+levels before claiming either accuracy or speed.
+
+#### Faster BF16 attention and fused projections
+
+Use local TP head counts and role-aware shared attention:
+
+```python
+from vllm.model_executor.layers.linear import QKVParallelLinear
+from vllm_omni.diffusion.attention.layer import Attention
+
+self.qkv_proj = QKVParallelLinear(
+    hidden_size=hidden_size,
+    head_size=head_dim,
+    total_num_heads=num_heads,
+    total_num_kv_heads=num_kv_heads,
+    bias=False,
+    quant_config=quant_config,
+    prefix=f"{prefix}.qkv_proj",
+)
+self.attn = Attention(
+    num_heads=self.qkv_proj.num_heads,
+    num_kv_heads=self.qkv_proj.num_kv_heads,
+    head_size=head_dim,
+    softmax_scale=head_dim**-0.5,
+    causal=False,
+    qkv_layout="BSND",
+    role="self",
+    prefix=prefix,
+)
+```
+
+Prove the selected backend from runtime logs/trace and compare it with dense
+SDPA. Packed metadata must express real sample boundaries; do not build masks
+that the selected backend ignores. A backend that is faster for one shape or
+platform is not a global default. See [performance patterns](references/performance-patterns.md)
+for QKV, SwiGLU, AdaLN, sparse attention, redundancy, and fusion examples.
+
+### Gate 3: Add online FP8 without breaking the loader
+
+Route the active config to each component, then pass a stable runtime prefix
+and config into every quantizable vLLM linear:
+
+```python
+def resolve_component_quant_config(quant_config, component):
+    return quant_config.resolve(component) if hasattr(quant_config, "resolve") else quant_config
+
+transformer_quant_config = resolve_component_quant_config(
+    od_config.quantization_config,
+    "transformer",
+)
+self.transformer = MyDiT(
+    od_config,
+    quant_config=transformer_quant_config,
+    prefix="transformer",
+)
+
+self.qkv_proj = QKVParallelLinear(
+    hidden_size=hidden_size,
+    head_size=head_dim,
+    total_num_heads=num_heads,
+    total_num_kv_heads=num_kv_heads,
+    quant_config=quant_config,
+    prefix=f"{prefix}.qkv_proj",
+)
+```
+
+Use `build_quant_config()` for direct construction/testing:
+
+```python
+from vllm_omni.quantization import build_quant_config
+
+quant_config = build_quant_config({
+    "transformer": {"method": "fp8"},
+    "text_encoder": None,
+    "vae": None,
+    "default": None,
+})
+```
+
+Preserve each parameter's vLLM `weight_loader`; perform checkpoint layout
+conversion before invoking it. Account for every QKV/gate-up source shard.
+Keep precision-sensitive norm/modulation/embedders in BF16 unless separately
+proven, using the current `safe_quant_config` pattern where applicable.
+
+Validate resident DiT FP8 first: prove which modules are quantized, HBM saved,
+BF16-vs-FP8 trajectory/artifact quality, latency, and throughput. Text encoder,
+VAE, other hardware, TP/cache/DLO combinations are separate rows.
+Pre-quantized, pruned, rotated, or adapter-modified checkpoints are separate
+model/loading lanes; compare them with the released BF16 model without calling
+the result a pure quantization ablation. Read
+[`../quantization/SKILL.md`](../quantization/SKILL.md) if present and prefer the
+target revision's implementation/docs when sibling guidance differs.
+
+### Gate 4: Make every component offloadable
+
+Declare topology instead of relying on heuristic discovery:
+
+```python
+from typing import ClassVar
+
+import torch.nn as nn
+
+from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
+from vllm_omni.diffusion.offloader import OffloadPlan
+
+class MyPipeline(nn.Module, SupportsComponentDiscovery):
+    _dit_modules: ClassVar[list[str]] = ["transformer"]
+    _encoder_modules: ClassVar[list[str]] = ["text_encoder"]
+    _vae_modules: ClassVar[list[str]] = ["vae"]
+    _resident_modules: ClassVar[list[str]] = []
+    _offload_plan: ClassVar[OffloadPlan] = OffloadPlan(
+        on_demand_component_paths=frozenset({"text_encoder", "vae"}),
+        block_attrs={"transformer": ("blocks",)},
+        offload_submodules={"token_refiner": "blocks"},
+        resident_dit_paths=frozenset({"transformer"}),
+        encoder_block_attrs={"text_encoder": ("encoder.layers",)},
+    )
+```
+
+Add a test that resolves every dotted component/block path and fails if it is
+missing or is not the expected module/container; discovery may otherwise warn
+and skip. Validate ordinary layerwise offload and distributed layerwise
+offload (DLO) independently.
+
+Keep component lifecycle and host-weight storage separate. `OffloadPlan`
+declares what can be streamed; the diffusion loader owns any `HostWeightPlan`
+and hands the exact prevalidated plan to DLO. Do not make the offloader rescan
+checkpoint files or duplicate loader name/shape/dtype decisions. Direct mmap is
+a fail-closed optimization: current preflight requires TP1, no HSDP, no online
+quantization, complete bindings, and represented transforms. Otherwise the
+ordinary loader remains authoritative.
+
+If the target revision supports Host Weight Runtime (HWR), treat it as a
+separate, exact-identity startup/cache path rather than generic offload or
+zero-copy execution. Qualify `preferred` population/fallback and `required`
+consume-only failure independently; verify immutable manifests, final-layout
+restore, lease cleanup, corruption/quarantine, and node-local sharing. Current
+model contracts may be BF16 no-AllGather-only, so reject AllGather, online
+quantization, HSDP, LoRA/adapted weights, and unrecognized load formats unless
+the exact target revision explicitly adds them.
+
+On small-HBM cards, record load/materialization peak, resident peak, encode,
+each denoise wave, decode, transient per-rank HBM, host PSS, H2D, and collective
+time. Test DLO AllGather and `--dlo-no-use-allgather` as different deployments.
+Sweep the supported resident-block policies and publish the non-dominated
+latency-HBM frontier; a single minimum-memory or fastest point is not the DLO
+trade-off. Keep request-specific preparation replica-local. Only weight
+materialization may enter the declared DLO collective group; never use WORLD
+for per-request work in a DP deployment.
+Direct checkpoint mmap preflight is limited to TP1 without HSDP or online
+quantization; TP>1 falls back to the ordinary TP-aware loader and can still feed
+DLO AllGather or no-AllGather after scoped E2E. HSDP remains incompatible with
+DLO AllGather. A target revision may allow finalized per-tensor online FP8
+through the ordinary loader and AllGather; keep every other online quantizer
+fail-closed, and require model/card/topology E2E before promotion.
+Use deploy YAML for DP under `--omni`; vLLM DP CLI flags are rejected.
+
+### Gate 5: Add request-scoped acceleration safely
+
+Server-wide Cache-DiT support does not imply a per-request quality API. Only
+add request policy after defining calibrated quality tiers and a safe hook
+lifecycle using `SupportsRequestScopedCacheDiT`, `CacheDiTRequestSpec`, and
+`RequestScopedCacheDiTRuntime`. Validate alternating and concurrent quality
+tiers, real cache hits, refresh/reset, success, error, disconnect, abort, and
+the next uncached request. Do not batch different quality policies unless the
+batch compatibility key and hook ownership make that safe.
+
+Then test Cache-DiT against FP8, DLO, TP/SP, compile, and batching one
+combination at a time. See [production feature patterns](references/feature-patterns.md)
+for the lifecycle skeleton and compatibility matrix.
+
+### Gate 6: Optimize only from profiles
+
+Freeze one canonical workload manifest before profiling. Keep lossless runtime
+A/Bs, accelerated paths with numerical/quality trade-offs, and production
+topology studies in separate result lanes. Reconcile the client boundary as
+queue + encode + denoise + video/audio decode + output transport/codec +
+residual; do not hide a material residual in another stage.
+
+Profile stage time, GPU kernels, CPU/GPU synchronization, allocations, H2D,
+output payload/copies, and collectives before choosing work. Apply and A/B one
+change at a time:
+
+1. Remove synchronization and redundant denoise-loop work: hoist static prompt
+   conditioning/token refinement, RoPE tables, masks/metadata, bounded AdaLN
+   schedule projections, and reference scans; remove per-layer `.item()` and
+   unused masks/allocations.
+2. Select the fastest correct BF16 attention backend for each role and shape.
+3. Prefer explicit fused QKV, gate-up + `SiluAndMul`, RMSNorm/RoPE, AdaLN, and
+   layout/residual operators with guarded native fallbacks. Preserve every
+   materialized dtype/rounding boundary consumed by the unfused graph.
+4. Enable sparse attention only when platform, shape, topology, realized
+   sparsity, quality, fallback, and end-to-end speedup are all proven.
+5. Profile USP all-to-all count/bytes, packing, contiguous copies, overlap, and
+   scaling efficiency. Audit replicated cross-attention before using
+   `skip_sequence_parallel`. Qualify regular and accelerated Ulysses transport
+   independently; activation logs, JIT/readiness time, workspace growth,
+   stream ownership, maximum-shape warmup, and numerical drift are gates.
+   Preserve the checkpoint's Q:KV head ratio when padding GQA for Ulysses, and
+   validate strided QKV staging rather than assuming packed contiguity.
+6. Evaluate text-encoder or VAE disaggregation only if stage share, transfer
+   volume, reuse/concurrency, and independent scaling justify it. Tiling,
+   offload, or patch parallelism is not disaggregation.
+
+Do not combine independent optimizations into one performance claim. Preserve
+raw before/after runs with fixed workload and warmup policy.
+
+### Gate 7: Make serving interruptible and load-stable
+
+Treat request-mode batching and step-wise execution as separate capabilities:
+
+- request batching requires `supports_request_batch = True` and
+  `DiffusionRequestBatch -> list[DiffusionOutput]`, exactly one output per
+  request;
+- step execution requires the complete `SupportsStepExecution` contract:
+  `prepare_encode`, `denoise_step`, `step_scheduler`, and `post_decode`.
+
+Keep all mutable scheduler, RNG, latent, mask, cache, and step state request
+scoped. First validate `--step-execution --max-num-seqs 1` and step-boundary
+abort; only then test heterogeneous multi-request waves. Step continuous
+batching is experimental and is not automatically compatible with Cache-DiT.
+Before recommending it, predeclare a useful case and success threshold, then
+A/B it against request mode under the same arrival process. Structural support
+or a generic bridge is not latency, throughput, or cancellation evidence for a
+model.
+
+Test queued and in-flight abort, client disconnect, timeout, OOM, worker error,
+and restart. Require one terminal result, idempotent cleanup, no post-decode
+after abort, no cache/temp/VRAM/request-ID leak, and a successful next request.
+Inject late worker results/exceptions after cancellation and prove the result
+pump stays alive instead of raising `InvalidStateError`. For large batched or
+long-window video outputs, freeze dtype, range, layout, contiguity, ownership,
+payload size, serializer/codec limits, and offline-versus-HTTP semantics.
+Validate device-side output preparation, D2H/IPC or shared handles, remote
+encoding, event-loop responsiveness, and client materialization as separate
+stages, including abort and consumer failure cleanup.
+A model/VAE callback that publishes ordered chunks is only a producer contract;
+it is not transport, backpressure, cancellation, public streaming, or
+time-to-first-frame evidence. Distinguish complete-response faster-than-playback
+(`client E2E / output duration <= 1`) from streaming and report first-chunk,
+steady cadence, finalization, and complete-artifact latency separately.
+Run below-, near-, and above-saturation mixed-RPS soaks and report success/error
+rate, throughput, queue time, memory slope, temp growth, and worker health.
+Report p50/p95/p99 only from a declared arrival model with enough measured
+requests for the claimed percentile. See
+[serving and CI validation](references/production-validation.md).
+
+### Gate 8: Publish recipes and four independent CI tracks
+
+For every target vendor/card/topology, publish exact environment, serve/deploy
+command, one complete JSON or multipart curl per supported task, expected
+output checks, benchmark command, warmup/repeat/concurrency policy, raw metrics,
+per-rank HBM/host PSS, accuracy artifact, and limitations. Mark each
+CUDA/ROCm/NPU/XPU row `validated`, `unsupported`, or `not tested`; never inherit
+support from the platform abstraction or another card. Re-run DLO on each
+small-HBM recipe.
+
+Maintain separate CI gates:
+
+1. Function: strict load, API/offline positive and negative contracts.
+2. Accuracy: fixed reference revision/assets/seed/schedule and scoped tolerance.
+3. Performance: fixed best deployment plus memory-constrained/DLO row, raw JSON,
+   explicit warmup, enough samples for each claimed statistic, and owned
+   regression thresholds.
+4. Reliability: long mixed-RPS soak, abort/disconnect/fault cleanup, memory and
+   temporary-resource trend.
+
+## Production Definition of Done
+
+A model is production-ready only when:
+
+- every official advertised task passes API parity, offline/online parity,
+  strict loading, fixed-reference quality, and negative validation;
+- the recommended deployment row passes correctness, isolation, abort/error
+  cleanup, benchmark, and soak gates on named hardware;
+- online FP8 and DLO are each implemented and either validated in scoped rows
+  or explicitly limited/unsupported with fail-fast behavior;
+- per-request cache quality tiers have measured speed/quality frontiers and
+  request isolation, if advertised;
+- CUDA/ROCm/NPU/XPU recipes describe evidence rather than inferred support;
+- Function, Accuracy, Performance, and Reliability CI have owners, artifacts,
+  and actionable failure output;
+- accelerated paths identify every precision/quality trade-off, and the raw
+  offline plus online encoded output contracts are versioned and tested;
+- all unverified task/backend/cache/quant/offload/topology/hardware combinations
+  remain `not tested` and are not presented as supported.
+
+Day-0 support is still a valid milestone, but title, docs, and matrix must say
+which representative task passed and which production gates remain.

.claude/skills/production-add-diffusion-model/agents/openai.yaml

@@ -0,0 +1,4 @@
+interface:
+  display_name: "Production Add Diffusion Model"
+  short_description: "Productionize vLLM-Omni diffusion models"
+  default_prompt: "Use $production-add-diffusion-model to take this integrated diffusion model from Day-0 support to evidence-backed production readiness."

.claude/skills/production-add-diffusion-model/references/api-and-recipes.md

@@ -0,0 +1,315 @@
+# API Contract and Hardware Recipes
+
+## Contents
+
+1. [Freeze the upstream contract](#freeze-the-upstream-contract)
+2. [Validate before side effects](#validate-before-side-effects)
+3. [Task request templates](#task-request-templates)
+4. [Serving benchmark template](#serving-benchmark-template)
+5. [Hardware recipe contract](#hardware-recipe-contract)
+6. [Best deployment selection](#best-deployment-selection)
+
+## Freeze the upstream contract
+
+Pin three revisions independently:
+
+- official source implementation commit or package version;
+- checkpoint/model-card revision and partition;
+- vLLM-Omni target commit.
+
+Build a table before editing code. One row is one official task/partition.
+
+| Field | Required content |
+|---|---|
+| Task and partition | Official task name, checkpoint subdirectory, task selection rules |
+| Call contract | Constructor and call arguments, types, required/optional, official defaults |
+| Prompt/input | Prompt count/length, modality, MIME, order, count, decoded shape |
+| Size/time | Width/height alignment, pixels, frames, duration, FPS, start offset |
+| Denoise | Scheduler, sigma schedule, steps, CFG/guidance, seed behavior |
+| Output | Count, format, codec/sample rate, return type |
+| API | Offline mapping, sync endpoint, async endpoint/job lifecycle |
+| Rejection | Status code and stable error class/message for every invalid boundary |
+
+Official API parity includes limits and defaults, not just names. Do not add a
+generic default that changes a distilled checkpoint's fixed schedule, or accept
+an input combination the official task rejects.
+
+For every accepted boundary, add the neighboring rejected values. Examples:
+`max_images` and `max_images + 1`, smallest/largest legal duration, aligned and
+misaligned dimensions, omitted and illegal steps, one and too many outputs.
+
+## Validate before side effects
+
+Use one normalization/validation path for offline, sync, and async requests.
+The ordering must be:
+
+```text
+parse -> normalize aliases -> validate task/source matrix -> validate declared
+limits -> stream with byte caps -> inspect/decode with frame/pixel/duration caps
+-> persist only if needed -> submit to engine
+```
+
+Fail before download, persistence, job creation, or engine submit whenever the
+available metadata is sufficient. For streamed uploads, enforce:
+
+- total reference count and per-modality count before reading bodies;
+- per-file bytes while streaming;
+- aggregate bytes across all files;
+- decoded frames, duration, pixels, and media type before engine submission.
+
+On rejection, close streams and remove every partial temporary resource. Use
+HTTP 400 for invalid task/field combinations and 413 for payload limits. Test
+both sync and async paths; an invalid async request must not create a job ID.
+
+Do not trust client MIME alone. Sniff/validate content and keep ordered repeated
+multipart fields ordered. Avoid logging prompts, raw references, signed URLs,
+or base64 payloads.
+
+## Task request templates
+
+These are endpoint-shape templates, not model recipes. Replace placeholders
+with the exact official defaults and validated assets for the target model.
+Every production recipe must include all advertised tasks and an expected
+output check.
+
+### Text to image: JSON
+
+```bash
+curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/images/generations \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "model": "<model>",
+    "prompt": "<validated prompt>",
+    "size": "1024x1024",
+    "num_inference_steps": 30,
+    "seed": 42,
+    "response_format": "b64_json"
+  }' | jq -er '.data[0].b64_json' | base64 -d > output.png
+file output.png
+```
+
+### Image edit: multipart
+
+```bash
+curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/images/edits \
+  -F 'model=<model>' \
+  -F 'prompt=<validated edit prompt>' \
+  -F 'image=@/path/to/reference.png;type=image/png' \
+  -F 'size=1024x1024' \
+  -F 'num_inference_steps=30' \
+  -F 'seed=42' \
+  -F 'output_format=png' \
+  | jq -er '.data[0].b64_json' | base64 -d > edited.png
+file edited.png
+```
+
+Use repeated `image` fields only if the model's official input matrix allows
+multiple ordered sources. Never describe a file upload as JSON-only.
+
+### Public request fields versus internal sampling fields
+
+Use the field exposed by the target revision's endpoint schema; do not rename
+it from the pipeline's internal attribute. For example, the current video JSON
+and multipart contracts expose `extra_params`, while serving translates that
+object into `DiffusionSamplingParams.extra_args`. Other request paths may
+expose canonical `extra_args` or nest it under `extra_body`. Inspect the exact
+Pydantic model/FastAPI form dependency and exercise the generated curl before
+publishing a recipe. A deprecation warning in a different endpoint is not
+evidence that an unrecognized multipart form field works here.
+
+Current endpoint map (re-check it at the target commit):
+
+| Endpoint | Public model-specific field |
+|---|---|
+| `/v1/videos`, `/v1/videos/sync` | Multipart `extra_params` JSON string |
+| `/v1/images/generations` | JSON `extra_params` object |
+| `/v1/chat/completions` diffusion | `extra_body.extra_args` |
+| `/v1/images/edits` | Declared typed form fields only |
+| `/v1/audio/generate` | Declared typed JSON fields only |
+
+Prefer declared top-level fields. Use a generic model-specific envelope only
+when the endpoint actually exposes it.
+
+### Text to video: multipart
+
+```bash
+curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/videos/sync \
+  -H 'Accept: video/mp4' \
+  -F 'model=<model>' \
+  -F 'prompt=<validated prompt>' \
+  -F 'size=1280x720' \
+  -F 'num_frames=81' \
+  -F 'fps=16' \
+  -F 'num_inference_steps=30' \
+  -F 'seed=42' \
+  -F 'extra_params={"task":"<official-task>"}' \
+  -o output.mp4
+ffprobe -v error -show_entries stream=codec_name,width,height,r_frame_rate \
+  -of json output.mp4
+```
+
+### Image/video to video: multipart
+
+```bash
+curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/videos/sync \
+  -H 'Accept: video/mp4' \
+  -F 'model=<model>' \
+  -F 'prompt=<validated prompt>' \
+  -F 'num_frames=81' \
+  -F 'fps=16' \
+  -F 'num_inference_steps=30' \
+  -F 'seed=42' \
+  -F 'extra_params={"task":"<official-task>","start_time_seconds":0}' \
+  -F 'input_reference=@/path/to/reference.png;type=image/png' \
+  -o i2v.mp4
+
+curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/videos/sync \
+  -H 'Accept: video/mp4' \
+  -F 'model=<model>' \
+  -F 'prompt=<validated prompt>' \
+  -F 'num_frames=81' \
+  -F 'num_inference_steps=30' \
+  -F 'seed=43' \
+  -F 'extra_params={"task":"<official-task>","start_time_seconds":0}' \
+  -F 'input_reference=@/path/to/reference.mp4;type=video/mp4' \
+  -o v2v.mp4
+```
+
+For multiple references, repeat the model's accepted multipart field in
+official order. Include complete downloadable or repository fixtures plus
+SHA256 hashes. A placeholder-only recipe is not runnable evidence.
+
+### Text to audio: JSON
+
+```bash
+curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/audio/generate \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "model": "<model>",
+    "input": "<validated audio prompt>",
+    "audio_length": 10.0,
+    "num_inference_steps": 100,
+    "seed": 42,
+    "response_format": "wav"
+  }' -o output.wav
+ffprobe -v error -show_entries stream=codec_name,sample_rate,channels \
+  -of json output.wav
+```
+
+Use `/v1/audio/generate` for general diffusion audio. Do not substitute the
+speech API unless the model is actually served through the speech contract.
+
+### Async video lifecycle
+
+Alongside `/v1/videos/sync`, include `POST /v1/videos`, status polling,
+content retrieval, cancellation/abort if public, and cleanup assertions. The
+same request must normalize to the same engine sampling parameters in both
+modes.
+
+## Serving benchmark template
+
+Run a smoke first, then fixed-concurrency and arrival-rate tests. Video task
+names for the harness are `t2v`, `i2v`, `ti2v`, or `v2v`; model-specific task
+selection belongs in `--extra-body` when required.
+
+```bash
+python benchmarks/diffusion/diffusion_benchmark_serving.py \
+  --base-url http://127.0.0.1:8091 \
+  --endpoint /v1/videos \
+  --model '<model>' \
+  --dataset random \
+  --task t2v \
+  --num-prompts 100 \
+  --width 1280 --height 720 \
+  --num-frames 81 --fps 16 \
+  --num-inference-steps 30 \
+  --request-rate 0.2 \
+  --max-concurrency 4 \
+  --warmup-requests 4 \
+  --warmup-concurrency 4 \
+  --extra-body '{"extra_params":"{\"task\":\"<official-task>\"}"}' \
+  --output-file result.json
+```
+
+The video backend currently stringifies each multipart value. Pre-serialize
+the nested `extra_params` object as above; passing it as a nested object would
+produce Python dict syntax rather than valid JSON. If the target revision
+changes the backend to JSON-encode dict/list values, update and smoke-test this
+command against that revision.
+
+For i2v/v2v, use the harness's validated dataset/trace path or provide the
+documented media source; do not silently benchmark t2v and label it i2v. Keep
+prompt, assets, seed, dimensions, frames, steps, guidance, warmup, concurrency,
+and arrival process identical across A/B runs.
+
+Keep three benchmark lanes distinct:
+
+1. lossless runtime A/B with the released schedule/precision/attention policy;
+2. accelerated-path A/B that changes one adapter, quantization, cache, or
+   sparsity policy and adds same-seed quality evidence;
+3. production-topology studies that vary placement or parallelism and report a
+   latency/throughput/memory frontier rather than a kernel speedup.
+
+For a fixed single-request A/B, retain every measured run and report median or
+mean with range after an explicit warmup; do not infer p95/p99 from a few
+repetitions. Tail latency requires a declared arrival process, request count,
+concurrency, and enough successful samples. Reconcile client E2E with queue,
+encoder, denoise, video/audio decode, D2H/IPC, codec, and residual timings.
+
+For generated media, define `real-time` rather than relying on the label. A
+complete-response claim uses `client E2E / validated output duration <= 1` and
+does not imply streaming or low time-to-first-frame. Report first chunk/fragment
+and cadence separately when a public streaming route exists. Never add gains
+across benchmark lanes with different schedules, prompts, seeds, artifacts, or
+topologies.
+
+The diffusion serving harness currently covers image/video task families, not
+`/v1/audio/generate`. For text-to-audio, use the target model's task-specific
+audio benchmark if one exists; otherwise add a serving harness that records
+request rate, concurrency, latency, generated-audio seconds per second, error
+rate, and output validation. Do not relabel the TTS harness or a video workload
+as diffusion-audio evidence.
+
+## Hardware recipe contract
+
+Create a separate row and runnable section for each vendor/card/topology:
+
+| Evidence | Required detail |
+|---|---|
+| Identity | Vendor, exact SKU, device count, HBM, interconnect |
+| Software | Driver, CUDA/ROCm/CANN/oneAPI, framework/container, dependency versions |
+| Source | vLLM-Omni commit, official/checkpoint revision and hashes |
+| Deployment | dtype, attention backend per role, TP/SP/CFG/HSDP/DP, cache, quant, offload |
+| Commands | Complete deploy YAML/serve command, every task curl, benchmark command |
+| Method | Warmup exclusion, repetitions or request count, concurrency, arrival distribution, duration, statistic definitions |
+| Results | Success rate, p50/p95/p99, RPS/throughput per device, stage latency |
+| Memory | Load/materialize, resident, encode, denoise, decode, transient per-rank HBM and host PSS |
+| Quality | Metric/tolerance, artifact links/hashes, temporal/audio checks |
+| Output | Raw/offline dtype, range, layout and ownership; online codec, payload bytes, transport route and client boundary |
+| Limits | `validated`/`limited`/`unsupported`/`not tested`, fallback/rejection behavior |
+
+Use current paths such as `recipes/<vendor>/`, `recipes/README.md`, supported
+models, and feature compatibility tables. Top-level documentation must link to
+the scoped matrix; it must not use an unqualified checkmark that hides a ROCm,
+NPU, XPU, task, topology, or dtype limitation.
+
+DLO evidence is per card, especially on small-HBM devices. A DLO result on one
+CUDA card does not establish another CUDA SKU, ROCm, NPU, or XPU support.
+
+## Best deployment selection
+
+Benchmark a correctness-approved candidate matrix, not every theoretical
+combination. Choose the recommended row by the production objective—latency,
+throughput, cost, or small-HBM fit—and document that objective.
+
+Do not recommend a row until it passes:
+
+- all advertised task/API contracts;
+- fixed-reference quality;
+- request isolation and abort/error cleanup;
+- sustained arrival-rate tests;
+- per-rank HBM and host PSS limits;
+- hardware-specific accuracy and performance CI.
+
+Keep the most memory-constrained DLO row even when it is not the fastest; it is
+a separate production recipe, not a failed latency candidate.

.claude/skills/production-add-diffusion-model/references/feature-patterns.md

@@ -0,0 +1,462 @@
+# Production Feature Patterns
+
+## Contents
+
+1. [Compatibility matrix](#compatibility-matrix)
+2. [Online FP8](#online-fp8)
+3. [Strict fused-weight loading](#strict-fused-weight-loading)
+4. [Distributed layerwise offload](#distributed-layerwise-offload)
+5. [Per-request Cache-DiT](#per-request-cache-dit)
+6. [Combination gates](#combination-gates)
+
+## Compatibility matrix
+
+Create rows at the granularity below. Add evidence links and a reason for every
+non-validated state.
+
+| Task/shape | Execution mode/capacity | Backend/layout | Quant | Cache policy | Offload | Topology | Hardware/dtype | Output contract/transport | State | Evidence |
+|---|---|---|---|---|---|---|---|---|---|---|
+| `<task>/<shape>/<schedule>` | serial request / 1 | SDPA, padded | BF16 | none | resident | 1 device | `<card>` BF16 | raw float / local | `not tested` | — |
+| `<task>/<shape>/<schedule>` | step / N | `<fast>`, packed | BF16 | none | resident | `<TP/SP>` | `<card>` BF16 | encoded / subprocess | `not tested` | — |
+| `<task>/<shape>/<schedule>` | step / N | `<fast>`, packed | FP8 | high | DLO no-AG | `<TP/SP>` | `<card>` | device uint8 / SHM | `not tested` | — |
+
+Initialize every row as `not tested`; the table is a work queue, not evidence.
+Promote the dense BF16 row to the oracle only after its scoped parity passes.
+Add one axis at a time. If the runtime cannot
+enforce a known-incompatible combination, add construction/admission
+validation before writing a recipe.
+
+## Online FP8
+
+### Component routing
+
+The public builder accepts one method or a longest-prefix component map:
+
+```python
+from vllm_omni.quantization import build_quant_config
+
+config = build_quant_config({
+    "transformer": {
+        "method": "fp8",
+        "ignored_layers": ["transformer.final_layer"],
+    },
+    "text_encoder": None,
+    "vae": None,
+    "default": None,
+})
+
+assert config.resolve("transformer.blocks.0.attn.qkv_proj") is not None
+assert config.resolve("text_encoder.layers.0.mlp") is None
+assert config.resolve("vae.decoder.conv_out") is None
+```
+
+Use actual runtime prefixes. Multi-DiT or nested pipelines may not use the
+generic names `transformer` and `vae`. A prefix that falls through to a default
+is not component validation. Add tests for every included and ignored prefix.
+
+For CLI serving, first validate resident DiT-only FP8:
+
+```bash
+vllm serve '<model>' --omni --quantization fp8 --max-num-seqs 1
+```
+
+For scoped routing:
+
+```bash
+vllm serve '<model>' --omni \
+  --diffusion-quantization-config \
+  '{"<dit-runtime-prefix>":{"method":"fp8","ignored_layers":["<exact-linear-prefix>"]},"<vae-runtime-prefix>":null,"default":null}' \
+  --max-num-seqs 1
+```
+
+Do not publish these placeholders. A recipe must contain the target model's
+verified prefixes.
+
+### Quantizable module wiring
+
+Pass `quant_config` and a stable prefix through every vLLM linear. Keep
+precision-sensitive modulation/norm layers safe unless proven otherwise.
+
+```python
+from vllm.model_executor.layers.activation import SiluAndMul
+from vllm.model_executor.layers.linear import (
+    MergedColumnParallelLinear,
+    QKVParallelLinear,
+    RowParallelLinear,
+)
+
+self.qkv_proj = QKVParallelLinear(
+    hidden_size=hidden_size,
+    head_size=head_dim,
+    total_num_heads=num_heads,
+    total_num_kv_heads=num_kv_heads,
+    bias=False,
+    quant_config=quant_config,
+    prefix=f"{prefix}.attn.qkv_proj",
+)
+self.gate_up_proj = MergedColumnParallelLinear(
+    hidden_size,
+    [intermediate_size, intermediate_size],
+    bias=False,
+    quant_config=quant_config,
+    prefix=f"{prefix}.mlp.gate_up_proj",
+)
+self.act_fn = SiluAndMul()
+self.down_proj = RowParallelLinear(
+    intermediate_size,
+    hidden_size,
+    bias=False,
+    input_is_parallel=True,
+    quant_config=quant_config,
+    prefix=f"{prefix}.mlp.down_proj",
+)
+```
+
+Set required checkpoint parameters to fail rather than silently initialize if
+the current loader supports `missing_param_init = "error"`.
+
+### FP8 evidence
+
+For each task/hardware row, capture:
+
+- startup proves the FP8 quant method is attached to intended linear modules;
+- no unexpected BF16 fallback and no meta/uninitialized parameters;
+- same-seed component, trajectory, and final-artifact comparison with BF16;
+- HBM at load, resident, encode, denoise, decode, and peak;
+- cold load time; fixed-work warm-latency raw runs; and serving p50/p95/p99 plus
+  throughput only from a declared arrival load with enough samples;
+- named ignored layers and why their BF16 retention is required.
+
+DiT FP8 evidence does not cover the text encoder or VAE. CUDA evidence does not
+cover ROCm/NPU/XPU. A pre-quantized checkpoint is a different loading path from
+online FP8 and needs its own state. If it is also pruned, rotated, distilled, or
+adapter-modified, report that model delta explicitly; BF16-versus-artifact
+quality and speed are deployment comparisons, not pure quantization ablations.
+
+## Strict fused-weight loading
+
+Never use `param.data.copy_()` for quantizable vLLM parameters. Transform the
+checkpoint layout before invoking the parameter loader so the online quant
+loader remains outermost:
+
+```python
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
+
+param = params[target_name]
+weight_loader = getattr(param, "weight_loader", default_weight_loader)
+
+if target_name.endswith(".qkv_proj.weight"):
+    packed_qkv = reorder_checkpoint_qkv(checkpoint_weight)
+    weight_loader(param, packed_qkv)
+elif target_name.endswith(".gate_up_proj.weight"):
+    gate, up = checkpoint_weight.chunk(2, dim=0)
+    weight_loader(param, gate, 0)
+    weight_loader(param, up, 1)
+else:
+    weight_loader(param, checkpoint_weight)
+```
+
+Track source shards per fused destination:
+
+```python
+expected = {
+    "blocks.0.attn.qkv_proj.weight": {"q", "k", "v"},
+    "blocks.0.mlp.gate_up_proj.weight": {0, 1},
+}
+seen = {name: set() for name in expected}
+
+# After mapping/loading each source:
+seen[target_name].add(shard_id)
+
+incomplete = {
+    name: sorted(shards - seen[name], key=str)
+    for name, shards in expected.items()
+    if shards - seen[name]
+}
+if incomplete:
+    raise RuntimeError(f"incomplete fused checkpoint parameters: {incomplete}")
+```
+
+Also compare retained model parameters with the returned loaded set. Explicit
+checkpoint ignores need a reason and test; an unknown or missing weight is not
+an ignore policy.
+
+Load-time adapter fusion creates a dedicated model, not a request-switchable
+LoRA. Pin the base and adapter revisions, validate every low-rank and full-rank
+delta target, freeze the adapter-declared schedule/task, and fuse before
+sharding only when the loader contract proves identical rank-local weights.
+Reject offload or host-weight paths that bypass that fusion rather than serving
+the unfused base model silently.
+
+## Distributed layerwise offload
+
+### Declare component topology
+
+Use `SupportsComponentDiscovery` and `OffloadPlan`. Validate all dotted paths
+at construction/test time because discovery may otherwise warn and skip.
+
+Treat a future/requested `--layerwise-offload-components` selector as a public
+API only if it exists in the target revision. Today, express component topology
+with `OffloadPlan` (`on_demand_component_paths`, `block_attrs`,
+`encoder_block_attrs`, `resident_dit_paths`, and `offload_submodules`). Do not
+invent a CLI flag or claim component-selective DLO from discovery declarations
+alone; validate the selector separately when it lands.
+
+```python
+from operator import attrgetter
+
+import torch
+
+def assert_offload_path(root, path):
+    value = attrgetter(path)(root)
+    if not isinstance(value, torch.nn.Module):
+        raise TypeError(f"offload path {path!r} is not an nn.Module")
+    return value
+
+for path in (
+    *pipeline._dit_modules,
+    *pipeline._encoder_modules,
+    *pipeline._vae_modules,
+    *pipeline._offload_plan.on_demand_component_paths,
+):
+    assert_offload_path(pipeline, path)
+```
+
+For `block_attrs`, resolve the DiT first and then each declared block attribute.
+Require an indexable block container with at least one `nn.Module`. Validate
+`offload_submodules`, `resident_dit_paths`, and `encoder_block_attrs` similarly.
+
+### Keep host storage loader-owned
+
+Current DLO accepts a loader-produced `HostWeightPlan`. Pipeline authors should
+not instantiate this plan or add ad hoc mmap capability flags. The loader must
+preflight complete runtime/checkpoint bindings, names, shapes, dtypes, persistent
+buffers, and represented transforms before it skips ordinary materialization.
+The exact accepted plan is consumed once by DLO; a rejected plan records an
+observable fallback reason and continues through the ordinary loader.
+
+Direct checkpoint mmap currently requires TP1 without HSDP or online
+quantization. TP>1 falls back to the ordinary TP-aware loader and may feed DLO
+AllGather or no-AllGather; the current design records a bounded DP2xTP2
+AllGather smoke, but that path does not receive direct-mmap host-page sharing.
+Per-tensor online FP8 must use the ordinary loader. Current revisions can admit
+its finalized weight/scale layout to DLO AllGather, including transposed runtime
+weights, while unsupported online quantizers still fail closed. Direct mmap
+remains unavailable, so capture the temporary full-model host materialization
+before rank sharding.
+
+Host Weight Runtime is a distinct opt-in path. Current final-layout consumers
+use no-AllGather DLO and exact model-declared BF16 representations. In
+`preferred` mode, consume an exact local hit or canonically load and publish for
+a future startup; in `required` mode, consume only and fail on a miss or unusable
+artifact. HWR must not interact with DLO AllGather, online quantization, HSDP,
+LoRA/adapted weights, or an unrecognized load format unless the target revision
+adds an exact producer/restore contract. It is immutable node-local CPU backing,
+not zero-copy GPU execution; the DLO transport still owns staging or direct H2D.
+Qualify identity inputs, cold population, warm reuse, atomic concurrent
+publication, source digests, corruption/quarantine, lease cleanup, capacity,
+node-local PSS, H2D, and E2E latency. A host-memory saving that materially
+regresses transfer or request latency is a separate memory-first deployment.
+
+### DLO deployment modes
+
+Treat these paths separately:
+
+| Path | Weight/runtime behavior | Required boundaries |
+|---|---|---|
+| DLO AllGather | Rank-sharded pinned host tensors; reconstruct layer through collective | TP>1 uses ordinary TP-aware loader output; HSDP and unsupported online quantizers are rejected; finalized per-tensor online FP8 is eligible only when recognized by the target revision; concurrent ranks must participate consistently |
+| DLO no-AllGather | Loader-approved checkpoint mmap or ordinary runtime tensors; each rank streams a complete block | Direct mmap is TP1/non-HSDP/non-online-quant only; TP/HSDP/online quant use ordinary runtime tensors and require scoped E2E |
+| SP + DLO | SP group can supply the DLO collective group | Packed boundaries and collective order require E2E |
+
+Single-stage examples:
+
+```bash
+# SP + DLO AllGather candidate; validate exact model/card before publishing.
+vllm serve '<model>' --omni \
+  --enable-distributed-layerwise-offload \
+  --usp 4 \
+  --max-num-seqs 1
+
+# TP + DLO rank-local candidate; not a support statement.
+vllm serve '<model>' --omni \
+  --tensor-parallel-size 2 \
+  --enable-distributed-layerwise-offload \
+  --dlo-no-use-allgather \
+  --max-num-seqs 1
+```
+
+Under `--omni`, configure DP in a deploy YAML rather than passing vLLM DP CLI
+flags:
+
+```yaml
+pipeline: <registered-pipeline>
+async_chunk: false
+data_parallel_size: 4
+stages:
+  - stage_id: 0
+    devices: "0,1,2,3"
+    max_num_seqs: 1
+    enable_distributed_layerwise_offload: true
+    dlo_use_allgather: true
+    parallel_config:
+      tensor_parallel_size: 1
+      sequence_parallel_size: 1
+```
+
+```bash
+vllm serve '<model>' --omni --deploy-config /path/to/dlo_dp4.yaml
+```
+
+Keep the YAML next to the validated recipe. Match its schema to the target
+revision; do not assume a model accepts another pipeline's deploy config.
+
+### DLO validation sequence
+
+1. Resident BF16 reference.
+2. Ordinary layerwise offload parity.
+3. DLO enable, warmup, same-seed generation, disable/restart.
+4. AllGather and no-AllGather separately.
+5. Concurrent requests with different prompts and identical collective-affecting
+   parameters; then reject or schedule incompatible steps/shapes safely.
+6. Queued/in-flight abort, error, idle rank, worker exit, and next request.
+7. TP, SP, cache, compile, online FP8 one axis at a time.
+8. Loader-plan selection/fallback reason, cold/warm storage lifecycle, and
+   corruption/stale-entry recovery when mmap/cache backing is advertised.
+9. Replica-local request preparation plus DLO-group-only weight collectives in
+   DP; exercise concurrent replicas to catch accidental WORLD rendezvous.
+10. Sweep resident block counts; publish raw latency/HBM pairs, dominated
+    points, and the non-dominated Pareto frontier.
+11. Per-rank HBM, host PSS for the full process tree, H2D and collective trace.
+
+On a small-HBM card, measure transient encode/VAE peaks as well as resident DiT
+weights. If a non-DiT component still exceeds HBM, combine only independently
+validated on-demand component staging, VAE tiling/patching, or another topology;
+do not hide the remaining peak behind the phrase “DLO enabled.”
+
+## Per-request Cache-DiT
+
+### Policy and lifecycle skeleton
+
+First verify that the target revision contains the shared protocol/runtime.
+If not, follow the nearest current model's lifecycle without inventing a new
+public API.
+
+The following is a **serial/exclusive request-mode sketch only**. Pipeline-wide
+hooks are mutable; concurrent requests need an admission/batching key and
+runner-owned transition serialization before this shape is safe.
+
+```python
+import torch.nn as nn
+
+from vllm_omni.diffusion.cache.cachedit import (
+    CacheDiTBackend,
+    CacheDiTRequestSpec,
+    RequestScopedCacheDiTRuntime,
+)
+
+class MyPipeline(nn.Module):
+    def __init__(self, *args, **kwargs):
+        super().__init__()
+        # ... components ...
+        self._cache_dit_runtime = RequestScopedCacheDiTRuntime(self)
+
+    def adopt_cache_dit_backend(self, backend: CacheDiTBackend) -> None:
+        self._cache_dit_runtime.adopt(
+            backend,
+            installation_key="my_model.generic",
+        )
+
+    def is_cache_dit_enabled(self) -> bool:
+        return self._cache_dit_runtime.is_enabled
+
+    def _cache_spec(self, quality, steps):
+        if quality == "lossless":
+            return None
+        if quality == "high":
+            return CacheDiTRequestSpec(
+                installation_key="my_model.high.v1",
+                cache_config=self._validated_high_config,
+                num_inference_steps=steps,
+            )
+        raise ValueError(f"unsupported quality tier: {quality!r}")
+
+    def forward(self, req):
+        quality, steps = validate_and_resolve_request(req)
+        self._cache_dit_runtime.prepare(self._cache_spec(quality, steps))
+        return self._run_validated_request(req)
+```
+
+The snippet shows the model-owned transition for exclusive execution. Put
+disable/restore handling in the runtime/runner lifecycle that owns success,
+exception, disconnect, and asynchronous abort; a `try/finally` inside
+`forward()` alone cannot observe every server cancellation boundary.
+
+The `installation_key` must include every policy dimension that changes hook
+installation; changes in step count can use runtime refresh. `None` disables
+installed hooks. Do not mutate the global startup cache config for a request.
+
+Centralize cleanup in the owning request/runner boundary. An exception, abort,
+disconnect, or admission failure must not leave a changed hook policy visible
+to the next request. The exact owner depends on the target revision; add a
+failure-injection test rather than relying on `forward()` alone to observe
+asynchronous abort.
+
+Example model-specific request form after calibration:
+
+```bash
+curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/videos/sync \
+  -F 'model=<model>' \
+  -F 'prompt=<prompt>' \
+  -F 'quality=lossless' \
+  -F 'num_inference_steps=<validated-steps>' \
+  -o lossless.mp4
+
+curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/videos/sync \
+  -F 'model=<model>' \
+  -F 'prompt=<prompt>' \
+  -F 'quality=high' \
+  -F 'num_inference_steps=<validated-steps>' \
+  -o high.mp4
+```
+
+Do not expose these tier names merely because another model uses them.
+
+### Cache-DiT evidence
+
+For every tier/task/shape/schedule:
+
+- show real cache hits/skipped blocks, not only “backend enabled” logs;
+- measure fixed-work speed and quality against lossless/native;
+- publish a speed/quality frontier and chosen tier mapping;
+- alternate lossless/high/lossless and vary steps to prove restore/refresh;
+- interleave tasks and shapes; run concurrent requests only when scheduling
+  guarantees exclusive compatible hook state;
+- inject validation error, denoise error, disconnect, and abort, then prove the
+  next request starts with its requested policy;
+- bound any cached tensors and clear request-local state.
+
+## Combination gates
+
+Start every combination as `not tested`.
+
+| Combination | Default production posture |
+|---|---|
+| Direct checkpoint mmap + TP>1/HSDP/online quant | Preflight falls back to the ordinary loader; do not claim direct-mmap savings |
+| Per-tensor online FP8 + DLO AllGather | Generic compatibility exists for finalized weights/scales; each model/card/topology remains a candidate until E2E, and other online quantizers fail closed |
+| Online FP8 + DLO no-AllGather | Candidate only after model/card/topology E2E |
+| TP>1 + DLO AllGather | Ordinary TP-aware loader only; bounded DP2xTP2 smoke exists, but each new model/card/topology still needs E2E |
+| TP>1 + DLO no-AllGather | Ordinary TP-aware loader only; candidate with limited generic coverage |
+| HSDP + DLO AllGather | Reject; it would double-shard HSDP-managed parameters |
+| HSDP + DLO no-AllGather | Configuration-compatible candidate with limited E2E coverage |
+| Cache-DiT + online FP8 | Separate trajectory, hit-rate, HBM, latency test |
+| Cache-DiT + DLO | Verify hooks, streamed blocks, memory peaks, abort cleanup |
+| Cache-DiT + step execution | Do not advertise until lifecycle/batching support is explicit |
+| Sparse attention + Ring | Verify backend execution; reject silently ignored sparse settings |
+| Load-time fused adapter + offload/HWR | Reject unless the selected loader path applies and validates the same fusion before sharding/storage |
+| Packed attention + any topology | Unequal-sample boundary parity is mandatory |
+| Device-side uint8 preparation + offline caller | Version the dtype/range/layout change; HTTP MP4 parity does not preserve the raw tensor contract |
+| Device-side preparation + remote codec | Validate route selection, strided/contiguous layouts, payload bytes, ordering, fallback, and encoded-media parity |
+
+A startup-only test never upgrades a combination. Require request output,
+parity/quality, actual fast-path evidence, resource cleanup, and a benchmark on
+the named hardware.

.claude/skills/production-add-diffusion-model/references/performance-patterns.md

@@ -0,0 +1,481 @@
+# Performance Patterns and Code Examples
+
+## Contents
+
+1. [Measurement loop](#measurement-loop)
+2. [Faster BF16 attention](#faster-bf16-attention)
+3. [Sparse attention](#sparse-attention)
+4. [Fusion-first operator patterns](#fusion-first-operator-patterns)
+5. [Remove redundant computation](#remove-redundant-computation)
+6. [USP communication optimization](#usp-communication-optimization)
+7. [Text encoder and VAE disaggregation](#text-encoder-and-vae-disaggregation)
+8. [Performance acceptance](#performance-acceptance)
+
+## Measurement loop
+
+Do not start with a list of optimizations. Start with a fixed correctness and
+performance workload:
+
+```text
+model/checkpoint revision + prompt/media hashes + seed + dimensions/frames +
+scheduler/sigma/steps/guidance + task + dtype + backend + topology + hardware
+```
+
+Capture:
+
+- stage time: load, encode, conditioning, denoise, VAE decode/postprocess;
+- GPU trace: kernel duration/count, launch gaps, allocations, H2D, collectives;
+- CPU trace: preprocessing, per-layer sync, Python dispatch, temporary files;
+- memory: per-rank HBM peaks and process-tree host PSS;
+- output: device preparation, D2H/IPC, payload bytes, codec wall/process CPU,
+  event-loop wait, client materialization, and signed residual;
+- serving: queue time, throughput, errors, request mix, and distribution samples.
+
+For a fixed-work single-request A/B, run one explicit warmup policy and at
+least three measured repetitions, keep every raw run, and report median or mean
+with range. Do not derive p95/p99 from three samples. For tail latency, declare
+the arrival model and collect enough successful requests for the percentile and
+confidence claimed. Implement one optimization at a time, re-run parity, and
+report both local kernel and end-to-end deltas. A kernel win that moves no E2E
+metric is not a production speed claim.
+
+Use the pipeline profiler already exposed by the target revision when
+available, and use PyTorch/CUDA/ROCm/NPU/XPU profiling tools appropriate to the
+platform. Do not force every platform through a CUDA-only workflow.
+
+## Faster BF16 attention
+
+### Shared attention wiring
+
+Use the shared role-aware attention layer and give it the true local head
+counts and QKV layout:
+
+```python
+import torch
+
+from vllm.model_executor.layers.linear import (
+    QKVParallelLinear,
+    RowParallelLinear,
+)
+from vllm_omni.diffusion.attention.layer import Attention
+
+class SelfAttention(torch.nn.Module):
+    def __init__(
+        self,
+        hidden_size,
+        head_dim,
+        num_heads,
+        num_kv_heads,
+        quant_config=None,
+        prefix="",
+    ):
+        super().__init__()
+        self.head_dim = head_dim
+        self.qkv_proj = QKVParallelLinear(
+            hidden_size=hidden_size,
+            head_size=head_dim,
+            total_num_heads=num_heads,
+            total_num_kv_heads=num_kv_heads,
+            bias=False,
+            quant_config=quant_config,
+            prefix=f"{prefix}.qkv_proj",
+        )
+        self.attention = Attention(
+            num_heads=self.qkv_proj.num_heads,
+            num_kv_heads=self.qkv_proj.num_kv_heads,
+            head_size=head_dim,
+            softmax_scale=head_dim**-0.5,
+            causal=False,
+            qkv_layout="BSND",
+            role="self",
+            prefix=prefix,
+        )
+        self.out_proj = RowParallelLinear(
+            num_heads * head_dim,
+            hidden_size,
+            bias=False,
+            input_is_parallel=True,
+            quant_config=quant_config,
+            prefix=f"{prefix}.out_proj",
+        )
+
+    def forward(self, x, attn_metadata=None):
+        qkv, _ = self.qkv_proj(x)
+        q_size = self.qkv_proj.num_heads * self.head_dim
+        kv_size = self.qkv_proj.num_kv_heads * self.head_dim
+        q, k, v = qkv.split([q_size, kv_size, kv_size], dim=-1)
+        q = q.unflatten(-1, (self.qkv_proj.num_heads, self.head_dim))
+        k = k.unflatten(-1, (self.qkv_proj.num_kv_heads, self.head_dim))
+        v = v.unflatten(-1, (self.qkv_proj.num_kv_heads, self.head_dim))
+        out = self.attention(q, k, v, attn_metadata)
+        out = out.flatten(-2)
+        out, _ = self.out_proj(out)
+        return out
+```
+
+Add Q/K norm and RoPE in the official order before the `Attention` call. Keep
+checkpoint mapping for the fused QKV and output projection complete.
+
+### Backend qualification
+
+For each role (`self`, cross-attention category, token refiner, etc.), record:
+
+- actual selected backend from logs/trace;
+- supported dtype, head size, QKV layout, mask/metadata, packed varlen behavior;
+- platform and compute capability;
+- TP/SP/Ring/AllGather-KV restrictions;
+- fallback behavior and whether fallback changes the claimed performance.
+
+Use unequal packed samples and compare with dense SDPA. Verify `cu_seqlens`,
+max sequence length, padding exclusion, modality boundaries, CFG ownership, and
+output slicing. Do not pass an `attn_mask` and assume it is consumed. Inspect
+the backend contract and trace.
+
+TRTLLM, Flash Attention, cuDNN, NPU varlen, and other backends are independent
+qualification rows. The shared `Attention` class does not make every backend
+packed-safe or faster automatically. Ring and hybrid paths require their own
+boundary parity. AllGather-KV and specific backend combinations may fail fast;
+keep that behavior visible in docs.
+
+## Sparse attention
+
+Sparse attention is a backend/model/platform optimization, not a synonym for
+low-precision dense attention. Enable it only after dense BF16 parity.
+
+For every candidate task/shape/topology:
+
+1. State the sparsity policy, selected backend/kernel, block/head dimensions,
+   supported hardware, external kernel/package revision, and fallback.
+2. Prove the sparse kernel executed and report realized sparsity; an enabled
+   flag is insufficient.
+3. Compare same-seed per-block outputs, denoise trajectory, and final artifact
+   with dense attention.
+4. Add modality-specific metrics: temporal consistency and audio/video sync for
+   video, spectral/audio quality for audio, and prompt/image metrics as relevant.
+5. Report attention time and full E2E time. Include preprocessing/index-build
+   overhead, peak memory, and the exact dense control.
+6. Validate each task and shape. Do not extrapolate a T2V result to I2V/Ref2V,
+   or NPU sparse support to CUDA/ROCm/XPU.
+
+If Ring bypasses the selected backend or a sparse setting would be silently
+ignored, reject the combination. Prefer a visible dense fallback only when the
+recipe labels the row `limited` and does not claim sparse speed.
+
+## Fusion-first operator patterns
+
+Prefer current shared/custom operators with explicit guards and native
+fallbacks. Prove parity at operator, block, trajectory, and artifact levels,
+then use a trace to prove fewer launches/bytes or lower latency.
+
+### Q/K RMSNorm + RoPE
+
+```python
+from vllm_omni.diffusion.layers.norm import RMSNorm
+from vllm_omni.diffusion.layers.rope import RotaryEmbedding, apply_rope_to_qk
+
+self.norm_q = RMSNorm(head_dim, eps=eps)
+self.norm_k = RMSNorm(head_dim, eps=eps)
+# Example only: set these booleans from the official checkpoint contract.
+use_neox_layout = False
+cos_sin_are_half_dim = True
+self.rope = RotaryEmbedding(
+    is_neox_style=use_neox_layout,
+    half_head_dim=cos_sin_are_half_dim,
+)
+
+q = self.norm_q(q)
+k = self.norm_k(k)
+q, k = apply_rope_to_qk(self.rope, q, k, (cos, sin))
+```
+
+Replace the illustrative booleans with the official layout in real code.
+Preserve partial rotary dimensions by splitting the rotated prefix and concatenating
+the pass-through suffix. If 3D cos/sin includes a batch axis, the current
+shared path assumes the values are shared across the batch; per-sample
+different positions require a proven representation/path.
+
+For packed `[tokens, heads, head_dim]` tensors with a non-interleaved
+`[cos | sin]` table, the current shared fused boundary is:
+
+```python
+from vllm_omni.diffusion.layers.fused_qk_norm_rope import fused_qk_norm_rope
+
+q, k = fused_qk_norm_rope(
+    q,
+    k,
+    self.norm_q.weight,
+    self.norm_k.weight,
+    rope_table,
+    self.norm_q.variance_epsilon,
+)
+```
+
+Its CUDA fast path currently specializes BF16 `head_dim=128` and
+`rotary_dim=96`; other valid inputs take the eager implementation. The separate
+shared RMSNorm and RotaryEmbedding still provide the general cross-platform
+baseline. Treat NPU/MUSA and other vendor fusions as independent qualification
+rows, and do not generalize a model-specific Triton kernel without compatible
+shape, layout, frequency partition, dtype, and platform evidence.
+
+### Fused QKV projection
+
+Use `QKVParallelLinear` as shown above. Qualification includes:
+
+- `num_heads` and `num_kv_heads` divisibility by TP;
+- local split sizes from the layer, never reusing global counts after TP;
+- Q/K/V checkpoint order conversion before the vLLM loader;
+- all Q, K, and V shards present;
+- quantization scale/loader mapping when FP8 or another method is enabled.
+
+### Gate-up projection + fused SwiGLU
+
+```python
+from vllm.model_executor.layers.activation import SiluAndMul
+from vllm.model_executor.layers.linear import (
+    MergedColumnParallelLinear,
+    RowParallelLinear,
+)
+
+self.gate_up_proj = MergedColumnParallelLinear(
+    input_size=hidden_size,
+    output_sizes=[intermediate_size, intermediate_size],
+    bias=False,
+    quant_config=quant_config,
+    prefix=f"{prefix}.gate_up_proj",
+)
+self.act_fn = SiluAndMul()
+self.down_proj = RowParallelLinear(
+    input_size=intermediate_size,
+    output_size=hidden_size,
+    bias=False,
+    input_is_parallel=True,
+    quant_config=quant_config,
+    prefix=f"{prefix}.down_proj",
+)
+
+gate_up, _ = self.gate_up_proj(x)
+x = self.act_fn(gate_up)
+x, _ = self.down_proj(x)
+```
+
+Map both checkpoint `gate_proj` and `up_proj` shards into the merged parameter
+and require both. A merged projection followed by `F.silu(gate) * up` is not
+the same claim as the `SiluAndMul` fused activation kernel; identify what the
+trace actually uses.
+
+### AdaLN
+
+Use a shared layer only when its mathematical/return contract matches the
+official model:
+
+```python
+from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNorm
+
+self.adaln = AdaLayerNorm(
+    hidden_size,
+    elementwise_affine=False,
+    eps=1e-6,
+)
+x = self.adaln(x, scale[:, None, :], shift[:, None, :])
+```
+
+`AdaLayerNormZero`, `AdaLayerNormZeroSingle`, and continuous variants include
+different projections, broadcasts, and tuple outputs; they are not drop-in
+replacements. A shared AdaLN may have an NPU fast path while using native ops on
+CUDA/HIP/XPU. Do not call it a universal fused AdaLN kernel.
+
+For sigma schedules with a small finite set of timesteps, profile whether
+precomputing the timestep/AdaLN projection once per request is worthwhile.
+Cache keys must include the exact schedule/config/device/dtype and stay bounded.
+
+### Layout, residual, and other candidates
+
+After profiling, inspect:
+
+- bias/dropout/add or gated residual chains;
+- norm + linear or norm + modulation boundaries;
+- repeated transpose/reshape/contiguous around attention;
+- VAE normalization/activation/convolution boundaries;
+- MoE routing and fused experts for models that actually use MoE.
+
+For fused residual-plus-norm or modulation chains, preserve the unfused
+materialization boundary. If the residual was stored in BF16 before RMSNorm,
+round to BF16 and promote again inside the fused kernel; using the unrounded
+FP32 temporary changes the denoising trajectory even when the stored residual
+looks correct.
+
+Keep DLO materialize/release hooks, process-group collectives, request cache
+transitions, and cleanup outside compiled/fused regions unless lifecycle safety
+is explicitly tested. Compile only stable regions and retain eager fallback.
+
+## Remove redundant computation
+
+### Hoist request invariants
+
+Audit the denoise loop for work that depends only on the request rather than
+the current timestep:
+
+```python
+# Before the denoise loop.
+condition = self.condition_proj(text_embeddings)
+refined_condition = self.token_refiner(condition, packed_metadata)
+rope_tables = self.build_rope_tables(position_ids)
+attn_metadata = self.build_attention_metadata(layout)
+
+for step, sigma in enumerate(sigmas):
+    timestep_state = self.prepare_timestep_state(sigma)
+    noise = self.transformer(
+        latents,
+        refined_condition,
+        rope_tables=rope_tables,
+        attn_metadata=attn_metadata,
+        timestep_state=timestep_state,
+    )
+    latents = self.scheduler_step(noise, sigma, latents)
+```
+
+Candidates include prompt conditioning, token-refiner output, reference
+encoding/scans, position IDs/RoPE tables, packed masks/metadata, static CFG
+branches, and finite-schedule AdaLN projections. First prove they are invariant
+for all official tasks, LoRA/adapters, prompt-update features, shapes, and
+schedules. In continuous batching, store request-specific results on request
+state rather than the pipeline singleton.
+
+### Eliminate per-layer synchronization
+
+Avoid device-to-host scalar extraction such as `.item()` inside every layer:
+
+```python
+# Bad: forces a GPU/CPU synchronization in every attention layer.
+max_len = int(cu_seqlens[-1].item())
+
+# Better: compute validated host metadata once during request packing and pass it.
+packed = PackedLayout(
+    cu_seqlens=cu_seqlens,
+    max_seqlen=max(sample_lengths),
+)
+for block in self.blocks:
+    hidden = block(hidden, packed_layout=packed)
+```
+
+Trace to verify the synchronization disappears. Do not replace a correct
+dynamic value with a stale constant merely to remove `.item()`.
+
+### Remove unused masks and allocations
+
+If packed attention consumes `cu_seqlens` and ignores a dense mask, do not
+construct the dense mask. Reuse immutable request metadata and preallocated
+buffers when safe. Prefer direct indexed scatter/gather over constructing large
+temporary tensors only when duplicate-index semantics and gradients (normally
+disabled for inference) remain correct.
+
+Any cross-request cache needs a bounded lifetime and a complete key containing
+shape, layout, device, dtype, schedule, task, adapter/LoRA, and any value that
+changes the tensor. Otherwise keep it request-scoped.
+
+### VAE and reference work
+
+Profile duplicate VAE decode/postprocess, repeated reference decoding/scans,
+unnecessary host-device round trips, and full-frame materialization. Apply VAE
+tiling, slicing, patch parallelism, or on-demand staging only with seam,
+ordering, color/range, and memory parity tests.
+
+## USP communication optimization
+
+“USP works” is not evidence that USP communication is optimized. For Ulysses,
+Ring, hybrid, and AllGather-KV separately, profile:
+
+- collective count, bytes, duration, stream, and overlap with compute;
+- Q/K/V and output all-to-all boundaries;
+- pack/unpack, padding, metadata broadcast, transpose, and contiguous copies;
+- head divisibility or advanced uneven-head support;
+- scaling efficiency from 1 to N devices on the same workload/interconnect;
+- load balance for unequal packed samples.
+
+Declare `_sp_plan` split/gather boundaries around meaningful module outputs,
+not arbitrary Python statements. Keep packed metadata sharded/gathered in the
+same ownership model as tokens.
+
+When the target revision offers accelerated Ulysses transport such as
+`--ulysses-a2a-permute`, qualify it separately from regular Ulysses. Prove the
+strict scatter-heads/gather-sequence layout selected the fast path, record
+one-time JIT/readiness cost, prewarm the maximum workspace shape before CUDA
+graph capture, keep the grow-only workspace on one stream, and exercise cleanup
+before process-group destruction. Compare collective and E2E deltas after the
+dense backend and topology are frozen; keep unsupported layouts on an explicit
+regular-Ulysses fallback.
+
+For GQA, validate `Q_heads % KV_heads == 0`, pad KV heads first, and derive Q
+padding from the original ratio. Exercise both contiguous and row-strided QKV
+layouts; a fast copy path must preserve bitwise values and packed boundaries.
+
+Cross-attention sometimes uses replicated text K/V while only video/audio Q is
+sequence-sharded. In that specific case, `skip_sequence_parallel=True` can
+avoid incorrect or wasteful Q/K/V redistribution:
+
+```python
+self.cross_attention = Attention(
+    num_heads=local_heads,
+    num_kv_heads=local_kv_heads,
+    head_size=head_dim,
+    softmax_scale=head_dim**-0.5,
+    causal=False,
+    role="cross",
+    skip_sequence_parallel=True,
+    prefix=f"{prefix}.cross_attn",
+)
+```
+
+Use it only after proving K/V are replicated and Q/output ownership is correct.
+Otherwise it disables needed communication. Report whether the optimization
+reduces collective bytes/time and improves E2E scaling.
+
+## Text encoder and VAE disaggregation
+
+Disaggregation is a measured architecture decision. First collect:
+
+| Question | Required measurement |
+|---|---|
+| Is the stage large enough? | Encode/decode share at target concurrency and shape |
+| Is transfer economical? | Tensor/media bytes, serialization, connector latency and bandwidth |
+| Is there reuse? | Prompt/reference reuse rate, cacheability, fan-out |
+| Can it scale independently? | Queue saturation and resource utilization per stage |
+| Is the contract stable? | Shape, dtype, mask/tag/position metadata, ordering |
+
+### Text encoder candidate
+
+Define an explicit tensor contract containing hidden states, masks, tags or
+position IDs, dtype, shape, task/checkpoint revision, and request ID. Support
+disabling the local encoder so the DiT cannot accidentally recompute it. Test
+all tasks, prompt lengths, partial TP group membership, retries, backpressure,
+abort, connector cleanup, and independent replica scaling.
+
+### VAE candidate
+
+Separate VAE encode and decode contracts if both exist. Define latent shape,
+scale/shift, dtype/range, tiling/chunk order, output media ordering, and stage
+metadata. Measure latent/media transfer against local decode cost. Test tiling
+seams, RNG/determinism where relevant, chunk ordering, codec/postprocess,
+backpressure, abort, and worker failure.
+
+Do not label VAE tiling, patch parallelism, compile, or offload as VAE
+disaggregation. Likewise, a separate text-encoder TP group is not a separate
+stage. If no reusable generic stage/connector exists in the target revision,
+produce a feasibility result or RFC and keep the capability `not tested` rather
+than inventing a model-local production architecture.
+
+## Performance acceptance
+
+For each accepted optimization, require:
+
+- same fixed workload and correctness artifacts before/after;
+- actual fast-path execution proven by logs/trace/counters;
+- local metric plus E2E latency/throughput and memory;
+- at least three raw fixed-work repetitions after explicit warmup, or a
+  declared arrival-load sample set large enough for every tail percentile;
+- named hardware, backend, topology, dtype, and task scope;
+- fallback/rejection tests for unsupported shapes/platforms;
+- no regression in abort/error cleanup or long-run memory trend.
+
+Update the feature matrix narrowly. Do not convert a profile-driven direction
+into a generic “supported” feature merely because a model-specific PR exists.

.claude/skills/production-add-diffusion-model/references/production-validation.md

@@ -0,0 +1,464 @@
+# Production Serving and Validation
+
+## Contents
+
+1. [Execution-mode decision](#execution-mode-decision)
+2. [Request-level batching](#request-level-batching)
+3. [Step execution and continuous batching](#step-execution-and-continuous-batching)
+4. [Abort and cleanup](#abort-and-cleanup)
+5. [Mixed-RPS soak](#mixed-rps-soak)
+6. [Four CI tracks](#four-ci-tracks)
+7. [PR evidence checklist](#pr-evidence-checklist)
+
+## Execution-mode decision
+
+Request batching and step-wise execution solve different problems:
+
+| Mode | Pipeline contract | Scheduling/abort boundary |
+|---|---|---|
+| Serial request | `forward(DiffusionRequestBatch)` for one request | Whole request; in-denoise preemption is unavailable |
+| Request batching | `supports_request_batch = True`; one forward consumes compatible independent requests | Whole batch/request |
+| Single-request step | Full `SupportsStepExecution`; `max_num_seqs=1` | Between denoise steps |
+| Step continuous batching | Full step contract plus heterogeneous batched-step evidence | Between denoise waves/steps |
+
+Do not expose four stub methods as an opt-out. The current protocol is
+structural; if a native pipeline is not step-capable, do not define the four
+step methods. If it is capable, implement and test all four.
+
+Treat structural support, a generic request-batch bridge, and model benefit as
+three separate claims. Before recommending step execution, name a hypothesis
+that request mode cannot satisfy, define a success threshold, and run a matched
+request-mode control. If throughput or latency regresses, retain step execution
+as functional/limited and keep request mode as the production recommendation.
+
+Choose compatibility keys from every value that changes tensor shapes,
+semantics, hooks, or scheduler behavior: task, output shape/count, frames,
+steps/schedule, guidance/CFG, dtype/backend, LoRA/adapter, cache quality policy,
+and model-specific conditions. Reject or schedule separately rather than
+silently using the first request's settings.
+
+## Request-level batching
+
+A request-batched pipeline returns exactly one `DiffusionOutput` per logical
+request. This is model-owned pseudocode: `validate_batch_compatibility()` and
+`_generate_batch()` are placeholders to implement from the official contract,
+not shared APIs to import.
+
+```python
+import torch
+
+from vllm_omni.diffusion.data import DiffusionOutput
+from vllm_omni.diffusion.worker.request_batch import (
+    DiffusionRequestBatch,
+    split_diffusion_output_by_request,
+)
+
+class MyPipeline(torch.nn.Module):
+    supports_request_batch = True
+
+    def forward(self, req: DiffusionRequestBatch) -> list[DiffusionOutput]:
+        validate_batch_compatibility(req.sampling_params_list, req.prompts)
+        common = req.sampling_params_list[0]
+
+        text_prompts: list[str] | None = [
+            prompt
+            if isinstance(prompt, str)
+            else (prompt.get("prompt") or "")
+            for prompt in req.prompts
+        ]
+
+        prompt_fields = DiffusionRequestBatch.collate_prompt_field_map(
+            req.prompts,
+            {"prompt_embeds": None, "prompt_mask": None},
+        )
+        if prompt_fields["prompt_embeds"] is not None:
+            text_prompts = None
+        generators = req.collate_request_generators(
+            common.num_outputs_per_prompt,
+            default_generator=None,
+        )
+        output = self._generate_batch(
+            prompt=text_prompts,
+            prompt_fields=prompt_fields,
+            generators=generators,
+            sampling=common,
+        )
+        return split_diffusion_output_by_request(
+            DiffusionOutput(output=output),
+            req,
+            num_outputs_per_prompt=common.num_outputs_per_prompt,
+        )
+```
+
+The prompt-field collator only handles tensor fields; extract raw text prompts
+and model-specific negative/alias fields separately, as current in-tree batched
+pipelines do. Set a raw prompt field to `None` when the matching precomputed
+embedding is supplied. The exact aliases and generator device depend on the
+model. Use current collators rather than manual `torch.cat` when they cover the
+contract; they reject mixed present/missing tensors and incompatible
+shape/dtype/device.
+
+Before reading `sampling_params_list[0]`, validate every field that must be
+common. Tests must cover:
+
+- two and N requests, independent seeds/generators, one output each;
+- mixed prompt lengths and packed/padded boundaries;
+- multiple outputs per prompt and correct result slicing;
+- compatible versus incompatible shapes, steps, guidance, LoRA, and quality;
+- one request failure without cross-request output or state corruption;
+- output ordering and request IDs under different completion times.
+
+After the model advertises support, candidate serving is:
+
+```bash
+vllm serve '<model>' --omni \
+  --max-num-seqs 4 \
+  --request-batch-max-wait-ms 20
+```
+
+The wait window trades admission latency for batch formation. Benchmark at 0
+and candidate nonzero values under the target arrival process.
+
+## Step execution and continuous batching
+
+### Full protocol skeleton
+
+Use the current shared state types and make every mutable value request-local:
+
+The helpers beginning with `_` and `validate_step_batch()` below are
+model-owned placeholders; scheduler initialization is scheduler-specific.
+
+```python
+import copy
+from collections.abc import Sequence
+from typing import Any, ClassVar
+
+import torch
+
+from vllm_omni.diffusion.data import DiffusionOutput
+from vllm_omni.diffusion.models.interface import SupportsStepExecution
+from vllm_omni.diffusion.worker.input_batch import InputBatch
+from vllm_omni.diffusion.worker.utils import StepRequestState
+
+class MyPipeline(torch.nn.Module, SupportsStepExecution):
+    supports_step_execution: ClassVar[bool] = True
+
+    def prepare_encode(
+        self,
+        state: StepRequestState,
+        **kwargs: Any,
+    ) -> StepRequestState:
+        prepared = self._prepare_request(state.prompt, state.sampling)
+        state.prompt_embeds = prepared.prompt_embeds
+        state.latents = prepared.latents
+        state.timesteps = prepared.timesteps
+        state.scheduler = copy.deepcopy(self.scheduler)
+        state.scheduler.set_begin_index(0)
+        state.step_index = 0
+        state.extra["task_state"] = prepared.task_state
+        return state
+
+    def denoise_step(
+        self,
+        input_batch: InputBatch,
+        *,
+        states: Sequence[StepRequestState] | None = None,
+        **kwargs: Any,
+    ) -> torch.Tensor | None:
+        validate_step_batch(input_batch, states)
+        return self.transformer(**build_batched_denoise_inputs(input_batch))
+
+    def step_scheduler(
+        self,
+        state: StepRequestState,
+        noise_pred: torch.Tensor,
+        **kwargs: Any,
+    ) -> None:
+        timestep = state.current_timestep
+        if timestep is None:
+            raise RuntimeError("step_scheduler called without a current timestep")
+        state.latents = state.scheduler.step(
+            noise_pred,
+            timestep,
+            state.latents,
+            return_dict=False,
+        )[0].to(state.latents.dtype)
+        state.step_index += 1
+
+    def post_decode(
+        self,
+        state: StepRequestState,
+        **kwargs: Any,
+    ) -> DiffusionOutput:
+        return DiffusionOutput(output=self.decode(state.latents))
+```
+
+This is a shape guide, not drop-in code: verify the target revision's
+`StepRequestState.extra`, scheduler signature, CFG handling, state fields, and
+decode contract. Follow a current native pipeline such as Qwen-Image for the
+exact revision.
+
+Never use the shared `self.scheduler` to advance a request. Store scheduler,
+latents, embeddings, timesteps, masks, RNG/generator, task metadata, cache
+policy, adapter, and output assembly state on `StepRequestState` or its
+request-local extension. Pipeline fields may hold immutable weights/config and
+bounded immutable caches only.
+
+### Qualification sequence
+
+1. Compare request-mode and step-mode at `max_num_seqs=1` for every task using
+   identical seed/schedule/shape and intermediate latents.
+2. Abort queued and in-flight requests at several step boundaries.
+3. Increase to two compatible requests; verify independent scheduler/RNG/state
+   and output order.
+4. Increase to the target capacity and mix prompt lengths.
+5. Exercise mixed shape, frames, steps/schedule, task, guidance, and quality.
+   Batch only compatible rows; otherwise split/reject predictably.
+6. Inject a failure in encode, denoise, scheduler, and decode independently.
+7. Run the production soak and fault matrix.
+
+Candidate commands only after capability validation:
+
+```bash
+# Conservative step execution and step-boundary abort.
+vllm serve '<model>' --omni \
+  --step-execution \
+  --max-num-seqs 1
+
+# Experimental batched-step candidate after multi-request E2E.
+vllm serve '<model>' --omni \
+  --step-execution \
+  --max-num-seqs 4
+```
+
+If an attention backend is only correct for single-request step mode, fail
+early or keep the recipe at `max_num_seqs=1`. Do not describe generic runtime
+support as model support.
+
+## Abort and cleanup
+
+### Required semantics
+
+For queued and in-flight abort/disconnect:
+
+- emit exactly one terminal aborted/error result;
+- do not run `post_decode` after abort unless a documented partial-output mode
+  explicitly requires it;
+- do not increment `step_index` or apply a scheduler result after cancellation;
+- remove scheduler/state cache entries and request IDs;
+- release latents, prompt/reference tensors, cache hooks, DLO buffers, connector
+  messages, uploaded/temp files, and output buffers;
+- keep other requests alive;
+- make cleanup idempotent when disconnect, timeout, and worker error race;
+- drop late results/exceptions for already-resolved or cancelled futures
+  without terminating the shared result-pump thread;
+- complete the next valid request with its requested cache/offload policy.
+
+Test cancellation before engine admission, while queued, after encode, during
+early/middle/final denoise steps, before decode, and while returning/persisting
+output. Include async job cancellation and client socket disconnect.
+
+### Failure injection matrix
+
+| Location | Injection | Assertions |
+|---|---|---|
+| Validation/upload | bad MIME, count/bytes overflow, truncated media | 400/413, no job/engine submit, no temp leak |
+| Load/start | missing fused shard, unsupported topology | startup fails, actionable error, no silent init |
+| Encode | exception/OOM/timeout | one error, request state/temp released |
+| Denoise | exception/OOM/worker exit | peers handled, hooks/buffers released, health behavior explicit |
+| Scheduler | invalid state/shape | no latent/step partial commit |
+| Decode/post | exception/codec error | no corrupt successful result, partial output removed |
+| Result/IPC | late result after cancel, large split tensor, orphan SHM | pump remains alive, one terminal result, exact ownership cleanup |
+| Connector/disagg | timeout/retry/duplicate | idempotency, backpressure, no orphan payload |
+
+After each injection, check server health and issue a known-good request. If
+the process must restart, document and test the restart policy; do not claim
+transparent recovery.
+
+### Large-output transport
+
+Treat worker-to-engine IPC and engine-to-client transport as separate hops.
+For batched video, prove that each logical output does not accidentally retain
+or serialize the complete batch storage. Exercise the target revision's shared
+memory or artifact-handle path above and below its threshold, including
+non-contiguous views, multiple outputs, consumer failure, timeout, abort, and
+worker exit. Verify every shared segment/file is released exactly once.
+
+Freeze the media contract at each hop: dtype, value range, shape/layout,
+contiguity, request ownership, encoding/color model, payload bytes, and pending
+consumers. Device-side float-to-uint8 preparation, D2H/IPC, planar/interleaved
+conversion, CPU encode/mux, and delivery off the asyncio event loop are
+different optimizations. Validate their route selection and fallbacks
+independently. Byte-identical HTTP MP4 output does not prove that a raw offline
+tensor kept its dtype or range.
+
+For long-window output, validate segment ordering and audio/video continuity,
+then exercise payloads around serializer, shared-memory, HTTP, and codec size
+limits. Chunking must reassemble exactly, reject malformed/missing chunks, and
+release partial state after abort, timeout, or peer failure.
+
+Qualify chunk production, transport, and public streaming as separate layers.
+A VAE callback must define owned representation, frame offsets, final marker,
+rank ownership, source/config gate, and collective-safe failure, but it does not
+prove SHM/ZMQ delivery, bounded backpressure, cancellation, codec reuse, or
+client streaming. Measure time to first owned chunk, first encoded fragment,
+steady cadence, mux finalization, and complete client artifact independently.
+
+Measure remote MP4 encoding, serialization/copies, network transfer, and client
+materialization separately from denoise/decode. An inference-kernel speedup does
+not establish serving throughput if output transport dominates or leaks.
+
+## Mixed-RPS soak
+
+Use a realistic request mix and arrival distribution. Include small/large
+shapes, short/long prompts, all advertised tasks, cache tiers, and a small rate
+of abort/disconnect/error injections. Run phases below, near, and above measured
+saturation so queue/backpressure behavior is visible.
+
+Example serving benchmark phase:
+
+```bash
+python benchmarks/diffusion/diffusion_benchmark_serving.py \
+  --base-url http://127.0.0.1:8091 \
+  --endpoint /v1/videos \
+  --model '<model>' \
+  --dataset trace \
+  --dataset-path /path/to/versioned-production-trace.jsonl \
+  --task t2v \
+  --num-prompts 500 \
+  --request-rate 0.5 \
+  --max-concurrency 8 \
+  --warmup-requests 8 \
+  --warmup-concurrency 8 \
+  --output-file soak-near-saturation.json
+```
+
+Use a task-appropriate trace/media dataset. Run the same phase at rates selected
+from an initial saturation curve; a literal `0.5` is illustrative only.
+
+On supported GPU hosts, wrap a test or soak with the repository monitor:
+
+```bash
+bash tests/dfx/stability/scripts/resource_monitor.sh run --backend gpu -- \
+  pytest -s -v tests/dfx/stability/scripts/test_stability_<model>.py -m slow
+```
+
+Check the target revision's monitor backends. Do not claim CPU/NPU monitoring
+from a reserved CLI choice unless collection is implemented and verified.
+
+Report:
+
+- offered and achieved RPS, success/error/abort rates;
+- p50/p95/p99 and queue/service/stage times, with the successful sample count
+  and arrival process used for each percentile;
+- throughput per device and batch/wave utilization;
+- per-rank HBM and process-tree PSS trend/slope;
+- cache size/hit rate, temp-file/disk growth, open FDs/handles;
+- outstanding shared-memory/artifact handles and encoded/network bytes;
+- worker/engine health, restarts, OOM/timeouts, backpressure response;
+- total duration, request count/mix, raw logs/JSON/monitor artifact hashes.
+
+A short burst or repeated serial curl loop is not a stability test.
+
+## Four CI tracks
+
+Keep the tracks independent so a failure answers a specific question. These
+four readiness tracks are not the repository's numbered CI levels. Map them
+onto the current L1-L5 taxonomy:
+
+| Readiness track | Repository CI placement |
+|---|---|
+| Function | L1 unit/logic plus L2 basic online/offline E2E; extend real-model scenarios in L3/L4 |
+| Accuracy | L3 real-model gates for important cases; deeper and broader nightly coverage in L4 |
+| Performance | Important thresholds in L3 where they fit the time budget; full baselines/regression jobs in nightly L4 |
+| Reliability | Cheap rejection/recovery assertions can run earlier, but long stability, fault injection, and reliability suites belong to weekly L5 |
+
+Distributed topology is a test case within the appropriate level, not a
+replacement definition for a level.
+
+### 1. Function CI
+
+Cover:
+
+- strict unquantized and advertised quantized loading;
+- fused source-shard completeness and prefix routing;
+- official task positive boundaries and neighboring negative cases;
+- offline, sync, async normalization/parity;
+- single-device reference plus every advertised topology smoke;
+- output type/count/order and request ID isolation;
+- feature rejection paths that must fail early.
+
+### 2. Accuracy CI
+
+Pin official implementation/checkpoint/vLLM-Omni revisions, prompts and media
+hashes, seed, scheduler/sigma/steps, dimensions/frames/FPS, guidance, dtype,
+backend, and topology. Compare at suitable levels:
+
+- component outputs or selected blocks;
+- one or more intermediate denoise latents;
+- final image/video/audio artifact metrics;
+- temporal consistency and audio/video synchronization where relevant.
+
+Set tolerance per modality/hardware/backend with rationale. Keep failure
+artifacts and metadata. A single visual inspection does not define CI.
+
+For quantization trajectory tooling, inspect the target revision's current
+`vllm_omni/quantization/tools/compare_diffusion_trajectory_similarity.py` and
+reuse it when compatible instead of creating an ad hoc comparator.
+
+### 3. Performance CI
+
+Pin workload and best deployment. Include a resident recommended row and a
+small-HBM/DLO row when both are production targets. Declare:
+
+- one explicit warmup exclusion and at least three measured fixed-work
+  repetitions;
+- raw result JSON and environment identity;
+- median/mean plus range for small fixed-work A/Bs; p50/p95/p99 only for an
+  arrival-load run with enough samples, plus throughput/RPS, stage time,
+  HBM/PSS;
+- regression metric, threshold, baseline owner, and update procedure;
+- correctness/accuracy check linked to the same configuration.
+
+Use the current harness config style, for example:
+
+```bash
+pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py \
+  --test-config-file tests/dfx/perf/tests/test_<model>_vllm_omni.json
+```
+
+Verify parser options on the target revision. Do not copy a JSON schema from a
+different model without matching its endpoint, task, dataset, and deploy path.
+
+### 4. Reliability CI
+
+Run a bounded CI soak plus a longer scheduled soak. Cover:
+
+- below/near/above-saturation arrival phases;
+- queued/in-flight abort and disconnect;
+- OOM/timeout and worker/process failure appropriate to the platform;
+- DLO/cache/connector cleanup;
+- health, fast-fail/backpressure, restart/recovery policy;
+- memory/temp/FD trend and successful known-good request after each fault.
+
+Keep reliability thresholds independent from latency thresholds. A fast server
+that leaks memory or hangs after cancellation fails production readiness.
+
+## PR evidence checklist
+
+The PR description must include or link:
+
+- official source/checkpoint revisions and API/limit matrix;
+- scoped feature compatibility matrix with the four allowed states;
+- implementation notes for FP8, DLO, request cache, attention/fusion, batching;
+- exact environment, deploy/serve, every task curl, benchmark and soak commands;
+- fixed accuracy inputs and artifacts;
+- raw before/after performance runs and memory traces;
+- output-contract manifests and stage accounting from device preparation
+  through the complete client artifact;
+- abort/disconnect/fault results and cleanup evidence;
+- Function/Accuracy/Performance/Reliability CI files and ownership;
+- limitations/TODOs that remain `not tested` rather than implied support.
+
+Separate the Day-0 milestone from production readiness. If the PR only closes
+some gates, title and description should say which scoped gates it closes.

.claude/skills/readme.md

@@ -26,6 +26,9 @@ self-check.
 
 - [`add-diffusion-model`](add-diffusion-model/SKILL.md): guides integration of
   a new diffusion model into `vllm-omni`
+- [`add-recipe`](add-recipe/SKILL.md): creates or updates in-repository model
+  recipes with verified task/input/output contracts, concrete hardware,
+  shared feature-guide links, exact commands, and qualification evidence
 - [`add-tts-model`](add-tts-model/SKILL.md): covers integration of new TTS
   models and related serving workflows
 - [`diffusion-perf-opt`](diffusion-perf-opt/SKILL.md): guides diffusion model
@@ -37,6 +40,10 @@ self-check.
 - [`precheck-pr`](precheck-pr/SKILL.md): self-checks a branch before creating a
   PR by validating title format, dead code, simplification opportunities,
   accuracy and performance claims, and merge readiness
+- [`production-add-diffusion-model`](production-add-diffusion-model/SKILL.md):
+  takes a working Day-0 diffusion integration through API/limit parity,
+  feature-combination evidence, quantization/offload, performance, production
+  serving, hardware recipes, and CI readiness
 - [`quantization`](quantization/SKILL.md): guides quantization method selection,
   model integration, checkpoint loading, and quality/performance validation
   for vLLM-Omni

README.md

@@ -16,6 +16,7 @@ Easy, fast, and cheap omni-modality model serving for everyone
 
 *Latest News* 🔥
 
+- [2026/08] We released [0.28.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.28.0), featuring production-ready [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) serving on GPU and NPU, a unified AR/DiT paged KV cache runtime, and enhanced realtime full-duplex serving for the [MiniCPM-o series](recipes/OpenBMB/MiniCPM-o-4_5.md).
 - [2026/08] [VeRL-Omni](https://github.com/verl-project/verl-omni) `v0.2.0` is released: faster diffusion RL powered by vLLM-Omni (request-level/step-wise batching with FA3), rebuilt Qwen3-Omni multimodal training (DPO & GSPO), plus LTX-2.3, Qwen-Image-Edit support and more. See the [release notes](https://github.com/verl-project/verl-omni/releases/tag/v0.2.0).
 - [2026/08] We released [0.26.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.26.0) - aligned with the vLLM 0.26 release line, featuring [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) joint video/audio generation, an experimental full-duplex realtime runtime for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md), distributed layerwise diffusion offload, and broader model, hardware, streaming, TTS, and quantization support.
 - [2026/07] We released [0.24.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.24.0) - aligned with the vLLM 0.24 release line, expanding production-ready coverage across TTS, speech, diffusion, image/video generation, and robot-policy serving, with major Omni stage runtime refactoring, diffusion request-level batching, async output materialization, quantization/cache/memory improvements, and broad CUDA/ROCm/XPU/NPU support.
@@ -52,14 +53,14 @@ vLLM-Omni is flexible and easy to use with:
 - Tensor, pipeline, data and expert parallelism support for distributed inference
 - Streaming outputs
 - OpenAI-compatible API server
-- Full-duplex realtime serving with streaming audio input and output (experimental)
+- [Full-duplex realtime serving](docs/serving/realtime_duplex_api.md) with streaming audio input and output
 
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
 - **Omni-modality models** (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
-- **TTS models** (e.g. Qwen3-TTS, VoxCPM2, Ming-Omni-TTS, CosyVoice3)
-- **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, Qwen-Image, Wan2.2, FLUX)
-- **Robot-policy and action models** (e.g. GR00T-N1.7, DreamZero-DROID, InternVLA-A1, Cosmos3 action policy)
+- **TTS models** (e.g. Qwen3-TTS, IndexTTS 2.5, dots.tts, CosyVoice3)
+- **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
+- **Robot-policy and action models** (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
 
 ## Getting Started
 
@@ -67,6 +68,7 @@ Visit our [documentation](https://vllm-omni.readthedocs.io/en/latest/) to learn
 
 - [Installation](https://vllm-omni.readthedocs.io/en/latest/getting_started/installation/)
 - [Quickstart](https://vllm-omni.readthedocs.io/en/latest/getting_started/quickstart/)
+- [Nightly CUDA Docker images](https://hub.docker.com/r/vllm/vllm-omni/tags?name=nightly) built from the latest `main`
 - [List of Supported Models](https://vllm-omni.readthedocs.io/en/latest/models/supported_models/)
 - [Deployment Recipes](https://recipes.vllm.ai) for vLLM-Omni model serving
 

benchmarks/kernels/benchmark_diffusion_lora_expand.py

@@ -0,0 +1,156 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Benchmark diffusion LoRA expand-and-accumulate implementations.
+
+This compares the previous functional expand path with direct accumulation into
+the selected output slice. It is a synthetic kernel microbenchmark and does not
+load a diffusion model.
+
+Example:
+    python benchmarks/kernels/benchmark_diffusion_lora_expand.py \
+        --m 1,16,128,512,4096,16384 \
+        --in-dim 4096 --out-dim 4096 --rank 64 \
+        --dtype bfloat16 --packed --warmup 30 \
+        --small-iterations 500 --large-iterations 100
+"""
+
+import argparse
+import math
+import statistics
+from collections.abc import Callable
+
+import torch
+
+MiB = 1024**2
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--m", default="1,16,128,512,4096,16384")
+    parser.add_argument("--in-dim", type=int, default=4096)
+    parser.add_argument("--out-dim", type=int, default=4096)
+    parser.add_argument("--rank", type=int, default=64)
+    parser.add_argument("--dtype", choices=("bfloat16", "float16"), default="bfloat16")
+    parser.add_argument(
+        "--packed",
+        action="store_true",
+        help="Accumulate into the second half of a packed output tensor.",
+    )
+    parser.add_argument("--warmup", type=int, default=30)
+    parser.add_argument("--small-iterations", type=int, default=500)
+    parser.add_argument("--large-iterations", type=int, default=100)
+    parser.add_argument("--large-m-threshold", type=int, default=1024)
+    return parser.parse_args()
+
+
+def percentile(samples: list[float], probability: float) -> float:
+    ordered = sorted(samples)
+    position = probability * (len(ordered) - 1)
+    lower = math.floor(position)
+    upper = math.ceil(position)
+    if lower == upper:
+        return ordered[lower]
+    weight = position - lower
+    return ordered[lower] * (1 - weight) + ordered[upper] * weight
+
+
+def measure_latency_ms(operation: Callable[[], None], warmup: int, iterations: int) -> list[float]:
+    for _ in range(warmup):
+        operation()
+    torch.accelerator.synchronize()
+
+    starts = [torch.cuda.Event(enable_timing=True) for _ in range(iterations)]
+    ends = [torch.cuda.Event(enable_timing=True) for _ in range(iterations)]
+    for start, end in zip(starts, ends, strict=True):
+        start.record()
+        operation()
+        end.record()
+    torch.accelerator.synchronize()
+    return [start.elapsed_time(end) for start, end in zip(starts, ends, strict=True)]
+
+
+def measure_extra_peak_mib(operation: Callable[[], None]) -> float:
+    torch.accelerator.synchronize()
+    torch.accelerator.reset_peak_memory_stats()
+    allocated = torch.accelerator.memory_allocated()
+    operation()
+    torch.accelerator.synchronize()
+    peak = torch.accelerator.max_memory_allocated()
+    return (peak - allocated) / MiB
+
+
+def print_header(args: argparse.Namespace, dtype: torch.dtype) -> None:
+    device_name = torch.cuda.get_device_name()
+    print(f"device={device_name} torch={torch.__version__} dtype={dtype}")
+    print(f"in_dim={args.in_dim} out_dim={args.out_dim} rank={args.rank} packed={args.packed} warmup={args.warmup}")
+    print("M       iters  old_p10  old_p50  old_p90  new_p10  new_p50  new_p90  speedup  old_MiB  new_MiB  max_abs")
+
+
+def benchmark_shape(m: int, args: argparse.Namespace, dtype: torch.dtype) -> None:
+    device = torch.device("cuda")
+    output_width = args.out_dim * (2 if args.packed else 1)
+    offset = args.out_dim if args.packed else 0
+    output_slice = slice(offset, offset + args.out_dim)
+    scale = 1 / math.sqrt(args.in_dim)
+
+    x = torch.randn(m, args.in_dim, device=device, dtype=dtype)
+    lora_a = torch.randn(args.rank, args.in_dim, device=device, dtype=dtype) * scale
+    lora_b = torch.randn(args.out_dim, args.rank, device=device, dtype=dtype) / math.sqrt(args.rank)
+    initial_output = torch.randn(m, output_width, device=device, dtype=dtype)
+
+    def baseline(output: torch.Tensor) -> None:
+        buffer = x @ lora_a.t()
+        delta = buffer @ lora_b.t()
+        selected = output[:, output_slice]
+        selected[:] = selected + delta
+
+    def optimized(output: torch.Tensor) -> None:
+        buffer = x @ lora_a.t()
+        selected = output[:, output_slice]
+        torch.addmm(selected, buffer, lora_b.t(), out=selected)
+
+    reference = initial_output.clone()
+    candidate = initial_output.clone()
+    baseline(reference)
+    optimized(candidate)
+    torch.testing.assert_close(candidate, reference, rtol=1e-2, atol=1e-2)
+    max_abs = (candidate - reference).abs().max().item()
+
+    old_output = initial_output.clone()
+    new_output = initial_output.clone()
+    iterations = args.large_iterations if m >= args.large_m_threshold else args.small_iterations
+
+    old_peak = measure_extra_peak_mib(lambda: baseline(old_output))
+    new_peak = measure_extra_peak_mib(lambda: optimized(new_output))
+    old_samples = measure_latency_ms(lambda: baseline(old_output), args.warmup, iterations)
+    new_samples = measure_latency_ms(lambda: optimized(new_output), args.warmup, iterations)
+
+    old_p50 = statistics.median(old_samples)
+    new_p50 = statistics.median(new_samples)
+    print(
+        f"{m:<7} {iterations:<6} "
+        f"{percentile(old_samples, 0.1):>8.5f} {old_p50:>8.5f} {percentile(old_samples, 0.9):>8.5f} "
+        f"{percentile(new_samples, 0.1):>8.5f} {new_p50:>8.5f} {percentile(new_samples, 0.9):>8.5f} "
+        f"{old_p50 / new_p50:>8.3f} {old_peak:>8.2f} {new_peak:>8.2f} {max_abs:>8.5f}"
+    )
+
+
+def main() -> None:
+    args = parse_args()
+    if not torch.cuda.is_available():
+        raise RuntimeError("This benchmark requires a CUDA GPU")
+    if args.warmup < 0 or args.small_iterations <= 0 or args.large_iterations <= 0:
+        raise ValueError("warmup must be non-negative and iteration counts must be positive")
+
+    dtype = getattr(torch, args.dtype)
+    torch.manual_seed(0)
+    print_header(args, dtype)
+    with torch.inference_mode():
+        for m in (int(value) for value in args.m.split(",")):
+            if m <= 0:
+                raise ValueError(f"M must be positive, got {m}")
+            benchmark_shape(m, args, dtype)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/tts/README.md

@@ -2,7 +2,7 @@
 
 A model-agnostic serving benchmark for TTS models in vllm-omni. One CLI
 (`bench_tts.py`) + one YAML registry (`model_configs.yaml`) drive perf and
-quality runs for every registered checkpoint: **Qwen3-TTS** (Base / CustomVoice)
+quality runs for every registered checkpoint: **Qwen3-TTS** (Base / CustomVoice / VoiceDesign)
 and **VoxCPM2** today, more to come.
 
 The same three task types — `voice_clone`, `default_voice`, `voice_design` —
@@ -21,6 +21,11 @@ The server auto-loads its Deploy YAML from `vllm_omni/deploy/qwen3_tts.yaml`
 (Pipeline + Deploy schema introduced in #2383). No explicit `--deploy-config`
 flag is needed for a registered model.
 
+Start the checkpoint that matches the benchmark task below: `-Base` for
+`voice_clone`, `-CustomVoice` for `default_voice`, or `-VoiceDesign` for
+`voice_design`. Restart the server with the matching checkpoint when switching
+tasks; changing only the benchmark's `--model` does not reload the server.
+
 ### 2. Run the benchmark (`vllm bench serve --omni`)
 
 The primary, directly-controllable path. Copy-paste one of these and tweak
@@ -62,12 +67,12 @@ vllm bench serve --omni \
     --save-result --result-dir ./results
 ```
 
-#### voice_design (Qwen3-TTS-CustomVoice, bundled seed_tts_design)
+#### voice_design (Qwen3-TTS-VoiceDesign, bundled seed_tts_design)
 
 ```bash
 vllm bench serve --omni \
     --host 127.0.0.1 --port 8000 \
-    --model Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
+    --model Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign \
     --backend openai-audio-speech \
     --endpoint /v1/audio/speech \
     --dataset-name seed-tts-design \
@@ -180,11 +185,11 @@ Outputs TTFP / RTF / throughput curves (and a markdown table) for every
 
 ## Task types
 
-| Task            | Dataset           | Request body                                        | Checkpoints that support it              |
-|-----------------|-------------------|-----------------------------------------------------|------------------------------------------|
-| `voice_clone`   | `seed-tts`        | `ref_audio` + `ref_text` + `task_type=Base`         | `Qwen3-TTS-*-Base`, `VoxCPM2`            |
-| `default_voice` | `seed-tts-text`   | `voice=Vivian` + `task_type=CustomVoice`            | `Qwen3-TTS-*-CustomVoice`                |
-| `voice_design`  | `seed-tts-design` | `instructions=<natural-language description>` + `task_type=VoiceDesign` | `Qwen3-TTS-*-CustomVoice` |
+| Task            | Dataset           | Request body                                                            | Checkpoints that support it   |
+| --------------- | ----------------- | ----------------------------------------------------------------------- | ----------------------------- |
+| `voice_clone`   | `seed-tts`        | `ref_audio` + `ref_text` + `task_type=Base`                             | `Qwen3-TTS-*-Base`, `VoxCPM2` |
+| `default_voice` | `seed-tts-text`   | `voice=Vivian` + `task_type=CustomVoice`                                | `Qwen3-TTS-*-CustomVoice`     |
+| `voice_design`  | `seed-tts-design` | `instructions=<natural-language description>` + `task_type=VoiceDesign` | `Qwen3-TTS-*-VoiceDesign`     |
 
 **`-CustomVoice` checkpoints do NOT ship `speaker_encoder` weights**, so
 voice_clone requests raise `ValueError` at model runtime. Use `-Base` for
@@ -210,12 +215,12 @@ Then add the model's Deploy YAML under `vllm_omni/deploy/<model>.yaml`
 
 ## Datasets
 
-| Dataset            | Bundled? | Format            | Source                                                         |
-|--------------------|----------|-------------------|----------------------------------------------------------------|
-| `seed-tts-design`  | ✅       | 5-field meta.lst  | `benchmarks/build_dataset/seed_tts_design/en/meta.lst` (20 prompts) |
-| `seed_tts_smoke`   | ✅       | 4-field meta.lst  | `benchmarks/build_dataset/seed_tts_smoke/en/meta.lst` (20 text-only) |
-| `seed-tts`         | ❌       | 4-field meta.lst + WAVs | Google-Drive: [BytedanceSpeech/seed-tts-eval][seedtts] (~1.2 GB) |
-| `seed-tts-text`    | ❌       | 4-field meta.lst  | Same archive as `seed-tts` (wav column unused)                 |
+| Dataset           | Bundled? | Format                  | Source                                                               |
+| ----------------- | -------- | ----------------------- | -------------------------------------------------------------------- |
+| `seed-tts-design` | ✅       | 5-field meta.lst        | `benchmarks/build_dataset/seed_tts_design/en/meta.lst` (20 prompts)  |
+| `seed_tts_smoke`  | ✅       | 4-field meta.lst        | `benchmarks/build_dataset/seed_tts_smoke/en/meta.lst` (20 text-only) |
+| `seed-tts`        | ❌       | 4-field meta.lst + WAVs | Google-Drive: [BytedanceSpeech/seed-tts-eval][seedtts] (~1.2 GB)     |
+| `seed-tts-text`   | ❌       | 4-field meta.lst        | Same archive as `seed-tts` (wav column unused)                       |
 
 [seedtts]: https://github.com/BytedanceSpeech/seed-tts-eval
 
@@ -227,11 +232,11 @@ For manual voice_clone / default_voice runs against the full corpus, follow
 
 `tests/dfx/perf/tests/test_tts.json` wires three perf regimes plus quality:
 
-| eval_phase    | concurrency | purpose                                                 | Baseline metrics                        |
-|---------------|-------------|---------------------------------------------------------|-----------------------------------------|
-| `latency`     | 1           | Single-request TTFP / RTF SLO                           | `median_audio_ttfp_ms`, `median_audio_rtf` |
-| `throughput`  | 8           | Codec-batching cliff sentinel (PDF #272 concurrency≥8)  | `median_audio_ttfp_ms`, `median_audio_rtf` |
-| `quality`     | 4           | WER / SIM / UTMOS regression (disabled in CI by default)| `mean_audio_rtf`                        |
+| eval_phase   | concurrency | purpose                                                  | Baseline metrics                           |
+| ------------ | ----------- | -------------------------------------------------------- | ------------------------------------------ |
+| `latency`    | 1           | Single-request TTFP / RTF SLO                            | `median_audio_ttfp_ms`, `median_audio_rtf` |
+| `throughput` | 8           | Codec-batching cliff sentinel (PDF #272 concurrency≥8)   | `median_audio_ttfp_ms`, `median_audio_rtf` |
+| `quality`    | 4           | WER / SIM / UTMOS regression (disabled in CI by default) | `mean_audio_rtf`                           |
 
 Why `median_*` for latency/throughput and `mean_*` for quality: latency
 distributions have cold-start tails that drag the mean; quality aggregates
@@ -245,10 +250,15 @@ PR #2558 — quality runs are manual / release-validation, not nightly).
 
 Observed on H20-3e, Qwen3-TTS-1.7B (measured pre-merge on this branch):
 
-| Task          | Model         | c=1    | c=4    | **c=8**    | c=16   | c=32   |
-|---------------|---------------|--------|--------|------------|--------|--------|
-| voice_clone   | 1.7B-Base     | RTF 0.15 / TTFP 165ms | 0.28 / 412ms | **0.49 / 1701ms** | 0.72 / 3355ms | 0.77 / 3772ms |
-| voice_design  | 1.7B-CustomVoice | RTF 0.08 / TTFP 53ms  | 0.11 / 154ms | **0.21 / 872ms**  | 0.33 / 1801ms | 0.38 / 1989ms |
+| Task         | Model            | c=1                   | c=4          | **c=8**           | c=16          | c=32          |
+| ------------ | ---------------- | --------------------- | ------------ | ----------------- | ------------- | ------------- |
+| voice_clone  | 1.7B-Base        | RTF 0.15 / TTFP 165ms | 0.28 / 412ms | **0.49 / 1701ms** | 0.72 / 3355ms | 0.77 / 3772ms |
+| voice_design | 1.7B-CustomVoice | RTF 0.08 / TTFP 53ms  | 0.11 / 154ms | **0.21 / 872ms**  | 0.33 / 1801ms | 0.38 / 1989ms |
+
+The historical `voice_design` row above lists a CustomVoice checkpoint, which
+current task validation rejects for VoiceDesign requests. It does not establish
+a VoiceDesign checkpoint baseline; rerun with `Qwen3-TTS-12Hz-1.7B-VoiceDesign`
+before comparing performance.
 
 Both models show a **4–6× TTFP jump from c=4 to c=8** while audio throughput
 saturates around c=4–8 — the codec-bs=1 bottleneck documented in
@@ -257,7 +267,7 @@ sentinel for regressions in this area.
 
 ## File layout
 
-```
+```text
 benchmarks/tts/
 ├── README.md                  (this file)
 ├── bench_tts.py               CLI — serve-mode benchmark driver

benchmarks/tts/model_configs.yaml

@@ -22,7 +22,7 @@ models:
   # NOT supported (an attempt raises ValueError from _extract_speaker_embedding
   # at model runtime).  Use -Base for voice_clone.
   Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice:
-    supported_tasks: [default_voice, voice_design]
+    supported_tasks: [default_voice]
     backend: openai-audio-speech
     endpoint: /v1/audio/speech
     trust_remote_code: true
@@ -31,6 +31,13 @@ models:
         voice: Vivian
         language: English
         task_type: CustomVoice
+
+  Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign:
+    supported_tasks: [voice_design]
+    backend: openai-audio-speech
+    endpoint: /v1/audio/speech
+    trust_remote_code: true
+    task_extra_body:
       voice_design:
         task_type: VoiceDesign
         language: English
@@ -132,3 +139,14 @@ models:
     endpoint: /v1/audio/speech
     task_extra_body:
       voice_design: {}
+
+  # OmniVoice serves plain text-to-speech over /v1/audio/speech; voice cloning
+  # and voice design are offline-only for this model, so default_voice is the
+  # only task the server can answer.
+  k2-fsa/OmniVoice:
+    supported_tasks: [default_voice]
+    backend: openai-audio-speech
+    endpoint: /v1/audio/speech
+    trust_remote_code: true
+    task_extra_body:
+      default_voice: {}

docker/Dockerfile.npu

@@ -30,6 +30,7 @@ RUN python3 -m pip install -v --no-deps \
       "stepaudio2-minicpmo>=0.1.1" \
       "step-audio2>=1.0.0"
 
+ENV VLLM_TARGET_DEVICE=npu
 ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
 
 ENTRYPOINT []

docker/Dockerfile.npu.a3

@@ -30,6 +30,7 @@ RUN python3 -m pip install -v --no-deps \
       "stepaudio2-minicpmo>=0.1.1" \
       "step-audio2>=1.0.0"
 
+ENV VLLM_TARGET_DEVICE=npu
 ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
 
 ENTRYPOINT []

docker/Dockerfile.npu.ci

@@ -65,6 +65,7 @@ RUN python3 -m pip install -v --no-deps \
       --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
       --default-timeout=60
 
+ENV VLLM_TARGET_DEVICE=npu
 ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
 
 ENTRYPOINT []

docker/Dockerfile.npu.ci.a3

@@ -66,6 +66,7 @@ RUN python3 -m pip install -v --no-deps \
       --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
       --default-timeout=60
 
+ENV VLLM_TARGET_DEVICE=npu
 ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
 
 ENTRYPOINT []

docs/.nav.yml

@@ -5,14 +5,23 @@ nav:
     - getting_started/quickstart.md
     - getting_started/installation/*
   - Serving:
-    - OpenAI-Compatible API:
-      - Chat Completions API: serving/chat_completions_api.md
-      - Image Generation: serving/image_generation_api.md
-      - Image Edit: serving/image_edit_api.md
-      - Text to Speech: serving/speech_api.md
-      - Videos: serving/videos_api.md
-      - Audio Generation: serving/audio_generate_api.md
-      - Streaming Video Input: serving/video_stream_api.md
+    - API Server:
+      - Overview: serving/README.md
+      - Core APIs:
+        - Chat Completions: serving/chat_completions_api.md
+        - Text to Speech: serving/speech_api.md
+        - Audio Generation: serving/audio_generate_api.md
+        - Image Generation: serving/image_generation_api.md
+        - Image Edit: serving/image_edit_api.md
+        - Videos: serving/videos_api.md
+      - Streaming and Realtime:
+        - Realtime Audio: serving/realtime_api.md
+        - Full Duplex: serving/full_duplex_api.md
+        - Realtime Duplex Reference: serving/realtime_duplex_api.md
+        - Streaming Video Input: serving/video_stream_api.md
+        - Streaming Video Output: serving/streaming_video_output_api.md
+        - OpenPI Robot Policy: serving/openpi_api.md
+      - Standalone Experimental Servers: serving/standalone_servers.md
   - Examples:
     - examples/README.md
     - Offline Inference:
@@ -57,13 +66,14 @@ nav:
       - ModelOpt: user_guide/quantization/modelopt.md
       - AutoRound: user_guide/quantization/autoround.md
       - msModelSlim: user_guide/quantization/msmodelslim.md
+      - TorchAO: user_guide/quantization/torchao.md
     - Diffusion Acceleration:
       - Overview and Compatibility:
         - Overview: user_guide/diffusion_features.md
         - Feature Compatibility: user_guide/feature_compatibility.md
       - CPU Offloading:
         - Overview: user_guide/diffusion/cpu_offload.md
-        - Model-Level: user_guide/diffusion/offloader/cpu_offload.md
+        - Model-Level: user_guide/diffusion/offloader/module_offload.md
         - Layerwise: user_guide/diffusion/offloader/layerwise_offload.md
         - Distributed Layerwise: user_guide/diffusion/offloader/distributed_layerwise_offload.md
       - Cache Acceleration:
@@ -142,7 +152,7 @@ nav:
           - Skip-Softmax: design/feature/skip_softmax.md
         - CPU Offloading:
           - Overview: design/feature/offloader/README.md
-          - Model-Level: design/feature/offloader/cpu_offload.md
+          - Model-Level: design/feature/offloader/module_offload.md
           - Layerwise: design/feature/offloader/layerwise_offload.md
           - Distributed Layerwise: design/feature/offloader/distributed_layerwise_offload.md
         - Cache-DiT: design/feature/cache_dit.md

docs/README.md

@@ -23,7 +23,6 @@ Easy, fast, and cheap omni-modality model serving for everyone
 <a class="github-button" href="https://github.com/vllm-project/vllm-omni/fork" data-show-count="true" data-icon="octicon-repo-forked" data-size="large" aria-label="Fork">Fork</a>
 </p>
 
-
 ## About
 
 [vLLM](https://github.com/vllm-project/vllm) was originally designed to support large language models for text-based autoregressive generation tasks. vLLM-Omni is a framework that extends its support for omni-modality model inference and serving:
@@ -52,14 +51,14 @@ vLLM-Omni is flexible and easy to use with:
 - Tensor, pipeline, data and expert parallelism support for distributed inference
 - Streaming outputs
 - OpenAI-compatible API server
-- Full-duplex realtime serving with streaming audio input and output (experimental)
+- [Full-duplex realtime serving](serving/realtime_duplex_api.md) with streaming audio input and output
 
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
 - Omni-modality models (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
-- TTS models (e.g. Qwen3-TTS, VoxCPM2, Ming-Omni-TTS, CosyVoice3)
-- Diffusion models — image, video, and audio generation (e.g. MiniMax H3, Qwen-Image, Wan2.2, FLUX)
-- Robot-policy and action models (e.g. GR00T-N1.7, DreamZero-DROID, InternVLA-A1, Cosmos3 action policy)
+- TTS models (e.g. Qwen3-TTS, IndexTTS 2.5, dots.tts, CosyVoice3)
+- Diffusion models — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
+- Robot-policy and action models (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
 
 For more information, checkout the following:
 

docs/api/README.md

@@ -15,6 +15,36 @@ Main entry points for vLLM-Omni inference and serving.
 - [vllm_omni.entrypoints.omni_base.OmniBase][]
 - [vllm_omni.entrypoints.pd_utils.PDDisaggregationMixin][]
 
+## Clients
+
+Client libraries that talk to a running vLLM-Omni server (the full-duplex Realtime client and its per-model session presets).
+
+- [vllm_omni.clients.duplex.AudioDelta][]
+- [vllm_omni.clients.duplex.AudioFormat][]
+- [vllm_omni.clients.duplex.ConnectionResumed][]
+- [vllm_omni.clients.duplex.DuplexClient][]
+- [vllm_omni.clients.duplex.DuplexClientError][]
+- [vllm_omni.clients.duplex.DuplexConnectionError][]
+- [vllm_omni.clients.duplex.DuplexEvent][]
+- [vllm_omni.clients.duplex.DuplexProtocolError][]
+- [vllm_omni.clients.duplex.DuplexSessionClosedError][]
+- [vllm_omni.clients.duplex.ErrorEvent][]
+- [vllm_omni.clients.duplex.EventCollector][]
+- [vllm_omni.clients.duplex.ListenDecision][]
+- [vllm_omni.clients.duplex.ReconnectPolicy][]
+- [vllm_omni.clients.duplex.ResponseCreated][]
+- [vllm_omni.clients.duplex.ResponseDone][]
+- [vllm_omni.clients.duplex.ResponseHandle][]
+- [vllm_omni.clients.duplex.SessionClosed][]
+- [vllm_omni.clients.duplex.SessionConfig][]
+- [vllm_omni.clients.duplex.SessionCreated][]
+- [vllm_omni.clients.duplex.SessionExpired][]
+- [vllm_omni.clients.duplex.SessionResumed][]
+- [vllm_omni.clients.duplex.SpeakDecision][]
+- [vllm_omni.clients.duplex.TextDelta][]
+- [vllm_omni.clients.duplex.TranscriptDelta][]
+- [vllm_omni.clients.duplex.WebSocketTransport][]
+
 ## Inputs
 
 Input data structures for multi-modal inputs.

docs/cli/bench/serve.md

@@ -1,11 +1,15 @@
 # vLLM-Omni Benchmark CLI Guide
+
 The vllm bench command launches the vLLM-Omni benchmark to evaluate the performance of multimodal models.
 
 ## Notes
+
 vLLM-Omni registers the `openai-chat-omni`, `openai-audio-speech`, `openai-image-edits-omni`, `daily-omni`, and `openai-realtime-duplex` serving benchmark backends. It also adds the `omniinteract` dataset.
 
 ## Basic Parameter Description
+
 You can use `vllm bench serve --omni --help=all` to get descriptions of all parameters. The commonly used parameters are described below:
+
 - `--omni`
   Enable Omni (multimodal) mode, supporting multimodal inputs and outputs such as images, videos, and audio.
 
@@ -77,8 +81,8 @@ Specify to save benchmark results to a json file
 - `--random-prefix-len`
   Number of fixed prefix tokens before the random context in a request.
   The total input length is the sum of random-prefix-len and a random
-  context length sampled from [input_len * (1 - range_ratio),
-  input_len * (1 + range_ratio)].Only the random and random-mm modes
+  context length sampled from [input_len *(1 - range_ratio),
+  input_len* (1 + range_ratio)].Only the random and random-mm modes
   support this parameter.
 
 - `--random-input-len`
@@ -91,7 +95,7 @@ Specify to save benchmark results to a json file
   Range ratio for sampling input/output length,
   used only for random sampling. Must be in the range [0, 1) to define
   a symmetric sampling range
-  [length * (1 - range_ratio), length * (1 + range_ratio)].
+  [length *(1 - range_ratio), length* (1 + range_ratio)].
   Only the random and random-mm modes support this parameter.
 
 - `--random-mm-base-items-per-request`
@@ -137,6 +141,7 @@ Specify to save benchmark results to a json file
 ## Usage Examples
 
 ### Online Benchmark
+
 <details class="admonition abstract" markdown="1">
 <summary>Show more</summary>
 
@@ -162,7 +167,9 @@ vllm bench serve \
   --dataset-path ShareGPT_V3_unfiltered_cleaned_split.json \
   --percentile-metrics ttft,tpot,itl,e2el
 ```
+
 If successful, you will see the following output:
+
 ```text
 ============ Serving Benchmark Result ============
 Successful requests:                     2
@@ -264,6 +271,7 @@ Median AUDIO_RTF:                        0.47
 P99 AUDIO_RTF:                           0.48
 ==================================================
 ```
+
 Notes:
 We use audio generation time / audio duration to calculate RTF.
 
@@ -301,12 +309,36 @@ options apply. Use `--omniinteract-require-response` only for functional E2E cas
 Each completed case writes `output.wav`, `wav_transcript.json`, `events.json`, `result.json`, and a final `.done` marker under
 `--omniinteract-output-dir`. The root also contains `batch_summary.json` and `official_eval_manifest.jsonl`; failed cases write
 `.failed.json`. Runs sharing one output root are serialized. Completion validates transport, response lifecycle, and artifacts,
-not answer accuracy. Transcript timestamps are serialized playback-queue times. Clipped or cancelled outputs are ineligible and
-omitted from the official manifest; `audio_clipped_bytes` records output beyond the rounded video horizon.
+not answer accuracy. Transcript timestamps are serialized playback-queue times. Playback ACKs report cumulative progress
+incrementally along that serialized clock, like a live listener; the first ack for a response goes out as soon as its audio
+arrives, checkpointing the response's history position so a later committed user input updates it in place. A residual
+`playback_ack_too_late` rejection is recorded as an artifact warning rather than failing the case. Clipped or cancelled
+outputs are ineligible and omitted from the official manifest; `audio_clipped_bytes` records output beyond the rounded video
+horizon.
 
 TTFT, TTFP, and RTF start at client receipt of `response.created`. TPOT/ITL use engine stage-0 timing; ITL is emitted only when
 every token interval is present.
 
+The checked-in local performance configuration measures four deterministic cases from each OmniInteract subset (12 videos
+total), with no benchmark warmups and a maximum concurrency of two. Each subset also sends one readiness request before its
+measured cases. Run it from the repository root with:
+
+```bash
+export HF_HOME=/path/to/persistent/huggingface-cache
+export BENCHMARK_DIR=tests/dfx/perf/results
+bash tools/nightly/run_nightly_jobs.sh \
+  --test-type local \
+  --model-type omni \
+  --label-substr minicpmo_4_5_omniinteract
+```
+
+The first run downloads the pinned OmniInteract archive into `HF_HOME`; later runs reuse that cache.
+
+It requires every case to commit its input, complete any emitted response lifecycles, and publish the expected WAV,
+transcript, event, and result artifacts without errors. A valid LISTEN-only case may have no response audio or transcript
+chunks. Official-manifest eligibility is reported separately because clipped or cancelled output is a benchmark-quality
+signal, not a transport failure. This local performance test does not score answer accuracy.
+
 ### Multi-Modal Benchmark
 
 <details class="admonition abstract" markdown="1">
@@ -319,8 +351,8 @@ Generate synthetic image、video、audio inputs alongside random text prompts to
 Notes:
 
 - Works only with online benchmark via:
-  - the OpenAI chat backend (`--backend openai-chat-omni`) and endpoint `/v1/chat/completions`.
-  - the OpenAI edit image backend (`--backend openai-image-edits-omni`) and endpoint `/v1/images/edits`.
+    - the OpenAI chat backend (`--backend openai-chat-omni`) and endpoint `/v1/chat/completions`.
+    - the OpenAI edit image backend (`--backend openai-image-edits-omni`) and endpoint `/v1/images/edits`.
 
 Start the server (example):
 
@@ -331,6 +363,7 @@ vllm serve Qwen/Qwen2.5-Omni-7B --omni
 It is recommended to use the flag `--ignore-eos` to simulate real responses. You can set the size of the output via the arg `random-output-len`.
 
 Then run the benchmarking script:
+
 ```bash
 vllm bench serve \
     --omni \
@@ -400,6 +433,7 @@ How sampling works:
 - If a modality (e.g., image) reaches its limit from `--random-mm-limit-mm-per-prompt`, all buckets of that modality are excluded and the remaining bucket probabilities are renormalized before continuing.
 This should be seen as an edge case, and if this behavior can be avoided by setting `--random-mm-limit-mm-per-prompt` to a large number. Note that this might result in errors due to engine config `--limit-mm-per-prompt`.
 - The resulting request contains synthetic image data in `multi_modal_data` (OpenAI Chat format). When `random-mm` is used with the OpenAI Chat backend, prompts remain text and MM content is attached via `multi_modal_data`.
+
 </details>
 
 ### Multi-Stage Benchmark
@@ -414,6 +448,7 @@ vllm serve --omni --port 29999 --model ~/Qwen3-Omni-30B-A3B-Instruct
 ```
 
 Then run the benchmarking script (remember to add the flag `--print-stage`):
+
 ```bash
 vllm bench serve --omni \
   --dataset-name random \
@@ -433,6 +468,7 @@ vllm bench serve --omni \
 ```
 
 Besides the "Serving Benchmark Result", there will be "Stage Benchmark Result" below:
+
 ```text
 ============= Stage Benchmark Result =============
 =============== Stage 0 (thinker) ================
@@ -489,7 +525,9 @@ Mean AUDIO_RTF:                          0.24
 Median AUDIO_RTF:                        0.26
 P99 AUDIO_RTF:                           0.27
 ```
+
 Explanation:
+
 - stage_gen_time: Time from submitting a request to a specific stage to that stage finishing generation.
 
 - Serving TTFC (Time to First Chunk): Time from the HTTP request being accepted by the serving frontend to

docs/configuration/README.md

@@ -2,7 +2,7 @@
 
 This section lists the most common options for running vLLM-Omni.
 
-For options within a vLLM Engine, please refer to the [vLLM 0.26 configuration guide](https://docs.vllm.ai/en/v0.26.0/configuration/index.html).
+For options within a vLLM Engine, please refer to the [vLLM 0.28 configuration guide](https://docs.vllm.ai/en/v0.28.0/configuration/index.html).
 
 Each model defines fixed topology in a registered `PipelineConfig` and runtime
 overrides in a deploy YAML.

docs/configuration/environment_variables.md

@@ -151,7 +151,7 @@ their keys only.
 ## Inherited vLLM variables
 
 vLLM-Omni also reads variables through its aligned vLLM dependency. Refer to
-the [vLLM 0.26 environment-variable reference](https://docs.vllm.ai/en/v0.26.0/configuration/env_vars.html)
+the [vLLM 0.28 environment-variable reference](https://docs.vllm.ai/en/v0.28.0/configuration/env_vars.html)
 for their definitions. This includes vLLM launch, cache, logging, plugin, ROCm,
 XPU, ModelScope, and FlashInfer workspace settings.
 
@@ -179,7 +179,7 @@ collection, examples, bug reports, or logs.
 
 ## Model-specific variables
 
-The audit found 56 variables read by a single model or pipeline family. They are
+The audit found 58 variables read by a single model or pipeline family. They are
 not listed as public usage options here because doing so would turn implementation
 escape hatches into an accidental compatibility contract.
 
@@ -188,7 +188,7 @@ Every audited model-specific name has a migration disposition in the
 
 | Disposition | Count | Required outcome |
 | --- | ---: | --- |
-| Promote | 33 | Move a stable setting into typed stage or model configuration. |
+| Promote | 35 | Move a stable setting into typed stage or model configuration. |
 | Request scope | 6 | Move request-varying behavior into a declared request-option schema. |
 | External | 0 | Retain only when a supported third-party library owns the contract. |
 | Internalize | 11 | Keep a debug or diagnostic switch out of public documentation and configuration. |

docs/contributing/ci/ci_settings.md

@@ -57,13 +57,11 @@ Canonical layout (prefer these paths for new changes):
 | Bootstrap vs test YAML | **Bootstrap** (`pipeline*.yml`) builds images and uploads **child** test pipelines. **Test** YAML (`test-*.yml`) lists pytest steps only. |
 | Register CI YAML in `skip_ci.py` | If you add a new whitelisted test pipeline file, update `L2_YAML_FILES`, `L3_YAML_FILES`, or `L45_YAML_FILES` in `.buildkite/common/scripts/skip_ci.py` so skip-ci paths stay correct. |
 
-There are still **legacy copies** at `.buildkite/*.yaml` (without the `cuda/` prefix). Treat `.buildkite/cuda/*` as source of truth.
-
 ## Platform comparison
 
 | Platform | Bootstrap entry | Test job files | Upload mechanism | Job hardware in YAML |
 | -------- | ---------------- | -------------- | ---------------- | -------------------- |
-| **CUDA** | `cuda/pipeline.yml` | `test-ready.yml`, `test-merge.yml`, `test-nightly.yml`, `test-weekly.yml` | `upload_pipeline.py --upload` (expands uploader-only keys) | `mirror_hardwares: <preset>` (string) |
+| **CUDA** | `cuda/pipeline.yml` | `test-ready.yml`, `test-merge.yml`, `test-nightly.yml`, `test-weekly.yml` | `upload_pipeline.py --upload` (expands uploader-only keys) | omit `mirror_hardwares` (from `-m`) / preset string + `MIRROR_HW` |
 | **NPU** | `npu/pipeline-npu.yml` | `test-npu-ready.yml`, `test-npu-nightly.yml` | `upload_pipeline.py --upload` | `mirror_hardwares: a2b3_npu_1` / `a2b3_npu_4` / `a3_npu_2` |
 | **AMD** | `amd/scripts/bootstrap-amd-omni.sh` | `test-amd-ready.yml`, `test-amd-merge.yml` | Jinja (`test-template-amd-omni.j2`) → `pipeline upload` | `agent_pool` + `mirror_hardwares: [amdproduction]` (array, template filter) |
 | **Intel** | `intel/scripts/bootstrap-intel-omni.sh` | `intel/pipeline-intel.yml` (steps inline) | Direct `pipeline upload` | Inline `agents.queue` on each step |
@@ -92,7 +90,23 @@ There are still **legacy copies** at `.buildkite/*.yaml` (without the `cuda/` pr
 
     **Upload:** `upload_pipeline.py --upload` expands uploader-only keys before Buildkite upload.
 
-    **Hardware in YAML:** `mirror_hardwares: <preset>` (string)—preset names in [`common/ci_mirror_hardwares.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/ci_mirror_hardwares.yml). Do **not** set `agents` / `plugins` on the same step.
+    **Hardware in YAML:** omit `mirror_hardwares` to compose `{chip}_{n}` from pytest `-m` (SKU + `cards_n`). Several positive `cards_*` take the **max** n (`cards_2 and cards_3` → `{chip}_3`). `not cards_1` with no positive `cards_*` uses that chip's **highest existing** preset in `ci_mirror_hardwares.yml` (L4 → `l4_4`, H100 → `h100_4`, B200 → `b200_4`). Set a **preset string** to force a pool (marks are ignored). Do **not** set `agents` / `plugins` on the same step as `mirror_hardwares`. `MIRROR_HW=b200` remaps inferred jobs whose `-m` includes `B200`, and omits CUDA `h100_*` / `l4_*` strings. Jobs whose `-m` does not match (unset: no H100/L4; set: no B200) are **skipped**.
+
+    ```yaml
+    # inferred: H100+B200 + cards_2 → h100_2, or b200_2 when MIRROR_HW=b200
+    commands:
+      - pytest -sv tests/e2e -m "full_model and H100 and B200 and omni and cards_2"
+    # no mirror_hardwares
+
+    # inferred: not cards_1 (no positive cards_*) → that chip's highest pool (l4_4 / h100_4 / b200_4)
+    commands:
+      - pytest -sv tests/e2e -m "full_model and diffusion and L4 and not cards_1"
+
+    # forced preset (ignores -m SKU/cards for pool selection)
+    mirror_hardwares: h100_1
+    ```
+
+    Inferred form: unset/`empty` `MIRROR_HW` matches `H100` or `L4` in `-m` (both present → H100); no match skips the step. `MIRROR_HW=b200` must appear in `-m` or the step is skipped. The uploader does **not** rewrite pytest `-m`; B200 collection comes from the YAML expression and from tests that declare `B200` in `hardware_test` / `hardware_marks` (for example `res={"cuda": ["H100", "B200"]}`). `MIRROR_HW` must be empty or `b200` (case-insensitive); unknown values (for example `b20o`) **fail the upload**. A CUDA preset string such as `mirror_hardwares: h100_4` is skipped when `MIRROR_HW=b200`. Unset `MIRROR_HW` keeps string presets. NPU presets ignore `MIRROR_HW`. A `mirror_hardwares` name that is not a key in `ci_mirror_hardwares.yml` **fails the upload**.
 
     **Conventions**
 
@@ -103,9 +117,9 @@ There are still **legacy copies** at `.buildkite/*.yaml` (without the `cuda/` pr
 
     **Adding a job**
 
-    1. Pick the level file (ready / merge / nightly / weekly).
+    1. Pick the level file (`test-ready.yml` / `test-merge.yml` / `test-nightly.yml` / `test-weekly.yml`).
     2. Add a step under the right **group** (usually **E2E Test** for model pytest).
-    3. Set `label`, `commands`, `mirror_hardwares`, `depends_on: upload-<level>-pipeline`.
+    3. Set `label`, `commands`, `mirror_hardwares`, `depends_on: upload-ready-pipeline` (or `upload-merge-pipeline` / `upload-nightly-pipeline` / `upload-weekly-pipeline`).
     4. For L2/L3 E2E, add `source_file_dependencies` (pytest + model + deploy YAML prefixes).
     5. Dry-run:
 
@@ -414,10 +428,12 @@ settings in `pyproject.toml`: online tests launch the server with
 `subprocess.Popen` and stop it with SIGTERM, and without those settings the XML
 reflects only the pytest parent process, not the server's code paths.
 
-The upload depends on `buildkite-agent` being callable inside the container. The
-`kubernetes` presets (`h100_*`, `*_npu_*`) provide it; the `docker` ones (`l4_*`)
-only do because they set `mount-buildkite-agent: true`. A new docker preset that
-runs a coverage job needs the same.
+The upload depends on `buildkite-agent` being callable inside the container.
+CUDA (`l4_*`, `h100_*`) and NPU (`*_npu_*`) presets all use the `kubernetes`
+plugin, which provides the agent. `l4_*` jobs run on the EKS `l4-k8s` queue
+(agent-stack-k8s); they no longer use the docker plugin or
+`mount-buildkite-agent`. A new **docker** preset that runs a coverage job
+still needs `mount-buildkite-agent: true`.
 
 List both `run_cov_split.sh` and `pyproject.toml` in every opted-in job's
 `source_file_dependencies` — both change what the job measures, so without them a

docs/contributing/ci/test_system_overview.md

@@ -136,7 +136,7 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
         <a href="test_writing_guide.md#l4-level-testing-full-functionality-performance-and-documentation-testing">L4</a><br>
         L4: Purpose, Test Content, Directory Location, Example
       </td>
-      <td>Nightly</td>
+      <td>Nightly / Days before Release<br>(additional GPU SKUs)</td>
       <td>GPU</td>
     </tr>
     <tr>
@@ -175,6 +175,15 @@ Before entering specific testing levels, the project establishes two common spec
 
 ## Notes
 
+### L4 cadence (nightly and pre-release)
+
+CUDA **L4** (`full_model`) jobs are defined once in [`.buildkite/cuda/test-nightly.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-nightly.yml). That file is reused at two cadences:
+
+- **Nightly:** scheduled `main` builds with `NIGHTLY=1` (and PR labels such as `nightly-test` / `omni-test`). Default fleet is typically H100.
+- **Pre-release:** the same L4 suite runs again on extra GPU SKUs before a release (for example B200 via `MIRROR_HW=b200`), so `full_model` coverage is not limited to the nightly machine type.
+
+How count-form `mirror_hardwares` and `MIRROR_HW` pick a SKU: [CI Settings](./ci_settings.md).
+
 ### L2 / L3 diff-aware CI (CUDA)
 
 On CUDA **L2** and **L3**, **E2E Test** Buildkite jobs may be omitted at pipeline upload when the PR diff does not touch their path prefixes; other groups still always upload. Full two-layer mechanics and YAML examples: [CI Settings — Diff-aware CI](./ci_settings.md#diff-aware-ci) ([step filtering](./ci_settings.md#step-filtering)).

docs/contributing/model/adding_omni_model.md

@@ -29,7 +29,7 @@ Each stage is implemented as a separate model class that can be configured indep
 
 When adding a new model, you'll need to create the following structure:
 
-```
+```text
 vllm_omni/model_executor/models/
 └── your_model_name/              # Model directory (e.g., qwen3_omni)
     ├── __init__.py               # Exports main model class
@@ -422,22 +422,25 @@ The stage transition process follows these steps:
 Understanding the data structures is crucial for implementing stage transitions:
 
 **Input to your function:**
+
 - `stage_list[source_stage_id].engine_outputs`: List of `EngineCoreOutput` objects
--   -  Each contains `outputs`: List of `RequestOutput` objects
-    - Each `RequestOutput` has:
--   -  - `token_ids`: Generated token IDs
-       - `multimodal_output`: Dict with keys like `"code_predictor_codes"`, etc.These are the hidden states or intermediate outputs from the model's forward pass
-       - `prompt_token_ids`: Original prompt token IDs
+    - Each contains `outputs`: List of `RequestOutput` objects
+        - Each `RequestOutput` has:
+            - `token_ids`: Generated token IDs
+            - `multimodal_output`: Dict with keys like `"code_predictor_codes"`, etc. These are the hidden states or intermediate outputs from the model's forward pass
+            - `prompt_token_ids`: Original prompt token IDs
 
 **Output from your function:**
+
 - Must return `list[OmniTokensPrompt]` where each `OmniTokensPrompt` contains:
--   - `prompt_token_ids`: List[int] - Token IDs for the next stage
+    - `prompt_token_ids`: List[int] - Token IDs for the next stage
     - `additional_information`: Dict[str, Any] - Optional metadata (e.g., embeddings, hidden states)
     - `multi_modal_data`: Optional multimodal data if needed
 
 ### How Model Outputs Are Stored
 
 The model's `forward()` method returns an `OmniOutput` object that contains:
+
 - `text_hidden_states`: Final hidden states for text generation
 - `multimodal_outputs`: Dict containing intermediate outputs
 
@@ -464,6 +467,7 @@ def forward(self, ...):
 ```
 
 These keys are then accessible in your stage transition function:
+
 ```python
 # In stage_input_processors/qwen3_omni.py
 thinker_prefill_embeddings = output.multimodal_output["0"]  # Access by key
@@ -493,7 +497,7 @@ thinker_hidden_states = output.multimodal_output["24"]
 Create stage transition processors in `vllm_omni/model_executor/stage_input_processors/your_model_name.py`. Each inter-stage edge should provide a **coherent processor set** rather than a single monolithic function:
 
 | Suffix | Role | Runs when |
-|--------|------|-----------|
+| -------- | ------ | ----------- |
 | `*_full_payload` | Worker-side payload producer | `async_chunk=false`; accumulates tensors and ships via connector |
 | `*_async_chunk` | Scheduler-side streaming producer | `async_chunk=true`; emits per-chunk payloads |
 | `*_token_only` | Orchestrator placeholder builder | `async_chunk=false`; allocates downstream prompt slots only |
@@ -554,7 +558,16 @@ For comprehensive testing guidelines, please refer to the [Test Writing Guide](.
 
 ## Adding a Model Recipe
 
-After implementing and testing your model, please add a model recipe to the [vllm-project/recipes](https://github.com/vllm-project/recipes) repository. This helps other users understand how to use your model with vLLM-Omni.
+After implementing and testing your model, add a recipe under this repository's
+`recipes/` directory. Follow the
+[`add-recipe` skill](https://github.com/vllm-project/vllm-omni/blob/main/.claude/skills/add-recipe/SKILL.md)
+and [`recipes/TEMPLATE.md`](https://github.com/vllm-project/vllm-omni/blob/main/recipes/TEMPLATE.md),
+then update the model row in
+[`recipes/README.md`](https://github.com/vllm-project/vllm-omni/blob/main/recipes/README.md).
+The external
+[vllm-project/recipes](https://github.com/vllm-project/recipes) repository can
+be used as a structural reference, but it is not the destination for a
+vLLM-Omni model recipe.
 
 ### What to Include
 
@@ -568,15 +581,10 @@ Your recipe should include:
 3. **Usage Examples**: Command-line examples demonstrating how to run the model
 4. **Configuration Details**: Important configuration parameters and their meanings
 
-### Example
-
-For reference, see the [LongCat recipe example](https://github.com/vllm-project/recipes/pull/179) which demonstrates the expected format and structure.
-
 ### Recipe Location
 
-Create your recipe file in the appropriate directory structure:
-- For organization-specific models: `OrganizationName/ModelName.md`
-- For general models: `ModelName.md`
+Create `recipes/<vendor>/<model-family>.md` and keep commands, feature status,
+hardware evidence, and limitations specific to the model family.
 
 The recipe should be a Markdown file that provides clear, reproducible instructions for users to get started with your model.
 
@@ -590,7 +598,7 @@ Adding a new model to vLLM-Omni involves:
 4. **Define pipeline topology and deploy defaults** in `pipeline.py` and `vllm_omni/deploy/`
 5. **Implement stage input processors** for stage transitions
 6. **Write tests** to verify functionality
-7. **Add model recipe** to the [vllm-project/recipes](https://github.com/vllm-project/recipes) repository (see [Adding a Model Recipe](#adding-a-model-recipe) section)
+7. **Add an in-repository model recipe** under `recipes/` and update its index (see [Adding a Model Recipe](#adding-a-model-recipe))
 
 ### Qwen3-Omni Reference Files
 
@@ -607,6 +615,7 @@ For a complete reference implementation, see:
 - **Testing**: `vllm_omni/tests/e2e/offline_inference/test_qwen3_omni.py`
 
 For more information, see:
+
 - [Architecture Overview](../../design/architecture_overview.md)
 - [Supported Models](../../models/supported_models.md)
 - [Pipeline and Deploy Configuration Guide](../../configuration/stage_configs.md)

docs/contributing/model/adding_tts_model.md

@@ -991,10 +991,13 @@ git push origin your-branch --force-with-lease
 
 ## Adding a Model Recipe
 
-After implementing and testing your model, add a model recipe to the
-[vllm-project/recipes](https://github.com/vllm-project/recipes) repository so users can
-get started quickly. See [Adding an Omni-Modality Model](./adding_omni_model.md#adding-a-model-recipe)
-for the expected format.
+After implementing and testing your model, add a recipe under this repository's
+`recipes/` directory. Follow the
+[`add-recipe` skill](https://github.com/vllm-project/vllm-omni/blob/main/.claude/skills/add-recipe/SKILL.md)
+and [`recipes/TEMPLATE.md`](https://github.com/vllm-project/vllm-omni/blob/main/recipes/TEMPLATE.md),
+update the model row in
+[`recipes/README.md`](https://github.com/vllm-project/vllm-omni/blob/main/recipes/README.md),
+and synchronize the relevant TTS example and speech API documentation.
 
 ## Summary
 
@@ -1011,7 +1014,7 @@ Adding a TTS model to vLLM-Omni involves:
 8. **Online serving** - add all 5 integration points to `serving_speech.py` in one commit
 9. **Tests** - cover single request, batching, and streaming
 10. **Pre-commit + DCO** - run `pre-commit` before pushing; sign every commit with `git commit -s`
-11. **Model recipe** - add to [vllm-project/recipes](https://github.com/vllm-project/recipes)
+11. **Model recipe** - add under the in-repository `recipes/` directory and update its index
 12. **Invariants** - re-check I1–I5 (streaming contract, consumer hygiene, hot-loop discipline, validation pyramid, per-request state) at the end of every phase
 
 ### Qwen3-TTS Reference Files

docs/design/architecture_overview.md

@@ -418,7 +418,7 @@ The public interfaces map onto the same engine and stage boundaries:
 flowchart LR
     offline["Offline Python<br/>Omni.generate()"] --> engine["AsyncOmniEngine"]
     online["OpenAI-compatible serving<br/>vllm serve ... --omni"] --> engine
-    duplex["Experimental duplex WebSocket<br/>/v1/duplex or realtime duplex"] --> engine
+    duplex["Duplex WebSocket<br/>/v1/realtime?duplex=1 or /v1/duplex"] --> engine
     engine --> stages["Configured AR and diffusion stages"]
     stages --> result["Streaming or final multimodal output"]
 ```

docs/design/feature/diffusion_continuous_batching.md

@@ -150,8 +150,17 @@ that do not belong in the shared contract should be stored in `state.extra`.
 Queueing and lifecycle metadata remain in the scheduler's request state.
 
 Current native pipelines that explicitly enable step execution include
-Qwen-Image, HunyuanImage3, and Helios. Step execution alone does not imply
-continuous-batching support: Qwen-Image accepts batched step states.
+Qwen-Image, BAGEL, HunyuanImage3, and Helios. Step execution alone does not
+imply continuous-batching support: Qwen-Image accepts batched step states.
+BAGEL accepts batched step states for image generation with its default,
+think-mode, and single-stage deploy configs. Only the diffusion stage uses the
+step protocol in two-stage deployments; explicit single-stage text output
+retains complete-request execution. BAGEL requires at least two inference steps
+because it schedules `num_inference_steps - 1` denoising updates, and its step
+path does not currently support sequence parallelism or a diffusion cache
+backend. When a finished stepwise request is followed by an explicit text
+fallback wave, the runner retires its step state and paged diffusion-KV rows
+before the full forward dispatch.
 HunyuanImage3 accepts batched step states only when its resolved self-attention
 backend is `TORCH_SDPA`; otherwise it rejects groups larger than one request.
 Configure `DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA` or

docs/design/feature/offloader/README.md

@@ -5,23 +5,27 @@ The three strategies have separate user and design pages:
 
 | Strategy | User guide | Design contract |
 | --- | --- | --- |
-| Model-level | [Guide](../../../user_guide/diffusion/offloader/cpu_offload.md) | [Design](cpu_offload.md) |
+| Model-level | [Guide](../../../user_guide/diffusion/offloader/module_offload.md) | [Design](module_offload.md) |
 | Layerwise | [Guide](../../../user_guide/diffusion/offloader/layerwise_offload.md) | [Design](layerwise_offload.md) |
 | Distributed layerwise | [Guide](../../../user_guide/diffusion/offloader/distributed_layerwise_offload.md) | [Design](distributed_layerwise_offload.md) |
 
 ## Strategy selection
 
 `OffloadConfig.from_od_config()` converts diffusion configuration into one
-`OffloadStrategy`. If callers set multiple flags, selection is deterministic:
+`OffloadStrategy`. The public config separates component selection from
+policy:
 
-1. `DISTRIBUTED_LAYERWISE`;
-2. `LAYERWISE`;
-3. `MODEL_LEVEL`; and
-4. `NONE`.
+- `mode="module"` selects model-level offload;
+- `mode="layer"` with rank-local transfers selects ordinary layerwise
+  offload; and
+- AllGather transfer or resident layers selects the distributed layerwise
+  backend that implements those capabilities.
 
-The factory logs when a higher-priority strategy disables another. It derives
-parallel and HSDP state from `DiffusionParallelConfig`; callers do not provide
-a separate offload group size.
+The compatibility boolean flags retain their historical priority (distributed
+layerwise, layerwise, then model-level). A compact config rejects a conflicting
+legacy strategy or non-default legacy DLO tuning. The factory derives parallel
+and HSDP state from `DiffusionParallelConfig`; callers do not provide a
+separate offload group size.
 
 `get_offload_backend()` then validates platform offload support, resolves the
 device, and creates exactly one backend. Returning `None` means offloading is
@@ -39,6 +43,11 @@ Every backend implements `OffloadBackend`:
 `disable()` does not promise to restore the pipeline's original device
 placement. The caller owns any subsequent rematerialization.
 
+An enable failure must remove every partially installed hook. Rank-local
+backends restore ordinary tensors so the pipeline can be retried. A failed
+multi-rank AllGather startup must not enter unmatched recovery collectives;
+that worker is safely discarded after local resource cleanup.
+
 Hooks are registered through `HookRegistry` and `ModelHook`; offload backends
 must use distinct hook names and remove only hooks they own.
 
@@ -55,13 +64,13 @@ Dotted paths are supported. Legacy pipelines may use the fallback scan of
 well-known attribute names, but new integrations should declare components
 explicitly.
 
-Ordinary layerwise offload obtains topology from declared
-`_layerwise_offload_blocks_attrs` (with a deprecated singular-name
-compatibility path). Distributed layerwise offload may instead use a pipeline
-`OffloadPlan`, then falls back to declared block attributes and its documented
-compatibility discovery. Discovery metadata describes structure only; the
-backend remains responsible for transfer, synchronization, and storage
-ownership.
+Both layerwise backends first consume pipeline `OffloadPlan` metadata. A
+plan's `block_attrs` maps each DiT path to its ordered block containers, while
+`encoder_block_attrs` declares streamable encoder stacks. DiTs absent from the
+plan fall back to `_layerwise_offload_blocks_attrs` (including the deprecated
+singular-name compatibility path). Discovery metadata describes structure
+only; the backend remains responsible for transfer, synchronization, and
+storage ownership.
 
 ## Cross-strategy invariants
 

docs/design/feature/offloader/distributed_layerwise_offload.md

@@ -1,10 +1,11 @@
 # Distributed Layerwise Offload
 
 This document describes distributed layerwise offload (DLO) for diffusion
-models. DLO keeps only a small number of DiT blocks on the accelerator and
-streams the remaining blocks from host memory. The distributed backend can
-either shard those host-side weights across an existing parallel group or keep
-complete rank-local block sources and avoid an additional collective.
+models. DLO keeps only a small number of selected DiT or text-encoder blocks
+on the accelerator and streams the remaining blocks from host memory. The
+distributed backend can either shard those host-side weights across an
+existing parallel group or keep complete rank-local block sources and avoid
+an additional collective.
 
 For user-facing commands, see the
 [distributed layerwise offloading guide](../../../user_guide/diffusion/offloader/distributed_layerwise_offload.md)
@@ -38,10 +39,11 @@ detailed contracts and validation boundaries.
 
 ## Status
 
-DLO is implemented for multi-device diffusion execution. The default
-AllGather path is the primary path for DP and SP deployments. The
-`--dlo-no-use-allgather` path streams complete blocks independently and adds no
-DLO weight collective.
+DLO is implemented for multi-device diffusion execution. The default public
+transfer is `rank-local`; setting a selected component's
+`layer_options.NAME.weight_transfer` to `allgather` enables weight sharding across a
+compatible DP or SP group. Transfers are configured independently for `dit`
+and `text_encoder`.
 
 Host storage is selected separately from the transfer protocol. The loader can
 produce a direct-checkpoint mmap plan for a proven-compatible runtime layout;
@@ -109,7 +111,7 @@ checkpoint adapter beside the ordinary loader.
 
 ### AllGather path
 
-With the default `dlo_use_allgather=True`, each rank stores approximately
+For a component configured with `weight_transfer="allgather"`, each rank stores approximately
 `1 / group_size` of each streamable block in pinned host memory. The next
 block's shard is copied to a device buffer and reconstructed with
 `all_gather_into_tensor` on a communication stream while the current block is
@@ -132,9 +134,8 @@ source used to prepare each rank's persistent shard. They can be closed after
 shard preparation. Across the AllGather group, those private shards total
 approximately one runtime model copy.
 
-An effective DLO group size of one performs no collective, even when
-`dlo_use_allgather=True`; it follows the rank-local transfer path described
-below.
+An effective DLO group size of one performs no collective; it follows the
+rank-local transfer path described below.
 
 When DP is greater than one, the engine can process one request per DP rank in
 the same denoising wave. Because AllGather is a collective, all participating
@@ -142,9 +143,10 @@ requests must take the same execution path at every denoising step.
 
 ### Rank-local path without DLO AllGather
 
-With `--dlo-no-use-allgather`, DLO forces its internal offload shard size to
-one and streams complete blocks using H2D copies only. The host backing may be
-either a loader-approved checkpoint mapping or ordinary runtime tensors.
+With `weight_transfer="rank-local"`, DLO forces the affected component's internal
+offload shard size to one and streams complete blocks using H2D copies only.
+The host backing may be either a loader-approved checkpoint mapping or
+ordinary runtime tensors.
 
 For direct mmap, each process retains immutable safetensors views and uses two
 bounded pinned host staging slots. Processes on the same node that map the same
@@ -204,7 +206,7 @@ adopt it.
 > optimizations without changing AllGather behavior.
 
 Final-layout Host Weight Runtime (HWR) backing is opt-in and currently applies
-only to no-AllGather DLO. Enable it with
+only to a selected rank-local DiT. Enable it with
 `--host-weight-runtime-mode preferred` (or `required`) and
 `--host-weight-runtime-root <node-local-root>`.
 
@@ -422,8 +424,8 @@ failure.
   collective. When the model declares the final-layout contract, HWR keys the
   reusable artifact by TP rank/size and SP layout.
 - **HSDP + SP:** the general parallel configuration permits HSDP over SP, but
-  DLO must use `--dlo-no-use-allgather`. HSDP remains responsible for weight
-  materialization and synchronization.
+  each selected component must use `weight_transfer="rank-local"`. HSDP remains
+  responsible for weight materialization and synchronization.
 - **HSDP + DP or TP:** rejected independently by the diffusion parallel
   configuration.
 
@@ -447,8 +449,9 @@ recorded layouts (they all keep plain 1-byte dtypes over ordinary strided
 views: online INT8 has an int8 weight plus fp32 scale, online MXFP8 has an
 fp8 weight plus e8m0 block scale, each with a contiguous or transposed-view
 finalized tensor, all covered by the same physical-order packing as online
-FP8). Other online methods must use `--dlo-no-use-allgather` or disable
-online quantization until their runtime layouts are validated.
+FP8). Other online methods must set
+`layer_options.dit.weight_transfer="rank-local"` or disable online quantization
+until their runtime layouts are validated.
 
 The Host Weight Runtime representation, publication, and no-AllGather consumer
 contracts are merged; see

docs/design/feature/offloader/layerwise_offload.md

@@ -36,12 +36,14 @@ dependencies, not incidental global synchronization, define correctness.
 ## Topology contract
 
 A DiT declares ordered block-container attributes using
-`_layerwise_offload_blocks_attrs` or an `OffloadPlan`. Each resolved item must
-be an executable `nn.Module`; ordering must match forward execution. Multiple
-containers are concatenated in execution order.
+`_layerwise_offload_blocks_attrs` or an `OffloadPlan`. A text encoder declares
+its block-list paths through `OffloadPlan.encoder_block_attrs`. Each resolved
+item must be an executable `nn.Module`; ordering must match forward execution.
+Multiple containers are streamed as separate hook rings.
 
-Non-block DiT modules, encoders, VAEs, and declared resident modules are not
-streamed by this backend.
+Non-block state of a selected DiT or text encoder remains device resident.
+Unselected components, VAEs, and declared resident modules are not streamed by
+this backend.
 
 ## Invariants and limitations
 
@@ -50,7 +52,8 @@ streamed by this backend.
 - Blocks with heterogeneous shapes and dtypes use their recorded metadata;
   implementations must not assume identical blocks.
 - The ring is initialized before the first forward so block zero is available.
-- This backend is single-device; distributed weight reconstruction belongs to
+- This backend uses rank-local host weights. Multi-device compute may remain
+  active, but cross-rank weight reconstruction belongs to
   [Distributed Layerwise Offload](distributed_layerwise_offload.md).
 
 Shared selection and lifecycle behavior is defined in the

docs/design/feature/offloader/module_offload.md

@@ -2,7 +2,7 @@
 
 Model-level offload enforces mutual exclusion between pipeline component
 groups. For commands and model support, see the
-[model-level user guide](../../../user_guide/diffusion/offloader/cpu_offload.md).
+[model-level user guide](../../../user_guide/diffusion/offloader/module_offload.md).
 
 ## Generic hook path
 
@@ -10,10 +10,15 @@ groups. For commands and model support, see the
 makes encoders and VAEs device resident for initialization, then installs
 `SequentialOffloadHook` instances:
 
-- each DiT hook offloads all encoders and other DiTs before loading itself;
-- each encoder hook offloads every DiT before loading itself; and
+- each DiT hook offloads selected encoders and other selected DiTs before
+  loading itself;
+- each encoder hook offloads selected DiTs before loading itself; and
 - VAEs are not part of the mutual-exclusion hook set.
 
+All execution-stage modules still receive a hook so an unselected resident
+stage can evict a selected component before it runs. Only components listed in
+`diffusion_offload_config.components` are eligible to move to CPU.
+
 The hook moves parameters and buffers without recursively calling
 `module.to()`. This avoids recursion through modules that retain references to
 other transformer blocks. Host tensors are pinned when configured; DTensors
@@ -36,7 +41,9 @@ contract:
 
 - only the executing component is device resident;
 - enable and disable are idempotent;
-- transfers use the supplied device, pinning, and HSDP settings; and
+- transfers use the supplied device, pinning, and HSDP settings;
+- explicit component selection is either honored or rejected before partial
+  setup; and
 - disabling removes all model-local offload hooks or contexts.
 
 The delegation is currently duck typed. If another independent implementation

docs/design/feature/quantization.md

@@ -84,8 +84,8 @@ dictionary narrows this global scope through runtime layer-prefix routing.
 
 The factory uses Omni-specific overrides for methods whose diffusion behavior
 or platform dispatch is not provided by the vLLM registry. The current override
-set includes INT8, BitsAndBytes, MXFP8, MXFP4, MXFP4 dual-scale, and INC/
-AutoRound. Other methods, such as FP8, GGUF, and ModelOpt, are resolved through
+set includes INT8, BitsAndBytes, MXFP8, MXFP4, MXFP4 dual-scale, INC/AutoRound,
+and TorchAO. Other methods, such as FP8, GGUF, and ModelOpt, are resolved through
 vLLM's registry when their configuration is compatible.
 
 ## Per-component routing

docs/design/feature/skip_softmax.md

@@ -1,90 +1,197 @@
 # Skip-Softmax
 
-Skip-Softmax is the sparse-attention mode of the `TRTLLM_ATTN` backend. Usage — the config keys
-and how to pick an operating point — is in
-[TRTLLM Attention](../../user_guide/diffusion/attention_backends/trtllm.md#skip-softmax).
-The shared selector contract is documented in
-[Diffusion Attention Backend Selection](attention_backend_selection.md). This
-page explains the algorithm.
+Skip-Softmax is the sparse-attention mode of the `TRTLLM_ATTN` backend. It implements
+[BLASST](https://arxiv.org/abs/2512.12087) (Dynamic BLocked Attention Sparsity via Softmax
+Thresholding), which adds a per-tile skip test to the FlashAttention main loop. This page describes the algorithm, how the user configuration is resolved
+into the value the kernel consumes, and how checkpoint calibration and timestep gating participate
+in that resolution. Configuration keys and operating-point guidance are covered in
+[TRTLLM Attention](../../user_guide/diffusion/attention_backends/trtllm.md#skip-softmax); the
+backend selection contract is covered in
+[Diffusion Attention Backend Selection](attention_backend_selection.md).
 
 ## Motivation
 
 In a long attention row (a video DiT can have tens of thousands of keys), the softmax weight
 concentrates on a small fraction of the keys; the rest receive near-zero weight and barely move the
 output. Computing softmax and the value-weighted sum over those keys is wasted work. Skip-Softmax
-detects, per block of keys, when a block cannot matter and skips its softmax and its value
-multiply.
+detects, per tile of keys, when a tile cannot matter and skips its softmax and its value multiply.
 
-It is approximate: a skipped block still carries a small non-zero contribution, so the mode is
+It is approximate: a skipped tile still carries a small non-zero contribution, so the mode is
 opt-in and off by default.
 
-## The online-softmax pass
+## Algorithm
 
-Attention is computed in a single streaming pass over the keys, in tiles of 128. Per query row the
+![The BLASST algorithm: FlashAttention with a per-tile skip test on the local maximum](../figures/skip_softmax/blasst_algorithm.jpg)
+
+FlashAttention computes attention in a single streaming pass over KV tiles. Per query row the
 kernel maintains three running values:
 
 - `m` — the largest score seen so far,
 - `l` — the running denominator `Σ exp(sⱼ − m)`,
 - `O` — the running numerator `Σ exp(sⱼ − m)·vⱼ`,
 
-and returns `O / l` at the end. For each key tile it computes the tile's scores `Q · K_jᵀ · scale`,
-updates `m`, and accumulates that tile's contribution into `l` and `O`. Rescaling `l` and `O` when
-`m` grows keeps the single pass numerically exact.
+and returns `O / l` at the end. For each KV tile it computes the scores `QK_j^T`, updates `m`, and
+accumulates the tile's contribution into `l` and `O`. Rescaling `l` and `O` when `m` grows keeps the
+dense pass numerically exact.
 
-## The skip test
-
-Once a tile's scores are known, its largest score `tile_max` is compared against the running
-maximum:
+A tile is a block of query rows against a block of keys, and the kernel processes it as a unit.
+BLASST inserts one test between computing a tile's scores and accumulating them. For each query
+row `i` in the tile, let `tile_max[i]` be the row's largest score within the tile, `running_max[i]`
+the row's running maximum, and `λ` the effective threshold:
 
 ```text
-if exp(tile_max − running_max) < threshold:
-    skip this tile          # do not compute its softmax weights or its P·V accumulation
+if all(exp(tile_max[i] - running_max[i]) < λ for every row i in the tile):
+    skip this tile          # do not compute its Softmax or PV contribution
 ```
 
-`exp(tile_max − running_max)` is an upper bound on the softmax weight any key in the tile can
-receive: if even the tile's best key is far below the current maximum, every key in the tile is
-negligible, and both the softmax (the exponentials) and the `P·V` accumulation for that tile can be
-dropped. The tile's contribution to `l` and `O` is simply skipped.
+`exp(tile_max[i] − running_max[i])` is an upper bound on the softmax weight any key in the tile can
+receive in row `i`: if even the row's best key in this tile is far below the row's current maximum,
+the tile is unimportant for that row. Only when this holds for every row in the tile can the
+Softmax and `PV` work for the tile be skipped, in which case its contribution to `l` and `O` is
+simply omitted. A larger `λ` makes the test more aggressive. The figure states the same test in
+log space, `m̃ − m < ln(λ)`, which is how the kernel evaluates it.
 
-## What this bounds
+The test needs the tile's scores, so `QK_j^T` always runs and only the Softmax and `PV` work can
+be skipped. Since the two matmuls have the same FLOP count, skipping every eligible tile removes at
+most roughly half of the attention arithmetic, and the kernel-level speedup stays well under 2×.
 
-Two properties of the test shape the achievable speedup:
+## From configuration to the kernel threshold
 
-- **`Q · K_jᵀ` always runs.** The test needs `tile_max`, which comes from the tile's scores, so the
-  score matmul is never skipped — only the softmax and the `P·V` accumulation are. The score matmul
-  and the value matmul are comparable in cost, so skipping every eligible tile removes at most
-  roughly half the attention work; the kernel-level speedup is bounded well under 2×, not unbounded.
-
-- **The decision is per tile, not per key.** A tile is skipped only when *all* of its keys are
-  collectively negligible; a single important key keeps the whole tile. How many tiles actually
-  qualify depends on the data and rounds down to tile granularity, so `target_sparsity` selects an
-  operating point on a calibrated curve — it is not a promise that a fixed fraction of tiles is
-  skipped.
-
-## The threshold
-
-The per-tile threshold is normalized by sequence length:
+The FlashInfer kernel does not take `λ` directly. It takes a `threshold_scale_factor` and divides
+it by the KV sequence length of the call:
 
 ```text
-threshold = factor / seqlen
+λ = threshold_scale_factor / max_kv_len
 ```
 
-`factor` comes from one of two sources:
+vLLM-Omni resolves the factor from one of two mutually exclusive user controls:
 
-- **Calibrated** — `factor = a · exp(b · target_sparsity)`, where `a` and `b` are fit per model (and
-  per expert, for a multi-expert model) so that a given `target_sparsity` lands near that fraction of
-  skipped tiles on the calibration data.
-- **Direct** — `factor = skip_softmax.threshold · seqlen`, the calibration-free path; the user sets
-  the threshold themselves.
+| Control | Factor passed to the kernel | Resulting `λ` |
+| --- | --- | --- |
+| `threshold` | `threshold * max_kv_len` | `threshold` |
+| `target_sparsity=s` | `a * exp(b * s)` | `a * exp(b * s) / max_kv_len` |
+
+`threshold` is therefore `λ` itself and is independent of sequence length: the same value yields
+the same per-tile test at any resolution or frame count. `threshold=0` skips no tiles. Because the
+left-hand side of the skip test lies in `(0, 1]`, values in `(0, 1)` are the meaningful range; the
+schema accepts any finite non-negative value.
+
+`target_sparsity` selects a point on a curve fitted per model by
+[NVIDIA ModelOpt](https://github.com/NVIDIA/Model-Optimizer/tree/main/examples/diffusers/sparsity),
+so that `s` lands near that fraction of skipped tiles on the calibration data. The coefficients
+`a` and `b` come from the checkpoint; the achieved sparsity on another prompt, shape, or layer can
+differ.
+
+### Skip-Softmax calibration config
+
+ModelOpt stores the calibration in each transformer component's `config.json` under
+`sparse_attention_config`. The following is taken from the `transformer/config.json` of
+[`nvidia/Wan2.2-T2V-A14B-Diffusers-FP8`](https://huggingface.co/nvidia/Wan2.2-T2V-A14B-Diffusers-FP8),
+with the `ignore` list abridged:
+
+```json
+{
+  "sparse_attention_config": {
+    "config_groups": {
+      "group_0": {
+        "algorithm": "skip_softmax",
+        "targets": ["WanAttention"],
+        "ignore": ["blocks.0.attn1", "blocks.0.attn2", "blocks.1.attn1", "..."],
+        "threshold_scale_factor": {
+          "formula": "a * exp(b * target_sparsity)",
+          "coefficients": {"a": 2822.6209952221557, "b": 4.388250623520793}
+        },
+        "target_sparsity": 0.7,
+        "disabled_until_timestep": 0.93
+      }
+    }
+  }
+}
+```
+
+A checkpoint with several transformer components carries one such config per component, each
+with its own coefficients; in this example, `transformer_2/config.json` holds the low-noise
+expert's curve.
+
+vLLM-Omni consumes `formula`, `coefficients`, and `ignore`, and ignores `targets`,
+`target_sparsity`, and `disabled_until_timestep`:
+
+- Only the formula `a * exp(b * target_sparsity)` is accepted; a checkpoint with any other formula
+  is rejected at startup. If a spec requests `target_sparsity` and the checkpoint carries no
+  calibration, startup fails and the error points to `threshold` as the calibration-free
+  alternative.
+- Each component's curve is applied to that component's attention layers. If the config for a
+  secondary component such as `transformer_2` cannot be read, that component stays dense and a
+  warning is logged.
+- `ignore` holds fnmatch patterns matched against both the full module name and the name relative
+  to the component, so `blocks.0.attn1` matches `transformer.blocks.0.attn1` as well as
+  `transformer_2.blocks.0.attn1`. Matching layers receive no coefficients and therefore stay dense
+  regardless of the user configuration.
+
+At each attention call, a layer with a `threshold` uses it directly; a layer with
+`target_sparsity` and stamped coefficients evaluates the curve; a layer with `target_sparsity` but
+no coefficients runs dense. The timestep gate below is applied to the result before it reaches the
+kernel.
 
 ## Timestep gating
 
-`disabled_until_timestep = D` keeps the mode off during the early, high-noise denoise steps and
-turns it on once the normalized timestep `t` drops to `t ≤ D` (`t` runs `1.0` → `0.0` over the
-schedule). The early steps set the global structure of the output and their errors propagate through
-every later step, so keeping them dense costs a few skipped-tile opportunities but protects fidelity.
+The early, high-noise denoising steps set the global structure of the output, and their errors
+propagate through every later step. Keeping just these steps dense markedly improves the quality
+of the generated video at a small cost in skipped work. `disabled_until_timestep` implements this:
+on each attention call the backend compares the current normalized timestep `t` against the
+configured cutoff and passes no skip factor to the kernel while `t` is above it. Skip-Softmax
+becomes active on the first step whose `t` is at or below the cutoff and stays active for the rest
+of the run, since `t` only decreases.
 
-`t` is the scheduler's own timestep divided by `num_train_timesteps`, published by the pipeline via
-`DenoiseProgressMixin.record_denoise_step`. It is deliberately not derived from the step index,
-because schedulers space their steps non-uniformly. A pipeline that does not publish a timestep
-stays dense when `disabled_until_timestep` is set.
+The default `disabled_until_timestep = 0` is a sentinel rather than a cutoff: the gate is off, the
+timestep is never read, and Skip-Softmax runs on every step. Any positive value enables the gate.
+`disabled_until_timestep = 1.0` therefore also yields no dense steps on a pipeline that publishes
+`t`, but on a pipeline that does not publish it the backend stays dense and logs a warning once
+rather than guessing from the step index.
+
+### Where `t` comes from
+
+The pipeline publishes `t` once per denoising step before running the transformer. It is the
+scheduler's own position in `[0, 1]`, decreasing from near `1.0` (pure noise) to `0.0` (clean
+sample): scheduler-based pipelines publish the scheduler timestep divided by `num_train_timesteps`,
+and rectified-flow pipelines publish the current sigma. `t` is deliberately not the step index
+divided by the step count. Schedulers place their steps non-uniformly in `t`, so expressing the
+cutoff in `t` ties it to the same noise level across step counts and schedulers.
+
+### Mapping the cutoff to denoising steps
+
+For a run whose published sequence is `t[0], ..., t[N-1]`, the number of dense steps is
+
+```text
+dense_steps = count(t[i] > disabled_until_timestep)
+skip_softmax_steps = N - dense_steps
+```
+
+This count **depends on the schedule**, and schedules differ from model to model: the
+scheduler family, the number of steps, and any time shift all change where the steps land in `t`.
+The same cutoff can therefore gate a very different fraction of the run on two models. Take
+MiniMax-H3 as an example. Its video branch uses a flow-shifted rectified-flow schedule that applies
+a shift `s` to `N + 1` uniform positions `u` from `1` to `0`,
+
+```text
+t = s * u / (1 + (s - 1) * u)
+```
+
+and a large `s` pushes most positions toward `t ≈ 1`. With the default `s = 12` and
+`num_inference_steps = 50` (50 sigma points, `N = 49` denoiser forwards), the published sequence
+starts `1.000, 0.998, 0.996, 0.995, 0.993, ...` and stays above `0.9` for 28 steps, giving:
+
+| `disabled_until_timestep` | Dense steps | Skip-Softmax steps |
+| :---: | ---: | ---: |
+| `1.00` | 0 | 49 |
+| `0.99` | 6 | 43 |
+| `0.97` | 14 | 35 |
+| `0.95` | 19 | 30 |
+| `0.90` | 28 | 21 |
+| `0.86` | 33 | 16 |
+
+These numbers hold only for that schedule. A model with a smaller shift, a different step count,
+or a scheduler that is not rectified flow produces a different sequence and a different table; a
+request that overrides `flow_shift` or the step count changes it for MiniMax-H3 too.
+Choose the cutoff by counting against the schedule actually served, not by reusing a value from
+another model.

docs/design/figures/magi2/dlo-pareto-1-step.svg

@@ -0,0 +1,59 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="760" height="460" viewBox="0 0 760 460" role="img" aria-labelledby="title desc">
+  <title id="title">MAGI-2 DLO one-step Pareto frontier</title>
+  <desc id="desc">Warm user latency in seconds versus throughput per device. DP1SP4 is 5.735 seconds and 0.0436 outputs per second per device, DP2SP2 is 6.785 and 0.0737, and DP4 is 9.997 and 0.1000.</desc>
+  <rect width="760" height="460" fill="#ffffff"/>
+  <style>
+    text { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; fill: #111827; }
+    .title { font-size: 20px; font-weight: 600; }
+    .axis-label { font-size: 14px; font-weight: 500; }
+    .tick { font-size: 12px; fill: #4b5563; }
+    .point-label { font-size: 13px; font-weight: 600; }
+    .value-label { font-size: 11px; fill: #4b5563; }
+    .grid { stroke: #e5e7eb; stroke-width: 1; }
+    .axis { stroke: #6b7280; stroke-width: 1.5; }
+    .frontier { fill: none; stroke: #2563eb; stroke-width: 3; }
+    .point { fill: #2563eb; stroke: #ffffff; stroke-width: 2; }
+  </style>
+
+  <text x="380" y="28" text-anchor="middle" class="title">One diffusion step</text>
+
+  <g aria-hidden="true">
+    <line x1="85" y1="309" x2="700" y2="309" class="grid"/>
+    <line x1="85" y1="226" x2="700" y2="226" class="grid"/>
+    <line x1="85" y1="143" x2="700" y2="143" class="grid"/>
+    <line x1="85" y1="61" x2="700" y2="61" class="grid"/>
+    <line x1="85" y1="40" x2="85" y2="350" class="axis"/>
+    <line x1="85" y1="350" x2="700" y2="350" class="axis"/>
+
+    <text x="73" y="313" text-anchor="end" class="tick">0.04</text>
+    <text x="73" y="230" text-anchor="end" class="tick">0.06</text>
+    <text x="73" y="147" text-anchor="end" class="tick">0.08</text>
+    <text x="73" y="65" text-anchor="end" class="tick">0.10</text>
+
+    <text x="85" y="370" text-anchor="middle" class="tick">5</text>
+    <text x="197" y="370" text-anchor="middle" class="tick">6</text>
+    <text x="309" y="370" text-anchor="middle" class="tick">7</text>
+    <text x="421" y="370" text-anchor="middle" class="tick">8</text>
+    <text x="532" y="370" text-anchor="middle" class="tick">9</text>
+    <text x="644" y="370" text-anchor="middle" class="tick">10</text>
+  </g>
+
+  <text x="392" y="414" text-anchor="middle" class="axis-label">Warm user latency (seconds) — lower is better</text>
+  <text x="20" y="195" transform="rotate(-90 20 195)" text-anchor="middle" class="axis-label">Throughput/device (outputs/s) — higher is better</text>
+
+  <polyline points="167,294 285,169 644,61" class="frontier"/>
+
+  <circle cx="167" cy="294" r="7" class="point"/>
+  <text x="179" y="288" class="point-label">DP1SP4</text>
+  <text x="179" y="305" class="value-label">5.735 s, 0.0436</text>
+
+  <circle cx="285" cy="169" r="7" class="point"/>
+  <text x="297" y="163" class="point-label">DP2SP2</text>
+  <text x="297" y="180" class="value-label">6.785 s, 0.0737</text>
+
+  <circle cx="644" cy="61" r="7" class="point"/>
+  <text x="632" y="84" text-anchor="end" class="point-label">DP4</text>
+  <text x="632" y="101" text-anchor="end" class="value-label">9.997 s, 0.1000</text>
+
+  <text x="700" y="444" text-anchor="end" class="tick">4x NVIDIA L20X · warm qualification run</text>
+</svg>

docs/design/figures/magi2/dlo-pareto-4-step.svg

@@ -0,0 +1,65 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="760" height="460" viewBox="0 0 760 460" role="img" aria-labelledby="title desc">
+  <title id="title">MAGI-2 DLO four-step Pareto frontier</title>
+  <desc id="desc">Warm user latency in seconds versus throughput per device. DP1SP4 is 8.647 seconds and 0.0289 outputs per second per device, DP2SP2 is 11.682 and 0.0428, and DP4 is 20.957 and 0.0477.</desc>
+  <rect width="760" height="460" fill="#ffffff"/>
+  <style>
+    text { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; fill: #111827; }
+    .title { font-size: 20px; font-weight: 600; }
+    .axis-label { font-size: 14px; font-weight: 500; }
+    .tick { font-size: 12px; fill: #4b5563; }
+    .point-label { font-size: 13px; font-weight: 600; }
+    .value-label { font-size: 11px; fill: #4b5563; }
+    .grid { stroke: #e5e7eb; stroke-width: 1; }
+    .axis { stroke: #6b7280; stroke-width: 1.5; }
+    .frontier { fill: none; stroke: #2563eb; stroke-width: 3; }
+    .point { fill: #2563eb; stroke: #ffffff; stroke-width: 2; }
+  </style>
+
+  <text x="380" y="28" text-anchor="middle" class="title">Four diffusion steps</text>
+
+  <g aria-hidden="true">
+    <line x1="85" y1="350" x2="700" y2="350" class="grid"/>
+    <line x1="85" y1="288" x2="700" y2="288" class="grid"/>
+    <line x1="85" y1="226" x2="700" y2="226" class="grid"/>
+    <line x1="85" y1="164" x2="700" y2="164" class="grid"/>
+    <line x1="85" y1="102" x2="700" y2="102" class="grid"/>
+    <line x1="85" y1="40" x2="700" y2="40" class="grid"/>
+    <line x1="85" y1="40" x2="85" y2="350" class="axis"/>
+    <line x1="85" y1="350" x2="700" y2="350" class="axis"/>
+
+    <text x="73" y="354" text-anchor="end" class="tick">0.025</text>
+    <text x="73" y="292" text-anchor="end" class="tick">0.030</text>
+    <text x="73" y="230" text-anchor="end" class="tick">0.035</text>
+    <text x="73" y="168" text-anchor="end" class="tick">0.040</text>
+    <text x="73" y="106" text-anchor="end" class="tick">0.045</text>
+    <text x="73" y="44" text-anchor="end" class="tick">0.050</text>
+
+    <text x="85" y="370" text-anchor="middle" class="tick">8</text>
+    <text x="173" y="370" text-anchor="middle" class="tick">10</text>
+    <text x="261" y="370" text-anchor="middle" class="tick">12</text>
+    <text x="349" y="370" text-anchor="middle" class="tick">14</text>
+    <text x="436" y="370" text-anchor="middle" class="tick">16</text>
+    <text x="524" y="370" text-anchor="middle" class="tick">18</text>
+    <text x="612" y="370" text-anchor="middle" class="tick">20</text>
+    <text x="700" y="370" text-anchor="middle" class="tick">22</text>
+  </g>
+
+  <text x="392" y="414" text-anchor="middle" class="axis-label">Warm user latency (seconds) — lower is better</text>
+  <text x="20" y="195" transform="rotate(-90 20 195)" text-anchor="middle" class="axis-label">Throughput/device (outputs/s) — higher is better</text>
+
+  <polyline points="113,302 247,129 654,69" class="frontier"/>
+
+  <circle cx="113" cy="302" r="7" class="point"/>
+  <text x="125" y="296" class="point-label">DP1SP4</text>
+  <text x="125" y="313" class="value-label">8.647 s, 0.0289</text>
+
+  <circle cx="247" cy="129" r="7" class="point"/>
+  <text x="259" y="123" class="point-label">DP2SP2</text>
+  <text x="259" y="140" class="value-label">11.682 s, 0.0428</text>
+
+  <circle cx="654" cy="69" r="7" class="point"/>
+  <text x="642" y="92" text-anchor="end" class="point-label">DP4</text>
+  <text x="642" y="109" text-anchor="end" class="value-label">20.957 s, 0.0477</text>
+
+  <text x="700" y="444" text-anchor="end" class="tick">4x NVIDIA L20X · warm qualification run</text>
+</svg>

docs/design/fullduplex-personaplex.md

@@ -23,16 +23,18 @@ The target is the engine-native path:
   -> response.audio.delta + response.audio_transcript.delta
 ```
 
-The standalone `/api/chat` and `/v1/audio/duplex` server remains a compatibility
-surface. It is not evidence that the unified engine path works.
+The standalone `/api/chat` and `/v1/audio/duplex` server that accompanied the
+original PR was demo-only and has been removed from the tree; the unified
+engine path above is the only serving surface. It was never evidence that the
+unified engine path works.
 
-## Why configuration-only enablement is invalid
+## Why configuration-only enablement was invalid
 
-The current staged pipeline is explicitly turn based. Its Talker reads
-`pplex_user_codes`, `pplex_prefill_text`, and `pplex_silence_codes`, but no
-production staged input path writes those fields. The voice prompt, persona
-prefill, and streaming Mimi state live only in the standalone
-`PersonaPlexEngine`.
+The staged pipeline as shipped by the original PR was explicitly turn based.
+Its Talker read `pplex_user_codes`, `pplex_prefill_text`, and
+`pplex_silence_codes`, but no production staged input path wrote those fields.
+The voice prompt, persona prefill, and streaming Mimi state lived only in the
+standalone `PersonaPlexEngine` (demo-only, since removed).
 
 Setting only the following fields would therefore advertise an endpoint whose
 model never receives the live microphone stream:
@@ -56,7 +58,12 @@ The unified implementation supports:
 - continuous user input while assistant audio is generated or played;
 - bundled `.pt` voice prompts and a session persona;
 - greedy text and depformer sampling, matching the current PersonaPlex port;
-- `/v1/realtime?duplex=1`;
+- `/v1/realtime?duplex=1` (the wire vocabulary is catalogued in the
+  [Realtime Duplex API](../serving/realtime_duplex_api.md) serving guide);
+- the public client preset
+  `vllm_omni.clients.personaplex.create_duplex_session_config()` (24 kHz
+  `pcm_f32le` input format, voice prompt, persona) as the canonical
+  session-config source for `DuplexClient` consumers;
 - engine lease close, disconnect cleanup, reconnect after cleanup, and explicit
   response cancellation without cross-session state reuse.
 
@@ -189,10 +196,17 @@ model-neutral native results:
 ```
 
 It owns per-request audio and text cursors so a cumulative output cannot replay
-old audio. PersonaPlex keeps one visible response open while continuous output
-arrives. Session close or cancellation terminates that response through the
-generic Realtime lifecycle; a codec segment finishing is not a conversational
-turn boundary.
+old audio. The generic projector turns each native result into the Realtime
+`response.audio.delta` / `response.audio_transcript.delta` pair (and
+`response.output_text.delta` for text) under one `response_id`; the full
+mapping is the name map in the
+[Realtime Duplex API](../serving/realtime_duplex_api.md) serving guide.
+PersonaPlex keeps one visible response open while continuous output arrives.
+Session close or cancellation terminates that response through the generic
+Realtime lifecycle; a codec segment finishing is not a conversational turn
+boundary. PersonaPlex advertises `supports_barge_in=false` and
+`supports_session_resume=false`, so `barge_in`, `turn_detection.server_vad`,
+and `session.resume` are rejected on this model.
 
 ## Error and lifecycle contracts
 
@@ -210,8 +224,8 @@ turn boundary.
 - New sessions cannot observe the previous voice, persona, PCM tail, Mimi
   convolution state, or Talker delayed code frame.
 
-The legacy `PersonaPlexDuplexRuntime.run()` and standalone server drain path
-must follow the same exception-safe rule while they remain in the tree.
+(The legacy `PersonaPlexDuplexRuntime.run()` and standalone server drain path,
+which followed the same exception-safe rule, have been removed from the tree.)
 
 ## Testing and acceptance
 

docs/design/fullduplex.md

@@ -1,5 +1,13 @@
 # MiniCPM-o 4.5 Full-Duplex Runtime Architecture
 
+> Relocation note: after the H20 validation recorded below, the full-duplex
+> implementation moved out of `vllm_omni/experimental/fullduplex/` into the
+> stable tree (`vllm_omni/engine/duplex/`, `vllm_omni/entrypoints/duplex/`,
+> `vllm_omni/model_executor/models/minicpmo_4_5/duplex/`, and companion
+> modules). Module paths in this document reflect the new layout; the runtime
+> evidence was produced on the pre-move tree, which is file-identical apart
+> from module paths.
+
 ## Purpose
 
 This document records the runtime architecture implemented by PR #3907, the
@@ -31,8 +39,8 @@ session was rejected and capacity returned after one session closed. The two
 accepted sessions independently completed their audio, transcript, response,
 playback, and close lifecycles. This is isolation and admission evidence for the
 tested two-session deployment, not a production fairness, arbitrary-capacity,
-or failure-recovery claim. The validated paths include the experimental runtime
-extension, typed resumable policy, extracted control plane and clients, request
+or failure-recovery claim. The validated paths include the MiniCPM duplex
+runtime extension, typed resumable policy, extracted control plane and clients, request
 preregistration, typed direct-output decision, engine-managed two-session
 capacity in the checked-in deploy profile, separate public/runtime configuration
 channels, private Session ledgers, ordered `session.update`, the MiniCPM runner
@@ -67,7 +75,9 @@ The checkpoint does not claim:
 - an acoustic-onset-to-cancel latency target across arbitrary client chunk sizes;
 - production multi-session admission, fairness, capacity, or failure recovery;
 - bounded long-session KV;
-- video input or audio/video synchronization.
+- audio/video synchronization beyond one camera frame per model unit (camera
+  frames ride the audio appends as `video_frames`; the wire contract is in the
+  [Realtime Duplex API](../serving/realtime_duplex_api.md) guide).
 
 ## Active Runtime Path
 
@@ -81,7 +91,7 @@ WebSocket
   -> AsyncOmni thin open/append/signal/close proxies
   -> DuplexControlClient over the engine-owned correlated RPC transport
   -> DuplexControlPlane + DuplexSessionRuntimeManager
-  -> experimental DuplexRuntimeExtension + engine session/stage bindings
+  -> MiniCPM DuplexRuntimeExtension + engine session/stage bindings
   -> StagePool
   -> resumable scheduler request
   -> MiniCPM Stage0
@@ -177,21 +187,21 @@ ordering guarantee.
 
 ### Extracted control ownership
 
-The generic engine mechanism and the experimental duplex plugin are split by
+The generic engine mechanism and the duplex plugin are split by
 responsibility:
 
 | Module | Responsibility |
 | --- | --- |
 | `engine.messages` | ordinary engine queue messages and collective RPC envelopes |
 | `engine.rpc_result_router` | generic correlation routing through each result message's `rpc_correlation_key` |
-| `experimental.fullduplex.engine.contracts` | immutable duplex DTOs, extension/stage protocols, typed decisions, request-ID codec |
-| `experimental.fullduplex.engine.lease` | duplex lease activity, TTL/grace configuration, and expiry records |
-| `experimental.fullduplex.engine.messages` | identity fence and duplex control message/result envelopes |
-| `experimental.fullduplex.engine.duplex_runtime` | extension loading, validation, and compatibility exports |
-| `experimental.fullduplex.engine.duplex_session` | session resources, fences, append reservations, idempotency cache, binding cleanup |
-| `experimental.fullduplex.engine.duplex_control_plane` | control algorithms and the narrow `DuplexStagePort` used to submit and clean up stage requests |
-| `experimental.fullduplex.engine.duplex_control_client` | typed control-message construction and calls through the engine-owned correlated RPC transport |
-| `experimental.fullduplex.request_client` | deterministic request identity, client-state preregistration, acknowledgement validation, data-plane collection, rollback |
+| `engine.duplex.contracts` | immutable duplex DTOs, extension/stage protocols, typed decisions, request-ID codec |
+| `engine.duplex.lease` | duplex lease activity, TTL/grace configuration, and expiry records |
+| `engine.duplex.messages` | identity fence and duplex control message/result envelopes |
+| `engine.duplex.runtime` | extension loading, validation, and compatibility exports |
+| `engine.duplex.session` | session resources, fences, append reservations, idempotency cache, binding cleanup |
+| `engine.duplex.control_plane` | control algorithms and the narrow `DuplexStagePort` used to submit and clean up stage requests |
+| `engine.duplex.control_client` | typed control-message construction and calls through the engine-owned correlated RPC transport |
+| `entrypoints.duplex_request_client` | deterministic request identity, client-state preregistration, acknowledgement validation, data-plane collection, rollback |
 
 `Orchestrator`, `AsyncOmniEngine`, and `AsyncOmni` retain explicit wiring and
 compatibility proxies. They no longer own the corresponding duplex algorithms.
@@ -362,33 +372,34 @@ vllm_omni.engine.rpc_result_router         # generic correlation routing
 vllm_omni.engine.async_engine_utils        # ordinary engine lifecycle helpers
 ```
 
-The full-duplex implementation remains experimental:
+The full-duplex implementation lives in dedicated duplex modules:
 
 ```text
-vllm_omni.experimental.fullduplex.engine.duplex_runtime
+vllm_omni.engine.duplex.runtime
                                            # extension loading and validation
-vllm_omni.experimental.fullduplex.engine.contracts
+vllm_omni.engine.duplex.contracts
                                            # immutable DTOs and narrow protocols
-vllm_omni.experimental.fullduplex.engine.lease
+vllm_omni.engine.duplex.lease
                                            # duplex lease primitives
-vllm_omni.experimental.fullduplex.engine.messages
+vllm_omni.engine.duplex.messages
                                            # duplex identity and queue envelopes
-vllm_omni.experimental.fullduplex.engine.duplex_session
+vllm_omni.engine.duplex.session
                                            # session transaction implementation
-vllm_omni.experimental.fullduplex.engine.duplex_control_plane
+vllm_omni.engine.duplex.control_plane
                                            # engine-side control algorithms
-vllm_omni.experimental.fullduplex.engine.duplex_control_client
+vllm_omni.engine.duplex.control_client
                                            # correlated control client
-vllm_omni.experimental.fullduplex.request_client
+vllm_omni.entrypoints.duplex_request_client
                                            # entrypoint request/output lifecycle
 ```
 
-Stable engine and entrypoint modules import only the kernel at module load.
-They load the experimental ControlPlane, ControlClient, request client, and
-runtime extension inside duplex-only construction or call paths. Importing
-`Orchestrator`, `AsyncOmniEngine`, or `AsyncOmni` for an ordinary deployment
-therefore does not import `vllm_omni.experimental.fullduplex`. Compatibility
-exports remain lazy and are not part of ordinary startup.
+Stable engine and entrypoint modules import the duplex kernel eagerly at
+module load (top-of-file imports). Duplex behavior remains opt-in: the
+ControlPlane, ControlClient, and request client are *constructed* only on
+duplex-enabled deployments, and the model runtime extension still loads only
+through the dotted-string plugin path. The client-side package
+(`vllm_omni/clients/`, home of the public `DuplexClient`) is never imported
+by runtime code.
 
 `PipelineConfig.duplex_control_enabled` enables the generic mechanism, while
 `PipelineConfig.duplex_runtime_extension` separately selects the model adapter.
@@ -452,7 +463,7 @@ sampling generation unchanged.
 `extra_body` or session payloads:
 
 | Field | Default | Owner | Lifecycle or resource effect |
-|---|---:|---|---|
+| --- | ---: | --- | --- |
 | `idle_ttl_s` | `300.0` | Engine lease manager | Expires an attached or detached session after inactivity; `null` disables only idle expiry. |
 | `disconnect_grace_s` | `30.0` | Engine lease manager and API attachment registry | Bounds how long a detached session may retain engine resources before cleanup. |
 | `reaper_interval_s` | `5.0` | Orchestrator | Sets the cadence for lease expiry and pending cleanup retries. |
@@ -467,6 +478,30 @@ Deploy profiles may lower or raise these values to match scheduler capacity,
 but clients cannot override them. The engine remains authoritative for
 admission and cleanup ownership even when the API performs an early check.
 
+### Client-side companion package
+
+The public client surface ships under `vllm_omni/clients/` and stays on the
+consumer side of the boundary above: `vllm_omni.clients.duplex` provides the
+model-agnostic `DuplexClient` (typed events, per-response demux via
+`ResponseHandle`, `session.resume` with acknowledgement-based replay dedup)
+and the `EventCollector` helper used for timing and request-metric summaries.
+Model-specific session defaults live in thin per-model preset modules —
+`vllm_omni.clients.minicpmo_4_5.create_duplex_session_config()` and
+`vllm_omni.clients.personaplex.create_duplex_session_config()` — so the core
+client never imports model packages and the model duplex plugin packages
+never import the client (both directions are enforced by tests:
+`tests/engine/test_duplex_import_boundary.py`,
+`tests/entrypoints/openai_api/test_duplex_capability.py`, and the subprocess
+purity guards in `tests/clients/test_model_session_configs.py`).
+`examples/online_serving/barge_in_client.py` (with its flow companion
+`barge_in_client_flow.md`) is the runnable reference for the client library
+against a live duplex deployment. The wire vocabulary the client speaks —
+every `/v1/realtime?duplex=1` event with its OpenAI-compatibility tier and a
+JSON example, plus the native-to-Realtime name map — is catalogued in the
+[Realtime Duplex API](../serving/realtime_duplex_api.md) serving guide; that
+page derives from the normative contract table above and must be updated
+with it.
+
 ## Generic-Path Cleanup
 
 Model-specific `MINICPMO45_PROFILE_LOGS` probes were removed from:
@@ -508,14 +543,16 @@ The MiniCPM Stage1 input processor consumes that capability for token-faithful
 delta transcripts. The generic orchestrator no longer installs or names a
 MiniCPM-specific bridge-state key.
 
-The stable API server inspects the deployment `session_mode` before importing or
-constructing the experimental duplex handler. Ordinary deployments do not load
-the full-duplex serving package or create its registries and background state.
+The stable API server imports the duplex serving package eagerly and inspects
+the deployment `session_mode` before constructing the duplex handler. Ordinary
+deployments do not create its registries and background state.
 For an enabled deployment, the client still selects the Realtime duplex route
 with `?duplex=1` (or an equivalent explicit true value). Model-name matching is
 not used for routing or native-runtime activation. MiniCPM clients explicitly
-set `extra_body.minicpmo45_native_duplex=true`; repository demos do so in their
-session payloads.
+set `extra_body.native_duplex=true`; the canonical way to do so is
+the `vllm_omni.clients.minicpmo_4_5.create_duplex_session_config()` preset,
+which the repository's barge-in example
+(`examples/online_serving/barge_in_client.py`) uses in its session payload.
 
 `DuplexRequestClient` derives the data-plane request identity from the accepted
 fence and preregisters a resumable `ClientRequestState` before the append crosses
@@ -623,7 +660,7 @@ continues to return `input_audio_buffer_empty`.
 The commit-to-response contract is:
 
 | Session state at `input_audio_buffer.commit` | Runtime action | Required event outcome |
-|---|---|---|
+| --- | --- | --- |
 | Auto-response disabled, valid idle input | Commit and retain the input item. | Emit `input_audio_buffer.committed`; emit no response until an explicit `response.create`. |
 | Auto-response enabled, valid idle speech | Submit the final native append for the current model turn. | Emit `input_audio_buffer.committed` before model output. Create a response only on the first visible text/audio output; a model listen decision may end with `response.listen` and no response. |
 | Auto-response enabled, short overlap during an active response | Discard the short overlap as an acknowledgement. | Emit the committed/no-response acknowledgement; do not create or defer a response. |
@@ -770,9 +807,12 @@ Graph capture. This is a model execution constraint, not an E2E-only override.
 
 ### Resource capabilities
 
-The checked-in MiniCPM duplex deploy profile sets Stage0 and Stage1
-`max_num_seqs` to two and configures the engine runtime manager with
-`max_sessions=2`. The Realtime capability response therefore advertises
+At the time of this snapshot the checked-in MiniCPM duplex deploy profile set
+Stage0 and Stage1 `max_num_seqs` to two and configured the engine runtime
+manager with `max_sessions=2`; the duplex settings have since been folded into
+`vllm_omni/deploy/minicpmo_4_5.yaml`, which admits four sessions
+(`duplex_session.max_sessions: 4`, matching its stage `max_num_seqs`). The
+Realtime capability response advertises
 multi-session and same-replica multi-session support with
 `session_admission_mode="engine_managed"`. Client session fields cannot raise
 this server-owned limit. Fresh E2E evidence covers two concurrent sessions in
@@ -1025,4 +1065,6 @@ Passing this checkpoint supports the statement:
 
 It does not support claims for an acoustic-onset barge-in latency target across
 arbitrary client chunk sizes, multi-session production concurrency, bounded
-long-session KV, scheduler-native append, or video input.
+long-session KV, or scheduler-native append. Camera-frame input
+(`video_frames`) landed after this snapshot and is covered by its own E2E
+test rather than by the evidence above.

docs/design/index.md

@@ -17,6 +17,8 @@ implementation contract; it is not, by itself, a general support claim.
 
 ### Runtime and stage execution
 
+- [Full-Duplex Runtime (MiniCPM-o 4.5)](fullduplex.md)
+- [Full-Duplex Runtime (PersonaPlex)](fullduplex-personaplex.md)
 - [Disaggregated Inference](feature/disaggregated_inference.md)
 - [Host Weight Runtime](feature/host_weight_runtime.md)
 - [Async Chunk](feature/async_chunk.md)
@@ -64,7 +66,7 @@ The design contracts separate selection mechanics from backend algorithms:
 #### CPU offloading
 
 - [Overview and Shared Contracts](feature/offloader/README.md)
-- [Model-Level Offload](feature/offloader/cpu_offload.md)
+- [Model-Level Offload](feature/offloader/module_offload.md)
 - [Layerwise Offload](feature/offloader/layerwise_offload.md)
 - [Distributed Layerwise Offload](feature/offloader/distributed_layerwise_offload.md)
 

docs/design/module/engine_orchestration.md

@@ -28,6 +28,9 @@ related_code_paths:
   - vllm_omni/engine/stage_pool.py
   - vllm_omni/outputs/output_processor.py
   - vllm_omni/distributed/omni_coordinator/**
+  # Duplex kernel design is owned by ../fullduplex.md; listed here because the
+  # orchestrator hosts the duplex control plane.
+  - vllm_omni/engine/duplex/**
 depends_on:
   - input_output_modality_contracts.md
   - error_contracts.md
@@ -44,7 +47,8 @@ validation_paths:
   - tests/engine/test_orchestrator_stage_input_bridge.py
   - tests/engine/test_cfg_companion_lifecycle.py
   - tests/engine/test_rpc_result_router.py
-  - tests/e2e/features/fullduplex/engine/**
+  - tests/engine/duplex/**
+  - tests/engine/test_duplex_import_boundary.py
 upstream_refs:
   - vllm.v1.engine.EngineCoreRequest
   - vllm.v1.engine.EngineCoreOutputs

docs/design/module/entrypoints.md

@@ -26,9 +26,18 @@ primary_code_paths:
   - vllm_omni/entrypoints/client_request_state.py
   - vllm_omni/entrypoints/stage_utils.py
   - vllm_omni/entrypoints/utils.py
+  - vllm_omni/entrypoints/duplex/**
+  - vllm_omni/entrypoints/duplex_request_client.py
+  - vllm_omni/clients/**
 primary_path_exceptions:
   - path: vllm_omni/entrypoints/openai/errors.py
     owner: error_contracts.md
+  - path: vllm_omni/entrypoints/duplex/**
+    owner: ../fullduplex.md
+  - path: vllm_omni/entrypoints/duplex_request_client.py
+    owner: ../fullduplex.md
+  - path: vllm_omni/clients/**
+    owner: ../fullduplex.md
 related_code_paths:
   - vllm_omni/errors.py
   - vllm_omni/inputs/**

docs/design/qwen3_omni_tts_performance_optimization.md

@@ -410,8 +410,8 @@ Notes:
 - `stages[].max_num_seqs` controls stage-level batching.
 - Thinker/Talker commonly use `enforce_eager: false` for CUDA Graph paths.
 - Code2Wav often remains eager (`enforce_eager: true`) depending on runtime behavior.
-- Qwen3-Omni defaults `VLLM_USE_FLASHINFER_MOE_FP16=0`. The Triton has been more stable & faster
- than the FlashInfer CUTLASS unquantized MoE backend on recent vLLM rebases.
+- The Qwen3-Omni deploy config pins `moe_backend: triton` for the Thinker and Talker stages. Triton has been more
+ stable and faster than the FlashInfer CUTLASS unquantized MoE backend on recent vLLM rebases.
 
 #### 2) Enable async chunk
 

docs/examples/README.md

@@ -3,4 +3,6 @@
 vLLM-Omni's examples are split into two categories:
 
 - If you are using vLLM-Omni from within Python code, see the *Offline Inference* section.
-- If you are using vLLM-Omni from an HTTP application or client, see the *Online Serving* section.
+- If you are using vLLM-Omni from an HTTP application or client, start with
+  the [API Server guide](../serving/README.md) to choose an endpoint, then use
+  the *Online Serving* section for model-specific examples.

docs/getting_started/installation/gpu.md

@@ -34,7 +34,7 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
 
 ### Pre-built wheels
 
-Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, 0.25.0, and 0.26.0; the 0.28.0 wheels are published together with the v0.28.0 release this branch produces. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
+Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, 0.25.0, 0.26.0, and 0.28.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
 
 === "NVIDIA CUDA"
 

docs/getting_started/installation/gpu/cuda.inc.md

@@ -5,7 +5,7 @@
 # --8<-- [end:requirements]
 # --8<-- [start:set-up-using-python]
 
-vLLM-Omni depends on the matching major/minor release of vLLM. The vLLM-Omni 0.26.x release line uses vLLM 0.26.x.
+vLLM-Omni depends on the matching major/minor release of vLLM. The vLLM-Omni 0.28.x release line uses vLLM 0.28.x.
 
 !!! note
     PyTorch installed via `conda` will statically link `NCCL` library, which can cause issues when vLLM tries to use `NCCL`. See <gh-issue:8420> for more details.
@@ -95,6 +95,13 @@ uv pip install --no-build-isolation --editable .
 
 vLLM-Omni offers an official docker image for deployment. These images are built on top of vLLM docker images and available on Docker Hub as [vllm/vllm-omni](https://hub.docker.com/r/vllm/vllm-omni/tags). The version of vLLM-Omni indicates which release of vLLM it is based on.
 
+Nightly CUDA images built from the latest `main` are also available:
+
+- [`vllm/vllm-omni:nightly`](https://hub.docker.com/r/vllm/vllm-omni/tags?name=nightly) is the rolling multi-architecture tag.
+- [`vllm/vllm-omni:nightly-<commit>`](https://hub.docker.com/r/vllm/vllm-omni/tags?name=nightly-) provides commit-pinned multi-architecture tags. The newest 14 are retained.
+
+Use a commit-pinned tag when you need a reproducible nightly environment.
+
 Here's an example deployment command that has been verified on 2 x H100's:
 ```bash
 docker run --runtime nvidia --gpus 2 \

docs/getting_started/installation/npu/npu.inc.md

@@ -9,8 +9,8 @@ The recommended way to use vLLM-Omni on NPU is through the vLLM-Ascend pre-built
 
 ```bash
 # vLLM-Ascend image
-# Atlas A2: quay.io/atlas-ci/vllm-ascend:v0.26.0
-# Atlas A3: quay.io/atlas-ci/vllm-ascend:v0.26.0-a3
+# Atlas A2: quay.io/atlas-ci/vllm-ascend:v0.28.0
+# Atlas A3: quay.io/atlas-ci/vllm-ascend:v0.28.0-a3
 docker run --rm \
     --name vllm-omni-npu \
     --shm-size=1g \
@@ -28,11 +28,11 @@ docker run --rm \
     -v /etc/ascend_install.info:/etc/ascend_install.info \
     -v /root/.cache:/root/.cache \
     -p 8000:8000 \
-    -it quay.io/atlas-ci/vllm-ascend:v0.26.0-a3 bash
+    -it quay.io/atlas-ci/vllm-ascend:v0.28.0-a3 bash
 
 # Inside the container, install vLLM-Omni from source
 cd /vllm-workspace
-git clone -b v0.26.0 https://github.com/vllm-project/vllm-omni.git
+git clone -b v0.28.0 https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
 pip install -v -e . --no-build-isolation
 # or VLLM_OMNI_TARGET_DEVICE=npu pip install -v -e .
@@ -50,21 +50,11 @@ We are keeping [issue #886](https://github.com/vllm-project/vllm-omni/issues/886
 
 # --8<-- [start:installation-main]
 
-You can also build vLLM-Omni from the latest main branch if you want to use the latest features or bug fixes. (But sometimes it will break for a while. You can check [issue #886](https://github.com/vllm-project/vllm-omni/issues/886) for the status of the latest commit of vLLM-Omni main branch on NPU.)
+You can also install vLLM-Omni from the latest main branch if you want to use the latest features or bug fixes. Use the aligned vLLM-Ascend v0.28.0 image described above, and check [issue #886](https://github.com/vllm-project/vllm-omni/issues/886) for the status of the latest vLLM-Omni commit on NPU.
 
 ```bash
-# Pin vLLM and vLLM-Ascend to the v0.26 release line
-git clone -b v0.26.0 https://github.com/vllm-project/vllm.git
-cd vllm
-VLLM_TARGET_DEVICE=empty pip install -v -e .
-cd ..
-
-git clone -b releases/v0.26.0rc https://github.com/vllm-project/vllm-ascend.git
-cd vllm-ascend
-pip install -v -e .
-cd ..
-
-# Install vLLM-Omni from the latest main branch
+# Inside the vLLM-Ascend v0.28.0 container, install vLLM-Omni from main
+cd /vllm-workspace
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
 pip install -v -e . --no-build-isolation
@@ -88,8 +78,8 @@ Supported images as following.
 Here's an example deployment command that has been verified on 4 x NPUs:
 
 ```bash
-# Atlas A2: quay.io/ascend/vllm-omni:v0.26.0
-# Atlas A3: quay.io/ascend/vllm-omni:v0.26.0-a3
+# Atlas A2: quay.io/ascend/vllm-omni:v0.28.0
+# Atlas A3: quay.io/ascend/vllm-omni:v0.28.0-a3
 docker run --rm \
     --name vllm-omni-a3 \
     --shm-size=4g \
@@ -114,7 +104,7 @@ docker run --rm \
     -v /etc/ascend_install.info:/etc/ascend_install.info \
     -v ~/.cache:/root/.cache \
     -p 8091:8091 \
-    -it quay.io/ascend/vllm-omni:v0.26.0-a3 bash
+    -it quay.io/ascend/vllm-omni:v0.28.0-a3 bash
 ```
 
 !!! tip

docs/getting_started/quickstart.md

@@ -91,7 +91,7 @@ if __name__ == "__main__":
 
 For more usages, please refer to [offline inference](../user_guide/examples/offline_inference/qwen2_5_omni.md)
 
-## Online Serving with OpenAI-Completions API
+## Online Serving with the API Server
 
 Text-to-image generation quickstart with vLLM-Omni:
 
@@ -100,20 +100,16 @@ vllm serve Tongyi-MAI/Z-Image-Turbo --omni --port 8091
 ```
 
 ```bash
-curl -s http://localhost:8091/v1/chat/completions \
+curl -s http://localhost:8091/v1/images/generations \
   -H "Content-Type: application/json" \
   -d '{
-    "messages": [
-      {"role": "user", "content": "a cup of coffee on the table"}
-    ],
-    "extra_body": {
-      "height": 1024,
-      "width": 1024,
-      "num_inference_steps": 50,
-      "guidance_scale": 4.0,
-      "seed": 42
-    }
-  }' | jq -r '.choices[0].message.content[0].image_url.url' | cut -d',' -f2 | base64 -d > coffee.png
+    "prompt": "a cup of coffee on the table",
+    "size": "1024x1024",
+    "response_format": "b64_json",
+    "seed": 42
+  }' | jq -r '.data[0].b64_json' | base64 -d > coffee.png
 ```
 
-For more details, please refer to [online serving](../user_guide/examples/online_serving/text_to_image.md).
+See the [API Server guide](../serving/README.md) to choose an endpoint. For
+model-specific details, refer to the [text-to-image online serving
+example](../user_guide/examples/online_serving/text_to_image.md).

docs/mkdocs/hooks/generate_api_readme.py

@@ -22,6 +22,13 @@ CATEGORIES = {
         "name": "Entry Points",
         "description": "Main entry points for vLLM-Omni inference and serving.",
     },
+    "clients": {
+        "name": "Clients",
+        "description": (
+            "Client libraries that talk to a running vLLM-Omni server "
+            "(the full-duplex Realtime client and its per-model session presets)."
+        ),
+    },
     "inputs": {
         "name": "Inputs",
         "description": "Input data structures for multi-modal inputs.",

docs/models/supported_models.md

@@ -18,10 +18,10 @@ th {
 </style>
 
 | Architecture | Models | Example HF Models | NVIDIA GPU | AMD GPU | Ascend NPU | Intel GPU | Recipe |
-|--------------|--------|-------------------|------------|---------|-----|-----------|--------|
+| -------------- | -------- | ------------------- | ------------ | --------- | ----- | ----------- | -------- |
 | `Qwen3OmniMoeForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen3-Omni.md) |
 | `Qwen2_5OmniForConditionalGeneration` | Qwen2.5-Omni | `Qwen/Qwen2.5-Omni-7B`, `Qwen/Qwen2.5-Omni-3B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `MingFlashOmniForConditionalGeneration` + `MingImagePipeline` | Ming-flash-omni-2.0 (omni-speech + imagegen<sup>1</sup>) | `Jonathan1909/Ming-flash-omni-2.0` | ✅︎ |   |   |   | — |
+| `MingFlashOmniForConditionalGeneration` + `MingImagePipeline` | Ming-flash-omni-2.0 (omni-speech + imagegen<sup>1</sup>) | `Jonathan1909/Ming-flash-omni-2.0` | ✅︎ | | | | — |
 | `BagelForConditionalGeneration` | BAGEL (DiT-only) | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `InternVLAA1Pipeline` | InternVLA-A1 | `InternRobotics/InternVLA-A1-3B` | ✅︎ | ✅︎ | | | — |
 | `Gr00tN1d7Pipeline` | GR00T N1.7 | `nvidia/GR00T-N1.7-3B` | ✅︎ | | | | — |
@@ -42,7 +42,7 @@ th {
 | `SanaWmPipeline` | SANA-WM | `BBBBruce/SANA-WM_bidirectional-stage1-diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/SANA-WM.md) |
 | `Wan22S2VPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
-| `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ |   |   | ✅︎ | — |
+| `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ | | | ✅︎ | — |
 | `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | — |
 | `LTX2TwoStagePipeline` | LTX-2 / LTX-2.3 ordinary two-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` + matching Lightricks LoRA and upsampler | ✅︎ | ✅︎ | | | — |
 | `LTX2DistilledOneStagePipeline` | LTX-2 / LTX-2.3 merged-distilled one-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` | ✅︎ | ✅︎ | | | — |
@@ -55,6 +55,7 @@ th {
 | `SanaVideoPipeline` | SANA-Video-2B T2V (native 480p/720p; Diffusers adapter validated at both resolutions) | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers`, `Efficient-Large-Model/SANA-Video_2B_720p_diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/SANA-Video-2B.md) |
 | `SanaImageToVideoPipeline` | SANA-Video-2B I2V (native and Diffusers adapter validated at 480p/720p) | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers`, `Efficient-Large-Model/SANA-Video_2B_720p_diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/SANA-Video-2B.md) |
 | `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | ✅︎ | | | [Published](https://recipes.vllm.ai/MiniMaxAI/MiniMax-H3) |
+| `Magi2Pipeline` | MAGI-2 native Preview-stage T2VA and I2VA (272p/540p) | `sand-ai/MAGI-2-preview` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SandAI/MAGI-2-preview-L20X.md) |
 | `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Pi0Pipeline` | π0 (Pi-Zero) VLA | `lerobot/pi0_base` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/lerobot/Pi0.md) |
 | `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | | — |
@@ -62,13 +63,13 @@ th {
 | `LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `LongCatVideoAvatarPipeline` | LongCat-Video-Avatar-1.5 A2V/AI2V (native single-speaker and multi-speaker AI2V/AVC) | `meituan-longcat/LongCat-Video-Avatar-1.5` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/meituan-longcat/LongCat-Video-Avatar-1.5.md) |
-| `BooguImagePipeline` | Boogu-Image | `Boogu/Boogu-Image-0.1-Base`, `Boogu/Boogu-Image-0.1-Edit` | ✅︎ | | | ✅︎ | — |
+| `BooguImagePipeline` | Boogu-Image | `Boogu/Boogu-Image-0.1-Base`, `Boogu/Boogu-Image-0.1-Edit`, `Boogu/Boogu-Image-0.1-Edit-Turbo` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Boogu/Boogu-Image.md) |
 | `StableDiffusionXLPipeline` | Stable-Diffusion-XL | `stabilityai/stable-diffusion-xl-base-1.0` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | ✅︎ | — |
 | `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | ✅︎ | — |
-| `DotsTTSForConditionalGeneration` | dots.tts | `rednote-hilab/dots.tts-soar` | ✅︎ | | | | — |
+| `DotsTTSForConditionalGeneration` | dots.tts | `dots-studio/dots.tts-soar` | ✅︎ | | | | — |
 | `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | | — |
 | `MammothModa2ForConditionalGeneration` | MammothModa2-Dev (AR-only image understanding) | `bytedance-research/MammothModa2-Dev` | ✅︎ | | | | — |
 | `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
@@ -95,17 +96,17 @@ th {
 | `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | | — |
 | `Flux2Pipeline` | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | ✅︎ | ✅︎ | | | — |
 | `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | ✅︎ | — |
-| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only) | `SenseNova/SenseNova-U1-8B-MoT` | ✅︎ | | | | — |
+| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only), SenseNova-U1.5 | `SenseNova/SenseNova-U1-8B-MoT`, `sensenova/SenseNova-U1.5-8B-MoT` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SenseNova/SenseNova-U1.5.md) |
 | `LancePipeline` | Lance | `bytedance-research/Lance` | ✅︎ | | | | — |
 | `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | | — |
 | `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | | — |
 | `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | | — |
 | `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | | — |
-|`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | ✅︎ | — |
+| `DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | ✅︎ | — |
 | `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | | — |
 | `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `GepardTalkerForConditionalGeneration` | Gepard-1.0 | `nineninesix/gepard-1.0` | ✅︎ | | | ✅︎ | — |
-|`HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | | — |
+| `HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | | — |
 | `HiDreamO1ImagePipeline` | HiDream-O1-Image | `HiDream-ai/HiDream-O1-Image` | ✅︎ | | | | — |
 
 The `Recipe` column links the corresponding deployment recipe. `Published` is

docs/serving/README.md

@@ -0,0 +1,186 @@
+# API Server
+
+vLLM-Omni exposes OpenAI-compatible HTTP APIs plus vLLM-Omni extensions for
+image, audio, video, realtime, and robot-policy workloads. Use this page to
+choose an endpoint; follow the linked reference page for model-specific fields,
+wire protocols, and response details.
+
+Each server instance hosts one model. An endpoint is usable only when the
+loaded model supports that task.
+
+This guide focuses on vLLM-Omni's primary task endpoints. Compatible models
+may also expose standard routes inherited from vLLM, such as
+`POST /v1/completions`; those routes remain model- and task-dependent.
+
+## Start and Verify the Server
+
+Start a model with the shared serving command:
+
+```bash
+vllm serve <model> --omni --port 8091
+```
+
+Some models require additional flags or a deployment configuration. Use the
+model-specific guide when one is provided.
+
+After the server starts, check its health and served model name:
+
+```bash
+export VLLM_OMNI_BASE_URL=http://localhost:8091
+
+curl "$VLLM_OMNI_BASE_URL/health"
+curl "$VLLM_OMNI_BASE_URL/v1/models" | jq .
+```
+
+Use `http://localhost:8091/v1` as the `base_url` for an OpenAI SDK client. If
+the server was started with `--api-key`, include an `Authorization` header with
+`Bearer <api-key>` in requests.
+
+## Choose a Core HTTP Endpoint
+
+Prefer a task-specific endpoint when one matches your workload. Use Chat
+Completions for conversational or heterogeneous omni pipelines, rather than as
+the default wrapper for every generation task.
+
+| Task | Endpoint | Request | Response | Details |
+| ------ | ---------- | --------- | ---------- | --------- |
+| Conversation or multimodal understanding/generation | `POST /v1/chat/completions` | JSON | OpenAI-style JSON or SSE | [Chat Completions](chat_completions_api.md) |
+| Text-to-speech | `POST /v1/audio/speech` | JSON | Audio bytes or SSE | [Speech](speech_api.md) |
+| Sound, music, or ambient audio generation | `POST /v1/audio/generate` | JSON | Audio bytes | [Audio Generation](audio_generate_api.md) |
+| Text-to-image | `POST /v1/images/generations` | JSON | Base64 JSON or an image file | [Image Generation](image_generation_api.md) |
+| Image editing | `POST /v1/images/edits` | Multipart form | Base64 JSON or SSE | [Image Edit](image_edit_api.md) |
+| Video generation (recommended) | `POST /v1/videos` | Multipart form | Asynchronous job JSON | [Videos](videos_api.md) |
+| Video generation (blocking) | `POST /v1/videos/sync` | Multipart form | Video bytes | [Videos](videos_api.md#synchronous-response) |
+
+`POST /v1/videos/sync` is intended for tests and simple scripts. Use the
+asynchronous job API for production requests so clients can poll, download,
+and delete generated videos independently.
+
+## Minimal Requests
+
+The examples below assume a compatible model is already running. TTS voices,
+media inputs, generation controls, and output capabilities vary by model.
+
+=== "Chat"
+
+    ```bash
+    curl "$VLLM_OMNI_BASE_URL/v1/chat/completions" \
+      -H "Content-Type: application/json" \
+      -d '{
+        "messages": [{"role": "user", "content": "Describe vLLM-Omni briefly."}],
+        "modalities": ["text"]
+      }'
+    ```
+
+=== "Speech"
+
+    ```bash
+    curl "$VLLM_OMNI_BASE_URL/v1/audio/speech" \
+      -H "Content-Type: application/json" \
+      -d '{
+        "input": "Hello from vLLM-Omni.",
+        "voice": "vivian"
+      }' \
+      --output speech.wav
+    ```
+
+=== "Audio"
+
+    ```bash
+    curl "$VLLM_OMNI_BASE_URL/v1/audio/generate" \
+      -H "Content-Type: application/json" \
+      -d '{
+        "input": "Ocean waves on a quiet beach",
+        "audio_length": 5.0
+      }' \
+      --output audio.wav
+    ```
+
+=== "Image"
+
+    ```bash
+    curl "$VLLM_OMNI_BASE_URL/v1/images/generations" \
+      -H "Content-Type: application/json" \
+      -d '{
+        "prompt": "A small robot reading beside a window",
+        "size": "1024x1024",
+        "response_format": "file"
+      }' \
+      --output image.png
+    ```
+
+=== "Image edit"
+
+    ```bash
+    curl "$VLLM_OMNI_BASE_URL/v1/images/edits" \
+      -F "image=@input.png" \
+      -F "prompt=Turn the background into a snowy mountain" \
+      | jq -r '.data[0].b64_json' \
+      | base64 --decode > edited.png
+    ```
+
+=== "Video"
+
+    ```bash
+    curl "$VLLM_OMNI_BASE_URL/v1/videos" \
+      -F "prompt=A cinematic tracking shot of a mountain lake at sunrise" \
+      -F "size=1280x720" \
+      -F "seconds=5"
+    ```
+
+    The response contains a video job `id`. Poll and download it with:
+
+    ```bash
+    curl "$VLLM_OMNI_BASE_URL/v1/videos/<video_id>"
+    curl -L "$VLLM_OMNI_BASE_URL/v1/videos/<video_id>/content" --output video.mp4
+    ```
+
+## Choose a Streaming or Realtime Endpoint
+
+These WebSocket APIs have different event schemas and cannot be used
+interchangeably.
+
+| Workload | Endpoint | Interaction model | Details |
+| ---------- | ---------- | ------------------- | --------- |
+| Incremental text input for speech synthesis | `WS /v1/audio/speech/stream` | Send text events and receive audio | [Streaming Text to Speech](speech_api.md#streaming-text-input-websocket) |
+| Live video understanding | `WS /v1/video/chat/stream` | Send video frames and receive text/audio | [Streaming Video Input](video_stream_api.md) |
+| Turn-based realtime audio | `WS /v1/realtime` | Stream one audio input and receive transcript/audio events | [Realtime Audio](realtime_api.md) |
+| Continuous speech-to-speech interaction | `WS /v1/realtime?duplex=1` or `WS /v1/duplex` | Listen and speak concurrently with session control | [Full Duplex](full_duplex_api.md) |
+| Generated video chunks | `WS /v1/realtime/video` | Start a diffusion request and receive fragmented MP4 | [Streaming Video Output](streaming_video_output_api.md) |
+| Robot policy inference | `WS /v1/realtime/robot/openpi` | Send MessagePack observations and receive action arrays | [OpenPI Robot Policy](openpi_api.md) |
+
+All six routes are model- or configuration-dependent. In particular,
+`/v1/realtime` is not full duplex unless the client sets `duplex=1` and the
+deployment explicitly enables duplex sessions. Clients should also verify the
+duplex capability payload because the query-parameter form falls back to the
+ordinary realtime handler when duplex is unavailable.
+
+## Related Endpoints
+
+| Purpose | Endpoints | Reference |
+| --------- | ----------- | ----------- |
+| Discovery and readiness | `GET /health`, `GET /v1/models` | This page |
+| Batched conversations | `POST /v1/chat/completions/batch` | [Batch requests](chat_completions_api.md#batch-requests) |
+| Batched speech | `POST /v1/audio/speech/batch` | [Batch speech generation](speech_api.md#batch-speech-generation) |
+| TTS voice management | `GET/POST /v1/audio/voices`, `DELETE /v1/audio/voices/{name}` | [Voices](speech_api.md#voices-endpoint) |
+| Video job lifecycle | `GET /v1/videos`, `GET/DELETE /v1/videos/{video_id}`, `GET /v1/videos/{video_id}/content` | [Video endpoints](videos_api.md#endpoints) |
+| Release and restore stage memory | `POST /v1/omni/sleep`, `POST /v1/omni/wakeup` | [Sleep Mode](../features/sleep_mode.md) |
+
+## Standalone Experimental Servers
+
+JoyVL provides a stateful interaction orchestrator in front of another model
+server. It is a separate process with its own routes; they are not additional
+paths on every `vllm serve --omni` instance. See [Standalone Experimental
+Servers](standalone_servers.md) before deploying it.
+
+## Compatibility Notes
+
+- A registered path does not mean every model supports it. Unsupported tasks
+  return an OpenAI-style error or a WebSocket `error` event.
+- Use JSON for Chat, Speech, Audio Generation, and Image Generation. Use
+  `multipart/form-data` when uploading images, videos, or other files.
+- Generation can take minutes for large diffusion models. Configure client
+  timeouts accordingly; prefer asynchronous video jobs over long blocking
+  requests.
+- Model-specific parameters are vLLM-Omni extensions. Check the endpoint
+  reference before assuming an OpenAI SDK exposes them directly.

docs/serving/audio_generate_api.md

@@ -4,14 +4,14 @@ vLLM-Omni provides an API for text-to-audio generation using diffusion-based mod
 
 Unlike the [Speech API](speech_api.md) which targets text-to-speech synthesis, the Audio Generate API is designed for general-purpose audio generation from text descriptions (sound effects, music, ambient soundscapes, etc.).
 
-Each server instance runs a single model (specified at startup via `vllm-omni serve <model> --omni`).
+Each server instance runs a single model (specified at startup via `vllm serve <model> --omni`).
 
 ## Quick Start
 
 ### Start the Server
 
 ```bash
-vllm-omni serve stabilityai/stable-audio-open-1.0 \
+vllm serve stabilityai/stable-audio-open-1.0 \
     --host 0.0.0.0 \
     --port 8091 \
     --gpu-memory-utilization 0.9 \
@@ -55,7 +55,7 @@ with open("cat.wav", "wb") as f:
 
 ### Endpoint
 
-```
+```text
 POST /v1/audio/generate
 Content-Type: application/json
 ```
@@ -63,7 +63,7 @@ Content-Type: application/json
 ### Request Parameters
 
 | Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
+| ----------- | ------ | --------- | ------------- |
 | `input` | string | **required** | Text prompt describing the audio to generate |
 | `model` | string | server's model | Model to use (optional, should match server if specified) |
 | `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, opus |
@@ -72,7 +72,7 @@ Content-Type: application/json
 #### Diffusion Parameters
 
 | Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
+| ----------- | ------ | --------- | ------------- |
 | `audio_length` | float | null | Audio duration in seconds (default value is the max ~47s for `stable-audio-open-1.0`) |
 | `audio_start` | float | 0.0 | Audio start time in seconds |
 | `negative_prompt` | string | null | Text describing what to avoid in generation |
@@ -85,7 +85,7 @@ Content-Type: application/json
 Returns binary audio data with the appropriate `Content-Type` header:
 
 | `response_format` | Content-Type |
-|--------------------|--------------|
+| -------------------- | -------------- |
 | `wav` | `audio/wav` |
 | `mp3` | `audio/mpeg` |
 | `flac` | `audio/flac` |
@@ -210,7 +210,7 @@ with open("thunder.wav", "wb") as f:
 Controls how closely the generated audio follows the text prompt.
 
 | Range | Behaviour |
-|-------|-----------|
+| ------- | ----------- |
 | 3 - 5 | More creative / varied output |
 | 7 (default) | Balanced adherence |
 | 10+ | Strict adherence to the prompt |
@@ -220,7 +220,7 @@ Controls how closely the generated audio follows the text prompt.
 Controls the number of denoising steps in the diffusion process.
 
 | Steps | Quality | Speed | Use Case |
-|-------|---------|-------|----------|
+| ------- | --------- | ------- | ---------- |
 | 50 | Good | Fast | Quick previews |
 | 100 | Very Good | Medium | General purpose |
 | 150+ | Excellent | Slow | Final / critical audio |
@@ -240,7 +240,7 @@ Describes characteristics to avoid. Common negative prompts include:
 ## Supported Models
 
 | Model | Description |
-|-------|-------------|
+| ----- | ----------- |
 | `stabilityai/stable-audio-open-1.0` | Open-source audio generation model, up to ~47 seconds, 44.1 kHz stereo |
 
 ## Error Responses
@@ -326,7 +326,7 @@ curl http://localhost:8091/health
 Enable debug logging:
 
 ```bash
-vllm-omni serve stabilityai/stable-audio-open-1.0 \
+vllm serve stabilityai/stable-audio-open-1.0 \
     --host 0.0.0.0 \
     --port 8091 \
     --gpu-memory-utilization 0.9 \

docs/serving/chat_completions_api.md

@@ -1,78 +1,110 @@
 # Chat Completions API
 
-vLLM-Omni supports generating and editing images via the `/v1/chat/completions`
-endpoint using diffusion models. This page explains how to pass generation
-parameters (such as `num_inference_steps`, `height`, `width`) to diffusion
-models through this endpoint.
+Use `/v1/chat/completions` for conversational and multimodal pipelines that
+follow the OpenAI Chat API schema. Depending on the loaded model, a request can
+contain text, image, audio, or video input and can return text, audio, images,
+or other model-specific output.
 
-!!! tip
-    For dedicated endpoints that accept generation parameters as top-level
-    fields, see [Image Generation API](image_generation_api.md) and
-    [Image Edit API](image_edit_api.md).
+Some diffusion pipelines also support image generation and editing through
+Chat Completions. Prefer the dedicated [Image Generation
+API](image_generation_api.md) or [Image Edit API](image_edit_api.md) for
+task-specific clients; their request fields and response contracts are more
+direct.
 
-## Passing Generation Parameters
-
-The `/v1/chat/completions` endpoint follows the OpenAI Chat API schema, which
-does not natively include diffusion-specific fields like `num_inference_steps`
-or `height`. How you pass these extra fields depends on your client.
-
-### curl / Python `requests`
-
-Wrap generation parameters inside an `"extra_body"` key in the JSON body:
+## Basic Request
 
 ```bash
-curl -s http://localhost:8091/v1/chat/completions \
+curl http://localhost:8091/v1/chat/completions \
   -H "Content-Type: application/json" \
   -d '{
     "messages": [
-      {"role": "user", "content": "A beautiful landscape painting"}
+      {"role": "user", "content": "Describe vLLM-Omni briefly."}
     ],
-    "extra_body": {
-      "num_inference_steps": 50,
-      "seed": 42
-    }
+    "modalities": ["text"]
   }'
 ```
 
-### OpenAI Python SDK
+Use `stream: true` for Server-Sent Events. Input media syntax, supported output
+`modalities`, and response choices depend on the model; see the
+[model-specific online serving examples](../user_guide/examples/online_serving/qwen3_omni.md).
 
-Use the `extra_body` **keyword argument**. The SDK automatically merges these
-fields into the top-level request body:
+!!! tip
+    Each server hosts one model. Query `GET /v1/models` for its served model
+    name when your client requires the `model` field.
 
-```python
-response = client.chat.completions.create(
-    model="Qwen/Qwen-Image",
-    messages=[{"role": "user", "content": "A beautiful landscape painting"}],
-    extra_body={
-        "num_inference_steps": 50,
-        "seed": 42,
-    },
-)
+## vLLM Extension Parameters
+
+Send standard and vLLM-specific Chat parameters as top-level fields in a
+direct HTTP request. When using the OpenAI Python SDK, pass vLLM-specific
+fields through the SDK's `extra_body` keyword; the SDK merges them into the
+top-level JSON sent to the server.
+
+=== "curl"
+
+    ```bash
+    curl http://localhost:8091/v1/chat/completions \
+      -H "Content-Type: application/json" \
+      -d '{
+        "messages": [{"role": "user", "content": "Write a short haiku."}],
+        "top_k": 40
+      }'
+    ```
+
+=== "OpenAI Python SDK"
+
+    ```python
+    from openai import OpenAI
+
+    client = OpenAI(base_url="http://localhost:8091/v1", api_key="none")
+
+    response = client.chat.completions.create(
+        model="your-served-model",
+        messages=[{"role": "user", "content": "Write a short haiku."}],
+        extra_body={"top_k": 40},
+    )
+    ```
+
+??? note "Diffusion compatibility"
+
+    Diffusion models exposed through Chat Completions can accept supported
+    fields such as `num_inference_steps`, `seed`, `height`, and `width`. Send
+    them at the top level with direct HTTP, or in the SDK's `extra_body`
+    keyword argument.
+
+    A literal nested `"extra_body"` JSON object is accepted for compatibility,
+    but it is not recommended for new direct HTTP clients. Do not provide the
+    same parameter in multiple locations; the server returns a `400` error for
+    duplicate diffusion parameters. Prefer the dedicated [Image Generation
+    API](image_generation_api.md) or [Image Edit API](image_edit_api.md) when
+    either endpoint matches the task.
+
+## Batch Requests
+
+`POST /v1/chat/completions/batch` accepts the same shared generation fields,
+but `messages` is a list of conversations. The response contains one choice
+per conversation in input order.
+
+```bash
+curl http://localhost:8091/v1/chat/completions/batch \
+  -H "Content-Type: application/json" \
+  -d '{
+    "messages": [
+      [{"role": "user", "content": "Summarize vLLM in one sentence."}],
+      [{"role": "user", "content": "Summarize vLLM-Omni in one sentence."}]
+    ],
+    "max_tokens": 64
+  }'
 ```
 
-!!! note "SDK `extra_body` vs. JSON `extra_body`"
-    These two `extra_body` usages look similar but work differently under the
-    hood. The SDK flattens the dict into the top-level request JSON, while the
-    curl/requests approach sends it as a nested `"extra_body"` key. Both are
-    handled correctly by the server.
-
-!!! note "About the `ignored fields` warning"
-    You may see a log message like:
-
-    ```
-    WARNING: The following fields were present in the request but ignored: {'height', 'width', ...}
-    ```
-
-    This is **harmless**. It is emitted by vLLM's request validation layer
-    because these fields are not part of the standard OpenAI
-    `ChatCompletionRequest` schema. The fields are still stored internally
-    and correctly forwarded to the diffusion pipeline.
+Batch Chat Completions does not support streaming, tools, beam search, or
+`n > 1`.
 
 ## Model-Specific Examples
 
-For complete examples with full request/response details, see the model-specific
-guides:
+For complete examples with model-specific inputs and outputs, see:
 
+- [Qwen3-Omni](../user_guide/examples/online_serving/qwen3_omni.md)
+- [Qwen2.5-Omni](../user_guide/examples/online_serving/qwen2_5_omni.md)
 - [Text-to-Image (Qwen-Image)](../user_guide/examples/online_serving/text_to_image.md)
 - [Image-to-Image (Qwen-Image-Edit, Qwen-Image-Layered)](../user_guide/examples/online_serving/image_to_image.md)
 - [GLM-Image](../user_guide/examples/online_serving/glm_image.md)

docs/serving/full_duplex_api.md

@@ -0,0 +1,122 @@
+# Full-Duplex WebSocket API
+
+vLLM-Omni provides a full-duplex runtime for models that can
+continue receiving speech while producing speech. It adds persistent session
+state, model-specific turn policy, overlap handling, playback acknowledgement,
+and optional session resume.
+
+Full duplex is distinct from the turn-based [Realtime Audio API](realtime_api.md).
+
+This page is the endpoint overview. For the complete wire contract, the
+`vllm_omni.clients.duplex.DuplexClient` Python library, and the per-model
+capability gates, see the [Realtime Duplex API guide](realtime_duplex_api.md).
+
+## Choose an Endpoint
+
+| Endpoint | Protocol | Recommended use |
+| --- | --- | --- |
+| `WS /v1/realtime?duplex=1` | OpenAI Realtime-style event projection | Applications and browser clients |
+| `WS /v1/duplex` | Native vLLM-Omni duplex events | Runtime integration and low-level testing |
+
+Both endpoints use the same duplex engine and require the same model-side
+adapter. Prefer the Realtime projection unless the native lifecycle events are
+specifically required.
+
+## Enable Full Duplex
+
+The route becomes usable only when the deployment configuration explicitly
+sets:
+
+```yaml
+session_mode: duplex
+```
+
+The selected model pipeline must also provide a duplex serving adapter.
+Model-native deployments configure the corresponding engine runtime extension
+and control plane as part of their registered pipeline.
+
+!!! warning
+
+    `WS /v1/duplex` fails with `Duplex API is not available` when duplex is not
+    enabled. By contrast, `/v1/realtime?duplex=1` falls back to the ordinary
+    turn-based realtime handler when no duplex handler exists. Confirm that
+    `session.created.session.capabilities` is present before treating the
+    connection as full duplex.
+
+The current unified-runtime integrations are:
+
+- MiniCPM-o 4.5, using `vllm_omni/deploy/minicpmo_4_5.yaml`
+  (`session_mode: duplex`);
+- PersonaPlex, whose default `vllm_omni/deploy/personaplex.yaml` enables duplex;
+- Nemotron VoiceChat, via its registered duplex plugin package.
+
+JoyVL is a separate HTTP interaction orchestrator and does not use these
+WebSocket endpoints. See [Standalone Experimental Servers](standalone_servers.md).
+
+## MiniCPM-o Quick Start
+
+Start the duplex deployment:
+
+```bash
+vllm serve openbmb/MiniCPM-o-4_5 --omni \
+  --deploy-config vllm_omni/deploy/minicpmo_4_5.yaml \
+  --trust-remote-code \
+  --port 8091
+```
+
+Stream a mono, PCM16, 16 kHz WAV file with the provided client:
+
+```bash
+python examples/online_serving/minicpmo/realtime_duplex_demo.py \
+  --url 'ws://localhost:8091/v1/realtime?duplex=1' \
+  --model openbmb/MiniCPM-o-4_5 \
+  --input-wav input_16k_mono.wav \
+  --ref-audio reference_voice.wav \
+  --output-dir /tmp/minicpmo-duplex
+```
+
+## Realtime Event Lifecycle
+
+A typical `/v1/realtime?duplex=1` session follows this lifecycle:
+
+1. Send `session.update` with the model, modalities, audio formats, and session
+   options.
+2. Wait for `session.created`; inspect `session.capabilities` instead of
+   assuming every model supports the same controls.
+3. Send `input_audio_buffer.append` events while microphone audio arrives.
+4. Send `input_audio_buffer.commit` at a user-turn boundary when required by
+   the model policy.
+5. Consume `response.created`, transcript deltas, `response.audio.delta`, and
+   `response.done` or `response.listen` events.
+6. Send `playback.ack` after audio has been played when the session advertises
+   playback acknowledgement support.
+7. Send `session.close` and wait for `session.closed`.
+
+Unlike the turn-based realtime endpoint, input may continue while a response
+is active. The server can emit `overlap.decision` to describe whether input was
+deferred, treated as a short acknowledgement, or used to interrupt output.
+
+## Capabilities and Model Differences
+
+The `session.created` payload includes capability fields such as
+`supports_barge_in`, `supports_playback_ack`, `supports_multi_session`,
+`supports_session_resume`, and `chunk_period_ms`. Treat this payload as the
+runtime contract.
+
+For example, PersonaPlex supports native overlapping speech but currently
+advertises `supports_barge_in=false`; destructive output interruption and
+model-state rewind have not been validated for that integration. Capacity and
+session-resume behavior also depend on the selected deployment configuration.
+
+## Native Protocol
+
+`WS /v1/duplex` exposes lower-level lifecycle names including
+`session.create`, `input_audio_buffer.append`, `turn.signal`, `playback.ack`,
+and `session.close`. It returns native session, input, response, overlap, and
+error events. This protocol is experimental and may evolve with the runtime;
+applications should use the provided Realtime client where possible.
+
+See the [MiniCPM-o example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/minicpmo),
+[PersonaPlex example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/personaplex),
+and the [full-duplex runtime design](../design/fullduplex.md)
+for model-specific validation and architecture details.

docs/serving/image_edit_api.md

@@ -30,7 +30,6 @@ curl -s -D >(grep -i x-request-id >&2) \
   -F "output_format=png"
 ```
 
-
 **Using OpenAI SDK:**
 
 ```python
@@ -72,7 +71,7 @@ with open("edit_out_http.jpeg", "wb") as f:
 
 ### Endpoint
 
-```
+```text
 POST /v1/images/edits
 Content-Type: multipart/form-data
 ```
@@ -82,7 +81,7 @@ Content-Type: multipart/form-data
 #### OpenAI Standard Parameters
 
 | Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
+| ----------- | ------ | --------- | ------------- |
 | `prompt` | string | **required** | A text description of the desired image |
 | `model` | string | server's model | Model to use (optional, should match server if specified) |
 | `image` | string or array | **required** | The image(s) to edit. |
@@ -98,7 +97,7 @@ Content-Type: multipart/form-data
 #### vllm-omni Extension Parameters
 
 | Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
+| ----------- | ------ | --------- | ------------- |
 | `url` | string or array | None | The image(s) to edit. |
 | `negative_prompt` | string | null | Text describing what to avoid in the image |
 | `num_inference_steps` | integer | model defaults | Number of diffusion steps |
@@ -212,7 +211,6 @@ Use the `ar_delta` events for progressive display of the AR-generated
 recaption. Decode the `b64_json` field from the final `image` event to get the
 edited image.
 
-
 ## Parameter Handling
 
 The API passes parameters directly to the diffusion pipeline without model-specific transformation:
@@ -266,6 +264,7 @@ curl http://localhost:8000/v1/images/edits \
 ### Out of Memory
 
 If you encounter OOM errors:
+
 1. Reduce image size: `"size": "512x512"`
 2. Reduce inference steps: `"num_inference_steps": 25`
 

docs/serving/image_generation_api.md

@@ -45,7 +45,6 @@ curl -o dragon.png -X POST http://localhost:8000/v1/images/generations \
   }'
 ```
 
-
 **Using Python:**
 
 ```python
@@ -123,7 +122,7 @@ response = client.images.generate(
 
 ### Endpoint
 
-```
+```text
 POST /v1/images/generations
 Content-Type: application/json
 ```
@@ -133,7 +132,7 @@ Content-Type: application/json
 #### OpenAI Standard Parameters
 
 | Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
+| ----------- | ------ | --------- | ------------- |
 | `prompt` | string | **required** | Text description of the desired image |
 | `model` | string | server's model | Model to use (optional, should match server if specified) |
 | `n` | integer | 1 | Number of images to generate (1-10) |
@@ -144,7 +143,7 @@ Content-Type: application/json
 #### vllm-omni Extension Parameters
 
 | Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
+| ----------- | ------ | --------- | ------------- |
 | `negative_prompt` | string | null | Text describing what to avoid in the image |
 | `num_inference_steps` | integer | model defaults | Number of diffusion steps |
 | `guidance_scale` | float | model defaults | Classifier-free guidance scale (typically 0.0-20.0) |
@@ -267,6 +266,7 @@ curl http://localhost:8000/v1/images/generations \
 ### Out of Memory
 
 If you encounter OOM errors:
+
 1. Reduce image size: `"size": "512x512"`
 2. Reduce inference steps: `"num_inference_steps": 25`
 3. Generate fewer images: `"n": 1`

docs/serving/openpi_api.md

@@ -0,0 +1,109 @@
+# OpenPI Robot Policy WebSocket API
+
+Use `WS /v1/realtime/robot/openpi` for low-latency robot-policy inference with
+OpenPI-compatible clients. A persistent WebSocket carries observations to the
+loaded policy model and returns action arrays.
+
+Despite the `/realtime` prefix, this endpoint does not use the OpenAI Realtime
+event schema. It uses binary MessagePack frames with NumPy extensions.
+
+## Availability
+
+The endpoint is available only when the loaded diffusion policy configuration
+contains `policy_server_config`. The server sends that model-specific
+configuration to the client as the first binary frame.
+
+For example, DreamZero declares:
+
+```yaml
+policy_server_config:
+  image_resolution: [180, 320]
+  n_external_cameras: 2
+  needs_wrist_camera: true
+  needs_stereo_camera: false
+  needs_session_id: true
+  action_space: joint_position
+```
+
+Without this configuration, the WebSocket returns `Robot policy not available`
+and closes.
+
+## DreamZero Quick Start
+
+Start the bundled DreamZero deployment:
+
+```bash
+vllm serve GEAR-Dreams/DreamZero-DROID --omni --port 8091 \
+  --served-model-name dreamzero-droid \
+  --deploy-config vllm_omni/deploy/dreamzero_tp1_cfg2.yaml \
+  --enforce-eager --disable-log-stats
+```
+
+Install the optional client dependencies, download the sample camera inputs,
+and run the client:
+
+```bash
+pip install openpi-client websockets opencv-python
+
+hf download YangshenDeng/vllm-omni-dreamzero-assets \
+  --repo-type dataset \
+  --local-dir outputs/dreamzero/assets
+
+python examples/online_serving/dreamzero/openpi_client.py \
+  --host 127.0.0.1 \
+  --port 8091 \
+  --video-dir outputs/dreamzero/assets
+```
+
+## Protocol
+
+The connection is request-response after an initial server handshake:
+
+```text
+connect
+  <- msgpack(policy_server_config)
+
+infer
+  -> msgpack({"endpoint": "infer", "session_id": "...", ...observation})
+  <- msgpack(ndarray | dict[str, ndarray])
+
+reset
+  -> msgpack({"endpoint": "reset"})
+  <- msgpack({"status": "reset successful"})
+```
+
+If `endpoint` is omitted, the server treats the message as `infer`. Each
+inference message produces one action response after the engine request
+completes; action tokens or intermediate tensors are not streamed.
+
+NumPy arrays use the marker format implemented by `openpi-client`. The server
+also accepts the legacy vLLM NumPy marker representation on input. JSON text
+frames are not observation messages.
+
+## Sessions and Reset
+
+- The API layer's current-session and first-call counters are scoped to each
+  WebSocket connection.
+- `session_id` identifies model-side state across observations on that
+  connection. If omitted, it defaults to `default`.
+- The first inference for a session is sent to the model with `reset=true`.
+- Changing `session_id`, or sending the `reset` command, causes the next
+  inference to start with `reset=true`.
+- The policy pipeline owns observation transforms and persistent model state,
+  normally keyed by `session_id`; the API layer forwards the raw observation
+  dictionary.
+
+## Limits and Errors
+
+- Maximum inbound payload size is 64 MiB.
+- The server closes an idle connection after 30 seconds.
+- Invalid binary input returns a MessagePack
+  `{"type":"error","message":"Invalid request payload"}` response.
+- Inference failures return a generic `Internal inference error` without
+  exposing an internal traceback.
+- Unsupported NumPy object, structured, and complex dtypes are rejected.
+
+Observation keys, camera layout, state tensors, and action shapes are defined
+by the loaded policy rather than by this transport. Read the handshake before
+constructing observations. See the [DreamZero example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/dreamzero)
+for a complete OpenPI client and DROID simulation loop.

docs/serving/realtime_api.md

@@ -0,0 +1,82 @@
+# Realtime Audio WebSocket API
+
+Use `WS /v1/realtime` to stream PCM audio into a compatible omni model and
+receive incremental transcription and synthesized response audio. This is a
+turn-based streaming API: it starts one generation over an audio stream and
+finishes after the client closes that input stream.
+
+For a model that listens while it is already speaking, use the
+[Full-Duplex API](full_duplex_api.md) instead.
+
+## Quick Start
+
+Start a compatible model. The currently documented example uses Qwen3-Omni:
+
+```bash
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
+```
+
+Run the provided client with a mono, 16-bit PCM, 16 kHz WAV file:
+
+```bash
+python examples/online_serving/qwen3_omni/openai_realtime_client.py \
+  --url ws://localhost:8091/v1/realtime \
+  --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
+  --input-wav input_16k_mono.wav \
+  --output-wav response.wav \
+  --output-text response.txt
+```
+
+The client dependency is `websockets`.
+
+## Protocol
+
+Messages in both directions are JSON text frames. Audio carried inside an
+event is base64-encoded raw PCM16.
+
+| Direction | Event | Purpose |
+| ----------- | ------- | --------- |
+| Server to client | `session.created` | Confirms that the WebSocket connection is ready |
+| Client to server | `session.update` | Selects and validates the served model |
+| Client to server | `input_audio_buffer.commit` with `final: false` | Starts generation over the incoming stream |
+| Client to server | `input_audio_buffer.append` | Appends base64 PCM16 audio |
+| Client to server | `input_audio_buffer.commit` with `final: true` | Marks the end of input |
+| Server to client | `transcription.delta` | Carries incremental response text |
+| Server to client | `transcription.done` | Carries final text and token usage |
+| Server to client | `response.audio.delta` | Carries incremental PCM16 response audio |
+| Server to client | `response.audio.done` | Marks the end of response audio |
+| Server to client | `error` | Reports an invalid event, model, or audio payload |
+
+A minimal client sends events in this order:
+
+```json
+{"type":"session.update","model":"Qwen/Qwen3-Omni-30B-A3B-Instruct"}
+{"type":"input_audio_buffer.commit","final":false}
+{"type":"input_audio_buffer.append","audio":"<base64-pcm16>"}
+{"type":"input_audio_buffer.commit","final":true}
+```
+
+The initial non-final commit intentionally starts generation before all audio
+has arrived. Continue sending `input_audio_buffer.append` events while the
+engine is consuming the stream.
+
+## Audio Handling
+
+- Input is mono PCM16 at 16 kHz for the Qwen3-Omni example.
+- `response.audio.delta.audio` contains base64-encoded PCM16 bytes.
+- Read `sample_rate_hz` from each audio event instead of assuming an output
+  rate. Qwen3-Omni output is typically 24 kHz.
+- Concatenate audio deltas in receive order to construct the output waveform.
+- Set `input_audio_buffer.commit.final` to `true` after the last input chunk so
+  the server can terminate the streaming request.
+
+## Availability and Limitations
+
+The path is registered on the unified API server, but it is usable only when
+the loaded pipeline implements realtime audio input and produces compatible
+audio output. Unsupported deployments return an `error` event. The endpoint
+does not provide duplex session resume, playback acknowledgement, overlap
+policy, or barge-in controls.
+
+See the [Qwen3-Omni online serving example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/qwen3_omni)
+for concurrency options, chunk pacing, and per-delta audio debugging.

docs/serving/speech_api.md

@@ -2,7 +2,7 @@
 
 vLLM-Omni provides an OpenAI-compatible API for text-to-speech (TTS) generation. Supported TTS models include:
 
-- **Qwen3-TTS** (`Qwen/Qwen3-TTS-12Hz-*`) -- Qwen3-based TTS with CustomVoice, VoiceDesign, and Base (voice cloning) task types. Output: 24 kHz.
+- **Qwen3-TTS** (`Qwen/Qwen3-TTS-12Hz-*`) -- Qwen3-based TTS with CustomVoice, VoiceDesign, and Base (voice cloning) task types. Native output: 24 kHz; API output can be resampled to 8 kHz.
 - **Fish Speech S2 Pro** (`fishaudio/s2-pro`) -- Dual-AR TTS with DAC codec. Supports text-to-speech and voice cloning via reference audio. Output: 44.1 kHz.
 - **Voxtral TTS** (`mistralai/Voxtral-4B-TTS-2603`) -- AR + FlowMatching TTS with preset voices. Output: 24 kHz.
 - **CosyVoice3** (`FunAudioLLM/Fun-CosyVoice3-0.5B-2512`) -- 2-stage talker + flow-matching code2wav. Voice cloning via `ref_audio` + `ref_text` (no presets). Output: 24 kHz.
@@ -95,7 +95,7 @@ response.stream_to_file("output.wav")
 
 ### Endpoint
 
-```
+```text
 POST /v1/audio/speech
 Content-Type: application/json
 ```
@@ -105,7 +105,7 @@ Content-Type: application/json
 #### OpenAI Standard Parameters
 
 | Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
+| ----------- | ------ | --------- | ------------- |
 | `input` | string | **required** | The text to synthesize into speech |
 | `model` | string | server's model | Model to use (optional, should match server if specified) |
 | `voice` | string | "vivian" | Speaker name (e.g., vivian, ryan, aiden) |
@@ -115,8 +115,9 @@ Content-Type: application/json
 #### vLLM-Omni Extension Parameters
 
 | Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `task_type` | string | "CustomVoice" | TTS task type: CustomVoice, VoiceDesign, or Base |
+| ----------- | ------ | --------- | ------------- |
+| `task_type` | string | null (inferred) | TTS task type: CustomVoice, VoiceDesign, or Base. For Qwen3-TTS, `ref_audio` or `ref_text` infers Base when this field is omitted, while an uploaded/precomputed voice always selects Base. Other omitted values select CustomVoice; VoiceDesign must be specified explicitly. |
+| `sample_rate` | integer | model native | Target output sample rate. Qwen3-TTS supports 8000 or 24000 Hz; the model remains native 24 kHz internally and is resampled before encoding. |
 | `language` | string | "Auto" | Language (see supported languages below) |
 | `instructions` | string | "" | Voice style/emotion instructions |
 | `max_new_tokens` | integer | 2048 | Maximum tokens to generate |
@@ -130,7 +131,7 @@ Content-Type: application/json
 #### Voice Clone Parameters (Base task)
 
 | Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
+| ----------- | ------ | --------- | ------------- |
 | `ref_audio` | string | null | Reference audio (HTTP URL, base64 data URL, or `file://` URI with `--allowed-local-media-path`). Local files fold `mtime_ns` and `size` into cache keys to automatically reload on-disk edits; HTTP URLs and base64 URIs remain cached by string locator. |
 | `ref_text` | string | null | Transcript of reference audio |
 | `x_vector_only_mode` | bool | null | Use speaker embedding only (no ICL) |
@@ -204,7 +205,7 @@ The `usage` object on `speech.audio.done` is the same shape returned per item by
 
 ### Voices Endpoint
 
-```
+```text
 GET /v1/audio/voices
 ```
 
@@ -229,7 +230,7 @@ Lists available voices for the loaded model.
 
 `uploaded_voices` is always present (empty list when no custom voices have been uploaded). Fields `ref_text` and `speaker_description` are omitted per-entry when not provided at upload time.
 
-```
+```text
 POST /v1/audio/voices
 Content-Type: multipart/form-data
 ```
@@ -239,7 +240,7 @@ Upload a new voice sample for voice cloning in Base task TTS requests.
 **Form Parameters:**
 
 | Parameter | Type | Required | Description |
-|-----------|------|----------|-------------|
+| ----------- | ------ | ---------- | ------------- |
 | `audio_sample` | file | Yes | Audio file (max 10MB, supported formats: wav, mp3, flac, ogg, aac, webm, mp4) |
 | `consent` | string | Yes | Consent recording ID |
 | `name` | string | Yes | Name for the new voice |
@@ -288,7 +289,7 @@ The `/v1/audio/speech/stream` WebSocket endpoint accepts text incrementally and
 Client -> Server:
 
 | Message | Description |
-|---------|-------------|
+| --------- | ------------- |
 | `{"type": "session.config", ...}` | Session configuration (first message; may be resent between utterances to change it) |
 | `{"type": "input.text", "text": "..."}` | Text chunk |
 | `{"type": "input.done"}` | End of utterance: flushes the buffer and keeps the connection open |
@@ -297,7 +298,7 @@ Client -> Server:
 Server -> Client:
 
 | Message | Description |
-|---------|-------------|
+| --------- | ------------- |
 | `{"type": "audio.start", "utterance_index": 0, "sentence_index": 0, "sentence_text": "...", "format": "pcm", "sample_rate": 24000}` | Audio generation starting for the buffered input |
 | Binary frame | Raw audio bytes (one or more PCM chunks when `stream_audio=true`) |
 | `{"type": "audio.done", "utterance_index": 0, "sentence_index": 0, "total_bytes": 96000, "error": false}` | Audio complete for the buffered input |
@@ -311,17 +312,17 @@ text, emits `session.done`, and then waits on the same connection for the next
 utterance, so a client that speaks repeatedly (for example one driven by an
 upstream LLM) pays the WebSocket handshake once instead of once per utterance.
 
-* The session config is sticky. Send `input.text` again straight after
+- The session config is sticky. Send `input.text` again straight after
   `session.done` to reuse it, or send another `session.config` first to change
   voice, format, or reference audio. A `session.config` sent while text is
   still buffered is rejected so no pending input is silently dropped.
-* An utterance is the flush unit, not a linguistic one: it is whatever text was
+- An utterance is the flush unit, not a linguistic one: it is whatever text was
   buffered when `input.done` arrived, of any length, synthesized as one request.
   `utterance_index` counts those flushes across the connection, so it tells you
   which `input.done` a frame belongs to. `sentence_index` counts within one
   flush and so pairs with `total_sentences`, which means every utterance reports
   `sentence_index: 0` of `total_sentences: 1` (or `0` for an empty buffer).
-* End the connection with `session.close`, or by closing the socket. An idle
+- End the connection with `session.close`, or by closing the socket. An idle
   connection is still closed after the server's idle timeout, which now also
   applies to the gap between utterances.
 
@@ -330,10 +331,9 @@ upstream LLM) pays the WebSocket handshake once instead of once per utterance.
 All REST API parameters are supported, plus:
 
 | Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
+| ----------- | ------ | --------- | ------------- |
 | `stream_audio` | bool | false | Stream one or more PCM chunks for the buffered input over WebSocket |
 
-
 ```bash
 DELETE /v1/audio/voices/{name}
 ```
@@ -343,7 +343,7 @@ Delete an uploaded voice sample.
 **Path Parameters:**
 
 | Parameter | Type | Required | Description |
-|-----------|------|----------|-------------|
+| ----------- | ------ | ---------- | ------------- |
 | `name` | string | Yes | Name of the voice to delete |
 
 **Response Example:**
@@ -433,6 +433,7 @@ curl -X POST http://localhost:8091/v1/audio/speech \
 ### Upload Voice
 
 Upload voice (speaker embedding only):
+
 ```bash
 curl -X POST http://localhost:8091/v1/audio/voices \
   -F "audio_sample=@/path/to/voice_sample.wav" \
@@ -441,6 +442,7 @@ curl -X POST http://localhost:8091/v1/audio/voices \
 ```
 
 Upload voice with transcript (in-context cloning, higher quality):
+
 ```bash
 curl -X POST http://localhost:8091/v1/audio/voices \
   -F "audio_sample=@/path/to/voice_sample.wav" \
@@ -450,6 +452,7 @@ curl -X POST http://localhost:8091/v1/audio/voices \
 ```
 
 ### Use Uploaded Voice
+
 ```bash
 curl -X POST http://localhost:8091/v1/audio/speech \
     -H "Content-Type: application/json" \
@@ -459,6 +462,11 @@ curl -X POST http://localhost:8091/v1/audio/speech \
     }' --output cloned.wav
 ```
 
+For Qwen3-TTS, an uploaded voice is a Base voice-cloning input. The server
+infers `task_type="Base"` when `voice` names an uploaded entry, so the request
+must be sent to a Base checkpoint. Built-in presets such as `vivian` and
+`ryan` remain CustomVoice speakers and require a CustomVoice checkpoint.
+
 ### Voice Storage & Caching
 
 Uploaded voices are persisted to disk as a single `.safetensors` file per voice
@@ -478,6 +486,9 @@ once.
 ### Precomputed Custom Voices
 
 Qwen3-TTS Base and VoxCPM2 can load offline-precomputed voices at startup.
+Qwen3-TTS precomputed voices follow the same Base task and checkpoint-matching
+rules as uploaded voices.
+
 Generate a directory containing `custom_voice_manifest.json` plus one
 `.safetensors` file per voice, then set the pipeline-wide deploy config field:
 
@@ -516,7 +527,7 @@ by `GET /v1/audio/voices`. Valid precomputed voices can be used in
 **Configuration (environment variables):**
 
 | Variable | Default | Description |
-|----------|---------|-------------|
+| ---------- | --------- | ------------- |
 | `SPEAKER_SAMPLES_DIR` | `~/.cache/vllm-omni/speakers` | Directory for persisted uploaded speakers (`.safetensors` files). |
 | `SPEAKER_MAX_UPLOADED` | `1000` | Maximum number of uploaded speakers kept on disk. Upload requests past the cap return 400. |
 
@@ -528,7 +539,7 @@ The batch endpoint synthesizes multiple texts in a single request, returning all
 
 ### Endpoint
 
-```
+```text
 POST /v1/audio/speech/batch
 Content-Type: application/json
 ```
@@ -536,7 +547,7 @@ Content-Type: application/json
 ### Request Parameters
 
 | Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
+| ----------- | ------ | --------- | ------------- |
 | `items` | array | **required** | List of items to synthesize (1–32) |
 | `model` | string | server's model | Model to use |
 | `voice` | string | null | Default voice for all items |
@@ -553,7 +564,7 @@ Content-Type: application/json
 Each item in the `items` array requires only `input` (the text). All other fields are optional and override the batch-level defaults when set:
 
 | Field | Type | Description |
-|-------|------|-------------|
+| ------- | ------ | ------------- |
 | `input` | string | **required** — text to synthesize |
 | `voice` | string | Override voice for this item |
 | `response_format` | string | Override format for this item |
@@ -685,7 +696,7 @@ for result in response.json()["results"]:
 ### Configuration
 
 | Parameter | Source | Default | Description |
-|-----------|--------|---------|-------------|
+| ----------- | -------- | --------- | ------------- |
 | `tts_batch_max_items` | engine kwarg | 32 | Maximum number of items per batch request |
 
 All items are fanned out to `generate()` concurrently. The engine's stage worker automatically batches them up to the configured `max_num_seqs` and queues the rest — no client-side throttling needed.
@@ -708,17 +719,17 @@ The bundled config also sets `initial_codec_chunk_frames: 1`. This emits only th
 ### Qwen3-TTS
 
 | Model | Task Type | Description |
-|-------|-----------|-------------|
+| ------- | ----------- | ------------- |
 | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | CustomVoice | Predefined speaker voices with optional style control |
 | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | VoiceDesign | Natural language voice style description |
-| `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | Base | Voice cloning from reference audio |
+| `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | Base | Voice cloning from reference audio or an uploaded/precomputed voice |
 | `Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice` | CustomVoice | Smaller/faster variant |
-| `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | Base | Smaller/faster variant for voice cloning |
+| `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | Base | Smaller/faster Base variant for voice cloning, including uploaded/precomputed voices |
 
 ### Fish Speech S2 Pro
 
 | Model | Description |
-|-------|-------------|
+| ------- | ------------- |
 | `fishaudio/s2-pro` | 4B dual-AR TTS with DAC codec (44.1 kHz). Supports text-to-speech and voice cloning. |
 
 Fish Speech uses `ref_audio` and `ref_text` for voice cloning (no `task_type` needed). The `voice` field should be set to `"default"`. See the [Fish Speech section of the online TTS hub](../user_guide/examples/online_serving/text_to_speech.md#fish-speech-s2-pro) for details.
@@ -726,31 +737,31 @@ Fish Speech uses `ref_audio` and `ref_text` for voice cloning (no `task_type` ne
 ### Voxtral TTS
 
 | Model | Description |
-|-------|-------------|
+| ------- | ------------- |
 | `mistralai/Voxtral-4B-TTS-2603` | 3B AR + FlowMatching TTS. Supports text-to-speech with preset voices. |
 
 ### CosyVoice3
 
 | Model | Description |
-|-------|-------------|
+| ------- | ------------- |
 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | Voice cloning from `ref_audio` + `ref_text`. No built-in voice presets — upload a voice or pass `ref_audio`/`ref_text` per request. |
 
 ### OmniVoice
 
 | Model | Description |
-|-------|-------------|
+| ------- | ------------- |
 | `k2-fsa/OmniVoice` | Pure-diffusion TTS. Supports voice cloning via `ref_audio` (with optional `ref_text`); no built-in voice presets. |
 
 ### VoxCPM2
 
 | Model | Description |
-|-------|-------------|
+| ------- | ------------- |
 | `openbmb/VoxCPM2` | TTS + voice cloning with built-in speaker presets and uploaded-voice support. Accepts `voice` (preset or uploaded) or `ref_audio` + optional `ref_text`. |
 
 ### MOSS-TTS-Nano
 
 | Model | Description |
-|-------|-------------|
+| ------- | ------------- |
 | `OpenMOSS-Team/MOSS-TTS-Nano` | Voice cloning only. Requires `ref_audio` (or an uploaded `voice`); no built-in voice presets. `ref_text` is accepted but ignored — upstream's `voice_clone` mode does not consume a transcript. |
 
 ## Error Responses
@@ -770,6 +781,19 @@ Invalid parameters:
 }
 ```
 
+Qwen3-TTS task/checkpoint mismatch:
+
+```json
+{
+    "error": {
+        "message": "Qwen3-TTS CustomVoice checkpoint does not support task_type='Base'. Use task_type='CustomVoice' or load the matching Base checkpoint.",
+        "type": "BadRequestError",
+        "param": null,
+        "code": 400
+    }
+}
+```
+
 ### 404 Not Found
 
 Model not found:
@@ -790,10 +814,15 @@ Model not found:
 ### "TTS model did not produce audio output"
 
 Ensure you're using the correct model variant for your task type:
+
 - CustomVoice task → CustomVoice model
 - VoiceDesign task → VoiceDesign model
 - Base task → Base model
 
+Uploaded and precomputed Qwen3-TTS voices select the Base task, even when the
+client supplies a different `task_type`; serve them with a Base checkpoint.
+Built-in Qwen3-TTS presets remain CustomVoice voices.
+
 ### Server Not Running
 
 ```bash
@@ -804,6 +833,7 @@ curl http://localhost:8091/v1/audio/voices
 ### Out of Memory
 
 If you encounter OOM errors:
+
 1. Use smaller model variant: `Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice`
 2. Reduce `--gpu-memory-utilization`
 
@@ -822,7 +852,7 @@ final-output drain to a condition-variable wakeup at the same time.
 **Configuration (environment variables):**
 
 | Variable | Default | Description |
-|----------|---------|-------------|
+| --- | --- | --- |
 | `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `0` (off) | Switches the orchestration loop and the final-output drain from the legacy 1 ms poll to event-driven wakeups. Enabled by `1`, `true`, `yes`, or `on`, matched case-insensitively after surrounding whitespace is stripped; any other value leaves it off. |
 
 Set it on the process that runs the orchestrator (stage 0 of an omni

docs/serving/standalone_servers.md

@@ -0,0 +1,67 @@
+# Standalone Experimental Servers
+
+Most vLLM-Omni endpoints run on the unified server started with
+`vllm serve <model> --omni`. The experimental package also contains one
+separate user-facing server process; its routes are not added to the unified
+API server.
+
+| Server | Transport | Role |
+| --- | --- | --- |
+| JoyVL interaction server | HTTP | Adds state, memory, proactive decisions, and delegation in front of a separate OpenAI-compatible model backend |
+
+PersonaPlex no longer ships a standalone compatibility server: it is served
+through the unified duplex path (`vllm_omni/deploy/personaplex.yaml`) over
+[`WS /v1/realtime?duplex=1`](full_duplex_api.md); see the
+[PersonaPlex example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/personaplex)
+for its clients.
+
+## JoyVL Interaction Server
+
+JoyVL is an orchestration layer, not a model-serving engine. It calls a
+separate OpenAI-compatible backend and adds per-session frame history, memory,
+persona policy, and optional delegation.
+
+Start the model backend and orchestrator separately:
+
+```bash
+vllm serve jdopensource/JoyAI-VL-Interaction-Preview \
+  --served-model-name JoyAI-VL-Interaction-Preview \
+  --port 8092 \
+  --max-model-len 131072 \
+  --enable-prefix-caching \
+  --limit-mm-per-prompt '{"image":256,"video":1}'
+
+python -m vllm_omni.experimental.fullduplex.joyvl.serving.server \
+  --port 8091 \
+  --main-backend-url http://127.0.0.1:8092/v1 \
+  --main-model JoyAI-VL-Interaction-Preview
+```
+
+| Method and route | Purpose |
+| ------------------ | --------- |
+| `GET /health` | Readiness check |
+| `GET /v1/models` | Reports the configured interaction model |
+| `POST /v1/chat/completions` | Processes one frame or interaction tick |
+| `POST /reset` | Resets a session |
+| `POST /v1/streaming/reset` | Alias for `/reset` |
+| `POST /v1/streaming/persona` | Changes the session persona |
+
+Send one multimodal Chat Completions request per video frame, normally around
+one frame per second, and identify the session with `x-session-id`. Responses
+include an `interaction` block whose action is `silence`, `response`, or
+`delegate`.
+
+```bash
+curl http://127.0.0.1:8091/v1/chat/completions \
+  -H 'content-type: application/json' \
+  -H 'x-session-id: demo' \
+  -d '{
+    "messages": [{"role": "user", "content": [
+      {"type": "text", "text": "Alert me if a fire appears"},
+      {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,..."}}
+    ]}]
+  }'
+```
+
+See the [JoyVL recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/JD/JoyAI-VL-Interaction.md)
+for memory, personas, delegation backends, and the browser UI.

docs/serving/streaming_video_output_api.md

@@ -0,0 +1,92 @@
+# Streaming Video Output WebSocket API
+
+Use `WS /v1/realtime/video` to receive fragmented MP4 chunks while a compatible
+diffusion video model is generating. This endpoint streams generated video
+output; it is unrelated to [`WS /v1/video/chat/stream`](video_stream_api.md),
+which streams live video input into a conversational model.
+
+## Quick Start
+
+Start a model that supports step execution and enable diffusion streaming
+output:
+
+```bash
+vllm serve BestWishYsh/Helios-Distilled --omni \
+  --diffusion-streaming-output \
+  --port 8091
+```
+
+Install the client dependency and run the provided client:
+
+```bash
+pip install av websockets
+
+python examples/online_serving/streaming_video_generation/streaming_video_client.py \
+  --host 127.0.0.1 \
+  --port 8091 \
+  --model BestWishYsh/Helios-Distilled \
+  --prompt "A serene lakeside sunrise with mist over the water" \
+  --output output.mp4
+```
+
+## Protocol
+
+Control events are JSON text frames. Each `video.chunk_metadata` event is
+immediately followed by one binary WebSocket frame containing fragmented MP4
+bytes.
+
+| Direction | Event or frame | Purpose |
+| ----------- | ---------------- | --------- |
+| Client to server | `session.start` | Starts generation with a prompt and video parameters |
+| Server to client | `video.start` | Confirms the request ID, format, and accepted configuration |
+| Server to client | `video.chunk_metadata` | Describes the next binary media frame |
+| Server to client | Binary frame | Carries one fragmented MP4 chunk |
+| Client to server | `session.interaction` | Queues an optional prompt update during generation |
+| Server to client | `session.interaction.queued` | Confirms that an interaction was queued |
+| Client to server | `session.ping` | Refreshes the stall clock |
+| Server to client | `session.pong` | Acknowledges a ping |
+| Client to server | `session.stop` | Cancels the active generation |
+| Server to client | `session.done` | Reports completion or cancellation and the chunk count |
+| Server to client | `error` | Reports validation, generation, or timeout failures |
+
+Start a minimal session with:
+
+```json
+{
+  "type": "session.start",
+  "model": "BestWishYsh/Helios-Distilled",
+  "prompt": "A serene lakeside sunrise",
+  "format": "m4s",
+  "width": 640,
+  "height": 384,
+  "fps": 16,
+  "num_frames": 99
+}
+```
+
+Only `m4s` output is currently accepted. Concatenate the binary frames in
+receive order, then remux the fragmented stream if the target player requires
+a progressive MP4 file.
+
+## Mid-Generation Interaction
+
+Models that implement interaction updates can change the active prompt:
+
+```json
+{
+  "type": "session.interaction",
+  "interaction": {
+    "event_id": "sunlight-update",
+    "event": {"prompt": "Sunlight breaks through the mist"},
+    "transition_chunks": 2
+  }
+}
+```
+
+This feature is pipeline-dependent. An enabled endpoint does not imply that
+the loaded model supports midway prompt updates.
+
+The default start timeout is 10 seconds and the server reports a stall after
+about 60 seconds without generation progress or a `session.ping`. See the
+[streaming video generation example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/streaming_video_generation)
+for prompt-update scheduling and browser playback.

docs/serving/video_stream_api.md

@@ -36,7 +36,7 @@ WebSocket /v1/video/chat/stream
 ### Protocol
 
 | Direction | Type | Required fields | Description |
-|-----------|------|-----------------|-------------|
+| ----------- | ------ | ----------------- | ------------- |
 | Client -> Server | `session.config` | none | First message. Configures output modalities, frame sampling, EVS, and prompts. |
 | Client -> Server | `video.frame` | `data` | Base64 JPEG/PNG frame. |
 | Client -> Server | `audio.chunk` | `data` | Base64 PCM16 16 kHz mono audio bytes. |
@@ -53,7 +53,7 @@ WebSocket /v1/video/chat/stream
 ### `session.config` Fields
 
 | Field | Type | Default | Description |
-|-------|------|---------|-------------|
+| ------- | ------ | --------- | ------------- |
 | `model` | string or null | null | Optional model name. Usually omitted because the server hosts one model. |
 | `modalities` | list[string] | `["text", "audio"]` | Output modalities. Use `["text"]`, `["audio"]`, or both. |
 | `num_frames` | integer, 1-128 | `4` | Number of buffered frames sampled for each query. |
@@ -69,7 +69,7 @@ WebSocket /v1/video/chat/stream
 The server accepts these legacy field names and rewrites them before validation. New clients should send the canonical names above.
 
 | Legacy field | Canonical field |
-|--------------|-----------------|
+| -------------- | ----------------- |
 | `num_sample_frames` | `num_frames` |
 | `evs_enabled` | `enable_frame_filter` |
 | `evs_threshold` | `frame_filter_threshold` |
@@ -77,7 +77,7 @@ The server accepts these legacy field names and rewrites them before validation.
 ### Environment Variables
 
 | Variable | Values | Default | Description |
-|----------|--------|---------|-------------|
+| -------- | ------ | ------- | ----------- |
 | `VLLM_VIDEO_ASYNC_CHUNK` | `on`, `off` | `on` | Wire-level streaming switch. `off` buffers server-side deltas and emits coalesced outputs at the end of a query. |
 | `VLLM_VIDEO_AUDIO_DELTA_MODE` | `fast`, `slow` | `fast` | Audio delta extraction strategy. `fast` emits only newly produced chunks; `slow` recomputes from accumulated audio and exists for A/B verification. |
 

docs/serving/videos_api.md

@@ -41,7 +41,7 @@ curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4
 ### Endpoints
 
 | Endpoint | Method | Description |
-|----------|--------|-------------|
+| ---------- | -------- | ------------- |
 | `/v1/videos` | `POST` | Create an asynchronous video generation job |
 | `/v1/videos/sync` | `POST` | Generate a video synchronously and return raw video bytes |
 | `/v1/videos/{video_id}` | `GET` | Retrieve job status and metadata |
@@ -56,7 +56,7 @@ curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4
 #### OpenAI-style fields
 
 | Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
+| ----------- | ------ | --------- | ------------- |
 | `prompt` | string | **required** | Text prompt for video generation |
 | `model` | string | server's model | Optional model name |
 | `seconds` | string | null | Requested clip duration in seconds |
@@ -66,7 +66,7 @@ curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4
 #### vLLM-Omni extension fields
 
 | Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
+| ----------- | ------ | --------- | ------------- |
 | `input_reference` | file | null | Uploaded reference image or video for image-to-video/video-to-video requests |
 | `image_reference` | string | null | JSON-encoded reference image payload; do not combine with `input_reference` or `video_reference` |
 | `video_reference` | string | null | JSON-encoded reference video payload; do not combine with `input_reference` or `image_reference` |
@@ -203,7 +203,6 @@ curl -s http://localhost:8091/v1/videos \
   -F "fps=16"
 ```
 
-
 ### Synchronous Generation
 
 ```bash

docs/usage/faq.md

@@ -18,4 +18,4 @@ A: AWQ is available through the inherited vLLM quantization registry, and vLLM-O
 
 > Q: Does vLLM-Omni support multimodal streaming input and output?
 
-A: Yes, for specific models and serving paths. Current capabilities include [streaming video input](../serving/video_stream_api.md), [streaming diffusion output](../user_guide/diffusion/execution_modes.md#streaming-output), and experimental full-duplex real-time audio described in the [project overview](../README.md). Check the model and API documentation because input and output support is model-specific.
+A: Yes, for specific models and serving paths. Current capabilities include [streaming video input](../serving/video_stream_api.md), [streaming diffusion output](../user_guide/diffusion/execution_modes.md#streaming-output), and [full-duplex realtime audio](../serving/realtime_duplex_api.md). Check the model and API documentation because input and output support is model-specific.

docs/user_guide/diffusion/attention_backends.md

@@ -20,7 +20,7 @@ For the internal selector, registry, and platform contract, see
 | Install and use SageAttention 2.2 or SageAttention3 | [SageAttention](attention_backends/sage.md) |
 | Match training or rollout kernels loaded from Hugging Face | [Hugging Face Hub Backends](attention_backends/huggingface_hub.md) |
 | Use block-sparse video attention on Ascend NPU | [RainFusion](attention_backends/rainfusion.md) |
-| Use FastVideo VSA with FastWan2.2-TI2V-5B on CUDA | [FastVideo VSA](attention_backends/fastvideo_vsa.md) |
+| Use FastVideo VSA with FastWan2.2-TI2V-5B or FastH3 MiniMax-H3 on CUDA | [FastVideo VSA](attention_backends/fastvideo_vsa.md) |
 
 ## Backend options
 
@@ -36,7 +36,7 @@ For the internal selector, registry, and platform contract, see
 | `FLASH_ATTN_HUB` | Hub kernel | FlashAttention 2 from Hugging Face `kernels` | [Hugging Face Hub Backends](attention_backends/huggingface_hub.md) |
 | `FLASH_ATTN_3_HUB` | Hub kernel | FlashAttention 3 from Hugging Face `kernels` on Hopper or newer | [Hugging Face Hub Backends](attention_backends/huggingface_hub.md) |
 | `RAINFUSION_ATTN` | Block sparse | MindIE-SD RainFusion video attention on Ascend NPU | [RainFusion](attention_backends/rainfusion.md) |
-| `FASTVIDEO_VSA` | Block sparse | FastVideo variable sparse self-attention for FastWan2.2-TI2V-5B on CUDA | [FastVideo VSA](attention_backends/fastvideo_vsa.md) |
+| `FASTVIDEO_VSA` | Block sparse | FastVideo variable sparse self-attention for FastWan2.2-TI2V-5B and FastH3 MiniMax-H3 on CUDA | [FastVideo VSA](attention_backends/fastvideo_vsa.md) |
 
 ## Configuration
 

docs/user_guide/diffusion/attention_backends/dense_backends.md

@@ -1,8 +1,9 @@
 # Dense Attention Backends
 
-Use these backends when you want dense attention without backend-specific
-sparsity. Start with `TORCH_SDPA` for correctness comparisons, then benchmark
-the fastest compatible kernel for your model, shape, and hardware.
+Use these backends to run dense attention. Optional quantized or sparse modes
+remain inactive unless they are explicitly configured. Start with
+`TORCH_SDPA` for correctness comparisons, then benchmark the fastest compatible
+kernel for your model, shape, and hardware.
 
 For selection precedence and per-role configuration, see the
 [attention backend overview](../attention_backends.md).
@@ -20,10 +21,11 @@ vllm-omni serve <model> --diffusion-attention-backend TORCH_SDPA
 ## `FLASH_ATTN`
 
 `FLASH_ATTN` uses the installed FlashAttention implementation. On Blackwell it
-prefers the CuTe-based FlashAttention 4 path and falls back to
-FlashAttention 3 or 2 when that path is unavailable. On Hopper, Ada, and
-Ampere it is the preferred automatic route when a compatible package is
-installed.
+is FA4 only (`flash_attn.cute` / `vllm-omni[fa4]`). If FA4 is unavailable,
+explicit `FLASH_ATTN` is rejected and automatic selection continues to another
+compatible Blackwell backend. Hopper-only FA2/FA3 wheels are not used, even if
+they import. On Hopper, Ada, and Ampere it is the preferred automatic route
+when a compatible package is installed.
 
 ```bash
 vllm-omni serve <model> --diffusion-attention-backend FLASH_ATTN
@@ -38,8 +40,25 @@ pip install 'vllm-omni[fa4]'
 ```
 
 Version `4.0.0b18` is required; earlier beta wheels had known JIT failures on
-Blackwell. If the CuTe path is unavailable, the backend falls back to the
-compatible FlashAttention 3 or 2 path.
+Blackwell. On Blackwell, `FLASH_ATTN` is FA4 only: Hopper-only FA2/FA3
+wheels are not used, even if they import. If FA4 is missing, explicit
+`FLASH_ATTN` is rejected and automatic selection continues to other
+Blackwell backends.
+
+## `TRTLLM_ATTN`
+
+`TRTLLM_ATTN` runs FlashInfer's trtllm-gen FMHA kernels and is the platform
+default on datacenter Blackwell for models that declare a compatible attention
+path. Selected without a `quant` or `skip_softmax` block, it runs dense BF16
+attention at FA4-level performance.
+
+```bash
+vllm serve <model> --omni \
+  --diffusion-attention-backend TRTLLM_ATTN
+```
+
+See [TRTLLM Attention](trtllm.md) for its requirements and for the optional
+SAGE quantization and Skip-Softmax modes.
 
 ## `CUDNN_ATTN`
 
@@ -73,6 +92,13 @@ DIFFUSION_ATTENTION_BACKEND=FLASHINFER_ATTN \
 option on CUDA platforms and an automatic Blackwell fallback when FlashInfer
 is installed but cuDNN is too old for `CUDNN_ATTN`.
 
+On Blackwell, `auto` resolves to FlashInfer cute-dsl, which cannot run a
+nontrivial custom mask. Automatic selection may fall back to SDPA for those
+masks. An explicit `FLASHINFER_ATTN` selection does not: sequence-parallel
+auto-padding that would create a padding mask is rejected at capability
+preflight. Use `TORCH_SDPA`, pin `quant.flashinfer_backend` to `fa2`/`fa3`
+where those kernels exist, or choose a mask-capable backend.
+
 ```bash
 vllm-omni serve <model> --diffusion-attention-backend FLASHINFER_ATTN
 ```

docs/user_guide/diffusion/attention_backends/fastvideo_vsa.md

@@ -7,11 +7,18 @@ attention only against the selected top-k blocks.
 
 The supported checkpoint provides both text-to-video and image-to-video modes through `Wan22Pipeline`; the separate Wan I2V-14B, S2V, and VACE pipelines are outside this backend's supported scope.
 
+MiniMax-H3 served with a FastH3 VSA adapter uses a second, H3-specific route
+through the same backend. See the
+[MiniMax-H3 recipe](https://recipes.vllm.ai/MiniMaxAI/MiniMax-H3) for its geometry,
+supported topologies, and commands.
+
 VSA is a CUDA-only, explicitly selected backend. It requires the
 `fastvideo-kernel` package and currently supports non-causal self-attention
 with equal query and key/value sequence lengths. Unsupported shapes, masks,
 dtypes, sequence-parallel execution, or kernel failures fall back to
-`TORCH_SDPA` and emit a warning with the reason.
+`TORCH_SDPA` and emit a warning with the reason. On the Wan route, an active
+sequence-parallel context is one of those fallbacks; the H3 route supports pure
+Ulysses and rejects ring or all-gather sequence parallelism at startup.
 
 ## Enable the backend
 
@@ -119,7 +126,8 @@ the backend guarantees sparse execution:
 - `route=SDPA` or `FASTVIDEO_VSA falling back to SDPA: ...` means dense SDPA
   executed; the warning includes the reason.
 
-The current kernel requires CUDA tensors in FP16 or BF16, 256-token blocks,
+The Wan route requires CUDA tensors in FP16 or BF16, 256-token blocks,
 standard `head_size**-0.5` scaling, equal Q/K/V head counts, no attention mask,
-and no active sequence-parallel context. NPU and XPU paths do not execute the
-FastVideo VSA CUDA kernel.
+and no active sequence-parallel context. The MiniMax-H3 route uses 64-token
+`(4, 4, 4)` blocks and supports pure Ulysses. NPU and XPU paths do not execute
+the FastVideo VSA CUDA kernel.

docs/user_guide/diffusion/attention_backends/huggingface_hub.md

@@ -12,12 +12,12 @@ implementations to reduce numerical drift and sampling divergence.
 ## Installation
 
 ```bash
-pip install kernels==0.14.1
+pip install kernels==0.15.2
 ```
 
-If `kernels` is unavailable, vLLM-Omni warns and falls back to the
-corresponding local FlashAttention implementation. On CUDA devices older than
-Hopper, `FLASH_ATTN_3_HUB` falls back to `FLASH_ATTN_HUB`.
+If `kernels` is unavailable, an explicit `FLASH_ATTN_HUB` or
+`FLASH_ATTN_3_HUB` selection raises. Automatic / `FLASH_ATTN` selection in the
+Diffusers adapter may still walk hub then local FlashAttention backends.
 
 ## Usage
 

docs/user_guide/diffusion/attention_backends/trtllm.md

@@ -1,103 +1,274 @@
 # TRTLLM Attention
 
-`TRTLLM_ATTN` runs FlashInfer's trtllm-gen FMHA kernels. It supports dense
-BF16 attention plus optional Skip-Softmax sparsity or SAGE quantization.
+`TRTLLM_ATTN` runs FlashInfer's trtllm-gen FMHA kernels on datacenter
+Blackwell GPUs. Selected on its own it computes dense BF16 attention, and its
+dense performance is on par with
+[FlashAttention-4](dense_backends.md#flashattention-4-on-blackwell). It also
+provides two opt-in, lossy acceleration modes that can be enabled independently
+or together:
 
-For common selection and per-role configuration, see the
-[attention backend overview](../attention_backends.md).
+| Mode | Config key | What it changes |
+| --- | --- | --- |
+| [Skip-Softmax](#skip-softmax) | `skip_softmax` | Skips the Softmax and `PV` work of KV tiles whose scores are too low to matter |
+| [SAGE quantization](#sage-quantization) | `quant` | Runs `QK^T` in INT8 or FP8 and `PV` in FP8 instead of BF16 |
+
+Both keys are set through `--diffusion-attention-config`, as JSON or as
+vLLM-style dotted flags; the
+[attention backend overview](../attention_backends.md#configuration) covers
+both syntaxes and per-role resolution. The examples on this page use JSON.
+
+Attention computes scores `S = QK^T`, probabilities `P = softmax(S)`, and the
+output `O = PV`. SAGE lowers the precision of both matrix multiplications.
+Skip-Softmax keeps `QK^T` dense and removes the Softmax and `PV` work for
+tiles that `QK^T` shows to be unimportant. The two modes therefore compose:
+SAGE makes every tile cheaper, and Skip-Softmax reduces the number of tiles
+that reach the second half of the kernel.
 
 ## Requirements
 
-Selecting `TRTLLM_ATTN` requires:
+`TRTLLM_ATTN` requires all of the following:
 
-- datacenter Blackwell, sm_100 or sm_103;
+- an `sm100a` or `sm103a` GPU (B200, B300, GB200, GB300); workstation Blackwell
+  (`sm120`/`sm121`) is not supported;
 - `head_dim=128`;
-- FlashInfer; and
-- a model-declared packed or mask-free compatible path.
+- a FlashInfer build that exposes the trtllm-gen kernels (0.6.16rc1 or newer
+  for SAGE);
+- an attention path that is mask-free or provides packed-padding metadata.
+  Structural suffix padding is expressed through that metadata rather than an
+  `attn_mask` tensor.
 
-It is not supported on workstation Blackwell (sm_120/sm_121). An incompatible
-explicit selection raises instead of silently changing the backend.
+An explicit selection that violates these requirements raises at startup
+instead of silently falling back to another backend.
 
-## Dense mode
+### Sequence parallelism
 
-Select `TRTLLM_ATTN` without `skip_softmax` or `quant` to run dense BF16:
+`TRTLLM_ATTN` runs under no sequence parallelism or under pure Ulysses.
+Ulysses redistributes the sequence and attention heads around the attention
+call, but the local computation still goes through the configured backend, so
+both optional modes work unchanged. Ring and AllGather-KV do not:
+
+- Ring runs its own distributed attention and bypasses the backend. Combining
+  Ring with a `skip_softmax` key raises; a `quant` key would be silently
+  ignored, so do not combine them either.
+- AllGather-KV changes the Q/KV distribution and is rejected when
+  `TRTLLM_ATTN` is selected.
+
+Configure eight-way Ulysses alone as:
 
 ```bash
-vllm-omni serve <model> --diffusion-attention-backend TRTLLM_ATTN
+--usp 8 --ring 1 --allgather-degree 1
+```
+
+A degree of `1` disables that sequence-parallel mode. It does not limit the
+server to one GPU or affect tensor, pipeline, or VAE parallelism.
+
+## Quick start
+
+On datacenter Blackwell the platform selects `TRTLLM_ATTN` by default when the
+model declares a compatible path. To select it explicitly, for dense BF16:
+
+```bash
+vllm serve <model> --omni \
+  --diffusion-attention-backend TRTLLM_ATTN
+```
+
+To enable both optimizations with a conservative Skip-Softmax setting and FP8
+SAGE:
+
+```bash
+vllm serve <model> --omni \
+  --diffusion-attention-config '{
+    "default": {
+      "backend": "TRTLLM_ATTN",
+      "quant": {"dtype_qk": "fp8_e4m3", "q_block_size": 1, "k_block_size": 16},
+      "skip_softmax": {"threshold": 0.05, "disabled_until_timestep": 0.97}
+    }
+  }'
+```
+
+The startup log reports the selection; look for one of:
+
+```text
+Defaulting to diffusion attention backend TRTLLM_ATTN (datacenter Blackwell ..., head_dim 128)
+Resolved diffusion attention backend 'TRTLLM_ATTN' for role='self' via attention_config.default
 ```
 
 ## Skip-Softmax
 
-Skip-Softmax trades some fidelity for speed by skipping low-contribution
-attention work. See the [feature design](../../../design/feature/skip_softmax.md)
-for the algorithm and implementation contract.
+Skip-Softmax (BLASST) skips the Softmax and `PV` work of KV tiles whose
+scores fall far below the running row maximum. `QK^T` always runs, and how
+many tiles qualify depends on the input. The
+[feature design](../../../design/feature/skip_softmax.md) gives the skip test.
 
-| Key | Valid values | Meaning |
+### Configuration keys
+
+| Key | Range | Meaning |
 | --- | --- | --- |
-| `target_sparsity` | finite, `[0, 1]` | Point on a model-calibrated curve |
-| `threshold` | finite, `>= 0` | Direct threshold; mutually exclusive with `target_sparsity` |
-| `disabled_until_timestep` | finite, `[0, 1]` | Keep early high-noise steps dense until normalized `t <= D` |
+| `threshold` | `>= 0`; useful values in `(0, 1)` | Skip threshold, independent of sequence length. Calibration-free. |
+| `target_sparsity` | `[0, 1]` | Requested operating point on the checkpoint's calibrated curve. Requires calibration metadata. |
+| `disabled_until_timestep` | `[0, 1]`; default `0` | Keeps attention dense while the normalized timestep `t > D`; `0` disables the gate. |
+
+`threshold` and `target_sparsity` are two ways to set the same kernel
+threshold; setting both is a configuration error. Exactly one of them enables
+Skip-Softmax.
+
+### Direct threshold
+
+Set `threshold` when the checkpoint carries no calibration. `threshold=0`
+skips nothing; larger values skip more tiles and lower output fidelity. Values
+around `0.05` are a reasonable first try for video DiTs; tune against dense
+output on the same prompt and seed. The value is independent of sequence
+length; see the
+[feature design](../../../design/feature/skip_softmax.md#from-configuration-to-the-kernel-threshold)
+for how it reaches the kernel.
 
 ```bash
-vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \
-  --diffusion-attention-config '{"default":{"backend":"TRTLLM_ATTN",\
-    "skip_softmax":{"target_sparsity":0.65,"disabled_until_timestep":0.86}}}'
+vllm serve <model> --omni \
+  --diffusion-attention-config \
+  '{"default":{"backend":"TRTLLM_ATTN","skip_softmax":{
+    "threshold":0.05,"disabled_until_timestep":0.97}}}'
 ```
 
-```python
-from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, SkipSoftmaxSpec
+### Calibrated target sparsity
 
-config = AttentionConfig(
-    default=AttentionSpec(
-        backend="TRTLLM_ATTN",
-        skip_softmax=SkipSoftmaxSpec(
-            target_sparsity=0.65,
-            disabled_until_timestep=0.86,
-        ),
-    ),
-)
+A fixed `threshold` does not produce a fixed fraction of skipped tiles because
+the score distribution changes with the model, prompt, and shape.
+[NVIDIA ModelOpt](https://github.com/NVIDIA/Model-Optimizer)
+can calibrate a per-model curve from sparsity to threshold and store it in the
+checkpoint's transformer `config.json` under `sparse_attention_config`.
+`target_sparsity` selects a point on that curve. The achieved sparsity still
+varies per prompt, layer, and denoising step; the calibration makes the
+requested value a meaningful target, not a guarantee.
+
+What vLLM-Omni takes from the checkpoint:
+
+- The curve coefficients. Only the `a * exp(b * target_sparsity)` form ModelOpt
+  writes is supported; another formula is rejected at startup.
+- The `ignore` list: attention layers matching those patterns stay dense.
+- For multi-expert Diffusers checkpoints, `transformer_2/config.json` is read
+  separately; if it is missing, `transformer_2` stays dense with a warning.
+
+Checkpoint-level `target_sparsity` and `disabled_until_timestep` defaults are
+not consumed; set them in the vLLM-Omni configuration. The
+[feature design](../../../design/feature/skip_softmax.md#skip-softmax-calibration-config)
+documents the checkpoint format and how the coefficients reach the kernel.
+
+To calibrate a checkpoint yourself, follow the
+[ModelOpt Skip-Softmax example](https://github.com/NVIDIA/Model-Optimizer/tree/main/examples/diffusers/sparsity),
+which exports a copy of the checkpoint with the coefficients embedded. The
+fitted curve depends on the attention statistics of the calibration shape, so
+calibrate at the resolution and frame count you will serve.
+
+Requesting `target_sparsity` for a checkpoint without calibration is a startup
+error that names the `threshold` alternative. The
+[ModelOpt Wan2.2 FP8 checkpoint](https://huggingface.co/nvidia/Wan2.2-T2V-A14B-Diffusers-FP8/blob/main/transformer/config.json)
+is a calibrated example:
+
+```bash
+vllm serve nvidia/Wan2.2-T2V-A14B-Diffusers-FP8 --omni \
+  --diffusion-attention-config \
+  '{"default":{"backend":"TRTLLM_ATTN","skip_softmax":{
+    "target_sparsity":0.75,"disabled_until_timestep":0.86}}}'
 ```
 
-Start with `target_sparsity=0.65` and `disabled_until_timestep=0.86`. Increase
-sparsity only after comparing quality against dense output at the same seed.
-The benefit grows with sequence length because only attention is accelerated.
+### Timestep gating
+
+The early, high-noise denoising steps fix the global layout of the output, and
+their errors propagate through every later step. `disabled_until_timestep`
+keeps those steps dense and enables Skip-Softmax once the normalized timestep
+`t` reaches the configured cutoff. The default `0` is a sentinel that turns the
+gate off: Skip-Softmax runs on every step and no timestep needs to be
+published. `1.0` also leaves no step dense, but goes through the gate and
+therefore requires the pipeline to publish `t`.
+
+`t` is the scheduler's own timestep normalized to `[0, 1]`, published by the
+pipeline for each denoising step; for rectified-flow models it is the current
+sigma. It runs from near `1.0` down to `0.0`, but not linearly in the step
+index: flow-shifted schedules spend many of their steps at high `t`. The number
+of dense steps a given cutoff produces therefore depends on the model's
+schedule and step count, and follows from the actual timestep sequence:
+
+```text
+dense_steps = count(t[i] > disabled_until_timestep)
+```
+
+Derive the cutoff from the schedule of the model you are serving rather than
+reusing another model's value; the
+[feature design](../../../design/feature/skip_softmax.md#mapping-the-cutoff-to-denoising-steps)
+works through an example.
+
+A pipeline that does not publish `t` stays dense whenever
+`disabled_until_timestep > 0` is set, and logs a warning once. Pipelines
+publish it through `DenoiseProgressMixin.record_denoise_step`.
 
 ## SAGE quantization
 
-TRTLLM SAGE quantizes Q/K per block and V per channel so both attention
-matrix multiplications use low precision.
+SAGE quantization runs both attention matrix multiplications in low precision:
+Q and K are quantized to INT8 or FP8 E4M3 for `QK^T`, and P and V use FP8 E4M3
+for `PV`. P is quantized inside the FMHA kernel; V is quantized per channel
+before the kernel call.
+vLLM-Omni exposes the Q/K dtype and the Q/K scale granularity. The P and V
+formats are fixed by the kernel, so the `quant` key has no V dtype for this
+backend. This mode is distinct from the standalone
+[SageAttention backends](sage.md), which use their own kernels.
 
-| Key | Valid values | Meaning |
+FP8 Q/K kernels exist on `sm100a` and `sm103a`; INT8 Q/K kernels exist on
+`sm100a` only.
+
+| Key | Values | Meaning |
 | --- | --- | --- |
-| `dtype_qk` | `int8`, `fp8_e4m3` | Q/K quantization dtype; absent means dense |
-| `q_block_size` | `1`, `4`, `16` | Q scale block size; default `1` |
-| `k_block_size` | `1`, `4`, `16` | K scale block size; default `16` |
+| `dtype_qk` | `int8`, `fp8_e4m3` | Q/K quantization dtype. Setting it enables SAGE. |
+| `q_block_size` | `1`, `4`, `16` | Consecutive query tokens sharing one Q scale; default `1` |
+| `k_block_size` | `1`, `4`, `16` | Consecutive key tokens sharing one K scale; default `16` |
 
-V is always quantized per channel to FP8 E4M3 and K-smoothing is internal.
-Only block sizes with compiled kernels are accepted.
+Start with the defaults, `q_block_size=1` and `k_block_size=16`, and try
+smaller K blocks only if output quality does not meet expectations; smaller
+blocks give finer scales at some cost in speed. When a KV sequence in a call is
+shorter than `k_block_size`, that call falls back to dense attention and a
+warning is logged once.
 
 ```bash
-vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \
-  --diffusion-attention-config '{"default":{"backend":"TRTLLM_ATTN",\
-    "quant":{"dtype_qk":"fp8_e4m3","q_block_size":1,"k_block_size":16}}}'
+vllm serve <model> --omni \
+  --diffusion-attention-config \
+  '{"default":{"backend":"TRTLLM_ATTN","quant":{
+    "dtype_qk":"fp8_e4m3","q_block_size":1,"k_block_size":16}}}'
 ```
 
-```python
-from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, AttnQuantSpec
+The `quant` key is shared with `FLASHINFER_ATTN`, but each backend validates
+its own fields: `float16`/`bfloat16` Q/K dtypes and `dtype_vo` are
+`FLASHINFER_ATTN` options and are rejected here.
 
-config = AttentionConfig(
-    default=AttentionSpec(
-        backend="TRTLLM_ATTN",
-        quant=AttnQuantSpec(
-            dtype_qk="fp8_e4m3",
-            q_block_size=1,
-            k_block_size=16,
-        ),
-    ),
-)
+## Composing both modes
+
+`skip_softmax` and `quant` may appear in the same `AttentionSpec`. Their
+quality effects compound, so establish a dense baseline, enable one mode at a
+time, and then evaluate the combination on the same prompts and seeds.
+
+Modes configured in `default` apply to every attention role that has no
+`per_role` entry. A `per_role` spec replaces the whole spec for that role and
+does not inherit `quant` or `skip_softmax` from `default`, so
+`{"backend":"TRTLLM_ATTN"}` is the way to keep a short or sensitive attention
+site dense while the long DiT sequence uses both modes:
+
+```bash
+vllm serve MiniMaxAI/MiniMax-H3 --omni \
+  --diffusion-attention-config '{
+    "default": {
+      "backend": "TRTLLM_ATTN",
+      "quant": {"dtype_qk": "fp8_e4m3", "q_block_size": 1, "k_block_size": 16},
+      "skip_softmax": {"threshold": 0.05, "disabled_until_timestep": 0.97}
+    },
+    "per_role": {
+      "minimax_h3.token_refiner": {"backend": "TRTLLM_ATTN"}
+    }
+  }'
 ```
 
-This path requires FlashInfer 0.6.16rc1 or newer. FP8 Q/K kernels are
-available on sm_100 and sm_103; INT8 Q/K kernels are compiled for sm_100 only.
-The shared `AttnQuantSpec` is backend typed: values intended for
-`FLASHINFER_ATTN` are rejected here and vice versa.
+Role names are declared by each model; the
+[attention backend overview](../attention_backends.md#configuration) covers
+the resolution order and the equivalent Python API.
+
+End-to-end speedup depends on the share of step time spent in attention, the
+sequence length, the chosen Q/K precision, and the tile sparsity the input
+actually yields. Benchmark the exact workload rather than extrapolating from
+another model.

docs/user_guide/diffusion/cpu_offload.md

@@ -11,7 +11,7 @@ For the shared factory, discovery, and lifecycle contract, see the
 
 | Strategy | Device residency | Parallel scope | Guide |
 | --- | --- | --- | --- |
-| Model-level (sequential) | One pipeline component group at a time | Single device | [Model-Level Offloading](offloader/cpu_offload.md) |
+| Model-level (sequential) | One pipeline component group at a time | Single device | [Model-Level Offloading](offloader/module_offload.md) |
 | Layerwise (blockwise) | One transformer block, with next-block prefetch | Single device | [Layerwise Offloading](offloader/layerwise_offload.md) |
 | Distributed layerwise | Fixed two-block device buffer; optional host-weight sharding and AllGather | Multiple GPU/NPU ranks | [Distributed Layerwise Offloading](offloader/distributed_layerwise_offload.md) |
 
@@ -27,7 +27,7 @@ higher-priority strategy is selected.
 
 ## Quick selection
 
-- Use [model-level offloading](offloader/cpu_offload.md) when swapping whole
+- Use [model-level offloading](offloader/module_offload.md) when swapping whole
   encoders and DiTs is enough to fit and phase-boundary transfers are
   acceptable.
 - Use [layerwise offloading](offloader/layerwise_offload.md) for compute-heavy video
@@ -63,7 +63,7 @@ These headings preserve links to sections that moved into dedicated guides.
 
 ## Model-level (Sequential) Offloading
 
-See [Model-Level Offloading](offloader/cpu_offload.md).
+See [Model-Level Offloading](offloader/module_offload.md).
 
 ## Layerwise (Blockwise) Offloading
 

docs/user_guide/diffusion/execution_modes.md

@@ -94,8 +94,16 @@ validating a model or debugging correctness, then increase it for
 multi-request throughput.
 
 Step execution is capability-based, not a generic switch for every diffusion
-model. Qwen-Image supports step-wise continuous batching. HunyuanImage3 also
-supports it, but only when its resolved self-attention backend is `TORCH_SDPA`;
+model. Qwen-Image supports step-wise continuous batching. BAGEL supports
+step-wise continuous batching for image generation with `bagel.yaml`,
+`bagel_think.yaml`, and `bagel_single_stage.yaml`. In the two-stage topologies,
+only the diffusion stage uses step execution; the autoregressive Thinker is
+unchanged. Explicit text-output requests in the single-stage topology retain
+the complete-request path. Use at least two inference steps for BAGEL image
+requests because BAGEL schedules `num_inference_steps - 1` denoising updates.
+BAGEL step execution does not currently support sequence parallelism or a
+diffusion cache backend. HunyuanImage3 also supports step execution, but only
+when its resolved self-attention backend is `TORCH_SDPA`;
 set `DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA` or configure
 `diffusion_attention_config.default.backend=TORCH_SDPA` before using
 `--max-num-seqs >1`. See the

docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md

@@ -1,11 +1,13 @@
 # Distributed Layerwise Offloading
 
 Distributed layerwise offloading (DLO) extends block streaming to multi-device
-deployments. With AllGather enabled, each rank stores roughly `1 / dp_size` of
-the host weights and reconstructs each layer at runtime. Without AllGather,
-each rank streams a complete block independently. Compatible TP1 deployments
-can share checkpoint-backed host pages among processes on the same node;
-otherwise DLO streams the ordinary loader's rank-local tensors.
+deployments. Each selected component chooses its transfer independently. With
+`allgather`, each rank stores roughly one shard per member of the effective DLO
+group (DP, or SP when DP is one) and reconstructs each layer at runtime. With
+`rank-local`, each rank streams a complete loader-produced block independently.
+Compatible TP1 DiT deployments can share checkpoint-backed host pages among
+processes on the same node; otherwise DLO streams the ordinary loader's
+rank-local tensors.
 
 See the [DLO feature design](../../../design/feature/offloader/distributed_layerwise_offload.md)
 for the implementation contract and compatibility matrix.
@@ -24,52 +26,120 @@ Slots:           slot 0: Layer N    slot 1: Layer N+1
 AllGather communicates only request-independent weight shards, so data-
 parallel ranks may process different requests concurrently.
 
+The public API describes intent in one nested configuration:
+
+| Level | Setting | Question answered |
+| --- | --- | --- |
+| Granularity | `mode` | move a complete component (`module`) or stream its blocks (`layer`) |
+| Selection | `components` list | whether `dit` and/or `text_encoder` may move to CPU |
+| Per-component layer options | `layer_options` | choose rank-local or AllGather weight transfer and optionally retain leading DiT blocks |
+
+`weight_transfer` applies only to model weights. It does not change encoder
+activations or the encoder's compute parallelism. `rank-local` copies each
+rank's loader-produced block to its device. `allgather` stores one shard of
+that block per rank, copies each shard to its device, and reconstructs the
+complete block with a device-side AllGather before compute.
+
+The historical `enable_cpu_offload`, `enable_layerwise_offload`, and
+`enable_distributed_layerwise_offload` fields remain compatibility aliases
+for existing callers and model-specific stage lifecycles. Existing
+combinations retain their priority (`distributed layerwise` > `layerwise` >
+`module`). A compact config rejects conflicting compatibility strategies and
+non-default legacy DLO tuning.
+`distributed-layerwise` remains an internal backend name; users select
+`mode="layer"`, and the runtime chooses the capable backend.
+
 ## Usage
 
 ```bash
-# Four ranks with sharded host weights and AllGather
-vllm serve /path/to/model --omni \
-  --enable-distributed-layerwise-offload \
-  --data-parallel-size 4
+# Four SP ranks; Wan declares replicated text-encoder weights, so both
+# selected components can use AllGather safely.
+vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --omni \
+  --diffusion-offload-config \
+  '{"mode":"layer","components":["dit","text_encoder"],"layer_options":{"dit":{"weight_transfer":"allgather"},"text_encoder":{"weight_transfer":"allgather"}}}' \
+  --usp 4
 
-# Standard-loader rank-local weights, without DLO AllGather
+# Mix transfers independently: sharded DiT, loader-produced encoder layout
 vllm serve /path/to/model --omni \
-  --enable-distributed-layerwise-offload \
-  --data-parallel-size 4 \
-  --dlo-no-use-allgather
+  --diffusion-offload-config \
+  '{"mode":"layer","components":["dit","text_encoder"],"layer_options":{"dit":{"weight_transfer":"allgather"},"text_encoder":{"weight_transfer":"rank-local"}}}' \
+  --usp 4
 
-# Sequence parallel deployment
+# Full-topology rank-local DLO for an existing model-specific lifecycle
 vllm serve /path/to/model --omni \
   --enable-distributed-layerwise-offload \
+  --dlo-no-use-allgather \
   --usp 4
 ```
 
+A compact layer config whose selected components are all `rank-local` and
+whose `resident_layers` is zero resolves to the ordinary layerwise backend. It
+uses the ordinary loader and does not request DLO's direct-checkpoint mmap.
+Keep the compatibility DLO flags for a model-specific full-topology rank-local
+lifecycle; do not add resident layers solely to force backend selection.
+
 ```python
 from vllm_omni import Omni
 
 omni = Omni(
     model="/path/to/model",
-    enable_distributed_layerwise_offload=True,
-    dlo_use_allgather=True,
+    diffusion_offload_config={
+        "mode": "layer",
+        "components": ["dit", "text_encoder"],
+        "layer_options": {
+            "dit": {
+                "weight_transfer": "rank-local",
+                "resident_layers": 20,
+            },
+            "text_encoder": {
+                "weight_transfer": "allgather",
+            },
+        },
+        "pin_memory": True,
+    },
 )
 ```
 
 ## Flags
 
-| Flag | Meaning | Default |
+| Setting | Meaning | Default |
 | --- | --- | --- |
-| `--enable-distributed-layerwise-offload` | Enable DLO | `false` |
-| `--data-parallel-size N` | DP ranks and AllGather weight-sharding group | `1` |
-| `--dlo-use-allgather` | Shard host weights and reconstruct with AllGather | `true` |
-| `--dlo-no-use-allgather` | Stream complete rank-local blocks without a DLO weight collective | `false` |
-| `--dlo-resident-layers N` | Keep N leading main-DiT blocks on device; requires no-AllGather and model-declared resident paths | `0` |
+| `diffusion_offload_config.mode` | `module` or `layer` granularity | required |
+| `diffusion_offload_config.components` | Non-empty list containing `dit`, `text_encoder`, or both | required |
+| `layer_options.NAME.weight_transfer` | `rank-local` or `allgather` | `rank-local` |
+| `layer_options.dit.resident_layers` | Leading main-DiT blocks kept on device; requires `rank-local` and model-declared resident paths | `0` |
+| `diffusion_offload_config.pin_memory` | Pin streamed host memory for faster H2D copies | `true` |
+| `--data-parallel-size N` | DP ranks; DP is the DLO group when greater than one, otherwise SP is used | `1` |
 | `--host-weight-runtime-mode {disabled,preferred,required}` | HWR policy: no interaction, populate on a miss, or require an exact hit | `disabled` |
 | `--host-weight-runtime-root PATH` | Writable node-local HWR store shared by workers in one storage domain; required for `preferred` and `required` | unset |
 | `--dlo-host-registration-limit-gib N` | Optional per-worker ceiling for registering an HWR mmap; zero adds no ceiling | `0` |
 
+HWR and host-registration tuning remain on the compatibility DLO interface and
+cannot be combined with `diffusion_offload_config` in this release.
+
+## Component and weight-transfer matrix
+
+| Component | `allgather` | `rank-local` |
+| --- | --- | --- |
+| DiT | Shard each loader-visible block across the DLO DP group, or the SP group when DP is one | Stream each rank's complete loader-produced block |
+| Text encoder | Same shard + AllGather path when the model declares identical encoder weights across the DLO group | Stream each rank's complete encoder block, including encoder-TP shards |
+
+The text encoder's compute group and weight-transfer group are separate
+concepts. An encoder TP group owns different parameter shards, so it is not a
+valid AllGather offload group. Models opt in through
+`OffloadPlan.encoder_dlo_weight_replication`; otherwise multi-rank encoder
+AllGather fails at startup with guidance to use `rank-local`. This makes SP
+AllGather available to replicated encoders without silently corrupting an
+encoder-TP layout.
+
+`resident_layers` remains a DiT-only setting and currently requires the DiT
+transfer to be `rank-local`.
+
 ## Host-weight loading
 
-The diffusion loader chooses host storage before DLO is enabled. It first
+When the distributed backend is selected by AllGather, resident DiT layers, or
+the compatibility DLO flag, the diffusion loader chooses host storage before
+DLO is enabled. It first
 attempts to build a complete, validated direct-checkpoint mmap plan. If names,
 coverage, shape, dtype, topology, or loader-callback compatibility cannot be
 proven, it runs the ordinary model loader instead. DLO consumes that result and
@@ -97,20 +167,20 @@ With direct checkpoint mmap, the loader:
 6. preserves `post_load_weights()` and `validate_loaded_weights()` lifecycle
    hooks.
 
-For AllGather with a group larger than one, each process copies only its
-persistent shard and then releases the source mapping. For no-AllGather, each
-process keeps the mapping open and packs complete blocks through two bounded
-pinned staging slots. Processes mapping the same files on one node share the
-immutable pages; no-AllGather still performs a complete-block H2D copy in each
-process.
+For a DiT using `allgather` with a group larger than one, each process copies
+only its persistent shard and then releases the source mapping. For a DiT using
+`rank-local`, each process keeps the mapping open and packs complete blocks
+through two bounded pinned staging slots. Processes mapping the same files on
+one node share the immutable pages; rank-local transfer still performs a
+complete-block H2D copy in each process.
 
-When the effective DLO group size is one, `dlo_use_allgather=True` does not
+When the effective DLO group size is one, `weight_transfer="allgather"` does not
 perform a collective and uses the same rank-local transfer behavior.
 
 ### Final-layout Host Weight Runtime
 
 HWR is an opt-in startup optimization for models that declare the final-layout
-BF16 restore contract. Use it only with no-AllGather DLO:
+BF16 restore contract. Use it only when the selected DiT uses `rank-local`:
 
 The validated BF16 model contracts currently cover MiniMax H3 and
 `black-forest-labs/FLUX.2-klein-4B`. FLUX.2-klein-9B shares the same model
@@ -204,7 +274,7 @@ H2D behavior.
 #### Registered direct H2D
 
 Registration is attempted automatically only for an eligible warm HWR hit when
-`pin_cpu_memory` is enabled. A successful path is:
+the existing `pin_cpu_memory` policy is enabled. A successful path is:
 
 ```text
 shared read-only HWR mmap -> existing rotating HBM block buffer -> GPU kernel
@@ -217,10 +287,10 @@ kernels access host memory directly.
 CUDA requires read-only host-registration capability for the immutable HWR
 mapping. Unsupported capability, a positive registration limit smaller than
 the complete page-aligned mapping, or a safely rolled-back registration error
-falls back to bounded staging. Programmatic configurations can set
-`pin_cpu_memory=False` to disable registration explicitly. A successful
-registration locks the complete mapped range in host memory for that worker's
-lifetime, so use
+falls back to bounded staging. Programmatic compatibility configurations can
+set `pin_cpu_memory=False` to disable registration explicitly.
+A successful registration locks the complete mapped range in host memory for
+that worker's lifetime, so use
 `--dlo-host-registration-limit-gib` when an operator-enforced ceiling is
 required.
 
@@ -228,8 +298,8 @@ Shutdown drains H2D work, releases hook/source references, unregisters every
 range, and only then closes the HWR lease. AllGather and direct checkpoint mmap
 retain their existing transfer paths.
 
-When HWR is disabled, DLO is disabled, or AllGather is enabled, the loader does
-not resolve HWR sources or construct its store.
+When HWR is disabled, DLO is disabled, or the DiT uses `allgather`, the loader
+does not resolve HWR sources or construct its store.
 
 ## Declarative topology
 
@@ -241,38 +311,54 @@ from vllm_omni.diffusion.offloader import OffloadPlan
 
 class MyPipeline(nn.Module):
     _dit_modules = ["transformer"]
+    _encoder_modules = ["prompt_model"]
     _offload_plan = OffloadPlan(
         block_attrs={"transformer": ("blocks",)},
         offload_submodules={"context_encoder": "layers"},
+        encoder_component_types={"prompt_model": "text_encoder"},
+        encoder_block_attrs={"prompt_model": ("encoder.layers",)},
+        encoder_dlo_weight_replication=frozenset({"prompt_model"}),
+        on_demand_component_paths=frozenset({"prompt_model"}),
     )
 ```
 
-When no plan exists, discovery falls back to
-`_layerwise_offload_blocks_attrs` and then heuristic attribute lookup.
+`encoder_component_types` maps arbitrary encoder paths to the public
+`text_encoder` selector. `encoder_dlo_weight_replication` is the explicit
+safety declaration for reusing the DiT DLO group. On-demand encoder non-block
+state is loaded and offloaded by its pipeline phase.
+
+When no plan exists, DiT discovery falls back to
+`_layerwise_offload_blocks_attrs` and then heuristic attribute lookup;
+undeclared auxiliary components remain resident.
 
 ## Data-parallel concurrency
 
 With `data_parallel_size > 1` and AllGather enabled, the scheduler can process
-up to `dp_size` requests per denoising step. Every concurrent request must set
-the same explicit `num_inference_steps`; `None` is rejected because every rank
-must enter each collective.
+up to `dp_size` requests per denoising step. Concurrent requests must resolve
+to the same denoise schedule, so every request must provide the same explicit
+`num_inference_steps`. Pipeline defaults are rejected because different request
+modes can resolve the same `None` value to different schedules.
 
 ## Limitations
 
 - Direct checkpoint mmap currently requires TP1. TP greater than one is
   outside the Phase A shared-mmap support scope and falls back before model
   mutation to the ordinary TP-aware loader. DLO can stream that runtime layout,
-  and eligible no-AllGather configurations may use HWR final-layout artifacts,
+  and eligible DiT rank-local configurations may use HWR final-layout artifacts,
   but direct checkpoint mmap provides no shared-mmap host-memory guarantee for
   TP greater than one.
-- HSDP plus AllGather is rejected to avoid double sharding. HSDP without
-  AllGather has limited end-to-end validation.
+- HSDP with `allgather` on any selected component is rejected to avoid double
+  sharding. HSDP with rank-local transfers has limited end-to-end validation.
+- With data parallelism, `allgather` cannot be combined with TeaCache or
+  Cache-DiT on any selected component. Prompt-embedding cache is also
+  incompatible when the text encoder uses `allgather`. Rank-local cache hits
+  could otherwise make ranks enter different weight collectives.
 - Per-tensor online FP8 linears use the ordinary loader and can run with either
-  DLO transfer path. With AllGather, every rank temporarily materializes the
-  complete FP8 model in host memory before DLO retains only its shard. Other
-  online quantization methods require no-AllGather until their runtime layouts
-  are validated.
-- Resident leading layers require `--dlo-no-use-allgather` and a model
+  DiT transfer path. With DiT `allgather`, every rank temporarily materializes
+  the complete FP8 model in host memory before DLO retains only its shard.
+  Other online quantization methods require rank-local transfer for the
+  affected component until their runtime layouts are validated.
+- Resident leading layers require DiT `rank-local` transfer and a model
   `OffloadPlan` that declares eligible `resident_dit_paths`.
 - DP concurrency requires an explicit, identical inference-step count.
 

docs/user_guide/diffusion/offloader/layerwise_offload.md

@@ -18,8 +18,9 @@ stream.
 | ... | ... | ... | ... |
 | last block | Prefetch block 0 | Compute last block | Free last block |
 
-Encoders, VAE modules, and non-block DiT modules such as embeddings and norms
-remain device resident.
+Selected, plan-declared text-encoder blocks can use the same rank-local
+streaming mechanism. Image encoders, VAEs, and non-block DiT modules remain
+device resident.
 
 ## Usage
 
@@ -28,15 +29,44 @@ from vllm_omni import Omni
 
 omni = Omni(
     model="Wan-AI/Wan2.2-T2V-A14B-Diffusers",
-    enable_layerwise_offload=True,
+    diffusion_offload_config={
+        "mode": "layer",
+        "components": ["dit"],
+    },
 )
 ```
 
 ```bash
 vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \
-  --omni --enable-layerwise-offload
+  --omni \
+  --diffusion-offload-config '{"mode":"layer","components":["dit"]}'
 ```
 
+## Component selection
+
+Add `dit`, `text_encoder`, or both to the `components` list. Omitting
+`layer_options` uses safe rank-local defaults:
+
+- `["dit"]` streams only DiT blocks.
+- `["text_encoder"]` streams only declared text-encoder blocks.
+- Listing both streams both components.
+
+```bash
+# DiT-only layer offload
+vllm serve /path/to/model --omni \
+  --diffusion-offload-config \
+  '{"mode":"layer","components":["dit"]}'
+
+# Stream a model-declared text encoder while keeping the DiT resident
+vllm serve /path/to/model --omni \
+  --diffusion-offload-config \
+  '{"mode":"layer","components":["text_encoder"]}'
+```
+
+Encoder categories are resolved from `OffloadPlan.encoder_component_types`
+first. A name-based fallback is retained for pipelines that predate
+`OffloadPlan`.
+
 ## Model integration
 
 Transformer classes declare containers of executable blocks:
@@ -53,13 +83,29 @@ class Flux2Transformer2DModel(nn.Module):
     ]
 ```
 
-See the [layerwise design](../../../design/feature/offloader/layerwise_offload.md) for
-the discovery and hook invariants. `OffloadPlan` is a separate declarative
-topology path for distributed layerwise offload.
+Auxiliary components use declarative pipeline metadata:
+
+```python
+from vllm_omni.diffusion.offloader import OffloadPlan
+
+
+class MyPipeline(nn.Module):
+    _encoder_modules = ["prompt_model"]
+    _offload_plan = OffloadPlan(
+        encoder_component_types={"prompt_model": "text_encoder"},
+        encoder_block_attrs={"prompt_model": ("encoder.layers",)},
+    )
+```
+
+See the [layerwise design](../../../design/feature/offloader/layerwise_offload.md)
+for the discovery and hook invariants. Both ordinary and distributed layerwise
+offload consume the same `OffloadPlan` metadata.
 
 ## Limitations
 
-- Single device only.
+- The default weight transfer is `rank-local`. Set a selected component's
+  `weight_transfer` to `allgather` to shard its host weights across a
+  compatible multi-device group; backend selection is automatic.
 - Setup consolidates and pins block parameters, increasing cold-start time.
 - Performance depends on block compute time and host-to-device bandwidth;
   lightweight blocks may not hide transfers.

docs/user_guide/diffusion/offloader/module_offload.md

@@ -2,14 +2,14 @@
 
 Model-level, or sequential, offloading keeps only the pipeline component group
 currently executing on the accelerator. It is the simplest offload strategy
-and is selected with `--enable-cpu-offload`.
+and is selected with `diffusion_offload_config.mode="module"`.
 
 ## How it works
 
 Pre-forward hooks enforce mutual exclusion between DiT and encoder modules:
 
-- before an encoder runs, the DiT moves to CPU;
-- before a DiT runs, encoders and other DiTs move to CPU; and
+- before an encoder runs, selected DiTs move to CPU;
+- before a DiT runs, selected encoders and other selected DiTs move to CPU; and
 - VAE modules remain on the accelerator.
 
 Pinned host memory reduces transfer overhead. Transfers occur at phase
@@ -22,15 +22,28 @@ from vllm_omni import Omni
 
 omni = Omni(
     model="Wan-AI/Wan2.2-T2V-A14B-Diffusers",
-    enable_cpu_offload=True,
+    diffusion_offload_config={
+        "mode": "module",
+        "components": ["dit", "text_encoder"],
+    },
 )
 ```
 
 ```bash
 vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \
-  --omni --enable-cpu-offload
+  --omni \
+  --diffusion-offload-config \
+  '{"mode":"module","components":["dit","text_encoder"]}'
 ```
 
+List a component to select it for offload. Module mode rejects `layer_options`
+such as `weight_transfer` and `resident_layers`. The
+`enable_cpu_offload=True` compatibility entry point remains supported. New
+integrations should prefer the explicit config; existing model-specific stage
+lifecycles do not need to migrate until equivalent component coverage exists.
+For example, MiniMax-H3's compatibility lifecycle also stages its VAEs, while
+the compact selector intentionally covers only `dit` and `text_encoder`.
+
 ## Model integration
 
 Pipelines should implement `SupportsComponentDiscovery`:
@@ -73,9 +86,10 @@ while reusing sequential `.to()` movers.
 
 ## Limitations
 
-- Single device only.
+- Transfers are rank-local; module mode does not shard host payloads or add a
+  weight AllGather across ranks.
 - Higher cold-start latency.
 - Transfers between encoder and denoising phases add latency.
 
-See the [model-level design](../../../design/feature/offloader/cpu_offload.md)
+See the [model-level design](../../../design/feature/offloader/module_offload.md)
 for lifecycle and extension invariants.

docs/user_guide/diffusion/parallelism/cfg_parallel.md

@@ -94,7 +94,7 @@ In `DiffusionParallelConfig`
 
 
 !!! info
-    Most models support `cfg_parallel_size=2` (positive branch on rank 0, negative branch on rank 1). **Bagel** is an exception: it supports `cfg_parallel_size=3`, which adds a third branch on rank 2 for full three-way CFG parallelism.
+    Most models support `cfg_parallel_size=2` (positive branch on rank 0, negative branch on rank 1). **Bagel** and **Boogu-Image-Edit double guidance** also support `cfg_parallel_size=3` for full three-way branch execution. Boogu double guidance additionally supports size 2 via round-robin assignment of its three branches.
 
 
 ---

docs/user_guide/diffusion_features.md

@@ -1,5 +1,7 @@
 # Diffusion Advanced Features
 
+<!-- markdownlint-disable MD028 MD060 -->
+
 ## Table of Contents
 
 - [Overview](#overview)
@@ -30,7 +32,6 @@ Cache methods trade minimal quality for significant speedup. Quality loss is typ
 | **[TeaCache](diffusion/cache_acceleration/teacache.md)** | Adaptive caching using modulated inputs | Quick setup, balanced quality/speed on single GPU |
 | **[Cache-DiT](diffusion/cache_acceleration/cache_dit.md)** | Multiple caching techniques: DBCache, TaylorSeer, SCM | Fine-grained control, tunable quality-speed tradeoff |
 
-
 #### Lossless Acceleration
 
 Parallelism methods distribute computation across GPUs without quality loss (mathematically equivalent to single-GPU).
@@ -72,7 +73,6 @@ Extension methods add specialized capabilities to diffusion models beyond standa
 | **[LoRA Inference](diffusion/lora.md)** | Enables inference with Low-Rank Adaptation (LoRA) adapters weights | Reinforcement learning extensions |
 | **[Frame Interpolation](diffusion/frame_interpolation.md)** | Inserts intermediate video frames after generation for smoother motion | Video generation pipelines that need higher temporal smoothness |
 
-
 ### Execution Modes
 
 Execution modes control how the diffusion pipeline processes requests and
@@ -116,7 +116,8 @@ The following tables show which models support each feature:
 
 | Model                    | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | 🔀Pipeline-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
 |--------------------------|:---------:|:----------:|:---------------------:|:--------------:|:-----------------:|:-------------------:|:------:|:-------------------------:|:--------------------:|:--------------:|:----------------:|
-| **Bagel**                |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **Bagel**                |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |          ❌           |       ❌        |        ✅         |
+| **Boogu-Image**          |     ❌     |     ❌      |           ❌           |       ✅        |         ❌         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **FLUX.1-dev**           |     ✅     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ✅        |        ❌         |
 | **FLUX.1-schnell**       |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ✅        |        ❌         |
 | **FLUX.2-klein**         |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ✅        |        ❌         |
@@ -138,7 +139,7 @@ The following tables show which models support each feature:
 | **Qwen-Image-Edit**      |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Qwen-Image-Edit-2509** |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |        ✅ (decode)         |          ✅           |       ❌        |        ❌         |
 | **Qwen-Image-Layered**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
-| **SenseNova-U1**         |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **SenseNova-U1 / U1.5**  |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **Stable-Diffusion-XL**  |     ❌     |     ❌      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Stable-Diffusion3.5**  |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Z-Image**              |     ✅     |     ✅      |           ✅           |       ❓        |   ✅ (TP=2 only)   |          ❌          |   ✅    |             ❌             |      ✅ (decode)      |       ✅        |        ❌         |
@@ -146,11 +147,13 @@ The following tables show which models support each feature:
 | **Cosmos3**              |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ✅        |        ❌         |
 
 > Notes:
+>
 > 1. Nextstep_1(T2I) does not support cache acceleration methods such as TeaCache or Cache-DiT.
 > 2. `Tongyi-MAI/Z-Image-Turbo` is a distilled model with minimal NFEs; CFG-Parallel is not necessary.
 > 3. Cosmos3 T2I uses `Cosmos3OmniDiffusersPipeline` with `modalities=["image"]`. Model-level CPU offload swaps the nested UND reasoner and GEN generator pathways; layerwise offload remains available for blockwise GEN/UND offload.
 > 4. Krea 2 currently supports single-GPU inference plus LoRA, Cache-DiT, HSDP, CPU/layerwise offload, and VAE-patch-parallel (decode). TP/SP/CFG-Parallel are not yet wired. The few-step distilled (Turbo) checkpoint uses `is_distilled=true` (fixed timestep shift `mu=1.15`); generate at 2048x2048 by default with `num_inference_steps≈8` and `guidance_scale=0`. The Raw checkpoint uses 1024x1024, `num_inference_steps=28`, and `guidance_scale=4.5`.
 > 5. HunyuanImage3 supports step execution. Multi-request step execution requires `TORCH_SDPA`; see [Diffusion Execution Modes](diffusion/execution_modes.md#step-execution).
+> 6. BAGEL step execution supports image generation with `bagel.yaml`, `bagel_think.yaml`, and `bagel_single_stage.yaml`; two-stage Thinker execution and explicit single-stage text output remain on their existing complete-request paths. Image requests require `num_inference_steps >= 2`. BAGEL step execution cannot currently be combined with sequence parallelism or a diffusion cache backend; see [Diffusion Execution Modes](diffusion/execution_modes.md#step-execution).
 
 ### VideoGen
 
@@ -162,24 +165,24 @@ The following tables show which models support each feature:
 | **LTX-2**                    |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **LTX-2.3**                  |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **LTX-2.5**                  |     ❌     | ❓ (one-stage) | ❓ (Ulysses only) | ✅ (Full only) |         ❓         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |    ❓ (FP8)     |        ❌         |
-| **SANA-Video-2B T2V I2V**   |     ❓     |     ❓      |           ❓           |       ❓        |         ❓         |         ❓         |   ❓    |             ❓             |          ❓           |       ❓        |        ❌         |
+|  **SANA-Video-2B T2V I2V**   |     ❌     |     ❌      |   ✅ (`--usp`, frame-sharded)   |       ✅       |  ✅ (TP=2 only)   |         ❌         |   ❌   |            ❌             |          ❌          |       ❌       |        ❌        |
 | **Helios**                   |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ✅*        |
 | **HunyuanVideo-1.5 T2V I2V** |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
 | **Cosmos3**                  |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
 | **LongCat-Video-Avatar-1.5** |     ❌     |     ❌      |           ❌           |       ❌        |         ❌         |         ❌         |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **MiniMax-H3**               | ✅ (FL2VA) |     ✅      |           ✅           |       ❌        |       ✅ (DiT/TE)  |         ❌         |   ✅    |             ✅             |       ✅ (tile)       |      ✅ (DiT)      |        ❌         |
+| **MAGI-2 Preview**           |     ❌     |     ✅      |      ✅ (Ulysses)       |    ✅ (2-way)   |         ✅         |         ❌         |   ✅    | ✅ (1-GPU/LW; DLO DP-AG/SP no-AG) |       ✅ (tile)       |       ❌        |        ❌         |
 | **SANA-WM**                  |     ❌     |     ❌      |          ❌<sup>5</sup> |       ✅        |         ✅         |         ❌         |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 
 > Notes:
 > 5. SANA-WM cannot support sequence parallelism: its bidirectional gated delta
->    recurrence carries state across frames, so a rank cannot denoise a slice of
->    the token sequence in isolation. Doing so would need a distributed scan or
->    an all-gather before every GDN block. The remaining ❌ columns are simply
->    unvalidated on this model, not known-broken.
+> recurrence carries state across frames, so a rank cannot denoise a slice of
+> the token sequence in isolation. Doing so would need a distributed scan or
+> an all-gather before every GDN block. The remaining ❌ columns are simply
+> unvalidated on this model, not known-broken.
 
 > **Step execution note:** Helios supports single-request step execution only;
 > use `max_num_seqs=1`.
-
 **Frame Interpolation Support**
 
 - **Supported**: Wan2.2 text-to-video, image-to-video, and TI2V pipelines; SANA-Video-2B native text-to-video and image-to-video pipelines
@@ -191,7 +194,6 @@ The following tables show which models support each feature:
 |-----------------------|:---------:|:----------:|:---------------------:|:--------------:|:-----------------:|:-------------------:|:------:|:-------------------------:|:--------------------:|:--------------:|:----------------:|
 | **Stable-Audio-Open** |     ✅     |     ❌      |           ❓           |       ❓        |         ❌         |          ❌          |   ✅    |             ✅             |          ❌           |       ✅        |        ❌         |
 
-
 ## Feature Compatibility
 
 **Legend:**
@@ -230,7 +232,6 @@ The following tables show which models support each feature:
     5. The compatibility matrix uses FP8 as the representative quantization method.
     6. Step Execution is not compatible with any diffusion cache backend. LoRA is supported, but each scheduled batch must use a single adapter (requests with different `lora_request` or `lora_scale` are kept in separate batches).
 
-
 ## Multi-Thread Weight Loading
 
 The loading guide now lives at [Diffusion Startup and

docs/user_guide/examples/online_serving/bagel.md

@@ -60,6 +60,50 @@ vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 \
 
 See [`bagel_single_stage.yaml`](https://github.com/vllm-project/vllm-omni/tree/main/vllm_omni/deploy/bagel_single_stage.yaml) for configuration. The `pipeline: bagel_single_stage` field selects the single-stage topology from the pipeline registry.
 
+### Step Execution and Continuous Batching
+
+Image generation supports step execution with all three BAGEL deploy configs.
+The global flag is applied only to diffusion stages:
+
+- With `bagel.yaml` or `bagel_think.yaml`, Stage 1 uses step execution and the
+  Stage 0 Thinker remains autoregressive.
+- With `bagel_single_stage.yaml`, image requests use step execution in Stage 0.
+  Explicit `text2text` and `img2text` requests use the same complete-request
+  text path as they do without `--step-execution`.
+
+Start with the deploy file's default diffusion capacity of one:
+
+```bash
+vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 \
+    --deploy-config vllm_omni/deploy/bagel.yaml \
+    --step-execution
+```
+
+To enable step-wise continuous batching, override `max_num_seqs` on the
+diffusion stage. Use Stage 1 for either two-stage config:
+
+```bash
+vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 \
+    --deploy-config vllm_omni/deploy/bagel.yaml \
+    --step-execution \
+    --stage-overrides '{"1":{"max_num_seqs":4}}'
+```
+
+Use Stage 0 for the single-stage config:
+
+```bash
+vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 \
+    --deploy-config vllm_omni/deploy/bagel_single_stage.yaml \
+    --step-execution \
+    --stage-overrides '{"0":{"max_num_seqs":4}}'
+```
+
+BAGEL image requests require `num_inference_steps >= 2`: the original BAGEL
+schedule uses `num_inference_steps - 1` denoising updates. The scheduler batches
+only requests with compatible effective image sizes and BAGEL CFG/renormalization
+settings. Sequence parallelism and diffusion cache backends such as TeaCache or
+Cache-DiT are not currently supported together with BAGEL step execution.
+
 ### Tensor Parallelism (TP)
 
 For larger models or multi-GPU environments, enable TP via CLI:
@@ -203,6 +247,13 @@ curl http://localhost:8091/v1/chat/completions \
   -d @payload.json
 ```
 
+BAGEL img2img derives its generated image size from the input image. It
+preserves the input aspect ratio, aligns dimensions to the latent stride, and
+applies the checkpoint size limit; explicit `height` and `width` values do not
+override this model-specific resize policy. In step mode, the effective size is
+resolved before scheduler admission so requests with different resulting
+shapes are not combined.
+
 ### Image to Text (img2text)
 
 **Python client:**
@@ -267,8 +318,8 @@ curl http://localhost:8091/v1/chat/completions \
 | `--server` / `-s` | `http://localhost:8091` | Server URL |
 | `--image-url` / `-i` | `None` | Input image URL or local path (img2img/img2text) |
 | `--modality` / `-m` | `text2img` | `text2img`, `img2img`, `img2text`, `text2text` |
-| `--height` | `512` | Image height in pixels |
-| `--width` | `512` | Image width in pixels |
+| `--height` | `512` | Text-to-image output height; BAGEL img2img derives its size from the input image |
+| `--width` | `512` | Text-to-image output width; BAGEL img2img derives its size from the input image |
 | `--steps` | `25` | Number of inference steps |
 | `--seed` | `42` | Random seed |
 | `--negative` | `None` | Negative prompt for CFG |
@@ -293,6 +344,7 @@ python openai_chat_client.py \
 | File | Description |
 | :--- | :---------- |
 | [`bagel.yaml`](https://github.com/vllm-project/vllm-omni/tree/main/vllm_omni/deploy/bagel.yaml) | Two-stage default (Thinker + DiT on GPU 0) |
+| [`bagel_think.yaml`](https://github.com/vllm-project/vllm-omni/tree/main/vllm_omni/deploy/bagel_think.yaml) | Two-stage think mode (Thinker + DiT) |
 | [`bagel_single_stage.yaml`](https://github.com/vllm-project/vllm-omni/tree/main/vllm_omni/deploy/bagel_single_stage.yaml) | Single-stage (DiT only) |
 
 ### Key Deploy YAML Fields

docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md

@@ -16,6 +16,7 @@ implemented with clear Diffusers or vLLM-Omni behavior.
 - Sequence parallel execution
 - TeaCache / Cache-DiT acceleration
 - Step-wise execution (continuous batching)
+- Component-selective `diffusion_offload_config` (the legacy whole-pipeline CPU-offload options remain supported)
 
 For these features, it is recommended to use natively supported pipelines instead.
 

docs/user_guide/examples/online_serving/qwen3_omni.md

@@ -17,9 +17,9 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 
 The default deployment configuration situated at `vllm_omni/deploy/qwen3_omni_moe.yaml` is resolved and loaded
 automatically via the model registry, obviating the necessity for the `--deploy-config` flag in standard deployment topologies.
-The bundled Qwen3-Omni setup defaults `VLLM_USE_FLASHINFER_MOE_FP16=0`. This keeps the Thinker & Talker on vLLM's
-Triton unquantized MoE path and avoids the performance regression observed with the FlashInfer CUTLASS unquantized MoE
-backend.
+The bundled Qwen3-Omni setup explicitly sets `moe_backend: triton` for the Thinker and Talker stages. This keeps
+both stages on vLLM's Triton unquantized MoE path and avoids the unstable FlashInfer CUTLASS unquantized MoE path on
+some GPUs.
 Asynchronous chunk streaming is **enabled by default** within the bundled configuration.
 
 To explicitly utilize a custom deployment YAML, specify the configuration path:
@@ -87,15 +87,9 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
     --stage-overrides '{"1": {"gpu_memory_utilization": 0.5}}'
 ```
 
-To experiment with the FlashInfer FP16 MoE path, set `VLLM_USE_FLASHINFER_MOE_FP16=1` before launching the server:
-```bash
-VLLM_USE_FLASHINFER_MOE_FP16=1 \
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
-```
-
-For the stage-based CLI, you usually do **not** need `--stage-overrides` for
-that kind of change. Since each command launches one stage, just pass the knob
-directly on that stage command:
+To experiment with another supported MoE backend, set `moe_backend` in the
+deployment YAML for both the Thinker and Talker stages. For the stage-based
+CLI, pass `--moe-backend` directly on each stage command.
 
 ```bash
 CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \

docs/user_guide/examples/online_serving/text_to_speech.md

@@ -2,7 +2,6 @@
 
 Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/text_to_speech>.
 
-
 vLLM-Omni exposes TTS models through the OpenAI-compatible
 [`POST /v1/audio/speech`](https://github.com/vllm-project/vllm-omni/tree/main/docs/serving/speech_api.md) endpoint,
 launched with `vllm serve <model> --omni`. Each TTS model has its own
@@ -16,7 +15,7 @@ For the full list of supported architectures across all modalities, see
 ## Supported Models
 
 | Model | HuggingFace repo | Voice cloning | Streaming | Voice presets / upload | Gradio demo |
-|---|---|---|---|---|---|
+| --- | --- | --- | --- | --- | --- |
 | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | — |
 | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
 | higgs-audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` | ✓ (`ref_audio`+`ref_text`) | ✓ (codec_streaming) | — | — |
@@ -104,6 +103,7 @@ For full request-shape documentation (all parameters, response formats, error co
 2-stage TTS (`talker` + flow-matching `code2wav`) at 24 kHz. Voice cloning only — every request needs `ref_audio` + `ref_text`; there are no built-in voice presets.
 
 ### Prerequisites
+
 ```bash
 huggingface-cli download FunAudioLLM/Fun-CosyVoice3-0.5B-2512
 ```
@@ -111,6 +111,7 @@ huggingface-cli download FunAudioLLM/Fun-CosyVoice3-0.5B-2512
 If your downloaded checkpoint lacks `config.json`, add one with `{"model_type": "cosyvoice3", "architectures": ["CosyVoice3Model"]}` (the loader reads `model_type` to select the class).
 
 ### Launch
+
 ```bash
 vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 --omni --port 8091 --trust-remote-code
 # or:
@@ -120,13 +121,16 @@ vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 --omni --port 8091 --trust-remot
 Streaming is on by default via `async_chunk: true` in `vllm_omni/deploy/cosyvoice3.yaml`. Pass `--no-async-chunk` (or `NO_ASYNC_CHUNK=1 ./cosyvoice3/run_server.sh`) for the legacy synchronous path.
 
 ### CLI client
+
 The client defaults to the official upstream zero-shot prompt, so it runs without extra flags:
+
 ```bash
 cd examples/online_serving/text_to_speech/cosyvoice3
 python speech_client.py --text "收到好友从远方寄来的生日礼物。"
 ```
 
 Pass your own reference clip and transcript for a different voice:
+
 ```bash
 python speech_client.py --text "Hello, this is a cloned voice." \
     --ref-audio /path/to/reference.wav \
@@ -134,6 +138,7 @@ python speech_client.py --text "Hello, this is a cloned voice." \
 ```
 
 Stream PCM instead of WAV:
+
 ```bash
 python speech_client.py --text "Hello world" --stream --output output.pcm
 ```
@@ -141,6 +146,7 @@ python speech_client.py --text "Hello world" --stream --output output.pcm
 The client supports `--api-base`, `--model`, `--text`, `--ref-audio`, `--ref-text`, `--response-format`, `--stream`, `--output`.
 
 ### Notes
+
 - Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform.
 - Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config <path>` to override.
 - For offline inference and the end-to-end script, see the [offline CosyVoice3 section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#cosyvoice3).
@@ -152,19 +158,23 @@ The client supports `--api-base`, `--model`, `--text`, `--ref-audio`, `--ref-tex
 4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec.
 
 ### Prerequisites
+
 ```bash
 pip install fish-speech
 ```
 
 ### Launch
+
 ```bash
 vllm serve fishaudio/s2-pro --omni --port 8091
 # or:
 ./fish_speech/run_server.sh
 ```
+
 The deploy config auto-loads from `vllm_omni/deploy/fish_qwen3_omni.yaml` (the HF `model_type` on the fishaudio checkpoint is `fish_qwen3_omni`).
 
 ### Voice cloning
+
 ```bash
 curl -X POST http://localhost:8091/v1/audio/speech \
     -H "Content-Type: application/json" \
@@ -177,6 +187,7 @@ curl -X POST http://localhost:8091/v1/audio/speech \
 ```
 
 ### CLI client
+
 ```bash
 cd examples/online_serving/text_to_speech/fish_speech
 python speech_client.py --text "Hello, how are you?"
@@ -184,12 +195,14 @@ python speech_client.py --text "Hello world" --stream --output output.pcm
 ```
 
 ### Gradio demo
+
 ```bash
 ./fish_speech/run_gradio_demo.sh             # launches server + Gradio
 python fish_speech/gradio_demo.py --api-base http://localhost:8091  # if server already running
 ```
 
 ### Notes
+
 - Output: 44.1 kHz mono.
 - Streaming PCM player command must use `-r 44100`.
 
@@ -208,12 +221,15 @@ pip install -U "transformers>=5.3.0"
 ```
 
 ### Launch
+
 ```bash
 GPUS=6,7 PORT=8094 bash examples/online_serving/text_to_speech/higgs_audio_v2/run_server.sh
 ```
+
 Deploy config auto-loads from `vllm_omni/deploy/higgs_audio_v2.yaml`.
 
 ### Sending requests
+
 ```bash
 # Plain TTS
 python higgs_audio_v2/batch_speech_client.py \
@@ -231,14 +247,17 @@ python higgs_audio_v2/batch_speech_client.py \
 ```
 
 ### Notes
+
 - Output: 24 kHz mono.
 - `--ref-text` must be the real transcript of `--ref-audio`; mismatched text degrades cloned-voice quality.
 - Out of scope (rejected with explicit 4xx): multi-speaker `[SPEAKERn]` tags inside `input`, `profile:` text-only speaker descriptions, the `ref_audio_in_system_message` system-block variant, chunked long-form generation, and per-request `voice` / `instructions` / `task_type` / `language` / `speed != 1.0` / `x_vector_only_mode` / `speaker_embedding`.
+
 ## GLM-TTS
 
 2-stage TTS (AR + DiT flow-matching) at 24 kHz. Every request requires `ref_audio` + `ref_text`.
 
 ### Launch
+
 ```bash
 vllm serve zai-org/GLM-TTS --omni --trust-remote-code --port 8091
 # or:
@@ -246,6 +265,7 @@ bash examples/online_serving/text_to_speech/glm_tts/run_server.sh /path/to/GLM-T
 ```
 
 ### Sending requests
+
 ```bash
 # Voice cloning (required)
 python examples/online_serving/text_to_speech/glm_tts/openai_speech_client.py \
@@ -262,11 +282,13 @@ python examples/online_serving/text_to_speech/glm_tts/openai_speech_client.py \
 ```
 
 ### Gradio demo
+
 ```bash
 bash examples/online_serving/text_to_speech/glm_tts/run_gradio_demo.sh
 ```
 
 ### Notes
+
 - Output: 24 kHz mono WAV via HiFT vocoder.
 - `ref_audio` + `ref_text` are **required** together on every request. Reference audio should be 3-10 seconds.
 - Voice cloning feature extraction (WhisperVQ, CampPlus, mel) runs on the model side — no external dependency on the serving layer.
@@ -278,12 +300,15 @@ bash examples/online_serving/text_to_speech/glm_tts/run_gradio_demo.sh
 Zero-shot multilingual TTS (600+ languages). Online serving currently exposes **auto voice** only; voice cloning and voice design are available offline.
 
 ### Prerequisites
+
 ```bash
 huggingface-cli download k2-fsa/OmniVoice
 ```
+
 Voice cloning (offline) needs `transformers>=5.3.0`; auto voice works with `transformers>=4.57.0`.
 
 ### Launch
+
 ```bash
 vllm serve k2-fsa/OmniVoice --omni --port 8091 --trust-remote-code
 # or:
@@ -291,6 +316,7 @@ vllm serve k2-fsa/OmniVoice --omni --port 8091 --trust-remote-code
 ```
 
 ### CLI client
+
 ```bash
 cd examples/online_serving/text_to_speech/omnivoice
 python speech_client.py --text "Hello, how are you?"
@@ -300,6 +326,7 @@ python speech_client.py --text "Bonjour, comment allez-vous?" --language French
 The client supports `--api-base`, `--model`, `--text`, `--response-format`, `--language`, `--output`.
 
 ### Notes
+
 - Voice cloning and voice design require offline inference; see the [offline OmniVoice section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#omnivoice).
 
 ---
@@ -309,7 +336,7 @@ The client supports `--api-base`, `--model`, `--text`, `--response-format`, `--l
 Three model variants exposed via separate checkpoints:
 
 | Variant | HF repo | Use |
-|---|---|---|
+| --- | --- | --- |
 | CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | Predefined speakers (`vivian`, `ryan`, …) with optional style instructions |
 | VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | Natural-language voice style description |
 | Base | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | Voice cloning from a reference audio |
@@ -317,6 +344,7 @@ Three model variants exposed via separate checkpoints:
 Each variant ships smaller `0.6B` companions where available.
 
 ### Launch
+
 ```bash
 vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni --port 8091
 # or:
@@ -326,27 +354,30 @@ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni --port 8091
 ```
 
 ### Executor backend
+
 Single-GPU serves now default to the uniproc executor (lower IPC overhead, the Base cloning use case from [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) / [#2604](https://github.com/vllm-project/vllm-omni/pull/2604)). `vllm_omni/deploy/qwen3_tts.yaml` is the only Qwen3-TTS deploy config; pass `--deploy-config <path>` to override.
 
 To opt out of chunked streaming, pass `--no-async-chunk` — the pipeline auto-dispatches to the end-to-end codec processor.
 
 ### Tuning stage 1 `max_num_seqs` per task type
+
 The bundled `qwen3_tts.yaml` ships stage 1 (Code2Wav) at `max_num_seqs: 10`, tuned for Base voice cloning: stage-1 lifetimes are long (~3 s/req), so admitting up to 10 concurrent codec sequences lets requests progress in parallel in the scheduler — ~2× TTFA p95 at c=4 / c=8 (1× H100, 1.7B-Base, seed-tts) at an 8–12 % audio-throughput cost.
 
 CustomVoice / VoiceDesign have much shorter stage-1 lifetimes (~50–200 ms) and are TTFA-optimal at `max_num_seqs: 1`. Override the default when serving those task types:
 
 ```bash
-vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base --omni \
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni \
     --stage-overrides '{"1": {"max_num_seqs": 1}}'
 ```
 
 ### Sending requests
+
 ```bash
 # CustomVoice with a predefined speaker
 python qwen3_tts/openai_speech_client.py \
     --model Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
     --text "今天天气真好" \
-    --voice ryan \
+    --speaker ryan \
     --instructions "用开心的语气说"
 
 # VoiceDesign with a style description
@@ -366,7 +397,9 @@ python qwen3_tts/openai_speech_client.py \
 ```
 
 ### Voices endpoint
+
 List available voices, or upload a custom one for Base cloning:
+
 ```bash
 # List
 curl http://localhost:8091/v1/audio/voices
@@ -379,10 +412,18 @@ curl -X POST http://localhost:8091/v1/audio/voices \
     -F "ref_text=The exact transcript of the audio sample." \
     -F "speaker_description=warm narrator"
 ```
-Uploaded voices are then usable as `voice="custom_voice_1"` on subsequent requests.
+
+For Qwen3-TTS, uploaded voices are Base voice-cloning inputs and require a Base
+checkpoint. When a request names an uploaded voice, the server infers
+`task_type="Base"`. Built-in presets such as `vivian` and `ryan` remain
+CustomVoice speakers and require a CustomVoice checkpoint.
 
 ### Precomputed custom voices
-For reused Base voice-cloning speakers, precompute the reference artifacts once and load them at server startup:
+
+For reused Base voice-cloning speakers, precompute the reference artifacts once
+and load them at server startup. Precomputed voices use the same Base task and
+checkpoint-matching rules as uploaded voices:
+
 ```bash
 python qwen3_tts/precompute_custom_voice.py \
     --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
@@ -392,11 +433,15 @@ python qwen3_tts/precompute_custom_voice.py \
     --mode icl \
     --output-dir /path/to/custom_voices
 ```
+
 `--mode icl` stores both `speaker_embedding` and `ref_code`; `--mode xvec` stores only the speaker embedding. Add the output directory to a deploy config:
+
 ```yaml
 custom_voice_dir: /path/to/custom_voices
 ```
+
 Then start the server with that config and call the Speech API with only the voice name:
+
 ```bash
 vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base --omni --deploy-config /path/to/qwen3_tts_custom_voice.yaml
 
@@ -407,6 +452,7 @@ curl -X POST http://localhost:8091/v1/audio/speech \
 ```
 
 ### Streaming PCM
+
 ```bash
 curl -X POST http://localhost:8091/v1/audio/speech \
     -H "Content-Type: application/json" \
@@ -419,16 +465,20 @@ curl -X POST http://localhost:8091/v1/audio/speech \
         "response_format": "pcm"
     }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 -
 ```
+
 Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` in the deploy config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming.
 
 ### Streaming WebSocket
+
 The `/v1/audio/speech/stream` endpoint accepts text incrementally and synthesizes the buffered text as one continuous request on `input.done`:
+
 ```bash
 python qwen3_tts/streaming_speech_client.py --text "Hello world. How are you? I am fine."
 python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-delay 0.1
 ```
 
 `input.done` flushes without closing, so repeating `--text` synthesizes several utterances over one connection:
+
 ```bash
 python qwen3_tts/streaming_speech_client.py \
     --text "First utterance." \
@@ -436,6 +486,7 @@ python qwen3_tts/streaming_speech_client.py \
 ```
 
 ### Gradio demos
+
 ```bash
 ./qwen3_tts/run_gradio_demo.sh                              # CustomVoice (default)
 ./qwen3_tts/run_gradio_demo.sh --task-type VoiceDesign
@@ -447,12 +498,15 @@ python qwen3_tts/gradio_fastrtc_demo.py --api-base http://localhost:8000
 ```
 
 ### Speaker embedding interpolation
+
 `qwen3_tts/speaker_embedding_interpolation.py` blends two predefined speakers' embeddings to produce intermediate voices. See the script for usage.
 
 ### Batch client
+
 `qwen3_tts/batch_speech_client.py` issues many concurrent requests for throughput measurement.
 
 ### Notes
+
 - Base voice cloning has uniproc-vs-mp tradeoffs depending on per-request reference audio cost; see the executor-backend section above.
 - With async chunking, Qwen3-TTS Base voice cloning sends the full reference context in the first Code2Wav packet, then caches that prefix on the Code2Wav stage for follow-up chunks in the same request.
 - `vllm_omni/deploy/qwen3_tts.yaml` is the default deploy config (loaded by HF `model_type`); per-stage runtime overrides are available via `--stage-N-<field> <value>`.
@@ -467,12 +521,15 @@ python qwen3_tts/gradio_fastrtc_demo.py --api-base http://localhost:8000
 Single-stage native AR TTS at 48 kHz.
 
 ### Launch
+
 ```bash
 vllm serve openbmb/VoxCPM2 --omni --host 0.0.0.0 --port 8000
 ```
+
 Deploy config auto-loads from `vllm_omni/deploy/voxcpm2.yaml`. Pass `--deploy-config <path>` to override or `--stage-N-<field> <value>` for per-stage runtime tweaks.
 
 ### Sending requests
+
 ```bash
 # Zero-shot synthesis
 python voxcpm2/openai_speech_client.py --text "Hello, this is VoxCPM2."
@@ -482,10 +539,13 @@ python voxcpm2/openai_speech_client.py \
     --text "This should sound like the reference speaker." \
     --ref-audio /path/to/reference.wav
 ```
+
 The `ref_audio` field accepts local file paths (auto-base64), HTTP URLs, or `data:audio/wav;base64,...` data URIs.
 
 ### Precomputed custom voices
+
 For repeated VoxCPM2 speakers, precompute the prompt cache and load it through `custom_voice_dir`:
+
 ```bash
 python voxcpm2/precompute_custom_voice.py \
     --model openbmb/VoxCPM2 \
@@ -495,16 +555,21 @@ python voxcpm2/precompute_custom_voice.py \
     --prompt-text "Original transcript of the reference audio" \
     --output-dir /path/to/custom_voices
 ```
+
 Add the output directory to the deploy config:
+
 ```yaml
 custom_voice_dir: /path/to/custom_voices
 ```
+
 After startup, `/v1/audio/voices` lists `alice`, and `/v1/audio/speech` can use `voice="alice"` without sending `ref_audio`.
 
 ### Gradio demo (gapless streaming via AudioWorklet)
+
 ```bash
 python voxcpm2/gradio_demo.py
 ```
+
 Uses an AudioWorklet-based player adapted from the Qwen3-TTS demo for gap-free playback. Raw PCM audio is streamed from the OpenAI Speech endpoint with `stream=true` and `stream_format="audio"`.
 
 ---
@@ -514,24 +579,31 @@ Uses an AudioWorklet-based player adapted from the Qwen3-TTS demo for gap-free p
 Voxtral-4B-TTS (Mistral). Uses the `mistral_common` `SpeechRequest` protocol; voice presets are model-specific.
 
 ### Prerequisites
+
 Latest `mistral_common` with `SpeechRequest` support:
+
 ```bash
 pip install -e /path/to/mistral-common  # or upgrade from PyPI when available
 ```
 
 ### Launch
+
 ```bash
 vllm serve mistralai/Voxtral-4B-TTS-2603 --omni --port 8091
 ```
+
 Deploy config auto-loads from `vllm_omni/deploy/voxtral_tts.yaml`.
 
 ### Gradio demo
+
 ```bash
 python voxtral_tts/gradio_demo.py
 ```
+
 The demo handles voice-preset selection and reference-audio upload. `voxtral_tts/text_preprocess.py` provides the text-normalization helpers used by the demo (also available for other clients).
 
 ### Notes
+
 - Voice presets are listed on the HF model card (`mistralai/Voxtral-4B-TTS-2603`).
 - Voice cloning is gated upstream and may require a recent `mistral_common`.
 - A standalone CLI client is not yet shipped; the gradio demo is the canonical reference for now.

docs/user_guide/fault_injection_reliability_matrix.md

@@ -46,20 +46,20 @@ More failure modes (for example, network jitter and network interruption) will b
 ## Fault Injection Scenario Matrix
 
 | Scenario | Fault Type | System Behavior | Current Status |
-|------|----------------------------|----------|----------|
-| No load | Send `SIGKILL` to Worker | Worker process is killed immediately; main service detects child-process loss and turns unavailable; API enters stable 5xx |  |
-| No load | Send `SIGTERM` to Worker | Worker exits after receiving termination signal; main service is marked unavailable; API enters stable 5xx |  |
+| ------ | ---------------------------- | ---------- | ---------- |
+| No load | Send `SIGKILL` to Worker | Worker process is killed immediately; main service detects child-process loss and turns unavailable; API enters stable 5xx | |
+| No load | Send `SIGTERM` to Worker | Worker exits after receiving termination signal; main service is marked unavailable; API enters stable 5xx | |
 | No load | Send `SIGKILL` to serve main process | serve main process exits instantly; request connections are interrupted; related child processes are cleaned up with no residue; GPU memory is released quickly | [#3725](https://github.com/vllm-project/vllm-omni/issues/3725) <br>[#43060](https://github.com/vllm-project/vllm/issues/43060) |
-| No load | Send `SIGTERM` to serve main process | serve enters graceful shutdown and stops serving; then exits and completes cleanup; GPU memory is released |  |
-| No load | Send `SIGINT` to serve main process (equivalent to `Ctrl+C`) | Triggers serve shutdown path; service stops responding and becomes unavailable; related child processes exit and resources are released |  |
-| No load | Send `SIGKILL` to all related processes | All related processes terminate immediately; service becomes unavailable at once; no residual processes remain; GPU memory is released quickly |  |
-| No load | Send `SIGTERM` to all related processes | All processes enter exit flow and complete shutdown; service becomes unavailable; resource release is completed |  |
+| No load | Send `SIGTERM` to serve main process | serve enters graceful shutdown and stops serving; then exits and completes cleanup; GPU memory is released | |
+| No load | Send `SIGINT` to serve main process (equivalent to `Ctrl+C`) | Triggers serve shutdown path; service stops responding and becomes unavailable; related child processes exit and resources are released | |
+| No load | Send `SIGKILL` to all related processes | All related processes terminate immediately; service becomes unavailable at once; no residual processes remain; GPU memory is released quickly | |
+| No load | Send `SIGTERM` to all related processes | All processes enter exit flow and complete shutdown; service becomes unavailable; resource release is completed | |
 | Under load | Send `SIGKILL` to Worker | In-flight requests are hard interrupted (5xx/connection drop); main service becomes unavailable | [#3683](https://github.com/vllm-project/vllm-omni/issues/3683) |
-| Under load | Send `SIGTERM` to Worker | In-flight requests are canceled or fail fast; main service becomes unavailable |  |
+| Under load | Send `SIGTERM` to Worker | In-flight requests are canceled or fail fast; main service becomes unavailable | |
 | Under load | Send `SIGKILL` to serve main process | serve is hard-killed and current connections are interrupted; in-flight requests fail; after cleanup there are no residual processes and GPU memory is released | [#3683](https://github.com/vllm-project/vllm-omni/issues/3683) |
 | Under load | Send `SIGTERM` to serve main process | serve stops accepting new requests and executes shutdown flow; in-flight requests fail; no residue remains and GPU memory is released | [#3683](https://github.com/vllm-project/vllm-omni/issues/3683) |
 | Under load | Send `SIGINT` to serve main process (equivalent to `Ctrl+C`) | `Ctrl+C`-style serve shutdown; in-flight requests fail (5xx/connection interruption); service unavailable; after exit there is no residue and GPU memory is released | [#3683](https://github.com/vllm-project/vllm-omni/issues/3683) |
-| Under load | Send `SIGKILL` to all related processes | All processes terminate instantly; all in-flight requests fail; service becomes unavailable immediately; GPU memory is released quickly |  |
+| Under load | Send `SIGKILL` to all related processes | All processes terminate instantly; all in-flight requests fail; service becomes unavailable immediately; GPU memory is released quickly | |
 | Under load | Send `SIGTERM` to all related processes | All processes exit gracefully; in-flight requests fail; service unavailable; GPU memory is released | [#3683](https://github.com/vllm-project/vllm-omni/issues/3683) |
 | OOM | Occupy all free GPU memory via an extra process | After OOM injection process starts, GPU memory is continuously saturated; service enters unavailable/degraded state and health check drops to 503; different request types (chat/speech, etc.) fail fast within a fixed time and return 500 (no hanging) | [#4285](https://github.com/vllm-project/vllm-omni/issues/4285) |
 

docs/user_guide/quantization/online.md

@@ -7,16 +7,16 @@ loading the model. Use it when you want memory savings without preparing a
 separate quantized checkpoint.
 
 This mode is different from pre-quantized checkpoint formats such as GGUF,
-AutoRound, msModelSlim, or serialized Int8 checkpoints. Those formats are
-prepared before serving and are documented in their method-specific guides.
-For MXFP8 and MXFP4, use this page for load-time quantization from BF16
-checkpoints, and use the method-specific pages for offline checkpoints produced
-by msModelSlim and the merge tools.
+AutoRound, msModelSlim, serialized TorchAO checkpoints, or serialized Int8
+checkpoints. Those formats are prepared before serving and are documented in
+their method-specific guides. For MXFP8 and MXFP4, use this page for load-time
+quantization from BF16 checkpoints, and use the method-specific pages for
+offline checkpoints produced by msModelSlim and the merge tools.
 
 ## Hardware Support
 
 | Device | FP8 W8A8 | Int8 W8A8 | MXFP8 W8A8 | MXFP4 W4A4 |
-|--------|----------|-----------|------------|------------|
+| -------- | ---------- | ----------- | ------------ | ------------ |
 | NVIDIA Blackwell GPU (SM 100+) | ✅ | ✅ | ⭕ | ⭕ |
 | NVIDIA Ada/Hopper GPU (SM 89+) | ✅ | ✅ | ⭕ | ⭕ |
 | NVIDIA Ampere GPU (SM 80+) | ✅ | ✅ | ⭕ | ⭕ |
@@ -33,7 +33,7 @@ MXFP4 are documented for the Ascend NPU path.
 ### Diffusion Model (Qwen-Image, Wan2.2)
 
 | Method | Guide | Example models | Status |
-|--------|-------|----------------|--------|
+| -------- | ------- | ---------------- | -------- |
 | FP8 W8A8 | [FP8](fp8.md) | Qwen-Image; Wan2.2 is not validated | Validated for Qwen-Image family and other DiT models |
 | Int8 W8A8 | [Int8](int8.md) | Qwen-Image; Wan2.2 is not validated | Validated for Qwen-Image and Z-Image |
 | MXFP8 W8A8 | [MXFP8](mxfp8.md) | Wan2.2-T2V-A14B, Wan2.2-I2V-A14B, Wan2.2-TI2V-5B | Validated on Ascend NPU and Intel XPU |
@@ -88,7 +88,7 @@ config = build_quant_config({
 ## Parameters
 
 | Parameter | Methods | Description |
-|-----------|---------|-------------|
+| ----------- | --------- | ------------- |
 | `method` | FP8, Int8, MXFP8, MXFP4 | Quantization method: `"fp8"`, `"int8"`, `"mxfp8"`, `"mxfp4"`, or `"mxfp4_dualscale"` |
 | `ignored_layers` | FP8, Int8, MXFP8, MXFP4 | Layer name patterns to keep in BF16/FP16 |
 | `activation_scheme` | FP8, Int8 | The runtime value `"dynamic"` selects online activation scaling |

docs/user_guide/quantization/overview.md

@@ -11,21 +11,21 @@ For the internal architecture and backend extension points, see the
 ## Quantization Modes
 
 | Mode | Guide | Description | Methods |
-|------|-------|-------------|---------|
+| ------ | ------- | ------------- | --------- |
 | Online quantization | [Online Quantization](online.md) | vLLM-Omni computes quantized weights and scales while loading the model. | FP8 W8A8, Int8 W8A8, BitsAndBytes W4, MXFP8 W8A8, MXFP4 W4A4 |
 | Runtime attention quantization | [Quantized KV Cache](quantized_kvcache.md) | vLLM-Omni dynamically quantizes eligible diffusion Flash Attention tensors during inference. | FP8 FA |
-| Pre-quantized checkpoints | Method-specific guides | The checkpoint or an offline quantizer provides quantized weights and scales before serving. | ModelOpt, AutoRound, msModelSlim, serialized Int8, offline MXFP8, offline MXFP4 DualScale |
+| Pre-quantized checkpoints | Method-specific guides | The checkpoint or an offline quantizer provides quantized weights and scales before serving. | ModelOpt, AutoRound, TorchAO, msModelSlim, serialized Int8, offline MXFP8, offline MXFP4 DualScale |
 
 ## Hardware Support
 
-| Device | FP8 W8A8 | Int8 W8A8 | BitsAndBytes W4 | ModelOpt | MXFP8 W8A8 | MXFP4 W4A4 | AutoRound | msModelSlim |
-|--------|----------|-----------|------------------|----------|------------|------------|-----------|-------------|
-| NVIDIA Blackwell GPU (SM 100+) | ✅ | ✅ | ✅ | ✅ | ⭕ | ⭕ | ✅ | ❌ |
-| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ | ✅ | ✅ | ✅ | ⭕ | ⭕ | ✅ | ❌ |
-| NVIDIA Ampere GPU (SM 80+) | ✅ | ✅ | ✅ | ⭕ | ⭕ | ⭕ | ✅ | ❌ |
-| AMD ROCm | ⭕ | ⭕ | ❌ | ⭕ | ⭕ | ✅ | ⭕ | ❌ |
-| Intel XPU | ⭕ | ⭕ | ❌ | ⭕ | ⭕ | ⭕ | ✅ | ❌ |
-| Ascend NPU | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ |
+| Device | FP8 W8A8 | Int8 W8A8 | BitsAndBytes W4 | ModelOpt | MXFP8 W8A8 | MXFP4 W4A4 | AutoRound | msModelSlim | TorchAO FP8 weight-only |
+| -------- | ---------- | ----------- | ------------------ | ---------- | ------------ | ------------ | ----------- | ------------- | --------- |
+| NVIDIA Blackwell GPU (SM 100+) | ✅ | ✅ | ✅ | ✅ | ⭕ | ⭕ | ✅ | ❌ | ⭕ |
+| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ | ✅ | ✅ | ✅ | ⭕ | ⭕ | ✅ | ❌ | ✅ |
+| NVIDIA Ampere GPU (SM 80+) | ✅ | ✅ | ✅ | ⭕ | ⭕ | ⭕ | ✅ | ❌ | ⭕ |
+| AMD ROCm | ⭕ | ⭕ | ❌ | ⭕ | ⭕ | ✅ | ⭕ | ❌ | ⭕ |
+| Intel XPU | ⭕ | ⭕ | ❌ | ⭕ | ⭕ | ⭕ | ✅ | ❌ | ⭕ |
+| Ascend NPU | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ⭕ |
 
 Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
 guide. FP8 on Ampere may use a weight-only path where available.
@@ -40,7 +40,7 @@ encoder, and VAE stay on the base checkpoint unless a method guide says
 otherwise.
 
 | Method | Guide | Mode | Example models | Status |
-|--------|-------|------|----------------|--------|
+| -------- | ------- | ------ | ---------------- | -------- |
 | FP8 W8A8 | [FP8](fp8.md) | Online W8A8 or checkpoint FP8 | Qwen-Image; Wan2.2 is not validated | Validated for Qwen-Image family and other DiT models |
 | Int8 W8A8 | [Int8](int8.md) | Online or serialized W8A8 | Qwen-Image; Wan2.2 is not validated | Validated for Qwen-Image and Z-Image |
 | BitsAndBytes W4 | [BitsAndBytes](bitsandbytes.md) | Online W4 weight-only | Z-Image-Turbo; Qwen-Image/Wan2.2 not validated | Validated for Z-Image-Turbo |
@@ -49,6 +49,7 @@ otherwise.
 | MXFP4 W4A4 | [MXFP4](mxfp4.md) | `mxfp4`: online single-scale only; `mxfp4_dualscale`: online or offline dual-scale (offline recommended) | Wan2.2-T2V-A14B, I2V-A14B | Ascend NPU only; validated for Wan2.2 A14B cascade models; TI2V-5B not supported; offline `mxfp4_dualscale` uses calibrated `mul_scale` for best accuracy |
 | AutoRound | [AutoRound](autoround.md) | Pre-quantized W4A16 checkpoints | FLUX.1-dev; Qwen-Image/Wan2.2 not validated | Checkpoint-driven |
 | msModelSlim | [msModelSlim](msmodelslim.md) | Pre-quantized Ascend checkpoints | Wan2.2 recipe; HunyuanImage-3.0 inference target | Ascend/NPU path |
+| TorchAO | [TorchAO](torchao.md) | Pre-quantized FP8 weight-only (W8A16) | Boogu-Image Base/Edit `-fp8` | Validated for the Boogu-Image diffusion transformer |
 
 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
 
@@ -58,13 +59,14 @@ checkpoint contains a supported `quantization_config`; the non-AR stages stay
 in BF16 unless the model guide explicitly adds support.
 
 | Method | Guide | Scope | Example models | Status |
-|--------|-------|-------|----------------|--------|
+| -------- | ------- | ------- | ---------------- | -------- |
 | ModelOpt | [ModelOpt](modelopt.md) | Thinker or language-model checkpoint config | Qwen3-Omni thinker | ModelOpt checkpoint path |
 | Int8 | [Int8](int8.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
 | BitsAndBytes | [BitsAndBytes](bitsandbytes.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
 | MXFP8 | [MXFP8](mxfp8.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
 | MXFP4 | [MXFP4](mxfp4.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
 | AutoRound | [AutoRound](autoround.md) | Thinker or language-model checkpoint config | Qwen2.5-Omni, Qwen3-Omni | Supported through AutoRound checkpoints |
+| TorchAO | [TorchAO](torchao.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
 | msModelSlim | [msModelSlim](msmodelslim.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
 
 ### Multi-Stage Diffusion Model (BAGEL, GLM-Image)
@@ -73,7 +75,7 @@ These models split generation across multiple stages. Quantization must be
 attached to the intended stage rather than applied globally.
 
 | Method | Guide | Scope | Example models | Status |
-|--------|-------|-------|----------------|--------|
+| -------- | ------- | ------- | ---------------- | -------- |
 | FP8 | [FP8](fp8.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Requires model-specific validation |
 | Int8 | [Int8](int8.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Requires model-specific validation |
 | BitsAndBytes | [BitsAndBytes](bitsandbytes.md) | Stage-specific transformer or DiT module | BAGEL, GLM-Image | Not validated |
@@ -81,6 +83,7 @@ attached to the intended stage rather than applied globally.
 | MXFP8 | [MXFP8](mxfp8.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Not validated |
 | MXFP4 | [MXFP4](mxfp4.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Not validated |
 | AutoRound | [AutoRound](autoround.md) | Checkpoint-defined stage | BAGEL, GLM-Image | No validated checkpoint listed |
+| TorchAO | [TorchAO](torchao.md) | Stage-specific serialized checkpoint | BAGEL, GLM-Image | Not validated |
 | msModelSlim | [msModelSlim](msmodelslim.md) | Ascend-generated stage weights | GLM-Image | Requires model-specific adaptation |
 
 !!! note
@@ -105,8 +108,8 @@ config = build_quant_config({
 ```
 
 | Component | Default quantized? | Notes |
-|-----------|--------------------|-------|
-| Diffusion transformer | Yes | Primary target for FP8, Int8, BitsAndBytes, ModelOpt, MXFP8, MXFP4, AutoRound, and msModelSlim |
+| ----------- | -------------------- | ------- |
+| Diffusion transformer | Yes | Primary target for FP8, Int8, BitsAndBytes, ModelOpt, MXFP8, MXFP4, AutoRound, TorchAO, and msModelSlim |
 | Text encoder | No | Keep BF16 unless a method-specific guide documents support |
 | VAE | No | Keep BF16; storage-only paths are method-specific |
 | Scheduler/tokenizer | No | Loaded from the base model repository |
@@ -114,7 +117,7 @@ config = build_quant_config({
 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
 
 | Component | Default quantized? | Notes |
-|-----------|--------------------|-------|
+| ----------- | -------------------- | ------- |
 | Thinker or AR language model | Yes, when checkpoint config is supported | ModelOpt FP8/NVFP4 or AutoRound checkpoint config |
 | Audio encoder | No | BF16 |
 | Vision encoder | No | BF16 |
@@ -124,7 +127,7 @@ config = build_quant_config({
 ### Multi-Stage Diffusion Model (BAGEL, GLM-Image)
 
 | Component | Default quantized? | Notes |
-|-----------|--------------------|-------|
+| ----------- | -------------------- | ------- |
 | Selected diffusion or transformer stage | Method-specific | Must be routed to the intended stage |
 | Other generation stages | No | Keep BF16 unless separately validated |
 | VAE, tokenizer, scheduler | No | Loaded from the base checkpoint |
@@ -204,7 +207,7 @@ The output JSON includes `output_metrics`, `reference_generation`, and
 `candidate_generation`.
 
 | Metric | Direction | Meaning |
-|--------|-----------|---------|
+| -------- | ----------- | --------- |
 | `cosine_similarity` | Higher is better | Vector direction similarity between output pixels or frames. Useful as a broad sanity check. |
 | `mae` | Lower is better | Mean absolute pixel or frame error. For decoded outputs, values are in uint8 pixel units. |
 | `mse` / `rmse` | Lower is better | Squared error and its square root. These penalize localized large differences more than `mae`. |
@@ -217,7 +220,7 @@ The output JSON includes `output_metrics`, `reference_generation`, and
 Recommended starting thresholds for same-seed diffusion comparisons:
 
 | Metric | Smoke threshold | Stricter target | Notes |
-|--------|-----------------|-----------------|-------|
+| -------- | ----------------- | ----------------- | ------- |
 | `psnr_db` | `>= 20.0` | `>= 25.0` | Good for quick image or frame regression checks. |
 | `mae` | `<= 12.0` | `<= 6.0` | Interpreted in decoded uint8 pixel units. |
 | `cosine_similarity` | `>= 0.98` | `>= 0.995` | Less sensitive to global scale than L2-style metrics. |

docs/user_guide/quantization/torchao.md

@@ -0,0 +1,94 @@
+# TorchAO Quantization
+
+## Overview
+
+[TorchAO](https://github.com/pytorch/ao) provides quantization tools for
+PyTorch and supports both pre-quantized checkpoint loading and runtime
+quantization.
+
+## Hardware Support
+
+| Device | Support |
+| -------- | --------- |
+| NVIDIA Blackwell GPU (SM 100+) | ⭕ |
+| NVIDIA Ada GPU (SM 89) | ✅ |
+| NVIDIA Hopper GPU (SM 90) | ⭕ |
+| NVIDIA Ampere GPU (SM 80+) | ⭕ |
+| AMD ROCm | ⭕ |
+| Intel XPU | ⭕ |
+| Ascend NPU | ⭕ |
+
+Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
+guide.
+
+## Model Type Support
+
+### Diffusion Model (Boogu-Image)
+
+| Model                 | Checkpoint                                                         | Scope                      | Scheme                  |
+| --------------------- | ------------------------------------------------------------------ | -------------------------- | ----------------------- |
+| Boogu-Image Base/Edit | `Boogu/Boogu-Image-0.1-Base-fp8`, `Boogu/Boogu-Image-0.1-Edit-fp8` | Diffusion transformer only | FP8 weight-only (W8A16) |
+
+## Configuration
+
+Using Boogu-Image as an example:
+
+```bash
+vllm serve Boogu/Boogu-Image-0.1-Base-fp8 \
+  --omni \
+  --port 8091 \
+  --diffusion-quantization-config \
+  '{"transformer":{"method":"torchao_float8_weight_only"}}'
+```
+
+The equivalent command using the complete serialized TorchAO configuration is:
+
+```bash
+vllm serve Boogu/Boogu-Image-0.1-Base-fp8 \
+  --omni \
+  --port 8091 \
+  --diffusion-quantization-config \
+  '{
+    "transformer": {
+      "method": "torchao",
+      "quant_type": {
+        "default": {
+          "_type": "Float8WeightOnlyConfig",
+          "_version": 2,
+          "_data": {
+            "weight_dtype": {
+              "_type": "torch.dtype",
+              "_data": "float8_e4m3fn"
+            },
+            "set_inductor_config": false
+          }
+        }
+      }
+    }
+  }'
+```
+
+## Parameters
+
+| Parameter    | Type | Default | Description                                                                                                                         |
+| ------------ | ---- | ------- | ----------------------------------------------------------------------------------------------------------------------------------- |
+| `method`     | str  | -       | Use `torchao_float8_weight_only` for the serialized FP8 weight-only shorthand, or `torchao` with a full `quant_type` configuration. |
+| `quant_type` | dict | -       | Complete serialized TorchAO configuration used with `method: "torchao"`.                                                            |
+
+See TorchAO's [vLLM integration guide](https://docs.pytorch.org/ao/0.17/eager_tutorials/torchao_vllm_integration.html#configuration-system)
+for the serialized form and [workflow configs](https://docs.pytorch.org/ao/0.17/api_reference/api_ref_quantization.html#workflow-configs)
+for available configuration classes and parameters.
+
+## Validation and Notes
+
+At load time, vLLM-Omni builds a `TorchAOConfig` from
+`--diffusion-quantization-config`. For the Boogu-Image checkpoints documented
+here, indexed PyTorch `.bin` shards are loaded through `pt_weights_iterator`.
+
+When a component map is used, only the components included in the
+configuration use TorchAO. Other components keep their own checkpoint and
+runtime settings.
+
+Use either the `torchao_float8_weight_only` shorthand or the equivalent full
+`quant_type` configuration shown above. The checkpoint must already contain
+weights quantized with TorchAO.

examples/offline_inference/image_to_video/README.md

@@ -1,7 +1,7 @@
 # Image-To-Video
 
 This shared example generates videos from images with VACE, Wan2.2, LTX-2,
-HunyuanVideo-1.5, SANA-Video, Cosmos3, and other compatible pipelines.
+HunyuanVideo-1.5, SANA-Video, Cosmos3, MAGI-2, and other compatible pipelines.
 
 - `image_to_video.py`: command-line script for single video generation with advanced options.
 
@@ -27,9 +27,14 @@ This folder provides a unified CLI script for image-to-video generation using vL
 | `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | 480 x 832 | 81 | 50 | 4.0 | Around 20–25 GiB BF16, smallest I2V model |
 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v` | 480 x 832 | 121 | 50 | 6.0 | Around 100 GiB at default settings; the example enables `--enable-cpu-offload` + VAE tiling/slicing to fit an 80 GiB card |
 | `Lightricks/LTX-2` | 512 x 768 | 121 | 40 | video 3.0 / audio 7.0 | Memory use depends on frame count and tensor parallelism |
+| `sand-ai/MAGI-2-preview` | 512 x 896 | 125 | 100 | Model-fixed | Native four-GPU TP/SP; resident SP4 default; DLO available |
 | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers` | 480 x 832 | 81 | 50 | 6.0 | Native `SanaImageToVideoPipeline`; Wan VAE |
 | `Efficient-Large-Model/SANA-Video_2B_720p_diffusers` | 704 x 1280 | 81 | 50 | 6.0 | Native `SanaImageToVideoPipeline`; LTX-2 Video VAE |
 
+MAGI-2 native Preview setup, four-GPU topology and DLO choices, I2VA commands,
+and eight-GPU validation status are documented in the
+[`MAGI-2 Preview L20X recipe`](../../../recipes/SandAI/MAGI-2-preview-L20X.md).
+
 !!! info
     Peak VRAM: based on basic single-card usage, batch size = 1, without any acceleration/optimization features. Some model weights cannot fit into one card with 80 GiB VRAM, which may need to use CPU offloading.