[WIP] Optimize Helios accuracy-maintained path

Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
2026-05-21 16:34:09 +00:00
528 changed files with 11048 additions and 63065 deletions
@@ -52,7 +52,6 @@ time timeout -k 30 30m docker run \
    pip install tblib==3.1.0
    cd /workspace/vllm-omni
    pytest -v -s -m "core_model and xpu and B60"
    pytest -v -s tests/diffusion/quantization/test_mxfp8_config.py
    export VLLM_XPU_USE_SAMPLER_KERNEL=0    # NOTE: Remove this after vLLM v0.21.1 is merged. Fixes Qwen2-5 omni-expansion tests.
    pytest -v -s -m "advanced_model and xpu and B60"
 '
@@ -248,47 +248,6 @@ steps:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate
      - label: "TTS · GLM-TTS Test"
        timeout_in_minutes: 20
        commands:
          - |
            timeout 20m bash -c "
              export VLLM_LOGGING_LEVEL=DEBUG
              pytest -s -v tests/e2e/online_serving/test_glm_tts.py tests/e2e/offline_inference/test_glm_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
            "
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 1
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate
      - label: "Omni · Qwen3-Omni Test"
        timeout_in_minutes: 30
@@ -389,31 +348,6 @@ steps:
              volumes:
                - "/fsx/hf_cache:/fsx/hf_cache"
      - label: "Omni · Higgs-Audio V2 Test"
        timeout_in_minutes: 30
        commands:
          - |
            timeout 30m bash -c "
              export VLLM_LOGGING_LEVEL=DEBUG
              export VLLM_WORKER_MULTIPROC_METHOD=spawn
              export VLLM_USE_DEEP_GEMM=0
              export VLLM_MOE_USE_DEEP_GEMM=0
              pytest -s -v tests/e2e/online_serving/test_higgs_audio_v2.py tests/e2e/offline_inference/test_higgs_audio_v2.py -m 'advanced_model and cuda' --run-level 'advanced_model'
            "
        agents:
          queue: "gpu_1_queue"
        plugins:
          - docker#v5.2.0:
              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
              always-pull: true
              propagate-environment: true
              shm-size: "8gb"
              environment:
                - "HF_HOME=/fsx/hf_cache"
                - "HF_TOKEN"
              volumes:
                - "/fsx/hf_cache:/fsx/hf_cache"
      - label: "Diffusion · Qwen Image Edit Test"
        timeout_in_minutes: 20
        commands:
@@ -305,51 +305,6 @@ steps:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate
      - label: ":full_moon: Omni · Perf Test · Multi-Replica"
        key: nightly-omni-performance-multi-replicas
        timeout_in_minutes: 180
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
            exit $$EXIT
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 3
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate
      - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100"
        timeout_in_minutes: 45
@@ -459,52 +414,6 @@ steps:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate
      - label: ":full_moon: VoxCPM2 · Perf Test"
        key: nightly-voxcpm2-performance
        timeout_in_minutes: 180
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_voxcpm2.json
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
            exit $$EXIT
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 1
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate
  # Diffusion X2I suite: x2i / x2a / x2t and related non-video paths; x2v is only in "Diffusion X2V Model Test" below.
  - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
    key: nightly-diffusion-x2iat-group
@@ -616,7 +525,6 @@ steps:
            tests/e2e/online_serving/test_longcat_image_expansion.py
            tests/e2e/online_serving/test_longcat_image_edit_expansion.py
            tests/e2e/online_serving/test_flux_2_dev_expansion.py
            tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py
            -m "full_model and diffusion and H100 and distributed_cuda"
            --run-level "full_model"
        agents:
@@ -670,7 +578,7 @@ steps:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
-                        nvidia.com/gpu: 2
+                        nvidia.com/gpu: 3
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
@@ -889,8 +797,8 @@ steps:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate
-      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511"
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2509"
-        key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511
+        key: nightly-diffusion-x2iat-performance-qwen-image-edit-2509
        timeout_in_minutes: 180
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
@@ -898,7 +806,7 @@ steps:
          - export CACHE_DIT_VERSION=1.3.0
          - |
            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
@@ -985,20 +893,26 @@ steps:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate
-      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · BAGEL"
+      - label: ":full_moon: Diffusion X2I(&A&T) · HunyuanImage3 · DiT Perf Test"
-        key: nightly-diffusion-x2iat-performance-bagel
+        key: nightly-hunyuan-image3-performance
-        timeout_in_minutes: 180
+        soft_fail: true
        timeout_in_minutes: 120
        if: build.env("RUN_HUNYUAN_IMAGE3_PERF") == "1"
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
          - export CACHE_DIT_VERSION=1.3.0
          - |
            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp4_fp8.json
-            EXIT=$$?
+            EXIT1=$$?
            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_sp2.json
            EXIT2=$$?
            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_cfgp2.json
            EXIT3=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
-            exit $$EXIT
+            exit $$((EXIT1 | EXIT2 | EXIT3))
        agents:
          queue: "mithril-h100-pool"
        plugins:
@@ -1008,7 +922,7 @@ steps:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
-                        nvidia.com/gpu: 1
+                        nvidia.com/gpu: 4
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
@@ -1045,7 +959,7 @@ steps:
      - label: ":full_moon: Diffusion X2V · Function Test"
        timeout_in_minutes: 90
        commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_hunyuan_video_15_expansion.py tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model"
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_hunyuan_video_15_expansion.py  -m "full_model and cuda" --run-level "full_model"
        agents:
          queue: "mithril-h100-pool"
        plugins:
@@ -1084,8 +998,6 @@ steps:
        timeout_in_minutes: 180
        commands:
          - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model
          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model
          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model
          - pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m full_model --run-level full_model
          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
        agents:
@@ -1174,8 +1086,9 @@ steps:
    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test"
    steps:
      - label: ":full_moon: Quantization Test with H100"
        timeout_in_minutes: 90
        commands:
-          - timeout 60m pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model"
+          - timeout 30m pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model"
        agents:
          queue: "mithril-h100-pool"
        plugins:
@@ -1276,16 +1189,14 @@ steps:
      - nightly-omni-performance-no-async-chunk
      - nightly-omni-performance-async-chunk
      - nightly-omni-performance-vllm-text
      - nightly-omni-performance-multi-replicas
      - nightly-tts-performance
      - nightly-diffusion-x2iat-performance-qwen-image
      - nightly-diffusion-x2iat-performance-qwen-image-edit
-      - nightly-diffusion-x2iat-performance-qwen-image-edit-2511
+      - nightly-diffusion-x2iat-performance-qwen-image-edit-2509
      - nightly-diffusion-x2iat-performance-qwen-image-layered
      - nightly-diffusion-x2iat-performance-bagel
      - nightly-diffusion-x2v-performance
      - nightly-testcase-statistics
-    if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1"
+    if: build.env("NIGHTLY") == "1"
    commands:
      - pip install openpyxl
      - export DEFAULT_INPUT_DIR=tests/dfx/perf/results
@@ -1294,12 +1205,10 @@ steps:
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-multi-replicas
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit
-      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2511
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2509
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-bagel
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance
      - buildkite-agent artifact download "tests/dfx/perf/results/*.html" . --step nightly-testcase-statistics
      - python tools/nightly/generate_nightly_perf_excel.py
@@ -457,6 +457,51 @@ steps:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate
      - label: "Diffusion · Helios Distilled Acc/Perf Compare"
        commands:
          - |
            timeout 90m bash -c "
              set -euxo pipefail
              export VLLM_IMAGE_FETCH_TIMEOUT=60
              pytest -s -v tests/e2e/accuracy/test_helios_distilled.py::test_helios_distilled_pr_matches_main_acc_perf \
                -m 'full_model and benchmark' \
                --run-level 'full_model'
            "
          - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/helios_distilled_acc_perf/result.json"
          - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/helios_distilled_acc_perf/artifacts/**/*.mp4"
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 1
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate
      - label: "Diffusion · Qwen Image Test"
        commands:
@@ -21,9 +21,6 @@ Before proposing changes, ask for the optimization scene if it is not already kn
 - Current enabled strategies: USP/SP, CFG parallel, HSDP/FSDP, VAE patch parallel, torch.compile, profiler options.
 - Optimization target: latency, throughput, memory, cost, or quality-preserving speed.
 - Precision/quality tolerance: bf16/fp8/quantization/sparsity/approximate attention allowed or not.
 - Quality validation method: exact-output comparison when feasible, or metrics
  such as SSIM, PSNR, LPIPS, cosine similarity, MAE, MSE, temporal flicker
  checks, baseline self-run variance, and reviewable output artifacts.
 ## Workflow
@@ -82,19 +79,7 @@ Before proposing changes, ask for the optimization scene if it is not already kn
   - Record latency, stage timings, memory, output quality, and logs.
   - Report comparison tables in ms. Include at least end-to-end client time, server `inference_time`, server stage generation time if available, `vae.encode`, `diffuse`, `vae.decode`, and peak memory.
-6. **Enforce the quality and precision gate for every optimization.**
+6. **Collect diagnostic trace only after narrowing hypotheses.**
   - Do not mark an optimization ready until quality is checked on the same
     workload: model, prompt/media, seed, shape, steps, scheduler, dtype,
     backend, parallelism, and output encoding.
   - For math-preserving changes, require exact/near-exact agreement or show
     that differences are within baseline self-run variance.
   - For precision, quantization, approximate attention, sparse/custom kernels,
     or backend changes, include reviewable artifacts and metrics such as SSIM,
     PSNR, LPIPS, cosine similarity, MAE, MSE, and temporal flicker checks.
   - If quality tolerance is not stated, default to quality-preserving behavior.
     Failed or inconclusive quality validation blocks a ready-to-merge claim.
 7. **Collect diagnostic trace only after narrowing hypotheses.**
   - Use torch profiler for a small number of requests.
   - Run two separate diagnostic traces instead of mixing concerns:
     - **Operator/shape trace**: enable `torch_profiler_record_shapes=True` and keep stack collection disabled. Use this to rank CUDA kernels, NCCL collectives, attention/MLP/norm/RoPE work, and shape-specific hot operators.
@@ -106,16 +91,15 @@ Before proposing changes, ask for the optimization scene if it is not already kn
   - Diagnostic reports must be written to disk and preserve: server command, profiler config, warmup/request/polling commands, trace artifact paths, rank analyzed, analyzer output or summary, and the decision about whether additional ranks are necessary.
   - Analyze both rank-level balance and device-level free bubbles when additional ranks are opened.
-8. **Analyze host, communication, and operators.**
+7. **Analyze host, communication, and operators.**
   - Find GPU idle/free intervals and map each large gap to the enclosing CPU/Python code.
   - Separate real GPU idle from profiler overhead such as CUPTI `Command Buffer Full`.
   - Compare NCCL kernel time to user annotations; annotations can overcount nested intervals.
   - Rank operator work by total CUDA time and by repeated small-kernel launch count.
-9. **Produce an optimization plan.**
+8. **Produce an optimization plan.**
   - Classify candidates as P0/P1/P2.
   - For each candidate, state necessity, expected benefit, implementation path, validation plan, and quality risk.
   - Include a concrete quality/precision validation gate for each candidate.
   - Do not implement high-risk operator rewrites before proving the operator is a bottleneck for the target shapes.
   - End the plan with a user-facing candidate selection table. The assistant
     should not automatically choose a risky optimization just because it is
@@ -140,13 +124,7 @@ Before proposing changes, ask for the optimization scene if it is not already kn
 - **P1:** meaningful code changes with contained risk. Examples: cross-attention KV caching, VAE gather/broadcast reduction, AdaLayerNorm/RMSNorm/RoPE fusion after trace evidence.
 - **P2:** high implementation or quality risk. Examples: FA to LA replacement, custom Triton/CUDA fused kernels, FP8/quantization, sparsity/Rainfusion-style acceleration.
-Every implemented optimization needs A/B validation and a passing
+Every implemented optimization needs A/B validation and quality regression. A/B means same workload and hardware before/after. Quality regression means checking generated image/video stability, artifacts, temporal flicker, and seed behavior when precision or approximate kernels change.
 quality/precision gate, including math-preserving changes such as padding trim,
 layout cleanup, cache reuse, or host/runtime cleanup. Use objective metrics
 when available, keep reviewable artifacts, and compare against baseline
 self-run variance for precision, quantization, approximate kernels, or backend
 changes. If quality validation fails or is inconclusive, do not present the
 optimization as ready to merge.
 ## Optimization Layers
@@ -270,6 +270,3 @@ vllm_omni/_version.py
 .worktrees/
 # CI overlay yamls materialized from tests/utils.py:_CI_OVERLAYS at test time
 tests/.ci_generated/
 # Local Lance precision-matching dump scripts (not part of the repo)
 lance_compare/
@@ -10,11 +10,6 @@ build:
  jobs:
    post_checkout:
      - git fetch --unshallow || true
    pre_install:
      # Pre-install CPU torch so the subsequent `pip install .[docs]` does not
      # pull ~2 GB of CUDA 13 libs (cuBLAS/cuDNN/NCCL/...) that exceed RTD's
      # 1 GB memory cap and time out the mkdocs build.
      - python -m pip install --index-url https://download.pytorch.org/whl/cpu torch
 mkdocs:
  configuration: mkdocs.yml
@@ -1,60 +1,44 @@
-# Benchmarks
+# Benchmarks Overview and Architecture
-This directory contains benchmark suites for evaluating different model families and infrastructure components in vLLM-Omni. Each subfolder targets a different benchmark family with its own scripts, configs, and metrics. See the per-subfolder READMEs for detailed usage.
+This document explains the benchmark architecture across all benchmark assets in this repo. It describes what we measure, and where to find or plug in new scenarios. Per-task details remain in subfolder READMEs (e.g., `benchmarks/<model>/README.md`).
-## Benchmark families
+## Scope and goals
 - Establish repeatable latency/throughput measurements for multimodal LLM pipelines.
 - Provide both HF Transformers (offline) and vLLM-Omni (multi-stage/pipeline) baselines.
 - Make it easy to plug in new datasets and models with minimal changes to the runner scripts.
-### [TTS](tts/README.md) — Text-to-Speech
+## Dataset and inputs
 - Default example: SeedTTS top-100 prompts (`benchmarks/build_dataset/top100.txt`) via `benchmarks/build_dataset/`.
 - Extensible: drop in new prompt files or modality-aligned payloads; keep the expected format for the consuming scripts (e.g., one prompt per line).
 - If you add a new dataset, document it under `benchmarks/<model>/README.md` and point scripts to your data path.
-Model-agnostic serving benchmarks for TTS models, including Qwen3-TTS and VoxCPM2.
+## Directory layout
 - `benchmarks/build_dataset/` — dataset prep utilities (e.g., SeedTTS top100).
 - `benchmarks/<model>/vllm_omni/` — vLLM-Omni pipeline benchmarks, logs, outputs.
 - `benchmarks/accuracy/` — accuracy benchmark integrations that adapt external
  benchmark suites to vLLM-Omni serving and evaluation flows.
 - Add new tasks under `benchmarks/<model>/...` with the same pattern: `transformers/`, `vllm_omni/`, task-specific README, and (optionally) dataset prep notes.
- **Layout**: `tts/bench_tts.py` (serving benchmark driver), `tts/model_configs.yaml` (model registry), `tts/plot_results.py` (result plotting)
+## Reference workflows
- **Dataset**: Seed-TTS full or text-only datasets, plus bundled smoke/design prompts under `build_dataset/`
+- **HF Transformers (offline, single process)**  
- **Key metrics**: TTFP (time to first audio packet), E2E latency, RTF (real-time factor), throughput (audio seconds / wall-clock second)
+  Script (example): `benchmarks/<model>/transformers/eval_qwen3_moe_omni_transformers.sh`  
  Outputs: `benchmark_results/perf_stats.json`, `benchmark_results/results.json`, `benchmark_results/audio/` (if audio is produced).
-### [Diffusion](diffusion/README.md) — Image and Video Generation
+- **vLLM-Omni end-to-end pipeline**  
  Script (example): `benchmarks/<model>/vllm_omni/eval_qwen3_moe_omni.sh`  
  Outputs: `vllm_omni/logs/*.stats.jsonl` (per-stage/overall latency & TPS), `vllm_omni/logs/stage*.log`, `vllm_omni/outputs/` (text/audio artifacts).
-Online-serving benchmark for diffusion image/video models, sending requests to the configured vLLM serving endpoint (`/v1/chat/completions`, `/v1/images/generations`, or `/v1/videos`, depending on backend/task).
+- **Adding a new task/model**  
  1) Create `benchmarks/<model>/transformers/` and/or `benchmarks/<model>/vllm_omni/` with scripts referencing your model and dataset.  
  2) Add a task README describing dataset, configs, and expected outputs.  
  3) Keep the output/log structure similar for easy comparison (perf_stats/results/audio or text outputs; stats.jsonl/logs for pipeline).
- **Tasks**: text-to-image, text-to-video, image-to-image, image-to-video, text+image-to-image, text+image-to-video
+## Metrics to watch
- **Datasets**: `vbench`, `trace`, `random`
+- **Throughput**: `overall_tps`, `*_tps_avg` per stage.
- **Key metrics**: request throughput, latency percentiles, optional SLO attainment
+- **Latency distribution**: look for long tails in `*.stats.jsonl`.
 - **Quality/completeness**: missing outputs or errors in stage logs indicate pipeline failures or misconfigurations.
-### [GLM-Image](glm_image/README.md) — Text-to-Image and Image-to-Image
+## Troubleshooting
-
+- Verify GPU/driver/FlashAttention2 requirements for your chosen model/config.
-Benchmarks for GLM-Image performance across HuggingFace baseline, vLLM-Omni offline inference, and vLLM-Omni online serving.
+- Ensure network access for dataset/model downloads (Google Drive, Hugging Face, etc.).
-
+- If outputs are missing or slow, inspect per-stage logs and `*.stats.jsonl` for errors, stragglers, or contention.
 - **Layout**: `glm_image/huggingface/` (HF baseline), `glm_image/vllm-omni/` (offline inference), `glm_image/benchmark_glm_image.py` (online serving)
 - **Tasks**: text-to-image and image-to-image
 - **Key metrics**: request/image throughput, latency percentiles, optional per-stage pipeline timings
 ### [Distributed](distributed/omni_connectors/README.md) — RDMA Connector Testing
 RDMA environment setup and transfer tests for `MooncakeTransferEngineConnector`, including pytest-based single-node checks and manual cross-node benchmarks.
 - **Transfer modes**: `copy`, `zerocopy`, `gpu` (GPUDirect)
 - **Supports**: single-node pytest suites and manual multi-node/cross-node transfer testing
 ### [Accuracy](accuracy/README.md) — Image Generation and Editing Quality
 Accuracy benchmarks for image generation/editing models, adapting external suites to vLLM-Omni serving and local judge-evaluation flows.
 - **Layout**: `accuracy/text_to_image/` (GEBench), `accuracy/image_to_image/` (GEdit-Bench)
 - **Method**: generation and judge scoring both run through local `vllm-omni serve` endpoints
 ### Common serving metrics framework
 `vllm_omni/benchmarks/` extends `vllm bench serve --omni` with Omni-specific datasets, backends, and multimodal metrics. Key metrics include:
 - **Text output**: TTFT (time to first token), TPOT (time per output token), ITL (inter-token latency)
 - **Audio output**: TTFP (time to first audio packet), E2E latency, RTF (real-time factor)
 - **Throughput**: request throughput, output token throughput, total token throughput, audio throughput
 See `vllm_omni/benchmarks/serve.py` for the `vllm bench serve --omni` runner wrapper and `vllm_omni/benchmarks/metrics/` for Omni metric definitions.
 ## Adding a new benchmark
 1. Create a subfolder under `benchmarks/<name>/` with scripts, configs if needed, and a `README.md`.
 2. If comparing against another runtime, use clear backend subfolders where applicable, such as `huggingface/` and `vllm-omni/`, or follow the shared TTS serving benchmark pattern in `tts/`.
 3. Add reusable dataset or prompt-building utilities to `build_dataset/` if applicable.
 4. Update this README with a link to the new benchmark family.
@@ -10,8 +10,6 @@ from typing import Any
 import aiohttp
 from tqdm import tqdm
 DEFAULT_EDITS_BOT_TASK = "think"
@dataclass
 class RequestFuncInput:
@@ -29,7 +27,6 @@ class RequestFuncInput:
    extra_body: dict[str, Any] = field(default_factory=dict)
    image_paths: list[str] | None = None
    request_id: str = field(default_factory=lambda: str(uuid.uuid4()))
    default_bot_task: str | None = DEFAULT_EDITS_BOT_TASK
@dataclass
@@ -56,95 +53,6 @@ def _encode_image_as_data_url(path: str) -> str:
    return f"data:{mime};base64,{encoded}"
 async def async_request_image_edits(
    input: RequestFuncInput,
    session: aiohttp.ClientSession,
    pbar: tqdm | None = None,
    enable_diffusion_pipeline_profiler: bool = False,
 ) -> RequestFuncOutput:
    """POST /v1/images/edits (multipart)."""
    del enable_diffusion_pipeline_profiler
    output = RequestFuncOutput()
    output.start_time = time.perf_counter()
    extra_body = dict(input.extra_body)
    width = input.width or extra_body.get("width") or 1024
    height = input.height or extra_body.get("height") or 1024
    edits_url = input.api_url
    form = aiohttp.FormData()
    form.add_field("model", input.model)
    form.add_field("prompt", input.prompt)
    form.add_field("size", f"{width}x{height}")
    form.add_field("response_format", "b64_json")
    if input.num_inference_steps is not None:
        form.add_field("num_inference_steps", str(input.num_inference_steps))
    elif extra_body.get("num_inference_steps") is not None:
        form.add_field("num_inference_steps", str(extra_body["num_inference_steps"]))
    if input.seed is not None:
        form.add_field("seed", str(input.seed))
    elif extra_body.get("seed") is not None:
        form.add_field("seed", str(extra_body["seed"]))
    if extra_body.get("guidance_scale") is not None:
        form.add_field("guidance_scale", str(extra_body["guidance_scale"]))
    if extra_body.get("negative_prompt") is not None:
        form.add_field("negative_prompt", str(extra_body["negative_prompt"]))
    if extra_body.get("true_cfg_scale") is not None:
        form.add_field("true_cfg_scale", str(extra_body["true_cfg_scale"]))
    if extra_body.get("sys_type") is not None:
        form.add_field("sys_type", str(extra_body["sys_type"]))
    if extra_body.get("system_prompt") is not None:
        form.add_field("system_prompt", str(extra_body["system_prompt"]))
    bot_task = extra_body.get("bot_task")
    if bot_task is None and input.default_bot_task is not None:
        bot_task = input.default_bot_task
    if bot_task is not None:
        form.add_field("bot_task", str(bot_task))
    assert input.image_paths is not None
    for img_path in input.image_paths:
        if not os.path.exists(img_path):
            output.error = f"Image file not found: {img_path}"
            output.success = False
            if pbar:
                pbar.update(1)
            return output
        with open(img_path, "rb") as img_f:
            image_bytes = img_f.read()
        form.add_field(
            "image",
            image_bytes,
            filename=os.path.basename(img_path),
            content_type=_guess_mime_type(img_path),
        )
    try:
        async with session.post(edits_url, data=form) as response:
            if response.status == 200:
                resp_json = await response.json()
                output.response_body = resp_json
                output.success = True
            else:
                output.error = f"HTTP {response.status}: {await response.text()}"
                output.success = False
    except Exception as e:
        output.error = str(e)
        output.success = False
    output.latency = time.perf_counter() - output.start_time
    if output.success and input.slo_ms is not None:
        output.slo_achieved = (output.latency * 1000.0) <= float(input.slo_ms)
    if pbar:
        pbar.update(1)
    return output
 async def async_request_chat_completions(
    input: RequestFuncInput,
    session: aiohttp.ClientSession,
@@ -370,7 +278,7 @@ async def async_request_v1_videos(
                return output
        # invoke a poll request (GET /v1/videos/{video_id})
-        poll_interval = 2.0  # Unit(s)
+        poll_interval = 0.2  # Unit(s)
        timeout_seconds = 600.0
        deadline = time.perf_counter() + timeout_seconds
        job_url = f"{input.api_url}/{job_id}"
@@ -470,7 +378,6 @@ backends_function_mapping = {
    "2i": {
        "/v1/chat/completions": (async_request_chat_completions, "/v1/chat/completions"),
        "/v1/images/generations": (async_request_openai_image_generations, "/v1/images/generations"),
        "/v1/images/edits": (async_request_image_edits, "/v1/images/edits"),
    },
    "2v": {
        "/v1/videos": (async_request_v1_videos, "/v1/videos"),
@@ -1,328 +0,0 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 """Standalone diagnostic for the diffusion attention backends added in #3079.
 Exercises the same synthetic attention shape through each backend and each
 torch SDPA sub-kernel, with and without an attn_mask, so we can pinpoint why
 CUDNN_ATTN underperforms on some SKUs. Two typical causes:
  (a) cuDNN lacks a tuned kernel for this (SM, head_dim, seq) combination
      and silently falls back to MATH.
  (b) attn_mask is non-None and the cuDNN SDPA dispatch rejects it,
      walking CUDNN -> FLASH -> MATH.
 Run:
    python benchmarks/diffusion/bench_attention_backends.py --preset hv15
    python benchmarks/diffusion/bench_attention_backends.py --preset wan22
    python benchmarks/diffusion/bench_attention_backends.py \
        --batch 1 --heads 24 --seq 14336 --head-dim 128
 Optional extras:
    pip install --pre flash-attn-4        # FA4 is currently pre-release only
    pip install -U flashinfer              # latest FlashInfer (0.6.9)
 The table at the end is the data we want on the PR — surface the row where a
 backend is >1.5x the SDPA baseline, that's the one to gate off the auto-route.
 """
 from __future__ import annotations
 import argparse
 import time
 import torch
 from torch.nn.attention import SDPBackend, sdpa_kernel
 # Presets mirror the DiT hot-path attention call (single-stream block) at the
 # shapes we run in the PR's validation configs.
 _PRESETS = {
    # HunyuanVideo-1.5 480p/33f: (30 * 52 * 9) latent + ~256 text tokens.
    "hv15": {"batch": 1, "heads": 24, "seq": 14336, "head_dim": 128},
    # Wan 2.2 480p/33f rough estimate; adjust when we measure real shapes.
    "wan22": {"batch": 1, "heads": 40, "seq": 16384, "head_dim": 128},
    # Smaller image-gen shape for quick smoke tests.
    "flux": {"batch": 1, "heads": 24, "seq": 4096, "head_dim": 128},
 }
 _SDPA_BACKENDS = [
    ("CUDNN_ATTENTION", [SDPBackend.CUDNN_ATTENTION]),
    ("FLASH_ATTENTION", [SDPBackend.FLASH_ATTENTION]),
    ("EFFICIENT_ATTENTION", [SDPBackend.EFFICIENT_ATTENTION]),
    ("MATH", [SDPBackend.MATH]),
    # The PR's CUDNN_ATTN impl uses this priority chain; keep it here to show
    # what torch actually picks when multiple are allowed.
    ("CUDNN_ATTN_CHAIN", [SDPBackend.CUDNN_ATTENTION, SDPBackend.FLASH_ATTENTION, SDPBackend.MATH]),
 ]
 def _env_report() -> None:
    if not torch.cuda.is_available():
        raise SystemExit("CUDA not available; this diagnostic needs a GPU.")
    cc = torch.cuda.get_device_capability()
    print("=" * 72)
    print(f"GPU           : {torch.cuda.get_device_name()}")
    print(f"SM capability : sm_{cc[0]}{cc[1]}")
    print(f"torch         : {torch.__version__}")
    print(f"cuDNN version : {torch.backends.cudnn.version()}")
    print("flashinfer    : ", end="")
    try:
        import flashinfer
        print(getattr(flashinfer, "__version__", "present"))
    except Exception as e:
        print(f"not installed ({type(e).__name__})")
    print("=" * 72)
 def _make_qkv(batch: int, heads: int, seq: int, head_dim: int, device: str, dtype: torch.dtype):
    # Layout: (B, S, H, D) — what the PR's backends accept as input.
    q = torch.randn(batch, seq, heads, head_dim, device=device, dtype=dtype)
    k = torch.randn(batch, seq, heads, head_dim, device=device, dtype=dtype)
    v = torch.randn(batch, seq, heads, head_dim, device=device, dtype=dtype)
    return q, k, v
 def _make_mask(batch: int, seq: int, device: str, dtype: torch.dtype, pad_tokens: int = 128) -> torch.Tensor:
    # Mirrors a text-encoder padding mask: last `pad_tokens` positions masked.
    mask = torch.zeros(batch, 1, seq, seq, device=device, dtype=dtype)
    if pad_tokens > 0:
        mask[..., -pad_tokens:] = float("-inf")
    return mask
 def _time_call(fn, *args, warmup: int = 3, iters: int = 10) -> tuple[float, str]:
    """Return (median ms, error string). Skips on any exception so that a
    backend that rejects our args (wrong dtype, missing JIT module, unsupported
    kwarg value) doesn't abort the whole sweep."""
    try:
        for _ in range(warmup):
            fn(*args)
        torch.accelerator.synchronize()
        times = []
        for _ in range(iters):
            torch.accelerator.synchronize()
            t0 = time.perf_counter()
            fn(*args)
            torch.accelerator.synchronize()
            times.append((time.perf_counter() - t0) * 1000.0)
        times.sort()
        return times[len(times) // 2], ""
    except Exception as e:  # noqa: BLE001 — probe script, keep the whole table going
        msg = str(e).split("\n", 1)[0][:60]
        return float("nan"), f"{type(e).__name__}: {msg}" if msg else type(e).__name__
 def _run_sdpa_variants(q, k, v, attn_mask, scale: float) -> list[tuple[str, float, str]]:
    # q/k/v come in (B, S, H, D); F.sdpa wants (B, H, S, D).
    q_t = q.permute(0, 2, 1, 3)
    k_t = k.permute(0, 2, 1, 3)
    v_t = v.permute(0, 2, 1, 3)
    rows: list[tuple[str, float, str]] = []
    for name, backends in _SDPA_BACKENDS:
        def _call(mask=attn_mask):
            with sdpa_kernel(backends):
                return torch.nn.functional.scaled_dot_product_attention(
                    q_t, k_t, v_t, attn_mask=mask, dropout_p=0.0, is_causal=False, scale=scale
                )
        ms, err = _time_call(_call)
        rows.append((name, ms, err))
    return rows
 def _run_flashinfer(q, k, v, scale: float, backend: str | None = None, attn_mask=None) -> tuple[float, str]:
    """Call FlashInfer's dense single-prefill.
    ``backend`` hints at cutlass/fa3/trtllm if the installed version exposes
    that kwarg. ``attn_mask`` is the same additive 4D mask we pass to SDPA —
    we convert it to the 2D boolean ``custom_mask`` form FlashInfer accepts.
    Per FlashInfer docs, ``custom_mask`` only applies when ``causal=False``.
    """
    try:
        import inspect
        from flashinfer.prefill import single_prefill_with_kv_cache
    except Exception as e:
        return float("nan"), f"import-{type(e).__name__}"
    kwargs: dict = {"sm_scale": scale, "causal": False, "return_lse": False}
    if backend is not None:
        sig = inspect.signature(single_prefill_with_kv_cache)
        if "backend" not in sig.parameters:
            return float("nan"), "no-backend-kwarg"
        kwargs["backend"] = backend
    if attn_mask is not None:
        # Collapse (B, 1, S, S) additive float mask to (S, S) boolean.
        # FlashInfer expects True = keep, False = masked out.
        mask_2d = attn_mask[0, 0]
        kwargs["custom_mask"] = mask_2d != float("-inf")
    def _call():
        out = single_prefill_with_kv_cache(q[0], k[0], v[0], **kwargs)
        return out.unsqueeze(0)
    return _time_call(_call)
 def _run_fa4(q, k, v, scale: float) -> tuple[float, str]:
    """Call FlashAttention-4 directly (``pip install flash-attn-4``). FA4
    ships a Blackwell-native kernel via CuTe-DSL; on sm_120 it should beat
    cuDNN by ~20%. API lives under ``flash_attn.cute``; the older
    ``flash_attn.flash_attn_func`` path is FA2/FA3 only."""
    try:
        from flash_attn.cute import flash_attn_func
    except Exception as e:
        return float("nan"), f"import-{type(e).__name__}"
    def _call():
        # FA4 accepts (B, S, H, D) directly — same layout the PR's backends use.
        return flash_attn_func(q, k, v, softmax_scale=scale, causal=False)
    return _time_call(_call)
 def _run_flashinfer_cudnn_batch(q, k, v, scale: float) -> tuple[float, str]:
    """Call FlashInfer's direct cuDNN wrapper. Bypasses PyTorch SDPA dispatch
    (which has a few hundred ns of overhead per call) and talks to cuDNN FMHA
    straight. Useful as a ceiling for 'pure cuDNN, no SDPA' on Blackwell."""
    try:
        from flashinfer.prefill import cudnn_batch_prefill_with_kv_cache
    except Exception as e:
        return float("nan"), f"import-{type(e).__name__}"
    b, s, h, d = q.shape
    qo_indptr = torch.tensor([0, s], dtype=torch.int32, device=q.device)
    kv_indptr = torch.tensor([0, s], dtype=torch.int32, device=q.device)
    def _call():
        return cudnn_batch_prefill_with_kv_cache(
            q.reshape(b * s, h, d),
            k.reshape(b * s, h, d),
            v.reshape(b * s, h, d),
            qo_indptr=qo_indptr,
            kv_indptr=kv_indptr,
            max_qo_len=s,
            max_kv_len=s,
            sm_scale=scale,
            causal=False,
        )
    return _time_call(_call)
 def _print_table(title: str, rows: list[tuple[str, float, str]], baseline_name: str | None = None) -> None:
    print(f"\n{title}")
    print("-" * 88)
    header = f"{'backend':<24} {'median (ms)':>14}    {'vs baseline':>12}    status"
    print(header)
    print("-" * 88)
    baseline_ms = None
    if baseline_name is not None:
        for name, ms, _ in rows:
            if name == baseline_name and ms == ms:  # not NaN
                baseline_ms = ms
                break
    for name, ms, err in rows:
        ms_str = f"{ms:>14.3f}" if ms == ms else f"{'n/a':>14}"
        if baseline_ms is not None and ms == ms:
            ratio = baseline_ms / ms
            ratio_str = f"{ratio:>11.2f}x"
        else:
            ratio_str = f"{'—':>12}"
        status = "ok" if not err else f"FAILED ({err})"
        print(f"{name:<24} {ms_str}    {ratio_str}    {status}")
 def _pick_winner(rows: list[tuple[str, float, str]]) -> tuple[str, float] | None:
    """Return (backend, ms) of the fastest non-failing row, or None."""
    ok_rows = [(n, m) for n, m, err in rows if not err and m == m]
    if not ok_rows:
        return None
    return min(ok_rows, key=lambda x: x[1])
 def _bench_one_shape(shape: dict, dtype: torch.dtype, device: str) -> tuple[list, list]:
    torch.manual_seed(0)
    q, k, v = _make_qkv(**shape, device=device, dtype=dtype)
    scale = 1.0 / (shape["head_dim"] ** 0.5)
    rows_nomask = _run_sdpa_variants(q, k, v, attn_mask=None, scale=scale)
    rows_nomask.append(("FLASHINFER (default)", *_run_flashinfer(q, k, v, scale)))
    # `trtllm-gen` has no sm_120 cubins (NVIDIA/TensorRT-LLM#11799) — skipped.
    for fi_backend in ("fa2", "fa3", "cutlass", "auto"):
        rows_nomask.append((f"FLASHINFER ({fi_backend})", *_run_flashinfer(q, k, v, scale, backend=fi_backend)))
    rows_nomask.append(("FLASHINFER (cudnn-batch)", *_run_flashinfer_cudnn_batch(q, k, v, scale)))
    rows_nomask.append(("FA4 (direct)", *_run_fa4(q, k, v, scale)))
    mask = _make_mask(shape["batch"], shape["seq"], device=device, dtype=dtype, pad_tokens=256)
    rows_mask = _run_sdpa_variants(q, k, v, attn_mask=mask, scale=scale)
    rows_mask.append(("FLASHINFER (dense)", *_run_flashinfer(q, k, v, scale, attn_mask=mask)))
    rows_mask.append(("FA4 (direct)", float("nan"), "mask-not-supported"))
    return rows_nomask, rows_mask
 def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("--preset", choices=list(_PRESETS.keys()), default="hv15")
    parser.add_argument("--sweep", action="store_true", help="Run all presets and print a ranking")
    parser.add_argument("--batch", type=int, default=None)
    parser.add_argument("--heads", type=int, default=None)
    parser.add_argument("--seq", type=int, default=None)
    parser.add_argument("--head-dim", type=int, default=None)
    parser.add_argument("--dtype", choices=["bf16", "fp16"], default="bf16")
    parser.add_argument("--device", default="cuda:0")
    args = parser.parse_args()
    dtype = torch.bfloat16 if args.dtype == "bf16" else torch.float16
    _env_report()
    print(f"dtype         : {args.dtype}")
    presets = list(_PRESETS.keys()) if args.sweep else [args.preset]
    summary: dict[str, tuple[str, float] | None] = {}
    for preset_name in presets:
        shape = dict(_PRESETS[preset_name])
        if not args.sweep:
            for k in ("batch", "heads", "seq", "head_dim"):
                v = getattr(args, k if k != "head_dim" else "head_dim")
                if v is not None:
                    shape[k] = v
        print("\n" + "=" * 88)
        print(
            f"Preset: {preset_name}  |  batch={shape['batch']} heads={shape['heads']} "
            f"seq={shape['seq']} head_dim={shape['head_dim']}"
        )
        print("=" * 88)
        rows_nomask, rows_mask = _bench_one_shape(shape, dtype, args.device)
        _print_table(f"[{preset_name}] No attention mask", rows_nomask, baseline_name="CUDNN_ATTENTION")
        _print_table(
            f"[{preset_name}] With attention mask (pad 256 tokens)", rows_mask, baseline_name="CUDNN_ATTENTION"
        )
        summary[preset_name] = _pick_winner(rows_nomask)
    if args.sweep or len(presets) > 1:
        print("\n" + "=" * 88)
        print("Winners per preset (no-mask path)")
        print("=" * 88)
        for preset_name, winner in summary.items():
            if winner is None:
                print(f"  {preset_name:<10} — no successful backend")
            else:
                name, ms = winner
                print(f"  {preset_name:<10} {name:<24} {ms:>8.3f} ms")
    print("\nNotes:")
    print("  * Ratios are relative to CUDNN_ATTENTION. >1.0x means faster than cuDNN.")
    print("  * Mask-path winner inherits CUDNN_ATTN's fallback in the PR's backends.")
    print("  * `trtllm-gen` and FA4 4.0.0b10 are known-broken on sm_120 as of Apr 2026.")
    print("  * For e2e timings, run text_to_video.py / text_to_image.py while varying DIFFUSION_ATTENTION_BACKEND.")
 if __name__ == "__main__":
    main()
@@ -1,108 +0,0 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 #
 # 2x2 timing matrix (backend x compile mode) for SAGE_ATTN_3 vs TORCH_SDPA,
 # on HunyuanVideo-1.5 480p T2V and Wan 2.2 TI2V-5B. Run from the repo root:
 #
 #     bash benchmarks/diffusion/bench_sage_attn3_matrix.sh
 #
 # Requires: sageattn3 installed (for SAGE_ATTN_3 rows) and a Blackwell GPU.
 set -u
 STEPS=${STEPS:-50}
 SEED=${SEED:-42}
 PROMPT=${PROMPT:-"A dog running across a field of golden wheat."}
 OUT_DIR=${OUT_DIR:-/tmp/bench_sage3}
 mkdir -p "$OUT_DIR"
 declare -A TOTAL PER_STEP TEXT_ENC TRANSFORMER VAE
 run() {
    local name="$1"; shift
    local model="$1"; shift
    local h="$1"; shift
    local w="$1"; shift
    local f="$1"; shift
    local g="$1"; shift
    local log="$OUT_DIR/${name}.log"
    echo "[$(date +%T)] Running $name ..."
    env "$@" python examples/offline_inference/text_to_video/text_to_video.py \
        --model "$model" \
        --prompt "$PROMPT" \
        --height "$h" --width "$w" --num-frames "$f" \
        --num-inference-steps "$STEPS" --seed "$SEED" --guidance-scale "$g" \
        --enable-diffusion-pipeline-profiler \
        --output "$OUT_DIR/${name}.mp4" > "$log" 2>&1 || {
            echo "  FAILED — see $log"
            TOTAL[$name]="FAIL"; PER_STEP[$name]="FAIL"
            TEXT_ENC[$name]="-"; TRANSFORMER[$name]="-"; VAE[$name]="-"
            return
        }
    local wait_ms
    wait_ms=$(grep "add_req_and_wait=" "$log" | tail -1 | sed -nE 's/.*add_req_and_wait=([0-9.]+) ms.*/\1/p')
    local total_s
    total_s=$(grep -oE "Total generation time: [0-9.]+ seconds" "$log" | tail -1 | sed -nE 's/.*: ([0-9.]+) seconds.*/\1/p')
    local text_enc_s
    text_enc_s=$(grep -oE "text_encoder.forward took [0-9.]+s" "$log" | awk '{sub(/s$/,"",$NF); sum+=$NF} END {printf "%.3f", sum+0}')
    local vae_s
    vae_s=$(grep -oE "vae.decode took [0-9.]+s" "$log" | awk '{sub(/s$/,"",$NF); sum+=$NF} END {printf "%.3f", sum+0}')
    local pipeline_s
    pipeline_s=$(grep -oE "Pipeline\.forward took [0-9.]+s" "$log" | tail -1 | sed -nE 's/.*took ([0-9.]+)s.*/\1/p')
    local transformer_s
    transformer_s=$(awk "BEGIN {printf \"%.3f\", ${pipeline_s:-0} - ${vae_s:-0} - ${text_enc_s:-0}}")
    TOTAL[$name]="${total_s:-0}"
    PER_STEP[$name]=$(awk "BEGIN {printf \"%.3f\", ${wait_ms:-0} / 1000 / $STEPS}")
    TEXT_ENC[$name]="$text_enc_s"
    TRANSFORMER[$name]="$transformer_s"
    VAE[$name]="$vae_s"
    echo "  total=${TOTAL[$name]}s  per-step=${PER_STEP[$name]}s/it  transformer=${TRANSFORMER[$name]}s  vae=${VAE[$name]}s"
 }
 # ========== HunyuanVideo-1.5 480p T2V (480x832x33, CFG=6.0) ==========
 HV_MODEL=hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v
 run hv15_sdpa_eager     "$HV_MODEL" 480 832 33 6.0 TORCH_COMPILE_DISABLE=1
 run hv15_sdpa_compiled  "$HV_MODEL" 480 832 33 6.0
 run hv15_sage3_eager    "$HV_MODEL" 480 832 33 6.0 TORCH_COMPILE_DISABLE=1 DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN_3
 run hv15_sage3_compiled "$HV_MODEL" 480 832 33 6.0                         DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN_3
 # ========== Wan 2.2 TI2V-5B (704x1280x33, CFG=5.0) ==========
 WAN_MODEL=Wan-AI/Wan2.2-TI2V-5B-Diffusers
 run wan22_sdpa_eager     "$WAN_MODEL" 704 1280 33 5.0 TORCH_COMPILE_DISABLE=1
 run wan22_sdpa_compiled  "$WAN_MODEL" 704 1280 33 5.0
 run wan22_sage3_eager    "$WAN_MODEL" 704 1280 33 5.0 TORCH_COMPILE_DISABLE=1 DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN_3
 run wan22_sage3_compiled "$WAN_MODEL" 704 1280 33 5.0                         DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN_3
 print_table() {
    local title="$1"; shift
    echo
    echo "$title (${STEPS} steps, seed=${SEED})"
    echo
    printf "| %-23s | %-9s | %-15s | %-12s | %-15s | %-14s |\n" \
        "Config" "Total (s)" "Per-step (s/it)" "Text Enc (s)" "Transformer (s)" "VAE Decode (s)"
    printf "| %-23s | %-9s | %-15s | %-12s | %-15s | %-14s |\n" \
        "-----------------------" "---------" "---------------" "------------" "---------------" "--------------"
    for row in "$@"; do
        case "$row" in
            *_sdpa_eager)     label="SDPA + Eager" ;;
            *_sdpa_compiled)  label="SDPA + Compiled" ;;
            *_sage3_eager)    label="SAGE_ATTN_3 + Eager" ;;
            *_sage3_compiled) label="SAGE_ATTN_3 + Compiled" ;;
        esac
        printf "| %-23s | %-9s | %-15s | %-12s | %-15s | %-14s |\n" \
            "$label" "${TOTAL[$row]}" "${PER_STEP[$row]}" "${TEXT_ENC[$row]}" "${TRANSFORMER[$row]}" "${VAE[$row]}"
    done
 }
 print_table "HunyuanVideo-1.5 480p T2V, 832x480x33" \
    hv15_sdpa_eager hv15_sdpa_compiled hv15_sage3_eager hv15_sage3_compiled
 print_table "Wan 2.2 TI2V-5B, 1280x704x33" \
    wan22_sdpa_eager wan22_sdpa_compiled wan22_sage3_eager wan22_sage3_compiled
 echo
 echo "Logs: $OUT_DIR/*.log"
@@ -7,13 +7,10 @@ Benchmark online serving for diffusion models (Image/Video Generation).
 If you want to use i2v, i2i dataset, you should `uv pip install gdown` first
 Supports multiple endpoints:
-    - /v1/chat/completions: OpenAI chat-compatible image requests (e.g. t2i, Qwen i2i)
+    - /v1/chat/completions: OpenAI chat-compatible image requests
    - /v1/images/edits: OpenAI image edit / IT2I (multipart; e.g. Hunyuan --bot-task think)
    - /v1/images/generations: OpenAI image generation requests
    - /v1/videos: Async video jobs
 Legacy --backend vllm-omni and openai are aliases for chat/completions and images/generations.
 Usage:
    # Video (/v1/videos endpoint)
    t2v:
@@ -42,19 +39,7 @@ Usage:
            {"width":1536,"height":1536,"num_inference_steps":35,"weight":0.15}
        ]'
-    ti2i (Hunyuan / OpenAI image edit API):
+    i2i:
    python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
        --endpoint /v1/images/edits --dataset random --task ti2i --num-prompts 10 \
        --bot-task think
    python benchmarks/diffusion/diffusion_benchmark_serving.py \
      --endpoint /v1/images/edits \
      --dataset custom \
      --dataset-path custom_requests.jsonl \
      --task ti2i \
      --bot-task think
    i2i (chat-based models such as Qwen-Image-Edit):
    python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
        --endpoint /v1/chat/completions --dataset vbench --task i2i --num-prompts 10
@@ -83,6 +68,7 @@ import glob
 import json
 import logging
 import os
 import pathlib
 import random
 import tempfile
 import time
@@ -147,7 +133,7 @@ class VBenchDataset(BaseDataset):
    def _load_data(self) -> list[dict[str, Any]]:
        if self.args.task == "t2v":
            return self._load_t2v_prompts()
-        elif self.args.task in ["i2v", "ti2v", "ti2i", "i2i", "it2i"]:
+        elif self.args.task in ["i2v", "ti2v", "ti2i", "i2i"]:
            return self._load_i2v_data()
        else:
            return self._load_t2v_prompts()
@@ -539,7 +525,7 @@ class TraceDataset(BaseDataset):
            single = row.get("image_path")
            image_paths = [single] if single else None
-        if not image_paths and self.args.task in ["i2v", "i2i", "ti2v", "ti2i", "it2i"]:
+        if not image_paths and self.args.task in ["i2v", "i2i", "ti2v", "ti2i"]:
            raise ValueError(
                f"Task {self.args.task} requires image input, but no image_path or image_paths found in trace row."
            )
@@ -573,135 +559,6 @@ class TraceDataset(BaseDataset):
        return [self[i] for i in range(len(self))]
 class CustomDataset(BaseDataset):
    """
    Custom dataset that loads requests from a JSONL file.
    Each line in the JSONL file should be a JSON object with the following fields:
    - prompt (required): The text prompt for the request
    - width (optional): Image/video width
    - height (optional): Image height
    - num_inference_steps (optional): Number of diffusion steps
    - seed (optional): Random seed
    - image_paths (optional): List of input image paths for i2i/i2v/ti2i tasks
    - image_urls (optional): List of input image URLs (alternative to image_paths)
    Example JSONL for ti2i:
    {"prompt": "Add sunset lighting", "width": 1024, "height": 1024, "image_urls": ["https://example.com/image.jpg"]}
    """
    def __init__(self, args, api_url: str, model: str):
        super().__init__(args, api_url, model)
        self.dataset_path = args.dataset_path
        if not self.dataset_path:
            raise ValueError("--dataset-path must be provided when using 'custom' dataset")
        self.load_data()
    def load_data(self) -> None:
        """Load data from JSONL file."""
        import pandas as pd
        if not self.dataset_path.endswith(".jsonl"):
            raise ValueError("Custom dataset must be a JSONL file")
        try:
            df = pd.read_json(path_or_buf=self.dataset_path, lines=True)
        except Exception as e:
            raise ValueError(f"Failed to load JSONL file {self.dataset_path}: {e}")
        if "prompt" not in df.columns:
            raise ValueError("JSONL file must contain a 'prompt' column")
        self.data = df.to_dict("records")
        print(f"Loaded {len(self.data)} requests from {self.dataset_path}")
    def _resolve_image_paths(self, item: dict) -> list[str] | None:
        # Handle local file paths
        if "image_paths" in item and item["image_paths"]:
            paths = item["image_paths"]
            if isinstance(paths, str):
                paths = [paths]
            # Verify all paths exist
            valid_paths = []
            for path in paths:
                if os.path.exists(path):
                    valid_paths.append(path)
                else:
                    raise ValueError(f"Image file not found: {path}")
            return valid_paths if valid_paths else None
        # Handle URLs - download to temp files
        if "image_urls" in item and item["image_urls"]:
            urls = item["image_urls"]
            if isinstance(urls, str):
                urls = [urls]
            downloaded_paths = []
            for url in urls:
                try:
                    response = requests.get(url, timeout=30)
                    response.raise_for_status()
                    # Create temp file with appropriate extension
                    suffix = os.path.splitext(url)[1] or ".png"
                    with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
                        tmp.write(response.content)
                        downloaded_paths.append(tmp.name)
                except Exception as e:
                    raise ValueError(f"Failed to download image from {url}: {e}")
            return downloaded_paths if downloaded_paths else None
        return None
    def __len__(self) -> int:
        return len(self.data)
    def __getitem__(self, idx: int) -> RequestFuncInput:
        item = self.data[idx]
        # Extract fields with fallback to args defaults
        prompt = item["prompt"]
        width = item.get("width", self.args.width)
        height = item.get("height", self.args.height)
        num_inference_steps = item.get("num_inference_steps", self.args.num_inference_steps)
        seed = item.get("seed", self.args.seed)
        reserved_keys = {"prompt", "width", "height", "num_inference_steps", "seed", "image_paths", "image_urls"}
        extra_body = {k: v for k, v in item.items() if k not in reserved_keys}
        # Handle image paths/URLs
        image_paths = self._resolve_image_paths(item)
        return RequestFuncInput(
            prompt=prompt,
            api_url=self.api_url,
            model=self.model,
            seed=seed,
            image_paths=image_paths,
            extra_body=extra_body,
            width=width,
            height=height,
            num_inference_steps=num_inference_steps,
        )
    def get_requests(self) -> list[RequestFuncInput]:
        """Get all requests, cycling through data if num_prompts > dataset size."""
        num_requests = getattr(self.args, "num_prompts", len(self.data))
        if num_requests <= len(self.data):
            return [self[i] for i in range(num_requests)]
        # Cycle through the dataset to reach num_prompts
        requests = []
        for i in range(num_requests):
            idx = i % len(self.data)
            requests.append(self[idx])
        return requests
 class RandomDataset(BaseDataset):
    def __init__(self, args, api_url: str, model: str, enable_negative_prompt: bool = False):
        super().__init__(args, api_url, model)
@@ -729,7 +586,7 @@ class RandomDataset(BaseDataset):
            self._sampled_requests = None
        # Random image generate
-        if self.args.task in ["i2v", "ti2v", "ti2i", "i2i", "it2i"]:
+        if self.args.task in ["i2v", "ti2v", "ti2i", "i2i"]:
            self._random_image_path = self._generate_random_image_paths()
        else:
            self._random_image_path = None
@@ -742,6 +599,13 @@ class RandomDataset(BaseDataset):
        if self.enable_negative_prompt:
            extra_body["negative_prompt"] = f"Negative prompt {idx} for benchmarking diffusion models"
        request_fields = {
            "width",
            "height",
            "num_frames",
            "num_inference_steps",
            "fps",
        }
        params = {
            "width": self.args.width,
            "height": self.args.height,
@@ -751,9 +615,11 @@ class RandomDataset(BaseDataset):
        }
        if self._sampled_requests:
            profile = self._sampled_requests[idx]
-            params.update(profile)
+            params.update({key: value for key, value in profile.items() if key in request_fields})
            extra_body.update({key: value for key, value in profile.items() if key not in request_fields})
        prompt = self.args.random_prompt or f"Random prompt {idx} for benchmarking diffusion models"
        return RequestFuncInput(
-            prompt=f"Random prompt {idx} for benchmarking diffusion models",
+            prompt=prompt,
            api_url=self.api_url,
            model=self.model,
            seed=self.args.seed,
@@ -1013,6 +879,35 @@ def calculate_metrics(
    return metrics
 def _response_suffix(endpoint: str) -> str:
    if endpoint == "/v1/videos":
        return ".mp4"
    return ".json"
 def _save_success_responses(
    outputs: list[RequestFuncOutput],
    requests_list: list[RequestFuncInput],
    output_dir: str,
    endpoint: str,
 ) -> list[str]:
    path = pathlib.Path(output_dir).expanduser().resolve()
    path.mkdir(parents=True, exist_ok=True)
    saved_paths: list[str] = []
    suffix = _response_suffix(endpoint)
    for index, (request, output) in enumerate(zip(requests_list, outputs, strict=False)):
        if not output.success:
            continue
        item_path = path / f"{index:05d}_{request.request_id}{suffix}"
        if isinstance(output.response_body, bytes):
            item_path.write_bytes(output.response_body)
        else:
            item_path.write_text(json.dumps(output.response_body, indent=2, sort_keys=True), encoding="utf-8")
        saved_paths.append(str(item_path))
    return saved_paths
 def wait_for_service(base_url: str, timeout: int = 120) -> None:
    print(f"Waiting for service at {base_url}...")
    start_time = time.time()
@@ -1035,9 +930,7 @@ def wait_for_service(base_url: str, timeout: int = 120) -> None:
 def _default_endpoint_for_task(task: str) -> str:
    if task in {"t2v", "i2v", "ti2v"}:
        return "/v1/videos"
-    if task in {"i2i", "ti2i", "it2i"}:
+    if task in {"t2i", "i2i", "ti2i"}:
        return "/v1/images/edits"
    if task == "t2i":
        return "/v1/chat/completions"
    raise ValueError(f"Unsupported task for endpoint resolution: {task}")
@@ -1048,7 +941,7 @@ async def benchmark(args):
        args.base_url = f"http://{args.host}:{args.port}"
    VIDEO_TASKS = {"t2v", "i2v", "ti2v"}
-    IMAGE_TASKS = {"t2i", "i2i", "ti2i", "it2i"}
+    IMAGE_TASKS = {"t2i", "i2i", "ti2i"}
    if args.task in VIDEO_TASKS:
        task_type = "2v"
@@ -1086,8 +979,6 @@ async def benchmark(args):
        dataset = TraceDataset(args, api_url, args.model)
    elif args.dataset == "random":
        dataset = RandomDataset(args, api_url, args.model, args.enable_negative_prompt)
    elif args.dataset == "custom":
        dataset = CustomDataset(args, api_url, args.model)
    else:
        raise ValueError(f"Unknown dataset: {args.dataset}")
@@ -1095,10 +986,6 @@ async def benchmark(args):
    requests_list = dataset.get_requests()
    print(f"Prepared {len(requests_list)} requests from {args.dataset} dataset.")
    if args.endpoint == "/v1/images/edits":
        for req in requests_list:
            req.default_bot_task = args.bot_task
    # Limit concurrency
    if args.max_concurrency is not None:
        semaphore = asyncio.Semaphore(args.max_concurrency)
@@ -1150,8 +1037,13 @@ async def benchmark(args):
    metrics["model"] = args.model
    metrics["dataset"] = args.dataset
    metrics["task"] = args.task
-    if args.endpoint == "/v1/images/edits":
+    if args.save_response_dir:
-        metrics["bot_task"] = args.bot_task
+        metrics["saved_responses"] = _save_success_responses(
            outputs,
            requests_list,
            args.save_response_dir,
            args.endpoint,
        )
    print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
@@ -1235,14 +1127,14 @@ if __name__ == "__main__":
        "--dataset",
        type=str,
        default="vbench",
-        choices=["vbench", "trace", "random", "custom"],
+        choices=["vbench", "trace", "random"],
        help="Dataset to use.",
    )
    parser.add_argument(
        "--task",
        type=str,
        default="t2v",
-        choices=["t2v", "i2v", "ti2v", "ti2i", "i2i", "it2i", "t2i"],
+        choices=["t2v", "i2v", "ti2v", "ti2i", "i2i", "t2i"],
        help="Task type.",
    )
    parser.add_argument(
@@ -1308,6 +1200,12 @@ if __name__ == "__main__":
    )
    parser.add_argument("--fps", type=int, default=None, help="FPS (for video).")
    parser.add_argument("--output-file", type=str, default=None, help="Output JSON file for metrics.")
    parser.add_argument(
        "--save-response-dir",
        type=str,
        default=None,
        help="Directory to save successful raw responses. Video responses are saved as MP4 files.",
    )
    parser.add_argument(
        "--slo",
        action="store_true",
@@ -1343,21 +1241,21 @@ if __name__ == "__main__":
            '{"width":768,"height":768,"num_inference_steps":20,"weight":0.85}]'
        ),
    )
    parser.add_argument(
        "--random-prompt",
        type=str,
        default=None,
        help="Prompt to use for every synthetic random request. Defaults to generated placeholder prompts.",
    )
    parser.add_argument(
        "--num-input-images",
        type=int,
        default=1,
        help=(
            "Number of synthetic input images to attach for image-conditioned tasks "
-            "(i2v, ti2v, ti2i, i2i, it2i) when using random dataset."
+            "(i2v, ti2v, ti2i, i2i) when using random dataset."
        ),
    )
    parser.add_argument(
        "--bot-task",
        type=str,
        default="think",
        help=("bot_task form field for --endpoint /v1/images/edits (think, recaption, think_recaption, vanilla)."),
    )
    args = parser.parse_args()
    asyncio.run(benchmark(args))
@@ -0,0 +1,222 @@
 # Helios-Distilled Serving Performance Dashboard
 This document describes how to deploy and benchmark **BestWishYsh/Helios-Distilled** using vLLM-Omni. It follows the same serving benchmark structure as the Qwen-Image and Wan2.2 dashboards.
 ---
 # 1. Overview
 Helios-Distilled is a text-to-video diffusion model. Helios uses an autoregressive chunking pattern that generates 33 frames per chunk. For stable comparisons, choose `num_frames` as a multiple of 33, such as 99.
 This document covers:
 * Service launch configuration
 * Benchmark scripts and usage
 * Dataset and workload settings
 * Performance and accuracy comparison methodology
 * Reproducibility guidelines
 ---
 # 2. Test Environment
 | Component | Specification |
 |------------|----------------|
 | GPU | NVIDIA H100 for CI accuracy/perf gate; local bring-up also used L20X |
 | Diffusion Attention Backend | FlashAttention |
 | Model | `BestWishYsh/Helios-Distilled` |
 ---
 # 3. Service Launch Configuration
 ## 3.1 Basic Serving Command
 ```bash
 vllm serve BestWishYsh/Helios-Distilled --omni \
    --port 8091
 ```
 ## 3.2 Key Parameters
 | Parameter | Description |
 | --- | --- |
 | `--port` | Serving port |
 | `--max-num-seqs` | Maximum number of active sequences |
 | `--enable-diffusion-pipeline-profiler` | Optional stage-level profiling |
 | `--profiler-config` | Optional torch profiler configuration |
 Record these parameters when reporting performance results.
 ---
 # 4. Benchmark Scripts
 ## 4.1 Online Serving Benchmark
 Use the shared diffusion serving benchmark, as with Qwen-Image and Wan2.2:
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
    --base-url http://localhost:8091 \
    --endpoint /v1/videos \
    --model BestWishYsh/Helios-Distilled \
    --dataset random \
    --task t2v \
    --num-prompts 1 \
    --max-concurrency 1 \
    --disable-tqdm \
    --random-request-config '[
        {
            "width": 640,
            "height": 384,
            "num_inference_steps": 50,
            "num_frames": 99,
            "fps": 16,
            "guidance_scale": 1.0,
            "is_enable_stage2": true,
            "pyramid_num_stages": 3,
            "pyramid_num_inference_steps_list": [1, 1, 1],
            "is_amplify_first_chunk": false,
            "weight": 1
        }
    ]'
 ```
 `--random-request-config` forwards standard fields such as `width`, `height`, `num_frames`, `fps`, and `num_inference_steps` as first-class request fields. Model-specific fields are forwarded in the request body, which is required for Helios stage-2 settings.
 ## 4.2 Warmed Serving Benchmark
 For local bring-up, use the same shared serving benchmark with one warmup request and one measured request:
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
    --base-url http://localhost:8091 \
    --endpoint /v1/videos \
    --model BestWishYsh/Helios-Distilled \
    --dataset random \
    --task t2v \
    --num-prompts 1 \
    --max-concurrency 1 \
    --warmup-requests 1 \
    --warmup-num-inference-steps 1 \
    --random-prompt "A cat wearing sunglasses dances on a beach at sunset, cinematic lighting." \
    --seed 42 \
    --disable-tqdm \
    --output-file metrics.json \
    --save-response-dir responses \
    --random-request-config '[
        {
            "width": 640,
            "height": 384,
            "num_inference_steps": 50,
            "num_frames": 99,
            "fps": 16,
            "guidance_scale": 1.0,
            "is_enable_stage2": true,
            "pyramid_num_stages": 3,
            "pyramid_num_inference_steps_list": [1, 1, 1],
            "is_amplify_first_chunk": false,
            "weight": 1
        }
    ]'
 ```
 ## 4.3 PR-vs-Main Accuracy and Performance Comparison
 The PR-vs-main accuracy/performance gate lives in the Helios accuracy pytest. It checks out `origin/main`, starts a server for each checkout, and drives both with `diffusion_benchmark_serving.py`:
 ```bash
 pytest -s -v tests/e2e/accuracy/test_helios_distilled.py::test_helios_distilled_pr_matches_main_acc_perf \
    -m 'full_model and benchmark' \
    --run-level 'full_model'
 ```
 ---
 # 5. Dataset & Workload Settings
 ## 5.1 Recommended Evaluation Configurations
 ### Dataset A (99 frames, 384x640)
 * Dataset: `random`
 * Task: t2v
 * Concurrency: 1
 * Frame count: 99 (`33 x 3`)
 * FPS: 16
 * Stage-2 steps: `[1, 1, 1]`
 ```json
 [
  {
    "width": 640,
    "height": 384,
    "num_inference_steps": 50,
    "num_frames": 99,
    "fps": 16,
    "guidance_scale": 1.0,
    "is_enable_stage2": true,
    "pyramid_num_stages": 3,
    "pyramid_num_inference_steps_list": [1, 1, 1],
    "is_amplify_first_chunk": false,
    "weight": 1
  }
 ]
 ```
 ---
 # 6. Performance Metrics
 | Metric | Description | Unit |
 | --- | --- | --- |
 | Mean Latency | Mean request latency | seconds |
 | P99 Latency | P99 request latency | seconds |
 | Generated FPS | `num_frames / measured_time_s` | frames/second |
 | Candidate/reference latency ratio | PR latency divided by reference latency | ratio |
 ---
 # 7. Accuracy Metrics
 The PR-vs-main gate compares encoded video frames extracted from MP4 outputs:
 | Metric | Threshold | Direction |
 | --- | --- | --- |
 | PSNR | `>= 35 dB` | Higher is better |
 | MAE | `<= 3.0 / 255` | Lower is better |
 | Candidate/reference latency ratio | `<= 1.03` | Lower is better |
 Local L20X bring-up for this PR produced:
 | Variant | Measured Latency (ms) | Generated FPS |
 | --- | --- | --- |
 | `origin/main` | 5317.95 | 18.62 |
 | Candidate PR | 5283.27 | 18.74 |
 Encoded-video similarity:
 | Metric | Value |
 | --- | --- |
 | PSNR | 38.66 dB |
 | MAE | 1.89 / 255 |
 | Cosine similarity | 0.99978 |
 ---
 # 8. Reproducibility Checklist
 To ensure consistent and comparable benchmark results:
 * Record GPU type
 * Record vLLM and vLLM-Omni versions
 * Record benchmark parameters: resolution, frame count, FPS, seed, concurrency, and stage-2 steps
 * Use frame counts that are multiples of 33
 * Keep prompt, seed, resolution, and sampling parameters identical when comparing PR vs main
 * Ensure no background workload on GPUs during testing
 * Use the same MP4 export and frame extraction path for both variants
 ---
 This document serves as the Helios-Distilled serving performance reference under vLLM-Omni.
@@ -34,16 +34,6 @@ Video example (text-to-video):
        --height 720 --width 1280 \
        --num-frames 81 --num-inference-steps 40 --seed 42
 LTX-2 example (text-to-video; audio output is generated but not scored by LPIPS):
    python benchmarks/diffusion/quantization_quality.py \
        --model Lightricks/LTX-Video-2 \
        --task t2v \
        --quantization fp8 int8 \
        --prompts \
            "A serene lakeside sunrise with mist over the water" \
        --height 704 --width 1216 \
        --num-frames 121 --num-inference-steps 40 --seed 42
 Multiple quantization methods:
    python benchmarks/diffusion/quantization_quality.py \
        --model Tongyi-MAI/Z-Image-Turbo \
@@ -214,19 +204,10 @@ def _generate_video(omni, args, prompt, seed):
        else:
            frames = inner
    elif hasattr(first, "images") and first.images:
-        frames = first.images[0]
+        frames = first.images
    else:
        raise ValueError("Could not extract video frames from output.")
    # LTX-2 (and similar audio+video models) may surface a dict or (video, audio) tuple
    if isinstance(frames, dict):
        frames = frames.get("video") or frames.get("frames")
    elif isinstance(frames, tuple) and len(frames) == 2:
        frames = frames[0]
    if frames is None:
        raise ValueError("Could not extract video frames from output.")
    if isinstance(frames, torch.Tensor):
        video = frames.detach().cpu()
        if video.dim() == 5:
@@ -244,11 +225,9 @@ def _generate_video(omni, args, prompt, seed):
    return frames_array, elapsed, peak_mem
-def _free_gpu_memory():
+def _unload_omni(omni):
-    """Force GC and release cached GPU memory.
+    """Delete Omni instance and free GPU memory."""
-
+    del omni
    Must be called AFTER the caller has dropped (i.e., via `del`)
    every reference to the Omni instance"""
    gc.collect()
    if torch.cuda.is_available():
        torch.accelerator.empty_cache()
@@ -288,9 +267,7 @@ def run_benchmark(args):
    bl_avg_time = np.mean([v[1] for v in baseline_outputs.values()])
    bl_mem = baseline_outputs[prompts[0]][2]  # use first prompt's memory
-    omni_bl.shutdown()
+    _unload_omni(omni_bl)
    del omni_bl
    _free_gpu_memory()
    # Save baseline outputs
    bl_dir = output_dir / "baseline"
@@ -330,9 +307,7 @@ def run_benchmark(args):
        qt_avg_time = np.mean([v[1] for v in qt_outputs.values()])
        qt_mem = qt_outputs[prompts[0]][2]
-        omni_qt.shutdown()
+        _unload_omni(omni_qt)
        del omni_qt
        _free_gpu_memory()
        # Save quantized outputs
        qt_dir = output_dir / config_label.replace(" ", "_")
@@ -40,7 +40,7 @@ vllm bench serve --omni \
    --num-prompts 20 --num-warmups 2 \
    --extra-body '{"task_type":"Base"}' \
    --max-concurrency 1 --request-rate inf \
-    --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration,audio_underrun \
+    --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration \
    --save-result --result-dir ./results
 ```
@@ -58,7 +58,7 @@ vllm bench serve --omni \
    --num-prompts 20 --num-warmups 2 \
    --extra-body '{"voice":"Vivian","language":"English","task_type":"CustomVoice"}' \
    --max-concurrency 1 --request-rate inf \
-    --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration,audio_underrun \
+    --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration \
    --save-result --result-dir ./results
 ```
@@ -76,28 +76,10 @@ vllm bench serve --omni \
    --num-prompts 20 --num-warmups 2 \
    --extra-body '{"task_type":"VoiceDesign","language":"English"}' \
    --max-concurrency 1 --request-rate inf \
-    --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration,audio_underrun \
+    --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration \
    --save-result --result-dir ./results
 ```
 #### Streaming continuity (`audio_underrun`)
 `audio_underrun` (seconds) is the per-request worst-case buffer deficit
 under a realtime-rate player simulation. The audio-speech backend records
 chunk arrival times during the SSE stream and surfaces:
 - `Mean / Median / P{50,99} AUDIO_UNDERRUN (s)` per the standard percentile
  output - any positive value means at least one inter-chunk gap was longer
  than the chunk's audio duration.
 - `Streaming continuity OK rate` - fraction of requests whose worst-case
  underrun stayed under the threshold (default 100 ms, the commonly cited
  "audible gap" budget). Override the threshold with
  `VLLM_OMNI_BENCH_AUDIO_CONTINUITY_THRESHOLD_S=<float>`.
 This captures the failure mode where `RTF_p50 < 1` (server keeps up in
 aggregate) but per-stream chunk arrival is bursty enough that listeners
 still hear gaps - common at high concurrency on streaming TTS pipelines.
 #### Add WER / SIM / UTMOS to any of the above
 Append `--seed-tts-wer-eval` (and optionally `SEED_TTS_EVAL_DEVICE=cuda:0`
@@ -233,6 +215,7 @@ sentinel for regressions in this area.
 benchmarks/tts/
 ├── README.md                  (this file)
 ├── bench_tts.py               CLI — serve-mode benchmark driver
 ├── bench_voxcpm_offline.py    CLI — offline VoxCPM benchmark (sync + streaming)
 ├── plot_results.py            Generate per-task / per-concurrency curves
 └── model_configs.yaml         Model registry (supported tasks + extra body)
 ```
@@ -118,7 +118,7 @@ def build_bench_args(
        "--num-warmups",
        "2",
        "--percentile-metrics",
-        "ttft,e2el,audio_rtf,audio_ttfp,audio_duration,audio_underrun",
+        "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
    ]
    if resolved_dataset_path:
@@ -130,9 +130,6 @@ def build_bench_args(
    if task_extra_body:
        cmd += ["--extra-body", json.dumps(task_extra_body, separators=(",", ":"))]
    if model_cfg.get("trust_remote_code"):
        cmd.append("--trust-remote-code")
    if concurrency is not None:
        cmd += ["--max-concurrency", str(concurrency), "--request-rate", "inf"]
@@ -0,0 +1,922 @@
 """Offline VoxCPM benchmark for vLLM Omni.
 Supports both:
 - sync one-shot (Omni.generate)
 - streaming (AsyncOmni.generate with async_chunk config)
 - text-only synthesis
 - voice cloning
 - text/clone batch inputs from txt or jsonl
 Usage::
    # Sync (default voice)
    python benchmarks/tts/bench_voxcpm_offline.py \\
        --model /path/to/VoxCPM \\
        --text "Hello world" \\
        --output-dir results/audio/
    # Streaming (async_chunk)
    python benchmarks/tts/bench_voxcpm_offline.py \\
        --model /path/to/VoxCPM \\
        --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \\
        --txt-prompts prompts.txt \\
        --output-dir results/audio/
    # Voice cloning batch via JSONL
    python benchmarks/tts/bench_voxcpm_offline.py \\
        --model /path/to/VoxCPM \\
        --jsonl-prompts prompts.jsonl \\
        --output-dir results/audio/
 """
 from __future__ import annotations
 import asyncio
 import json
 import logging
 import os
 import tempfile
 import time
 import uuid
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 import torch
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 from vllm_omni import AsyncOmni, Omni
 def _find_repo_root(start: Path) -> Path:
    """Walk up from ``start`` until a repo marker is found.
    Falls back to ``parents[2]`` for backwards compatibility if no marker hits
    (which can only happen in unusual checkouts — the tree should always have
    pyproject.toml + vllm_omni/ at the top level).
    """
    for candidate in [start, *start.parents]:
        if (candidate / "pyproject.toml").is_file() and (candidate / "vllm_omni").is_dir():
            return candidate
    return start.parents[2]
 REPO_ROOT = _find_repo_root(Path(__file__).resolve())
 DEFAULT_STAGE_ASYNC = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm_async_chunk.yaml"
 DEFAULT_STAGE_SYNC = REPO_ROOT / "vllm_omni" / "deploy" / "voxcpm.yaml"
 logger = logging.getLogger(__name__)
@dataclass(frozen=True, slots=True)
 class PromptSpec:
    text: str
    label: str
    ref_audio: str | None = None
    ref_text: str | None = None
 def _require_soundfile():
    try:
        import soundfile as sf  # type: ignore
    except ModuleNotFoundError as exc:
        raise RuntimeError(
            "soundfile is required to write VoxCPM benchmark WAV outputs. Install it with: pip install soundfile"
        ) from exc
    return sf
 def _build_prompt(
    args,
    *,
    text: str,
    ref_audio: str | None = None,
    ref_text: str | None = None,
    global_request_id: str | None = None,
 ) -> dict[str, Any]:
    additional_information: dict[str, list[Any]] = {
        "text": [text],
        "cfg_value": [args.cfg_value],
        "inference_timesteps": [args.inference_timesteps],
        "min_len": [args.min_len],
        "max_new_tokens": [args.max_new_tokens],
    }
    if args.streaming_prefix_len is not None:
        additional_information["streaming_prefix_len"] = [args.streaming_prefix_len]
    if ref_audio:
        additional_information["ref_audio"] = [ref_audio]
    if ref_text:
        additional_information["ref_text"] = [ref_text]
    if global_request_id is not None:
        additional_information["global_request_id"] = [global_request_id]
    return {
        "prompt_token_ids": [1],
        "additional_information": additional_information,
    }
 def _extract_audio_tensor(mm: dict[str, Any]) -> torch.Tensor:
    audio = mm.get("audio", mm.get("model_outputs"))
    if audio is None:
        raise ValueError("No audio output found in multimodal output.")
    if isinstance(audio, list):
        parts = [torch.as_tensor(a).float().cpu().reshape(-1) for a in audio]
        audio = torch.cat(parts, dim=-1) if parts else torch.zeros(0)
    if not isinstance(audio, torch.Tensor):
        audio = torch.as_tensor(audio)
    return audio.float().cpu().reshape(-1)
 def _extract_sample_rate(mm: dict[str, Any]) -> int:
    sr_raw = mm.get("sr", 24000)
    if isinstance(sr_raw, list) and sr_raw:
        sr_raw = sr_raw[-1]
    if hasattr(sr_raw, "item"):
        return int(sr_raw.item())
    return int(sr_raw)
 def _emit_offline_metrics(
    *,
    request_id: str,
    elapsed_s: float,
    first_audio_elapsed: float | None,
    audio_duration_s: float,
 ) -> None:
    metrics = {
        "request_id": request_id,
        "ttfp_ms": round(first_audio_elapsed * 1000.0, 3) if first_audio_elapsed is not None else None,
        "audio_duration_s": round(audio_duration_s, 6),
        "rtf": round(elapsed_s / audio_duration_s, 6) if audio_duration_s > 0 else None,
    }
    print(f"[OfflineMetrics] {metrics}")
 def _write_audio_tensor(output_path: Path, audio_tensor: Any, sample_rate: int) -> None:
    sf = _require_soundfile()
    if isinstance(audio_tensor, torch.Tensor):
        audio_np = audio_tensor.float().cpu().clamp(-1.0, 1.0).numpy()
    else:
        audio_np = torch.as_tensor(audio_tensor).float().cpu().clamp(-1.0, 1.0).numpy()
    sf.write(
        output_path,
        audio_np,
        sample_rate,
        format="WAV",
        subtype="PCM_16",
    )
 def _save_wav(mm: dict[str, Any], output_dir: Path, request_id: str) -> Path:
    output_dir.mkdir(parents=True, exist_ok=True)
    output_path = output_dir / f"output_{request_id}.wav"
    _write_audio_tensor(output_path, _extract_audio_tensor(mm), _extract_sample_rate(mm))
    return output_path
 def _iter_request_multimodal_outputs(request_output: Any):
    outputs = getattr(request_output, "outputs", None)
    if outputs:
        for output in outputs:
            mm = getattr(output, "multimodal_output", None)
            if isinstance(mm, dict):
                yield mm
    mm = getattr(request_output, "multimodal_output", None)
    if isinstance(mm, dict):
        yield mm
 def _read_non_empty_lines(path: str) -> list[str]:
    with open(path, encoding="utf-8") as f:
        return [line.strip() for line in f if line.strip()]
 def _load_prompt_specs(args) -> list[PromptSpec]:
    specs: list[PromptSpec] = []
    if args.txt_prompts is not None:
        texts = _read_non_empty_lines(args.txt_prompts)
        if not texts:
            raise ValueError(f"No prompts found in {args.txt_prompts}")
        for idx, text in enumerate(texts, start=1):
            specs.append(
                PromptSpec(
                    text=text,
                    label=f"item{idx:03d}",
                    ref_audio=args.ref_audio,
                    ref_text=args.ref_text,
                )
            )
        return specs
    if args.jsonl_prompts is not None:
        with open(args.jsonl_prompts, encoding="utf-8") as f:
            for line_no, raw_line in enumerate(f, start=1):
                line = raw_line.strip()
                if not line:
                    continue
                try:
                    item = json.loads(line)
                except json.JSONDecodeError as exc:
                    raise ValueError(f"{args.jsonl_prompts}:{line_no} is not valid JSON: {exc}") from exc
                if not isinstance(item, dict):
                    raise ValueError(f"{args.jsonl_prompts}:{line_no} must be a JSON object")
                text = item.get("text")
                if not isinstance(text, str) or not text.strip():
                    raise ValueError(f"{args.jsonl_prompts}:{line_no} requires non-empty string field 'text'")
                ref_audio = item.get("ref_audio", args.ref_audio)
                ref_text = item.get("ref_text", args.ref_text)
                if (ref_audio is None) != (ref_text is None):
                    raise ValueError(
                        f"{args.jsonl_prompts}:{line_no} must provide both 'ref_audio' and 'ref_text' together"
                    )
                specs.append(
                    PromptSpec(
                        text=text.strip(),
                        label=f"item{len(specs) + 1:03d}",
                        ref_audio=ref_audio,
                        ref_text=ref_text,
                    )
                )
        if not specs:
            raise ValueError(f"No prompts found in {args.jsonl_prompts}")
        return specs
    specs.append(
        PromptSpec(
            text=args.text,
            label="item001",
            ref_audio=args.ref_audio,
            ref_text=args.ref_text,
        )
    )
    return specs
 def _build_prompt_for_spec(args, spec: PromptSpec, *, global_request_id: str | None = None) -> dict[str, Any]:
    return _build_prompt(
        args,
        text=spec.text,
        ref_audio=spec.ref_audio,
        ref_text=spec.ref_text,
        global_request_id=global_request_id,
    )
 def _count_voice_clone_prompts(prompt_specs: list[PromptSpec]) -> int:
    return sum(1 for spec in prompt_specs if spec.ref_audio is not None)
 def _get_warmup_specs(prompt_specs: list[PromptSpec]) -> list[PromptSpec]:
    return prompt_specs[:1]
 def _extract_stream_finished(stage_output: Any) -> bool:
    request_output = getattr(stage_output, "request_output", None)
    request_finished = getattr(request_output, "finished", None)
    if request_finished is not None:
        return bool(request_finished)
    return bool(getattr(stage_output, "finished", False))
 def _build_profiled_stage_config(
    stage_configs_path: str,
    profiler_dir: str,
 ) -> str:
    stage_config_path = Path(stage_configs_path)
    yaml_text = stage_config_path.read_text(encoding="utf-8")
    injected_lines: list[str] = []
    injected_count = 0
    for line in yaml_text.splitlines():
        injected_lines.append(line)
        if line.strip() != "engine_args:":
            continue
        indent = line[: len(line) - len(line.lstrip())]
        child_indent = indent + "  "
        grandchild_indent = child_indent + "  "
        injected_lines.extend(
            [
                f"{child_indent}profiler_config:",
                f'{grandchild_indent}profiler: "torch"',
                f'{grandchild_indent}torch_profiler_dir: "{profiler_dir}"',
                f"{grandchild_indent}torch_profiler_with_stack: true",
            ]
        )
        injected_count += 1
    if injected_count == 0:
        raise ValueError(f"No engine_args block found in stage config: {stage_configs_path}")
    tmp = tempfile.NamedTemporaryFile(
        mode="w",
        encoding="utf-8",
        delete=False,
        suffix=".yaml",
        prefix=f"{stage_config_path.stem}_profile_",
    )
    tmp.write("\n".join(injected_lines) + "\n")
    tmp.close()
    return tmp.name
 def parse_args():
    parser = FlexibleArgumentParser(
        description="Offline split-stage VoxCPM inference with vLLM Omni (auto sync/streaming by stage config)"
    )
    parser.add_argument(
        "--model",
        type=str,
        default=os.environ.get("VOXCPM_MODEL"),
        help="Local VoxCPM model directory. Defaults to $VOXCPM_MODEL.",
    )
    parser.add_argument(
        "--text",
        type=str,
        default="This is a split-stage VoxCPM synthesis example running on vLLM Omni.",
        help="Text to synthesize. Ignored when --txt-prompts or --jsonl-prompts is used.",
    )
    parser.add_argument(
        "--txt-prompts",
        type=str,
        default=None,
        help="Path to a .txt file with one synthesis text per line.",
    )
    parser.add_argument(
        "--jsonl-prompts",
        type=str,
        default=None,
        help=(
            "Path to a .jsonl file. Each line must contain at least {'text': ...}; "
            "clone rows can also set ref_audio/ref_text, and ref_text must be the "
            "real transcript of ref_audio."
        ),
    )
    parser.add_argument(
        "--ref-audio",
        type=str,
        default=None,
        help=(
            "Optional reference audio path for voice cloning. With --txt-prompts, "
            "the same reference is applied to every line."
        ),
    )
    parser.add_argument(
        "--ref-text",
        type=str,
        default=None,
        help=(
            "Real transcript of the reference audio. Placeholder text or mismatched "
            "text will usually produce noisy/electronic clone audio."
        ),
    )
    parser.add_argument(
        "--stage-configs-path",
        type=str,
        default=str(DEFAULT_STAGE_SYNC),
        help="Stage config YAML path. Routing is selected only from this path.",
    )
    parser.add_argument(
        "--cfg-value",
        type=float,
        default=2.0,
        help="Classifier-free guidance value for VoxCPM.",
    )
    parser.add_argument(
        "--inference-timesteps",
        type=int,
        default=10,
        help="Number of inference timesteps.",
    )
    parser.add_argument(
        "--min-len",
        type=int,
        default=2,
        help="Minimum generated token length.",
    )
    parser.add_argument(
        "--max-new-tokens",
        type=int,
        default=4096,
        help="Maximum generated token length.",
    )
    parser.add_argument(
        "--streaming-prefix-len",
        type=int,
        default=None,
        help="VoxCPM streaming window (optional, streaming mode only).",
    )
    parser.add_argument(
        "--output-dir",
        type=str,
        default=None,
        help="Directory for output WAV files.",
    )
    parser.add_argument(
        "--stage-init-timeout",
        type=int,
        default=600,
        help="Stage initialization timeout in seconds.",
    )
    parser.add_argument(
        "--log-stats",
        dest="log_stats",
        action="store_true",
        help="Enable vLLM Omni stats logging.",
    )
    parser.add_argument(
        "--no-log-stats",
        dest="log_stats",
        action="store_false",
        help="Disable vLLM Omni stats logging.",
    )
    parser.set_defaults(log_stats=True)
    parser.add_argument(
        "--num-runs",
        type=int,
        default=1,
        help="Number of full inference runs (same prompt each time). Default 1.",
    )
    parser.add_argument(
        "--warmup-runs",
        type=int,
        default=0,
        help=(
            "Optional number of warmup passes before measured runs. Warmup uses only "
            "the first prompt and does not save outputs."
        ),
    )
    parser.add_argument(
        "--enable-profiler",
        action="store_true",
        help=(
            "Enable torch profiler for the configured stages. A temporary profiled "
            "stage config is generated automatically."
        ),
    )
    parser.add_argument(
        "--profiler-dir",
        type=str,
        default=None,
        help="Directory for profiler traces. Defaults to <output-dir>/profiler when profiling is enabled.",
    )
    parser.add_argument(
        "--profiler-stages",
        type=int,
        nargs="*",
        default=None,
        help="Optional stage ids to profile. Defaults to all stages that have profiler_config.",
    )
    parser.add_argument(
        "--profiler-wait-seconds",
        type=float,
        default=30.0,
        help="Seconds to wait after stop_profile for trace files to flush.",
    )
    args = parser.parse_args()
    if not args.model:
        parser.error("--model is required unless $VOXCPM_MODEL is set")
    if args.txt_prompts is not None and args.jsonl_prompts is not None:
        parser.error("--txt-prompts and --jsonl-prompts are mutually exclusive")
    if (args.ref_audio is None) != (args.ref_text is None):
        parser.error("--ref-audio and --ref-text must be provided together")
    if args.num_runs < 1:
        parser.error("--num-runs must be >= 1")
    if args.warmup_runs < 0:
        parser.error("--warmup-runs must be >= 0")
    if args.output_dir is None:
        args.output_dir = (
            "output_audio_streaming" if _is_streaming_stage_config(args.stage_configs_path) else "output_audio"
        )
    if args.enable_profiler and args.profiler_dir is None:
        args.profiler_dir = str(Path(args.output_dir) / "profiler")
    try:
        args.prompt_specs = _load_prompt_specs(args)
    except ValueError as exc:
        parser.error(str(exc))
    return args
 def _is_streaming_stage_config(stage_configs_path: str) -> bool:
    cfg_name = Path(stage_configs_path).name.lower()
    return "async_chunk" in cfg_name
 async def _collect_streaming_audio(
    omni: AsyncOmni,
    args: Any,
    spec: PromptSpec,
    request_id: str,
    *,
    phase_label: str,
    prompt_index: int,
    prompt_count: int,
    print_prompt: bool = False,
 ) -> tuple[torch.Tensor, int, float, float | None]:
    prompt = _build_prompt_for_spec(args, spec, global_request_id=request_id)
    delta_chunks: list[torch.Tensor] = []
    sample_rate = 24000
    chunk_i = 0
    prev_total_samples = 0
    t_start = time.perf_counter()
    first_audio_elapsed: float | None = None
    if print_prompt:
        print(f"---prompt---:{prompt}")
    async for stage_output in omni.generate(prompt, request_id=request_id):
        mm = getattr(stage_output, "multimodal_output", None)
        if not isinstance(mm, dict):
            ro = getattr(stage_output, "request_output", None)
            if ro is None:
                continue
            mm = getattr(ro, "multimodal_output", None)
            if not isinstance(mm, dict) and getattr(ro, "outputs", None):
                seq = ro.outputs[0]
                mm = getattr(seq, "multimodal_output", None)
        if not isinstance(mm, dict):
            continue
        sample_rate = _extract_sample_rate(mm)
        try:
            w = _extract_audio_tensor(mm)
            n = int(w.numel())
            if n == 0:
                continue
            finished = _extract_stream_finished(stage_output)
            if n > prev_total_samples:
                delta = w.reshape(-1)[prev_total_samples:]
                prev_total_samples = n
            elif finished and n == prev_total_samples:
                delta = w.reshape(-1)[:0]
            else:
                delta = w.reshape(-1)
                prev_total_samples += int(delta.numel())
            if int(delta.numel()) > 0:
                delta_chunks.append(delta)
            if first_audio_elapsed is None and int(delta.numel()) > 0:
                first_audio_elapsed = time.perf_counter() - t_start
            logger.info(
                "%s prompt=%d/%d chunk=%d delta_samples=%d buf_len=%d finished=%s",
                phase_label,
                prompt_index + 1,
                prompt_count,
                chunk_i,
                int(delta.numel()),
                n,
                finished,
            )
            chunk_i += 1
        except ValueError:
            if not _extract_stream_finished(stage_output):
                logger.debug("skip non-audio partial output chunk=%d", chunk_i)
    if not delta_chunks:
        raise RuntimeError("No audio chunks received; check stage config and logs.")
    audio_cat = torch.cat([c.reshape(-1) for c in delta_chunks], dim=0)
    elapsed = time.perf_counter() - t_start
    return audio_cat, sample_rate, elapsed, first_audio_elapsed
 async def _abort_streaming_residual_work(
    omni: AsyncOmni,
    request_id: str,
    *,
    settle_seconds: float = 0.1,
 ) -> None:
    """Stop any late stage-0 work once the final audio has been collected."""
    await omni.engine.abort_async([request_id])
    if settle_seconds > 0:
        await asyncio.sleep(settle_seconds)
 async def _run_streaming_single(
    omni: AsyncOmni,
    args: Any,
    spec: PromptSpec,
    output_dir: Path,
    request_id: str,
    *,
    run_index: int,
    num_runs: int,
    prompt_index: int,
    prompt_count: int,
 ) -> Path:
    audio_cat, sample_rate, elapsed, first_audio_elapsed = await _collect_streaming_audio(
        omni,
        args,
        spec,
        request_id,
        phase_label=f"run={run_index + 1}/{num_runs}",
        prompt_index=prompt_index,
        prompt_count=prompt_count,
        print_prompt=(run_index == 0 and prompt_index == 0),
    )
    await _abort_streaming_residual_work(omni, request_id)
    output_path = output_dir / f"output_run{run_index + 1}_{spec.label}.wav"
    _write_audio_tensor(output_path, audio_cat, sample_rate)
    audio_duration_s = float(audio_cat.numel()) / float(sample_rate) if sample_rate > 0 else 0.0
    ttfp_text = f", ttfp={first_audio_elapsed:.2f}s" if first_audio_elapsed is not None else ""
    rtf_text = f", rtf={elapsed / audio_duration_s:.3f}" if audio_duration_s > 0 else ""
    print(
        f"Saved (streaming) run {run_index + 1}/{num_runs}, "
        f"prompt {prompt_index + 1}/{prompt_count}: {output_path} ({elapsed:.2f}s{ttfp_text}{rtf_text})"
    )
    _emit_offline_metrics(
        request_id=request_id,
        elapsed_s=elapsed,
        first_audio_elapsed=first_audio_elapsed,
        audio_duration_s=audio_duration_s,
    )
    return output_path
 async def _run_streaming_warmup(args, omni: AsyncOmni) -> None:
    if args.warmup_runs == 0:
        return
    warmup_specs = _get_warmup_specs(args.prompt_specs)
    print(
        f"Warmup: {args.warmup_runs} run(s) using the first prompt "
        f"({len(warmup_specs)} prompt(s)); outputs will be discarded."
    )
    for warmup_index in range(args.warmup_runs):
        t_warmup = time.perf_counter()
        tasks = []
        request_ids: list[str] = []
        for prompt_index, spec in enumerate(warmup_specs):
            request_id = f"warmup_stream_{warmup_index + 1}_{spec.label}_{uuid.uuid4().hex[:8]}"
            request_ids.append(request_id)
            tasks.append(
                _collect_streaming_audio(
                    omni,
                    args,
                    spec,
                    request_id,
                    phase_label=f"warmup={warmup_index + 1}/{args.warmup_runs}",
                    prompt_index=prompt_index,
                    prompt_count=len(warmup_specs),
                )
            )
        results = await asyncio.gather(*tasks)
        for request_id in request_ids:
            await _abort_streaming_residual_work(omni, request_id)
        total_samples = sum(int(audio.numel()) for audio, _, _, _ in results)
        warmup_ttfps = [ttfp for _, _, _, ttfp in results if ttfp is not None]
        ttfp_text = f", ttfp={min(warmup_ttfps):.2f}s" if warmup_ttfps else ""
        print(
            f"Warmup (streaming) {warmup_index + 1}/{args.warmup_runs} finished: "
            f"{len(results)} prompt(s), {total_samples} sample(s) "
            f"({time.perf_counter() - t_warmup:.2f}s{ttfp_text})"
        )
 async def _run_streaming(args) -> list[Path]:
    output_dir = Path(args.output_dir)
    output_dir.mkdir(parents=True, exist_ok=True)
    omni = AsyncOmni(
        model=args.model,
        stage_configs_path=args.stage_configs_path,
        log_stats=args.log_stats,
        stage_init_timeout=args.stage_init_timeout,
    )
    await _run_streaming_warmup(args, omni)
    profiler_started = False
    if args.enable_profiler:
        profile_prefix = f"voxcpm_streaming_{int(time.time())}"
        stages_text = args.profiler_stages if args.profiler_stages is not None else "all-configured"
        print(f"Starting profiler (streaming): stages={stages_text}, dir={args.profiler_dir}")
        await omni.start_profile(profile_prefix=profile_prefix, stages=args.profiler_stages)
        profiler_started = True
    t_total = time.perf_counter()
    total_elapsed = 0.0
    paths: list[Path] = []
    prompt_specs: list[PromptSpec] = args.prompt_specs
    try:
        for run in range(args.num_runs):
            for prompt_index, spec in enumerate(prompt_specs):
                request_id = f"stream_{run + 1}_{spec.label}_{uuid.uuid4().hex[:8]}"
                paths.append(
                    await _run_streaming_single(
                        omni,
                        args,
                        spec,
                        output_dir,
                        request_id,
                        run_index=run,
                        num_runs=args.num_runs,
                        prompt_index=prompt_index,
                        prompt_count=len(prompt_specs),
                    )
                )
        total_elapsed = time.perf_counter() - t_total
    finally:
        if profiler_started:
            print("Stopping profiler (streaming)...")
            await omni.stop_profile(stages=args.profiler_stages)
            if args.profiler_wait_seconds > 0:
                print(f"Waiting {args.profiler_wait_seconds:.1f}s for profiler traces to flush...")
                await asyncio.sleep(args.profiler_wait_seconds)
    print(
        f"All streaming runs finished: {args.num_runs} run(s), "
        f"{len(prompt_specs)} prompt(s), {len(paths)} file(s) in {total_elapsed:.2f}s total"
    )
    return paths
 def _run_sync(args) -> list[Path]:
    output_dir = Path(args.output_dir)
    omni = Omni(
        model=args.model,
        stage_configs_path=args.stage_configs_path,
        log_stats=args.log_stats,
        stage_init_timeout=args.stage_init_timeout,
    )
    def _run_sync_single(
        spec: PromptSpec,
        *,
        request_prefix: str,
        save_outputs: bool,
        run_index: int | None = None,
    ) -> tuple[list[Path], int, float | None, float, float, str]:
        global_request_id = f"{request_prefix}_{spec.label}"
        prompt = _build_prompt_for_spec(args, spec, global_request_id=global_request_id)
        if save_outputs and run_index == 0 and spec.label == "item001":
            print(f"---prompt---:{prompt}")
        saved_paths: list[Path] = []
        output_count = 0
        first_audio_elapsed: float | None = None
        total_audio_duration_s = 0.0
        metrics_request_id = global_request_id
        t_start = time.perf_counter()
        for stage_outputs in omni.generate(prompt):
            request_output = stage_outputs.request_output
            if request_output is None:
                continue
            request_output_id = getattr(request_output, "request_id", None)
            if isinstance(request_output_id, str) and request_output_id:
                metrics_request_id = request_output_id
            for j, mm in enumerate(_iter_request_multimodal_outputs(request_output)):
                output_count += 1
                if first_audio_elapsed is None:
                    try:
                        audio_tensor = _extract_audio_tensor(mm)
                        if int(audio_tensor.numel()) > 0:
                            first_audio_elapsed = time.perf_counter() - t_start
                        total_audio_duration_s += float(audio_tensor.numel()) / float(_extract_sample_rate(mm))
                    except ValueError:
                        pass
                else:
                    try:
                        audio_tensor = _extract_audio_tensor(mm)
                        total_audio_duration_s += float(audio_tensor.numel()) / float(_extract_sample_rate(mm))
                    except ValueError:
                        pass
                if not save_outputs:
                    continue
                save_stem = f"run{run_index + 1}_{spec.label}" if j == 0 else f"run{run_index + 1}_{spec.label}_{j}"
                saved_paths.append(_save_wav(mm, output_dir, save_stem))
        if output_count == 0:
            raise RuntimeError("No output from Omni.generate")
        elapsed_s = time.perf_counter() - t_start
        return saved_paths, output_count, first_audio_elapsed, elapsed_s, total_audio_duration_s, metrics_request_id
    if args.warmup_runs:
        warmup_specs = _get_warmup_specs(args.prompt_specs)
        print(
            f"Warmup: {args.warmup_runs} run(s) using the first prompt "
            f"({len(warmup_specs)} prompt(s)); outputs will be discarded."
        )
        for warmup_index in range(args.warmup_runs):
            t_warmup = time.perf_counter()
            _, output_count, first_audio_elapsed, elapsed_s, audio_duration_s, _ = _run_sync_single(
                warmup_specs[0],
                request_prefix=f"warmup_sync{warmup_index + 1}",
                save_outputs=False,
            )
            ttfp_text = f", ttfp={first_audio_elapsed:.2f}s" if first_audio_elapsed is not None else ""
            rtf_text = f", rtf={elapsed_s / audio_duration_s:.3f}" if audio_duration_s > 0 else ""
            print(
                f"Warmup (sync) {warmup_index + 1}/{args.warmup_runs} finished: "
                f"{output_count} output(s) ({time.perf_counter() - t_warmup:.2f}s{ttfp_text}{rtf_text})"
            )
    profiler_started = False
    if args.enable_profiler:
        profile_prefix = f"voxcpm_sync_{int(time.time())}"
        stages_text = args.profiler_stages if args.profiler_stages is not None else "all-configured"
        print(f"Starting profiler (sync): stages={stages_text}, dir={args.profiler_dir}")
        omni.start_profile(profile_prefix=profile_prefix, stages=args.profiler_stages)
        profiler_started = True
    t_total = time.perf_counter()
    total_elapsed = 0.0
    saved_paths: list[Path] = []
    prompt_specs: list[PromptSpec] = args.prompt_specs
    try:
        for run in range(args.num_runs):
            t_run = time.perf_counter()
            run_paths: list[Path] = []
            for prompt_index, spec in enumerate(prompt_specs):
                prompt_paths, _, first_audio_elapsed, elapsed_s, audio_duration_s, metrics_request_id = (
                    _run_sync_single(
                        spec,
                        request_prefix=f"sync_run{run + 1}_{prompt_index + 1:03d}",
                        save_outputs=True,
                        run_index=run,
                    )
                )
                run_paths.extend(prompt_paths)
                ttfp_text = f", ttfp={first_audio_elapsed:.2f}s" if first_audio_elapsed is not None else ""
                rtf_text = f", rtf={elapsed_s / audio_duration_s:.3f}" if audio_duration_s > 0 else ""
                print(
                    f"Saved (sync) run {run + 1}/{args.num_runs}, "
                    f"prompt {prompt_index + 1}/{len(prompt_specs)}: {len(prompt_paths)} file(s){ttfp_text}{rtf_text}"
                )
                _emit_offline_metrics(
                    request_id=metrics_request_id,
                    elapsed_s=elapsed_s,
                    first_audio_elapsed=first_audio_elapsed,
                    audio_duration_s=audio_duration_s,
                )
            saved_paths.extend(run_paths)
            print(
                f"Run {run + 1}/{args.num_runs} finished: {len(run_paths)} file(s) ({time.perf_counter() - t_run:.2f}s)"
            )
            for path in run_paths:
                print(f"  {path}")
        total_elapsed = time.perf_counter() - t_total
    finally:
        if profiler_started:
            print("Stopping profiler (sync)...")
            omni.stop_profile(stages=args.profiler_stages)
            if args.profiler_wait_seconds > 0:
                print(f"Waiting {args.profiler_wait_seconds:.1f}s for profiler traces to flush...")
                time.sleep(args.profiler_wait_seconds)
    print(
        f"All sync runs finished: {args.num_runs} run(s), "
        f"{len(prompt_specs)} prompt(s), {len(saved_paths)} file(s) in {total_elapsed:.2f}s total"
    )
    return saved_paths
 def main(args) -> int:
    logging.basicConfig(level=logging.INFO)
    profiled_stage_config_path: str | None = None
    original_stage_config_path = args.stage_configs_path
    if args.enable_profiler:
        Path(args.profiler_dir).mkdir(parents=True, exist_ok=True)
        profiled_stage_config_path = _build_profiled_stage_config(
            args.stage_configs_path,
            str(Path(args.profiler_dir).resolve()),
        )
        args.stage_configs_path = profiled_stage_config_path
    is_streaming = _is_streaming_stage_config(args.stage_configs_path)
    voice_clone_count = _count_voice_clone_prompts(args.prompt_specs)
    print(f"Model: {args.model}")
    print(f"Stage config: {original_stage_config_path}")
    print(f"Route: {'streaming' if is_streaming else 'sync'} (from stage-configs-path)")
    print(f"Prompt count: {len(args.prompt_specs)}")
    print("Batch mode: sequential (aligned with native VoxCPM)")
    print(f"Warmup runs: {args.warmup_runs}")
    print(f"Voice cloning prompts: {voice_clone_count}/{len(args.prompt_specs)}")
    if args.enable_profiler:
        print(f"Profiler: enabled (dir={args.profiler_dir}, stages={args.profiler_stages or 'all-configured'})")
        print(f"Profiled stage config: {args.stage_configs_path}")
    if voice_clone_count:
        print("Voice cloning note: --ref-text/ref_text must match the spoken content of the reference audio.")
    print(f"Num runs: {args.num_runs}")
    try:
        if is_streaming:
            asyncio.run(_run_streaming(args))
        else:
            _run_sync(args)
    finally:
        if profiled_stage_config_path is not None and os.path.exists(profiled_stage_config_path):
            os.unlink(profiled_stage_config_path)
    return 0
 if __name__ == "__main__":
    os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
    raise SystemExit(main(parse_args()))
@@ -32,11 +32,8 @@ models:
        task_type: Base
  openbmb/VoxCPM2:
-    supported_tasks: [voice_clone, default_voice]
+    supported_tasks: [voice_clone]
    trust_remote_code: true
    backend: openai-audio-speech
    endpoint: /v1/audio/speech
    task_extra_body:
      voice_clone: {}
      default_voice:
        voice: default
@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai:v0.21.0
+ARG BASE_IMAGE=vllm/vllm-openai:v0.20.0
 FROM ${BASE_IMAGE}
 ARG COMMON_WORKDIR=/app
@@ -133,11 +133,6 @@ ENTRYPOINT []
 FROM ${VLLM_BASE} AS vllm-omni
 RUN apt-get update && \
    apt-get install -y espeak-ng ffmpeg git jq && \
    apt-get clean && \
    rm -rf /var/lib/apt/lists/*
 WORKDIR /workspace/vllm-omni
 COPY . .
@@ -80,8 +80,6 @@ nav:
      - Quantized KV Cache: user_guide/quantization/quantized_kvcache.md
      - FP8 W8A8: user_guide/quantization/fp8.md
      - Int8 W8A8: user_guide/quantization/int8.md
      - MXFP8 W8A8: user_guide/quantization/mxfp8.md
      - MXFP4 W4A4: user_guide/quantization/mxfp4.md
      - ModelOpt: user_guide/quantization/modelopt.md
      - GGUF: user_guide/quantization/gguf.md
      - AutoRound: user_guide/quantization/autoround.md
@@ -121,6 +121,7 @@ Configuration classes.
 - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLTextConfig][]
 - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLVisionConfig][]
 - [vllm_omni.transformers_utils.configs.omnivoice.OmniVoiceConfig][]
 - [vllm_omni.transformers_utils.configs.voxcpm.VoxCPMConfig][]
 - [vllm_omni.transformers_utils.configs.voxcpm2.VoxCPM2Config][]
 ## Workers
@@ -39,34 +39,11 @@ Committers have write access and merge rights. They typically have deep expertis
 - [@linyueqian](https://github.com/linyueqian): TTS and Omni Support
 - [@lishunyang12](https://github.com/lishunyang12): Quantization and Configuration
 - [@princepride](https://github.com/princepride): Diffusion and Omni Support
- [@RuixiangMa](https://github.com/RuixiangMa): Diffusion models, parallel, cache, and docs
+- [@SamitHuang](https://github.com/SamitHuang): RL and Diffusion
 - [@SamitHuang](https://github.com/SamitHuang): RL, Diffusion, and cache
 - [@tzhouam](https://github.com/tzhouam): Engine and New Model Support
- [@wtomin](https://github.com/wtomin): Diffusion models, parallel, and docs
+- [@wtomin](https://github.com/wtomin): Diffusion and Parallelism
 - [@ZeldaHuang](https://github.com/ZeldaHuang): Omni Support
- [@ZJY0516](https://github.com/ZJY0516): Diffusion attention backend, kernel fusion, and CustomOp
+- [@ZJY0516](https://github.com/ZJY0516): Diffusion and CustomOp
 - [@yuanheng-zhao](https://github.com/yuanheng-zhao): Diffusion cache, offload, and Omni Support
 ## Diffusion Workload Division
 This section breaks down diffusion-related responsibilities. If you have PRs touching these areas, please ping the listed owners for review.
 ### Runtime-related
 - Diffusion models: @RuixiangMa, @wtomin
 ### Optimization
 - Parallel: @RuixiangMa, @wtomin
 - Attention backend: @ZJY0516
 - Cache: @yuanheng-zhao, @RuixiangMa, @SamitHuang
 - Offload: @yuanheng-zhao
 - Quantization: @lishunyang12, @david6666666
 - Kernel fusion / communication-computation: @ZJY0516
 ### Docs & Test
 - Diffusion docs: @RuixiangMa, @wtomin
 ## Meetings
@@ -83,10 +83,7 @@ There are two important details:
 Today that compatibility rule is still shape-sensitive. `height`, `width`,
 `num_frames`, and CFG-related fields remain part of the key, so different
-resolutions or incompatible guidance settings do **not** co-batch yet. The
+resolutions or incompatible guidance settings do **not** co-batch yet.
 key also covers LoRA identity (`lora_int_id`, `lora_scale`), so requests
 targeting different adapters or scales run in separate batches and the
 worker can activate exactly one adapter per step.
 The current batching unit is one `OmniDiffusionRequest`. Requests with
 multiple prompts do not participate in batching today.
@@ -9,7 +9,6 @@ Backend-specific setup lives in separate docs:
 - [MooncakeStoreConnector](omni_connectors/mooncake_store_connector.md)
 - [MooncakeTransferEngineConnector](omni_connectors/mooncake_transfer_engine_connector.md)
 - [YuanrongConnector](omni_connectors/yuanrong_connector.md)
 - [YuanrongTransferEngineConnector](omni_connectors/yuanrong_transfer_engine_connector.md)
 ## Overview
@@ -24,7 +23,6 @@ Current connectors operate in D2H2D (device to host to device) mode.
 | Multi node (Mooncake Store) | MooncakeStoreConnector | TCP-based, requires Mooncake Master + metadata server. |
 | Multi node (Mooncake RDMA) | MooncakeTransferEngineConnector | RDMA/TCP direct transfer with managed memory pool. Fastest. |
 | Multi node (Yuanrong) | YuanrongConnector | Requires Yuanrong Datasystem + etcd. |
 | Ascend NPU P2P (Yuanrong TE) | YuanrongTransferEngineConnector | Uses Yuanrong TransferEngine directly. Configure NPU device IPv4 and `memory_pool_device: "npu"`. |
 ## Core API
@@ -1,211 +0,0 @@
 # YuanrongTransferEngineConnector
 ## When to Use
 Use `YuanrongTransferEngineConnector` for high-performance peer-to-peer KV cache
 transfer on Ascend NPU. It is different from `YuanrongConnector`:
 - `YuanrongConnector` uses Yuanrong Datasystem as a distributed KV store and
  requires Datasystem workers plus etcd.
 - `YuanrongTransferEngineConnector` uses Yuanrong TransferEngine directly. It
  registers local memory in each stage worker and lets the receiver pull data
  from the sender with TransferEngine.
 The implementation currently lives under
 `vllm_omni/platforms/npu/omni_connectors/` because the supported backend is
 Ascend NPU only. The connector is still registered with the generic
 OmniConnector factory under the name `YuanrongTransferEngineConnector`.
 For Ascend P2P transfer, the connector should use NPU memory as its transfer
 pool. The Mooncake-style CPU pool is not supported for this connector.
 ## Prerequisites
 Install Yuanrong Datasystem/TransferEngine Python bindings in the runtime
 environment. The packaged install is:
 ```bash
 pip install openyuanrong-datasystem
 ```
 To build from source, follow the upstream project instructions:
 - Source repository: [openeuler/yuanrong-datasystem](https://atomgit.com/openeuler/yuanrong-datasystem)
 - Installation guide: [Yuanrong Datasystem Linux installation](https://pages.openeuler.openatom.cn/openyuanrong-datasystem/docs/zh-cn/latest/installation/installation_linux.html)
 The Ascend driver, CANN runtime, HCCL/HCCP, and NPU device network must also be
 configured on every machine participating in the transfer.
 ## Check NPU Device IPv4
 Yuanrong TransferEngine with `protocol: "ascend"` uses the NPU device network,
 not only the host management IP. Each participating NPU must have an IPv4 HCCN
 device IP.
 Check the device IP, link, and health state:
 ```bash
 for i in {0..7}; do
  echo "===== NPU $i ====="
  hccn_tool -i "$i" -ip -g
  hccn_tool -i "$i" -link -g
  hccn_tool -i "$i" -net_health -g
 done
 ```
 Expected output for each NPU should include an IPv4 address and healthy link:
 ```text
 ipaddr:10.86.1.80
 netmask:255.255.255.0
 link status: UP
 net health status: Success
 ```
 If the output says `Get ipconf failed, because no ip was preset there!`, the NPU
 device IP is not configured and Ascend P2P will fail. If the address is IPv6,
 configure IPv4 for this connector path or validate that the underlying Yuanrong
 TransferEngine build supports the IPv6 HCCN path before using it.
 ## Check Device-to-Device Connectivity
 After collecting the target-stage NPU IPv4 addresses, ping them from each
 source-stage NPU:
 ```bash
 # Example: AR uses NPU 0..3 and DiT uses NPU 4..7.
 # Replace the destination IPs with the values reported by hccn_tool -i 4..7 -ip -g.
 for src in 0 1 2 3; do
  for dst_ip in 10.86.1.78 10.86.1.79 10.86.1.80 10.86.1.81; do
    echo "===== NPU $src -> $dst_ip ====="
    hccn_tool -i "$src" -ping -g address "$dst_ip" || true
  done
 done
 ```
 The expected result is `0.00% packet loss` for every required source/target pair.
 This verifies basic HCCN IPv4 connectivity. TransferEngine can still fail later
 if RDMA/QP/HCCL setup is broken, but failed ping means the device network must be
 fixed first.
 ## YAML Configuration
 For Ascend P2P, configure the connector with `protocol: "ascend"` and an NPU
 memory pool:
 ```yaml
 runtime:
  enabled: true
  defaults:
    window_size: -1
    max_inflight: 1
  connectors:
    yuanrong_te_connector:
      name: YuanrongTransferEngineConnector
      extra:
        host: "auto"
        zmq_port: 50051
        rpc_port: "auto"
        protocol: "ascend"
        device_name: "auto"
        memory_pool_size: 1073741824
        memory_pool_device: "npu"
  edges:
    - from: 0
      to: 1
      window_size: -1
 ```
 Important fields:
 | Parameter | Recommended Value | Notes |
 |---|---|---|
 | `host` | `"auto"` | Advertises a routable local host IP for ZMQ metadata exchange. |
 | `zmq_port` | `50051` | Base ZMQ port. The runtime applies stage/rank offsets. |
 | `rpc_port` | `"auto"` | Lets TransferEngine choose an available RPC port. |
 | `protocol` | `"ascend"` | Uses the Ascend TransferEngine backend. |
 | `device_name` | `"auto"` | Each TP worker resolves to its local logical NPU, for example `npu:0`, `npu:1`, etc. |
 | `memory_pool_device` | `"npu"` | Required for the Ascend fast path. This registers NPU memory with TransferEngine. |
 | `memory_pool_size` | `1073741824` | 1 GiB per worker is a practical starting point for KV-cache transfer validation. Increase if the transfer pool is exhausted. |
 `memory_pool_device: "cpu"` is rejected for
 `YuanrongTransferEngineConnector` with `protocol: "ascend"`. CPU pool registration
 can lead to Ascend P2P failures such as HCCL RA init errors, QP timeouts, or
 FFTS/SDMA runtime errors. `memory_pool_device: "cpu"` is common for
 `MooncakeTransferEngineConnector`, but Yuanrong TE on Ascend only supports
 `"npu"`.
 ## 4-Card AR-to-DiT TP Example
 For AR 4-way TP sending KV cache to DiT 4-way TP, set the stage TP topology in
 both stage `omni_kv_config` blocks:
 ```yaml
 stage_args:
  - stage_id: 0
    runtime:
      devices: "0,1,2,3"
    engine_args:
      tensor_parallel_size: 4
      omni_kv_config:
        need_send_cache: true
        rank_mapping:
          from_tp: 4
          to_tp: 4
    output_connectors:
      to_stage_1: yuanrong_te_connector
  - stage_id: 1
    runtime:
      devices: "4,5,6,7"
    engine_args:
      parallel_config:
        tensor_parallel_size: 4
      omni_kv_config:
        need_recv_cache: true
        rank_mapping:
          from_tp: 4
          to_tp: 4
    input_connectors:
      from_stage_0: yuanrong_te_connector
 ```
 In this topology each DiT rank receives the corresponding AR rank shard:
 ```text
 DiT rank 0 <- AR rank 0
 DiT rank 1 <- AR rank 1
 DiT rank 2 <- AR rank 2
 DiT rank 3 <- AR rank 3
 ```
 ## Validate a Successful Run
 Search the inference log for Yuanrong TE receive-side messages:
 ```bash
 grep -E "\[YR GET\]|Successfully received KV cache|transfer_engine get failed" inference.log
 ```
 A successful Ascend fast-path transfer contains lines like:
 ```text
 [YR GET] ... fast_path, zero-copy
 Successfully received KV cache ... across 1 key(s)
 Applied CFG KV caches
 [KV Reuse] ...
 ```
 Sender-side `KV transfer OK` only means that the sender registered the payload
 and metadata. The actual P2P data transfer happens on the receiver side during
 `[YR GET]`.
 ## Troubleshooting
 | Symptom | Likely Cause | Action |
 |---|---|---|
 | `IPv4 device IP not found` | NPU HCCN IPv4 is missing. | Run `hccn_tool -i <id> -ip -g` and configure IPv4 for every participating NPU. |
 | `P2PCommInitRootInfo failed`, `ra init failed`, `RA_QP_STATUS_TIMEOUT` | Ascend P2P/RDMA/QP setup failed. | First verify device IPv4, link health, and device-to-device ping. Then test a minimal Yuanrong TE pair outside vLLM. |
 | `fftsplus sdma error`, `context is abort` | Runtime SDMA/FFTS failure, often after an invalid or failed P2P transfer. | Use `memory_pool_device: "npu"` for Ascend TE, reduce `memory_pool_size` if OOM occurs, and isolate the failing NPU pair with a Yuanrong TE demo. |
 | Sender logs `KV transfer OK` but receiver never logs `[YR GET]` | Sender `put()` succeeded, receiver P2P read failed or timed out. | Check receiver logs and search for `transfer_engine get failed`. |
 | Pool allocation fails or OOM | Transfer pool is too small or too large for available NPU memory. | Start with 1 GiB (`1073741824`), reduce to 512 MiB for validation, or increase for larger KV payloads. |
@@ -169,13 +169,15 @@ class DiffusionRequestStatus(enum.IntEnum):
@dataclass
 class DiffusionRequestState:
-    request_id: str
+    sched_req_id: str
    req: OmniDiffusionRequest
    status: DiffusionRequestStatus = DiffusionRequestStatus.WAITING
 ```
 **Design Features**:
 - **Scheduler-owned ID**: Each `OmniDiffusionRequest` is tracked by an internal `sched_req_id`, separated from public `request_id` values.
 - **Explicit lifecycle**: Requests move through waiting, running, optional preemption, and terminal states.
 - **Centralized error handling**: Completion, abort, and error states are all normalized in the scheduler layer.
@@ -186,6 +188,7 @@ class DiffusionRequestState:
 class _BaseScheduler(SchedulerInterface):
    def __init__(self) -> None:
        self._request_states = {}
        self._request_id_to_sched_req_id = {}
        self._waiting = deque()
        self._running = []
        self._finished_req_ids = set()
@@ -196,8 +199,7 @@ class _BaseScheduler(SchedulerInterface):
 - **Common state storage**: Shared request maps and waiting/running sets live in the base class.
- **Shared cleanup logic**: Duplicate-request checks, finish handling, and state removal are centralized instead of
+- **Shared cleanup logic**: Request-id registration, finish handling, and state removal are centralized instead of duplicated in each policy.
  duplicated in each policy.
 - **Current constraint boundary**: `_BaseScheduler` derives `max_num_running_reqs` from `max_num_seqs`, but request-mode diffusion is still clamped back to `1` by the engine. The step-wise path can keep this above `1` for compatible-request batching.
@@ -217,12 +219,12 @@ class RequestScheduler(_BaseScheduler):
 - **Single-request admission**: `RequestScheduler` still admits one active request at a time because request-mode execution completes a whole request per dispatch.
- **Executor result feedback**: `update_from_output()` converts executor output into `FINISHED_COMPLETED` or `FINISHED_ERROR` and returns finished request ids.
+- **Executor result feedback**: `update_from_output()` converts executor output into `FINISHED_COMPLETED` or `FINISHED_ERROR` and returns finished scheduler ids.
 #### 2.5 Engine-Driven Execution Loop
 ```python
-request_id = scheduler.add_request(request)
+sched_req_id = scheduler.add_request(request)
 while True:
    sched_output = scheduler.schedule()
    output = executor.add_req(req)
@@ -103,10 +103,10 @@ docker run --runtime nvidia --gpus 2 \
    -p 8091:8091 \
    --ipc=host \
    vllm/vllm-omni:v0.21.0 \
-    vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
+    --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 8091
 ```
 !!! tip
-    The CUDA image does not define a default entrypoint, so include `vllm serve ... --omni` after the image name.
+    You can use this docker image to serve models the same way you would with in vLLM! To do so, make sure you overwrite the default entrypoint (`vllm serve --omni`) which works only for models supported in the vLLM-Omni project.
 # --8<-- [end:pre-built-images]
@@ -16,7 +16,6 @@ ROOT_DIR_RELATIVE = "../../../../.."
 EXAMPLE_DIR = ROOT_DIR / "examples"
 EXAMPLE_DOC_DIR = ROOT_DIR / "docs/user_guide/examples"
 NAV_FILE = ROOT_DIR / "docs/.nav.yml"
 MAX_INLINE_MATERIAL_SIZE = 8 * 1024
 def fix_case(text: str) -> str:
@@ -192,21 +191,18 @@ class Example:
        return re.sub(link_pattern, replace_link, content)
    def github_url(self, path: Path) -> str:
        url = "https://github.com/vllm-project/vllm-omni/"
        url += "tree/main" if path.is_dir() else "blob/main"
        return f"{url}/{path.relative_to(ROOT_DIR).as_posix()}"
    def generate(self) -> str:
        content = f"# {self.title}\n\n"
-        content += f"Source <{self.github_url(self.path)}>.\n\n"
+        url = "https://github.com/vllm-project/vllm-omni/"
        url += "tree/main" if self.path.is_dir() else "blob/main"
        content += f"Source <{url}/{self.path.relative_to(ROOT_DIR)}>.\n\n"
        # Use long code fence to avoid issues with
        # included files containing code fences too
        code_fence = "``````"
        if self.is_code:
-            main_file_rel = self.main_file.relative_to(ROOT_DIR).as_posix()
+            main_file_rel = self.main_file.relative_to(ROOT_DIR)
            content += f'{code_fence}{self.main_file.suffix[1:]}\n--8<-- "{main_file_rel}"\n{code_fence}\n'
        else:
            with open(self.main_file, encoding="utf-8") as f:
@@ -220,15 +216,10 @@ class Example:
        content += "## Example materials\n\n"
        for file in sorted(self.other_files):
-            content += f'??? abstract "{file.relative_to(self.path).as_posix()}"\n'
+            content += f'??? abstract "{file.relative_to(self.path)}"\n'
            if file.stat().st_size > MAX_INLINE_MATERIAL_SIZE:
                content += (
                    f"    Large file omitted from the rendered docs. View it on GitHub: <{self.github_url(file)}>.\n\n"
                )
                continue
            if file.suffix != ".md":
                content += f"    {code_fence}{file.suffix[1:]}\n"
-            content += f'    --8<-- "{file.relative_to(ROOT_DIR).as_posix()}"\n'
+            content += f'    --8<-- "{file.relative_to(ROOT_DIR)}"\n'
            if file.suffix != ".md":
                content += f"    {code_fence}\n"
@@ -300,7 +291,7 @@ def update_nav_file(examples: list[Example]):
        for example in category_examples:
            doc_path = EXAMPLE_DOC_DIR / example.category / f"{example.path.stem}.md"
            rel_path = doc_path.relative_to(ROOT_DIR / "docs")
-            category_items.append({example.title: rel_path.as_posix()})
+            category_items.append({example.title: str(rel_path)})
        if category_items:
            # Format category name (e.g., "offline_inference" -> "Offline Inference")
@@ -1,29 +0,0 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 """Reduce browser search-index memory for generated API reference pages."""
 from material.plugins.search.plugin import SearchIndex
 _add_entry_from_context = SearchIndex.add_entry_from_context
 def add_entry_from_context(self: SearchIndex, page) -> None:
    search = page.meta.get("search") or {}
    if search.get("exclude"):
        return
    if str(page.url).startswith("api/"):
        entry = {
            "location": page.url,
            "title": str(page.meta.get("title", page.title)),
            "text": "",
        }
        if "boost" in search:
            entry["boost"] = search["boost"]
        self.entries.append(entry)
        return
    _add_entry_from_context(self, page)
 SearchIndex.add_entry_from_context = add_entry_from_context
@@ -1,44 +1,20 @@
-// Enables MathJax rendering only on pages that contain math.
+// Enables MathJax rendering
 window.MathJax = {
-  tex: {
+    tex: {
-    inlineMath: [["\\(", "\\)"]],
+      inlineMath: [["\\(", "\\)"]],
-    displayMath: [["\\[", "\\]"]],
+      displayMath: [["\\[", "\\]"]],
-    processEscapes: true,
+      processEscapes: true,
-    processEnvironments: true,
+      processEnvironments: true
-  },
+    },
-  options: {
+    options: {
-    ignoreHtmlClass: ".*|",
+      ignoreHtmlClass: ".*|",
-    processHtmlClass: "arithmatex",
+      processHtmlClass: "arithmatex"
-  },
+    }
-};
+  };
-let mathJaxLoading;
+  document$.subscribe(() => {
-
+    MathJax.startup.output.clearCache()
-function loadMathJax() {
+    MathJax.typesetClear()
-  if (window.MathJax.typesetPromise) {
+    MathJax.texReset()
-    return Promise.resolve();
+    MathJax.typesetPromise()
-  }
+  })
  if (!mathJaxLoading) {
    mathJaxLoading = new Promise((resolve, reject) => {
      const script = document.createElement("script");
      script.src = "https://unpkg.com/mathjax@3.2.2/es5/tex-mml-chtml.js";
      script.async = true;
      script.onload = resolve;
      script.onerror = reject;
      document.head.appendChild(script);
    });
  }
  return mathJaxLoading;
 }
 document$.subscribe(() => {
  if (!document.querySelector(".arithmatex")) {
    return;
  }
  loadMathJax().then(() => {
    MathJax.startup.output.clearCache();
    MathJax.typesetClear();
    MathJax.texReset();
    MathJax.typesetPromise();
  });
 });
@@ -1,2 +0,0 @@
 // Opt out of bfcache so large generated API pages are released after navigation.
 window.addEventListener("unload", () => {});
@@ -1,42 +1,21 @@
-// Load Mermaid only on pages that contain diagrams.
+// Initialize Mermaid for diagram rendering
-let mermaidLoading;
+mermaid.initialize({
  startOnLoad: false,
  theme: 'default',
  securityLevel: 'loose',
  flowchart: {
    useMaxWidth: true,
    htmlLabels: true
  }
 });
-function loadMermaid() {
+// Render Mermaid diagrams when page content is ready
-  if (window.mermaid) {
+document$.subscribe(() => {
-    return Promise.resolve();
+  const mermaidElements = document.querySelectorAll('.mermaid');
  if (mermaidElements.length > 0) {
    mermaid.run({
      querySelector: '.mermaid',
      nodes: mermaidElements
    });
  }
  if (!mermaidLoading) {
    mermaidLoading = new Promise((resolve, reject) => {
      const script = document.createElement("script");
      script.src = "https://unpkg.com/mermaid@10/dist/mermaid.min.js";
      script.async = true;
      script.onload = () => {
        mermaid.initialize({
          startOnLoad: false,
          theme: "default",
          securityLevel: "loose",
          flowchart: {
            useMaxWidth: true,
            htmlLabels: true,
          },
        });
        resolve();
      };
      script.onerror = reject;
      document.head.appendChild(script);
    });
  }
  return mermaidLoading;
 }
 document$.subscribe(() => {
  if (!document.querySelector(".mermaid")) {
    return;
  }
  loadMermaid().then(() => {
    mermaid.run({
      querySelector: ".mermaid",
      nodes: document.querySelectorAll(".mermaid"),
    });
  });
 });
@@ -19,7 +19,6 @@ th {
 |--------------|--------|-------------------|------------|---------|-----|-----------|
 | `Qwen3OmniMoeForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Qwen2_5OmniForConditionalGeneration` | Qwen2.5-Omni | `Qwen/Qwen2.5-Omni-7B`, `Qwen/Qwen2.5-Omni-3B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `MingFlashOmniForConditionalGeneration` + `MingImagePipeline` | Ming-flash-omni-2.0 (omni-speech + imagegen<sup>1</sup>) | `Jonathan1909/Ming-flash-omni-2.0` | ✅︎ |   |   |   |
 | `BagelForConditionalGeneration` | BAGEL (DiT-only) | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ | | ✅︎ |
 | `InternVLAA1Pipeline` | InternVLA-A1 | `InternRobotics/InternVLA-A1-3B` | ✅︎ | ✅︎ | | |
 | `HunyuanImage3ForCausalMM` | HunyuanImage3.0 (DiT-only) | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
@@ -47,7 +46,7 @@ th {
 | `LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ |
-| `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ |
+| `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | |
 | `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | |
 | `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | |
@@ -59,7 +58,6 @@ th {
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `GLMTTSForConditionalGeneration` | GLM-TTS | `zai-org/GLM-TTS` | ✅︎ | | | |
 | `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ |
 | `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | |
 | `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | |
@@ -73,6 +71,4 @@ th {
 | `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | |
 |`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | |
 | `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 |`HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | |
 ✅︎ indicates the model is supported on that backend. Empty cells mean not listed as supported on that backend.
@@ -13,7 +13,7 @@ For example...
 ```bash
 # Qwen-Image
 vllm serve Qwen/Qwen-Image-Edit-2511 --omni --port 8000
-```
+
 ### Generate Images
@@ -53,7 +53,7 @@ result = client.images.edit(
    output_format='jpeg',
    # url格式
    extra_body={
-        "url": [input_image_url,input_image_url],
+        "url": [input_image_url1,input_image_url],
        "num_inference_steps": 50,
        "guidance_scale": 1,
        "seed": 777,
@@ -92,8 +92,7 @@ Content-Type: multipart/form-data
 | `user` | string | null | User identifier for tracking |
 | `output_format` | string | "png" | The format in which the generated images are returned. Must be one of "png", "jpg", "jpeg", "webp". |
 | `output_compression` | integer | 100 | The compression level (0-100%) for the generated images. |
-| `background` | string or null | "auto" | Allows to set transparency for the background of the generated image(s). |
+| `background` | string or null | "auto" | Allows to set transparency for the background of the generated image(s).
 | `stream` | boolean | false | Return Server-Sent Events instead of a single JSON response. Currently supported only by multi-stage image edit pipelines that produce AR text before the final image, such as HunyuanImage3 IT2I AR+DiT. |
 #### vllm-omni Extension Parameters
@@ -110,8 +109,6 @@ Content-Type: multipart/form-data
 ### Response Format
 When `stream=false` or omitted, the endpoint returns the standard image edit response:
 ```json
 {
  "created": 1701234567,
@@ -123,56 +120,10 @@ When `stream=false` or omitted, the endpoint returns the standard image edit res
    }
  ],
  "output_format": null,
-  "size": null
+  "size": null,
 }
 ```
 ### Streaming Response Format
 Set `stream=true` when you want to receive the HunyuanImage3 IT2I AR
 recaption text before the final edited image is ready. Streaming is only
 available for multi-stage image edit pipelines where stage 0 produces AR text
 and a later diffusion stage produces the image. Single-stage diffusion image
 edit pipelines reject `stream=true` with HTTP 400.
 The response uses Server-Sent Events with `Content-Type: text/event-stream`.
 Each event is sent as one `data:` line. The stream order is:
 1. One or more AR text delta chunks.
 2. One final image chunk.
 3. `data: [DONE]`.
 AR text chunks expose only the generated text delta and the AR completion
 `index`. They do not expose token ids, ratio tokens, KV cache data, or internal
 prompt token ids.
 Example AR delta event:
 ```text
 data: {"object":"image.edit.chunk","type":"ar_delta","delta":"A close-up product photo...","index":0,"created":1701234567,"model":"tencent/HunyuanImage-3.0-Instruct"}
 ```
 Example final image event:
 ```text
 data: {"object":"image.edit.chunk","type":"image","data":[{"b64_json":"<base64-encoded PNG>","url":null,"revised_prompt":null}],"output_format":"png","size":"1024x1024","created":1701234567,"model":"tencent/HunyuanImage-3.0-Instruct"}
 ```
 Terminal event:
 ```text
 data: [DONE]
 ```
 If the engine fails after the SSE response has started, the stream emits an
 error chunk followed by `[DONE]`:
 ```text
 data: {"object":"error","created":1701234567,"model":"tencent/HunyuanImage-3.0-Instruct","error":{"message":"<error message>","type":"server_error","code":500}}
 data: [DONE]
 ```
 ## Examples
 ### Multiple Images input
@@ -183,28 +134,12 @@ curl -s -D >(grep -i x-request-id >&2) \
  -X POST "http://localhost:8000/v1/images/edits" \
  -F "model=xxx" \
  -F "image=@xx.png" \
-  -F "image=@xx.png" \
+  -F "image=@xx.png"
  -F "prompt='this bear is wearing sportwear. holding a basketball, and bending one leg.'" \
  -F "size=1024x1024" \
  -F "output_format=png"
 ```
 ### Streaming HunyuanImage3 IT2I AR Text
 ```bash
 curl -N -X POST "http://localhost:8000/v1/images/edits" \
  -F "model=tencent/HunyuanImage-3.0-Instruct" \
  -F "image=@./input.png" \
  -F "prompt=Turn the product into a clean studio advertisement." \
  -F "size=1024x1024" \
  -F "output_format=png" \
  -F "stream=true"
 ```
 Use the `ar_delta` events for progressive display of the AR-generated
 recaption. Decode the `b64_json` field from the final `image` event to get the
 edited image.
 ## Parameter Handling
@@ -5,9 +5,6 @@ vLLM-Omni provides an OpenAI-compatible API for text-to-speech (TTS) generation.
 - **Qwen3-TTS** (`Qwen/Qwen3-TTS-12Hz-*`) -- Qwen3-based TTS with CustomVoice, VoiceDesign, and Base (voice cloning) task types. Output: 24 kHz.
 - **Fish Speech S2 Pro** (`fishaudio/s2-pro`) -- Dual-AR TTS with DAC codec. Supports text-to-speech and voice cloning via reference audio. Output: 44.1 kHz.
 - **Voxtral TTS** (`mistralai/Voxtral-4B-TTS-2603`) -- AR + FlowMatching TTS with preset voices. Output: 24 kHz.
 - **CosyVoice3** (`FunAudioLLM/Fun-CosyVoice3-0.5B-2512`) -- 2-stage talker + flow-matching code2wav. Voice cloning via `ref_audio` + `ref_text` (no presets). Output: 24 kHz.
 See the [Supported Models](#supported-models) section below for the full list, including OmniVoice, VoxCPM2, and MOSS-TTS-Nano.
 Each server instance runs a single model (specified at startup via `vllm serve <model> --omni`).
@@ -29,10 +26,6 @@ vllm serve fishaudio/s2-pro --omni --port 8091
 # Voxtral TTS
 vllm serve mistralai/Voxtral-4B-TTS-2603 --omni --port 8091
 # CosyVoice3 (voice cloning only — supply ref_audio + ref_text per request)
 vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 \
    --omni --port 8091 --trust-remote-code
 ```
 ### Generate Speech
@@ -1,26 +1,28 @@
 # Diffusion Attention Backends
-This document describes the diffusion attention backends available in vLLM-Omni, how to select them globally and per-role, the per-platform defaults, and how to use SageAttention.
+This document describes the diffusion attention backends available in vLLM-Omni, how to select them globally and per-role, and how to use SageAttention.
 ## Overview
 Diffusion attention backend selection is resolved in `vllm_omni.diffusion.attention.selector`. It looks up the backend from a structured `AttentionConfig` carried on `OmniDiffusionConfig` and falls back to the platform default when nothing is configured.
-This backend is used by diffusion attention layers such as the DiT attention in video and image generation models. It does **not** affect autoregressive (LLM) attention paths — those go through vLLM's own attention backend selector.
+This backend is used by diffusion attention layers such as the DiT attention in video and image generation models.
-The full set of backends and their platform defaults is in the **Backend Options** and **Platform Defaults** sections below. If no attention backend is configured, vLLM-Omni asks the current platform to choose the default.
+On CUDA, the practical choices today are:
 - `FLASH_ATTN`: FlashAttention backend. This is the default on supported CUDA systems when FlashAttention is installed.
 - `TORCH_SDPA`: PyTorch `scaled_dot_product_attention`.
 - `SAGE_ATTN`: SageAttention backend, if `sageattention` is installed.
 If no attention backend is configured, vLLM-Omni asks the current platform to choose the default. On CUDA, that normally means `FLASH_ATTN` when available, otherwise `TORCH_SDPA`.
 ## Backend Options
 | Value | Notes |
 |---|---|
-| `FLASH_ATTN` | Wraps FlashAttention 2. Default on Hopper / Ada / Ampere when `flash-attn` is installed. |
+| `FLASH_ATTN` | Default on CUDA when FlashAttention is available. Good default for most diffusion workloads. |
-| `CUDNN_ATTN` | Pins `sdpa_kernel([CUDNN_ATTENTION])`. Default on Blackwell (sm_10x / sm_12x) with cuDNN ≥ 9.5. Wins on mask-heavy DiTs (HunyuanVideo-1.5: 2× e2e vs SDPA). |
+| `TORCH_SDPA` | Most conservative fallback. Useful for debugging or compatibility. |
-| `FLASHINFER_ATTN` | Calls FlashInfer's dense `single_prefill_with_kv_cache` directly with `custom_mask` for non-causal masked attention. Used as Blackwell fallback when cuDNN is unavailable. Requires `flashinfer`. |
+| `SAGE_ATTN` | Requires `sageattention`. Can improve performance on some workloads, but output quality must be validated model-by-model. |
 | `TORCH_SDPA` | PyTorch `scaled_dot_product_attention` with the default backend dispatcher. Most conservative; always available. |
 | `SAGE_ATTN` | SageAttention 2.2 — INT8-quantized attention with FP16 accumulation. Lossy but typically visually indistinguishable on diffusion outputs. Requires `sageattention`. |
 | `SAGE_ATTN_3` | Requires `sageattn3` from `SageAttention/sageattention3_blackwell`. CUDA only, intended for Blackwell GPUs, with GQA/MQA requests falling back to PyTorch SDPA. |
 ## Configuration
@@ -93,80 +95,6 @@ config = OmniDiffusionConfig(
 A plain dict is also accepted and normalized to `AttentionConfig`.
 ## Platform Defaults
 ### Blackwell (sm_100 / sm_103 / sm_120 / sm_121)
 Auto-route preference, in order:
 1. `CUDNN_ATTN` — when cuDNN ≥ 9.5 is available (ships in PyTorch 2.5+ wheels)
 2. `FLASHINFER_ATTN` — when `flashinfer` is installed but cuDNN < 9.5
 3. `FLASH_ATTN` — when `flash-attn` is installed with the Blackwell CUTE kernel
 4. `TORCH_SDPA` — last resort
 The startup log line `Defaulting to diffusion attention backend CUDNN_ATTN (Blackwell sm_120, cuDNN 91002)` confirms the route.
 **Why CUDNN_ATTN by default**: on mask-heavy diffusion models (HunyuanVideo-1.5, Qwen-Image), cuDNN's pinned FMHA kernel sidesteps a PyTorch SDPA dispatch quirk where the unpinned dispatcher picks `EFFICIENT_ATTENTION` (~25 ms) for masked calls instead of cuDNN (~11 ms). The pin gives 2× e2e on HV-1.5 with no regression on lighter models.
 ### Hopper (sm_90) / Ada (sm_89) / Ampere (sm_80–sm_86)
 Auto-route preference:
 1. `FLASH_ATTN` — when `flash-attn` is installed
 2. `TORCH_SDPA` — fallback
 `CUDNN_ATTN` and `FLASHINFER_ATTN` are still selectable via env var on these GPUs but are not in the auto-route — FlashAttention 2 is the well-tuned path on pre-Blackwell hardware.
 ## End-to-End Benchmark (BF16, sm_120 RTX Pro 6000 Blackwell)
 Same prompt and seed across runs. `Total generation time` from `text_to_video.py` / `text_to_image.py`.
 | Model | Shape | TORCH_SDPA | CUDNN_ATTN | FLASHINFER_ATTN |
 |---|---|---|---|---|
 | HunyuanVideo-1.5 (T2V) | 480p / 33f / 50 steps | 147.05 s | **73.02 s** | 127.84 s |
 | Wan 2.2 14B (T2V) | 480p / 33f / 40 steps | 117.75 s | 117.17 s | **115.07 s** |
 | Qwen-Image (T2I) | 1024² / 50 steps | 17.41 s | **15.14 s** | 16.02 s |
 | FLUX.2-dev (T2I) | 1024² / 50 steps, TP=2 | 53.62 s | **53.30 s** | 54.94 s |
 Pattern: mask-heavy DiTs (HV-1.5, Qwen-Image) favor `CUDNN_ATTN`; lighter-mask DiTs and TP-saturated configs (Wan 2.2, FLUX.2 TP=2) tie within noise.
 ## Known Limitations
 ### LTX-2.0: `CUDNN_ATTN` crashes under torch.compile
 LTX-2's audio attention has a symbolic head_dim under torch.compile tracing. cuDNN's SDPA backend selector rejects symbolic dims and Dynamo aborts compilation. Tracked in [#3121](https://github.com/vllm-project/vllm-omni/issues/3121).
 **Workaround**: explicitly select `FLASHINFER_ATTN` or `TORCH_SDPA` for LTX-2.0:
 ```bash
 DIFFUSION_ATTENTION_BACKEND=FLASHINFER_ATTN python examples/offline_inference/text_to_video/text_to_video.py \
    --model Lightricks/LTX-2 ...
 ```
 ### FA4 not yet integrated
 FlashAttention-4 (released March 2026) targets Blackwell natively and reportedly beats cuDNN by ~20% on B200. As of this writing the `flash-attn-4 4.0.0b10` wheel crashes with `AttributeError: 'NoneType' object has no attribute '_trait'` during JIT on sm_120. Not yet wired into vLLM-Omni; revisit when stable lands.
 ## Choosing a Backend Manually
 ### When to override the default
 - **Quality validation**: compare a new backend against `TORCH_SDPA` as the reference, since SDPA's default dispatcher is the most extensively tested.
 - **Lossy speedup hunting**: try `SAGE_ATTN` (INT8 quantized) on diffusion outputs — typically indistinguishable visually but always validate.
 - **Workaround for known issues**: see Known Limitations above.
 ### Verifying which backend is in use
 The startup log prints one of:
 ```
 Using diffusion attention backend 'CUDNN_ATTN'           # explicit override
 Defaulting to diffusion attention backend CUDNN_ATTN ... # auto-route
 Defaulting to diffusion attention backend SDPA           # nothing else available
 ```
 If you don't see one of these, the model didn't reach diffusion stage init — check earlier logs for failures.
 ## SageAttention Installation
 vLLM-Omni expects SageAttention to be installed into the same Python environment as vLLM-Omni.
@@ -187,56 +115,11 @@ Quick check:
 python -c "import sageattention; print(sageattention.__file__)"
 ```
-## SageAttention3 Installation
+## Usage
-vLLM-Omni expects SageAttention3 to be installed into the same Python environment as vLLM-Omni.
+### Enable SageAttention
-Build from source:
+Example: HunyuanVideo-1.5 text-to-video
 ```bash
 git clone https://github.com/thu-ml/SageAttention.git
 cd SageAttention/sageattention3_blackwell
 python setup.py install
 ```
 Quick check:
 ```bash
 python -c "import sageattn3; print(sageattn3.__file__)"
 ```
 Notes:
 - `SAGE_ATTN_3` is only selected on CUDA when `sageattn3` is importable and the GPU is Blackwell-class.
 - SageAttention3's Blackwell kernel assumes `Hq == Hkv`. In vLLM-Omni, GQA/MQA diffusion requests fall back to PyTorch SDPA for correctness.
 ## Usage Examples
 ### Default (auto-route)
 ```bash
 python examples/offline_inference/text_to_video/text_to_video.py \
    --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v \
    --prompt "A dog running across a field of golden wheat." \
    --height 480 --width 832 --num-frames 33 \
    --num-inference-steps 50 --seed 42 --guidance-scale 6.0 \
    --output hv15.mp4
 ```
 On Blackwell this picks `CUDNN_ATTN` automatically. Check the log for the `Defaulting to ...` line.
 ### Explicit backend selection
 ```bash
 DIFFUSION_ATTENTION_BACKEND=FLASHINFER_ATTN python examples/offline_inference/text_to_video/text_to_video.py \
    --model Lightricks/LTX-2 \
    --prompt "A dog running across a field of golden wheat." \
    --height 480 --width 832 --num-frames 33 \
    --num-inference-steps 40 --seed 42 --guidance-scale 4.0 \
    --output ltx2.mp4
 ```
 ### SageAttention (lossy)
 ```bash
 DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN python examples/offline_inference/text_to_video/text_to_video.py \
@@ -245,7 +128,7 @@ DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN python examples/offline_inference/text_to_
    --height 480 --width 832 --num-frames 33 \
    --num-inference-steps 30 --seed 42 --guidance-scale 6.0 \
    --tensor-parallel-size 2 \
-    --output hv15_sage.mp4
+    --output ../tmp/hv15_modelopt_sage.mp4
 ```
 Example: Wan2.2 TI2V 5B
@@ -260,20 +143,6 @@ DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN python examples/offline_inference/text_to_
    --output outputs/wan22_sage.mp4
 ```
 ### Enable SageAttention3
 Example:
 ```bash
 DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN_3 python examples/offline_inference/text_to_video/text_to_video.py \
    --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v \
    --prompt "A dog running across a field of golden wheat." \
    --height 480 --width 832 --num-frames 33 \
    --num-inference-steps 30 --seed 42 --guidance-scale 6.0 \
    --tensor-parallel-size 2 \
    --output outputs/hv15_sage3.mp4
 ```
 ### Mixed backends across roles
 Use `FLASH_ATTN` for self-attention and `TORCH_SDPA` for cross-attention:
@@ -304,13 +173,13 @@ python examples/offline_inference/text_to_video/text_to_video.py \
 ## Validation Guidance
-Don't assume a faster attention backend is numerically interchangeable with `TORCH_SDPA`.
+Do not assume that a faster attention backend is numerically interchangeable with `FLASH_ATTN`.
 Always compare:
 - End-to-end runtime
- Diffusion-stage runtime (`add_req_and_wait` line in DiffusionEngine.step breakdown)
+- DiT / diffusion stage runtime
- Output quality against a known-good baseline (CLIP similarity, frame-level diff, or visual review)
+- Output quality against a known-good baseline
 At minimum, keep the same:
@@ -318,19 +187,6 @@ At minimum, keep the same:
 - prompt
 - seed
 - resolution
- frame count / step count
+- frame count
- parallel config (TP / CFG-parallel / Ulysses degrees)
+- inference steps
-
+- parallel config
 ## Reproducing the Benchmark Table
 The end-to-end numbers above were collected by running `text_to_video.py` /
 `text_to_image.py` with the same prompt and seed while varying
 `DIFFUSION_ATTENTION_BACKEND`. For a quick kernel-level comparison of the
 backends without loading a model:
 ```bash
 python benchmarks/diffusion/bench_attention_backends.py --preset hv15
 ```
 It runs all three BF16 backends on representative DiT attention shapes and
 prints a ranking table at the end.
@@ -60,9 +60,6 @@ batching only becomes relevant when `--max-num-seqs > 1`.
 - `cache_backend` is not supported together with step execution.
 - Unsupported pipelines fail early during model loading.
 - Request-mode extras such as KV transfer are not wired into step mode yet.
 - LoRA is supported in step mode, but each batch must use a single adapter:
  requests with different `lora_request` or `lora_scale` are scheduled into
  separate batches.
 ## When To Use It
@@ -113,9 +113,8 @@ The following tables show which models support each feature:
 | **FLUX.1-schnell**       |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ✅        |        ❌         |
 | **FLUX.2-klein**         |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ✅        |        ❌         |
 | **FLUX.1-Kontext-dev**   |     ❌     |     ❌      |           ❌           |       ❌        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ❌        |        ❌         |
-| **FLUX.2-dev**           |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ❌        |        ❌         |
+| **FLUX.2-dev**           |     ✅     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **GLM-Image**            |     ❌     |     ❌      |           ❌           |       ✅        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **Hidream-I1-Full**        |     ❌     |     ❌      |           ❌           |       ❌        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **HunyuanImage3**        |     ❌     |     ✅      |           ❌           |       ❌        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ✅        |        ❌         |
 | **LongCat-Image**        |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **LongCat-Image-Edit**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
@@ -129,7 +128,7 @@ The following tables show which models support each feature:
 | **Qwen-Image-Edit**      |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Qwen-Image-Edit-2509** |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |        ✅ (decode)         |          ✅           |       ❌        |        ❌         |
 | **Qwen-Image-Layered**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
-| **SenseNova-U1**         |     ❌     |     ✅      |           ❌           |       ❌        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **SenseNova-U1**         |     ❌     |     ❌      |           ❌           |       ❌        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **Stable-Diffusion3.5**  |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **Z-Image**              |     ✅     |     ✅      |           ✅           |       ❓        |   ✅ (TP=2 only)   |          ❌          |   ✅    |             ❌             |      ✅ (decode)      |       ✅        |        ❌         |
 | **ERNIE-Image**          |     ❌     |     ✅      |           ✅           |       ❓        |         ✅         |          ❌          |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
@@ -145,9 +144,9 @@ The following tables show which models support each feature:
 | **Wan2.2**                   |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ✅         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ❌        |        ❌         |
 | **Wan2.1-VACE**              |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **LTX-2**                    |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
-| **LTX-2.3**                  |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
+| **LTX-2.3**                  |     ❌     |     ✅      |           ✅           |       ❌        |         ✅         |         ❌         |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **Helios**                   |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
-| **HunyuanVideo-1.5 T2V I2V** |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
+| **HunyuanVideo-1.5 T2V I2V** |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |       ✅        |        ❌         |
 | **DreamID-Omni**             |     ❌     |     ❌      |           ❌           |       ✅        |         ❌         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
 **Frame Interpolation Support**
@@ -185,7 +184,7 @@ The following tables show which models support each feature:
 | **💾VAE Patch Parallel** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | | | |
 | **💾FP8 Quant** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❓ | ❓ | ✅ | ✅ | ✅ | | | |
 | **🔧LoRA Inference** | ❓ | ❓ | ❓ | ❓ | ❓ | ❓ | ❓ | ❓ | ❓ | ❓ | ❓ | ❓ | | |
-| **🔄Step Execution** | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❓ | ❓ | ✅ | ❓ | ✅ | ✅ | ✅ | |
+| **🔄Step Execution** | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❓ | ❓ | ✅ | ❓ | ✅ | ✅ | ❌ | |
 !!! info
@@ -194,7 +193,7 @@ The following tables show which models support each feature:
    3. CPU Offloading (Layerwise) and CPU Offloading (Module-wise) are not compatible.
    4. CPU Offloading (Layerwise) supports single-card for now.
    5. Using FP8-Quant as an example of qunatization methods.
-    6. Step Execution is not compatible with cache backends (TeaCache, Cache-DiT). LoRA is supported, but each scheduled batch must use a single adapter (requests with different `lora_request` or `lora_scale` are kept in separate batches).
+    6. Step Execution is not compatible with cache backends (TeaCache, Cache-DiT) or LoRA.
 ## Multi-Thread Weight Loading
@@ -17,13 +17,12 @@ list of supported architectures across all modalities, see
 | Model | HuggingFace repo | Stages | Voice cloning | Streaming | Special modes | Sample rate |
 |---|---|---|---|---|---|---|
 | VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | — | continuation (`--ref-audio` + `--ref-text`) | 48 kHz |
 | VoxCPM | local model dir | split | ✓ | ✓ (`voxcpm_async_chunk.yaml`) | — | 24 kHz |
 | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ (`async_chunk: true` default) | — | 22.05 kHz |
 | Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ (`--streaming`) | — | 44.1 kHz |
 | GLM-TTS | `zai-org/GLM-TTS` | 2 (AR + DiT) | ✓ (required) | ✓ | — | 24 kHz |
 | OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design (`--instruct`), language (`--lang`) | 24 kHz |
 | Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | 2 (talker + code2wav) | ✓ (Base) | ✓ | 3 task variants (`--query-type`) | 24 kHz |
 | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | varies | ✓ | ✓ | voice presets (`--voice`) | 24 kHz |
 | higgs-audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` (+ `k2-fsa/OmniVoice/audio_tokenizer/` codec) | 2 (talker + code2wav, DualFFN) | ✓ (`--ref-audio` + `--ref-text`) | — | — | 24 kHz |
 ## Common Quick Start
@@ -73,6 +72,61 @@ python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
 ---
 ## VoxCPM
 Split-stage TTS. The hub example covers single TTS, single voice cloning, and streaming. Use `benchmarks/voxcpm/` for warmup, batch JSONL prompts, profiler injection, and offline TTFP / RTF measurement.
 ### Prerequisites
 ```bash
 pip install voxcpm soundfile
 # or use a local source tree:
 export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
 ```
 If the native VoxCPM `config.json` does not contain HF metadata such as `model_type`, prepare a persistent HF-compatible config directory and point the stage configs to it via `VLLM_OMNI_VOXCPM_HF_CONFIG_PATH`:
 ```bash
 export VOXCPM_MODEL=/path/to/voxcpm-model
 export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config
 mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"
 cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json"
 cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true
 python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)'
 ```
 ### Quick start
 ```bash
 python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
    --model "$VOXCPM_MODEL" \
    --text "This is a split-stage VoxCPM synthesis example running on vLLM Omni."
 ```
 ### Voice cloning
 ```bash
 python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
    --model "$VOXCPM_MODEL" \
    --text "This sentence is synthesized with a cloned voice." \
    --ref-audio /path/to/reference.wav \
    --ref-text  "The exact transcript spoken in reference.wav."
 ```
 ### Streaming
 Pass the async-chunk stage config:
 ```bash
 python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
    --model "$VOXCPM_MODEL" \
    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
    --text "This is a split-stage VoxCPM streaming example running on vLLM Omni."
 ```
 ### Notes
 - `voxcpm.yaml` is the default non-streaming stage config; `voxcpm_async_chunk.yaml` enables streaming.
 - Streaming is currently single-request oriented.
 - `--ref-text` must be the real transcript of `--ref-audio`; mismatched text degrades quality.
 - For online serving, see [`examples/online_serving/voxcpm`](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/voxcpm/README.md). For benchmark reporting, see [`benchmarks/voxcpm`](https://github.com/vllm-project/vllm-omni/tree/main/benchmarks/voxcpm/README.md).
 ---
 ## CosyVoice3
 2-stage TTS pipeline (`talker` + `code2wav`) at 22.05 kHz.
@@ -160,35 +214,6 @@ Streaming requires `async_chunk: true` in the stage config.
 ---
 ## GLM-TTS
 2-stage TTS pipeline (AR + DiT flow-matching) at 24 kHz. Every request requires reference audio and its transcript for zero-shot voice cloning.
 ### Quick start
 ```bash
 python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
    --model zai-org/GLM-TTS \
    --text "你好，这是语音合成测试。" \
    --ref-audio /path/to/reference.wav \
    --ref-text "这是参考音频的文本内容。" \
    --output-dir ./output
 ```
 ### Architecture
 ```
 Text → [Stage 0: AR] → Speech Tokens → [Stage 1: DiT + HiFT] → Audio (24 kHz)
        (Llama-based)    (32k vocab)      (Flow Matching)
 ```
 ### Notes
 - `--ref-audio` and `--ref-text` are **required** together; GLM-TTS does not support text-only synthesis.
 - Reference audio should be 3-10 seconds.
 - First run may be slow due to lazy loading of WhisperVQ tokenizer and CampPlus ONNX speaker embedder.
 - Default sampling: temperature=1.0, top_k=25, top_p=0.8 (RAS method).
 - The `--model` path should point to the repository root (not `llm/` subdirectory).
 ---
 ## OmniVoice
 Zero-shot multilingual TTS supporting 600+ languages, with three modes (auto / clone / design).
@@ -334,46 +359,6 @@ Available voice presets are listed on the HF model card (`mistralai/Voxtral-4B-T
 - `--concurrency M` requires `--streaming` and must evenly divide `--num-prompts`.
 - Run `--help` for the full argument surface.
 ---
 ## higgs-audio v2
 2-stage TTS at 24 kHz: a vLLM-native Llama-3.2-3B talker with a DualFFN audio expert (Stage 0) feeding a HiggsAudio codec decoder (Stage 1) over the shared-memory connector. Stage 1 builds on the HiggsAudio decoder kernel at `vllm_omni/model_executor/models/higgs_audio_v2/higgs_audio_decoder.py`.
 ### Prerequisites
 Voice clone uses HF's `HiggsAudioV2TokenizerModel`, instantiated from `k2-fsa/OmniVoice/audio_tokenizer/` — the boson-ai standalone tokenizer Hub repo's `model.safetensors` is actually the 3B talker LM, so we point HF at k2's repackaged codec weights instead. Only the `audio_tokenizer/` subdir (~806 MB) is downloaded.
 ```bash
 pip install -U "transformers>=5.3.0"
 ```
 ### Quick start (plain TTS)
 ```bash
 python examples/offline_inference/text_to_speech/higgs_audio_v2/end2end.py \
    --texts "Hello world." "The quick brown fox jumps over the lazy dog." \
    --output-dir results/wavs
 ```
 ### Voice cloning
 Pass both `--ref-audio` and `--ref-text` together:
 ```bash
 python examples/offline_inference/text_to_speech/higgs_audio_v2/end2end.py \
    --texts "Hello, this is a cloned voice." \
    --ref-audio /path/to/reference.wav \
    --ref-text  "Exact transcript spoken in reference.wav." \
    --output-dir results/wavs
 ```
 ### Notes
 - Deploy config auto-loads from `vllm_omni/deploy/higgs_audio_v2.yaml`.
 - `--ref-text` must be the real transcript of `--ref-audio`; mismatched text degrades cloned-voice quality.
 - Out of scope (rejected with 4xx by the request validator): multi-speaker `[SPEAKERn]` dialogue, `profile:` text-only speaker descriptions, the `ref_audio_in_system_message` system-block variant, chunked long-form generation, and per-request `voice` / `instructions` / `task_type` / `language` / `speed != 1.0` / `x_vector_only_mode` / `speaker_embedding`.
 ## Example materials
 ??? abstract "cosyvoice3/end2end.py"
@@ -384,13 +369,6 @@ python examples/offline_inference/text_to_speech/higgs_audio_v2/end2end.py \
    ``````py
    --8<-- "examples/offline_inference/text_to_speech/fish_speech/end2end.py"
    ``````
 ??? abstract "higgs_audio_v2/end2end.py"
    ``````py
    --8<-- "examples/offline_inference/text_to_speech/higgs_audio_v2/end2end.py"
 ??? abstract "glm_tts/end2end.py"
    ``````py
    --8<-- "examples/offline_inference/text_to_speech/glm_tts/end2end.py"
    ``````
 ??? abstract "omnivoice/end2end.py"
    ``````py
    --8<-- "examples/offline_inference/text_to_speech/omnivoice/end2end.py"
@@ -403,6 +381,10 @@ python examples/offline_inference/text_to_speech/higgs_audio_v2/end2end.py \
    ``````py
    --8<-- "examples/offline_inference/text_to_speech/qwen3_tts/end2end.py"
    ``````
 ??? abstract "voxcpm/end2end.py"
    ``````py
    --8<-- "examples/offline_inference/text_to_speech/voxcpm/end2end.py"
    ``````
 ??? abstract "voxcpm2/end2end.py"
    ``````py
    --8<-- "examples/offline_inference/text_to_speech/voxcpm2/end2end.py"
@@ -17,16 +17,14 @@ For the full list of supported architectures across all modalities, see
 | Model | HuggingFace repo | Voice cloning | Streaming | Voice presets / upload | Gradio demo |
 |---|---|---|---|---|---|
 | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | — |
 | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
 | higgs-audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` | ✓ (`ref_audio`+`ref_text`) | ✓ (codec_streaming) | — | — |
 | GLM-TTS | `zai-org/GLM-TTS` | ✓ (`ref_audio`+`ref_text`, required) | ✓ (PCM stream) | — | ✓ |
 | OmniVoice | `k2-fsa/OmniVoice` | (offline only) | — | — | — |
 | Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | ✓ (Base) | ✓ (PCM + WebSocket) | ✓ (presets + `/v1/audio/voices` upload) | ✓ (standard + FastRTC) |
 | VoxCPM | local model dir | ✓ | ✓ (PCM stream) | — | — |
 | VoxCPM2 | `openbmb/VoxCPM2` | ✓ | ✓ (AudioWorklet via gradio) | — | ✓ |
 | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✓ (gated upstream) | ✓ | ✓ (presets) | ✓ |
-For offline inference of any of these models, see the [offline TTS hub](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md).
+CosyVoice3 is intentionally absent: no online example exists for it yet. See its [offline section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#cosyvoice3) instead.
 ## Common Quick Start
@@ -98,54 +96,6 @@ For full request-shape documentation (all parameters, response formats, error co
 ---
 ## CosyVoice3
 2-stage TTS (`talker` + flow-matching `code2wav`) at 24 kHz. Voice cloning only — every request needs `ref_audio` + `ref_text`; there are no built-in voice presets.
 ### Prerequisites
 ```bash
 huggingface-cli download FunAudioLLM/Fun-CosyVoice3-0.5B-2512
 ```
 If your downloaded checkpoint lacks `config.json`, add one with `{"model_type": "cosyvoice3", "architectures": ["CosyVoice3Model"]}` (the loader reads `model_type` to select the class).
 ### Launch
 ```bash
 vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 --omni --port 8091 --trust-remote-code
 # or:
 ./cosyvoice3/run_server.sh
 ```
 Streaming is on by default via `async_chunk: true` in `vllm_omni/deploy/cosyvoice3.yaml`. Pass `--no-async-chunk` (or `NO_ASYNC_CHUNK=1 ./cosyvoice3/run_server.sh`) for the legacy synchronous path.
 ### CLI client
 The client defaults to the official upstream zero-shot prompt, so it runs without extra flags:
 ```bash
 cd examples/online_serving/text_to_speech/cosyvoice3
 python speech_client.py --text "收到好友从远方寄来的生日礼物。"
 ```
 Pass your own reference clip and transcript for a different voice:
 ```bash
 python speech_client.py --text "Hello, this is a cloned voice." \
    --ref-audio /path/to/reference.wav \
    --ref-text "Transcript of the reference audio."
 ```
 Stream PCM instead of WAV:
 ```bash
 python speech_client.py --text "Hello world" --stream --output output.pcm
 ```
 The client supports `--api-base`, `--model`, `--text`, `--ref-audio`, `--ref-text`, `--response-format`, `--stream`, `--output`.
 ### Notes
 - Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform.
 - Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config <path>` to override.
 - For offline inference and the end-to-end script, see the [offline CosyVoice3 section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#cosyvoice3).
 ---
 ## Fish Speech S2 Pro
 4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec.
@@ -194,84 +144,6 @@ python fish_speech/gradio_demo.py --api-base http://localhost:8091  # if server
 ---
 ## higgs-audio v2
 2-stage TTS at 24 kHz: a vLLM-native Llama-3.2-3B talker with a DualFFN audio expert (Stage 0) feeding a HiggsAudio codec decoder (Stage 1) that streams chunks back to the client.
 ### Prerequisites
 Voice clone uses HF's `HiggsAudioV2TokenizerModel` loaded from `k2-fsa/OmniVoice/audio_tokenizer/` (~806 MB subdir; the boson-ai standalone tokenizer Hub repo's `model.safetensors` is the 3B talker LM, not the codec):
 ```bash
 pip install -U "transformers>=5.3.0"
 ```
 ### Launch
 ```bash
 GPUS=6,7 PORT=8094 bash examples/online_serving/text_to_speech/higgs_audio_v2/run_server.sh
 ```
 Deploy config auto-loads from `vllm_omni/deploy/higgs_audio_v2.yaml`.
 ### Sending requests
 ```bash
 # Plain TTS
 python higgs_audio_v2/batch_speech_client.py \
    --base-url http://localhost:8094 \
    --output-dir /tmp/higgs_out \
    --prompts "Hello world." "The quick brown fox jumps over the lazy dog."
 # Voice cloning — pass a reference clip and its transcript together
 python higgs_audio_v2/batch_speech_client.py \
    --base-url http://localhost:8094 \
    --output-dir /tmp/higgs_clone \
    --ref-audio /path/to/reference.wav \
    --ref-text  "Exact transcript spoken in reference.wav." \
    --prompts "Hello, this is a cloned voice."
 ```
 ### Notes
 - Output: 24 kHz mono.
 - `--ref-text` must be the real transcript of `--ref-audio`; mismatched text degrades cloned-voice quality.
 - Out of scope (rejected with explicit 4xx): multi-speaker `[SPEAKERn]` tags inside `input`, `profile:` text-only speaker descriptions, the `ref_audio_in_system_message` system-block variant, chunked long-form generation, and per-request `voice` / `instructions` / `task_type` / `language` / `speed != 1.0` / `x_vector_only_mode` / `speaker_embedding`.
 ## GLM-TTS
 2-stage TTS (AR + DiT flow-matching) at 24 kHz. Every request requires `ref_audio` + `ref_text`.
 ### Launch
 ```bash
 vllm serve zai-org/GLM-TTS --omni --trust-remote-code --port 8091
 # or:
 bash examples/online_serving/text_to_speech/glm_tts/run_server.sh /path/to/GLM-TTS
 ```
 ### Sending requests
 ```bash
 # Voice cloning (required)
 python examples/online_serving/text_to_speech/glm_tts/openai_speech_client.py \
    --text "你好，这是语音克隆测试。" \
    --ref-audio file:///path/to/ref.wav \
    --ref-text "这是参考音频的文本内容。"
 # Custom format
 python examples/online_serving/text_to_speech/glm_tts/openai_speech_client.py \
    --text "Hello, this is a voice cloning test." \
    --ref-audio file:///path/to/ref.wav \
    --ref-text "Transcript of the reference audio." \
    --response-format mp3 -o output.mp3
 ```
 ### Gradio demo
 ```bash
 bash examples/online_serving/text_to_speech/glm_tts/run_gradio_demo.sh
 ```
 ### Notes
 - Output: 24 kHz mono WAV via HiFT vocoder.
 - `ref_audio` + `ref_text` are **required** together on every request. Reference audio should be 3-10 seconds.
 - Voice cloning feature extraction (WhisperVQ, CampPlus, mel) runs on the model side — no external dependency on the serving layer.
 ---
 ## OmniVoice
 Zero-shot multilingual TTS (600+ languages). Online serving currently exposes **auto voice** only; voice cloning and voice design are available offline.
@@ -324,20 +196,10 @@ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni --port 8091
 ./qwen3_tts/run_server.sh Base
 ```
-### Executor backend
+### Choosing an executor backend (uniproc vs mp)
-Single-GPU serves now default to the uniproc executor (lower IPC overhead, the Base cloning use case from [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) / [#2604](https://github.com/vllm-project/vllm-omni/pull/2604)). `vllm_omni/deploy/qwen3_tts.yaml` is the only Qwen3-TTS deploy config; pass `--deploy-config <path>` to override.
+Stage configs ship with the chunked-streaming default. To use the uniproc executor (lower IPC overhead for the Base cloning task), pass `--stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts_uniproc.yaml`. See [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) and [#2604](https://github.com/vllm-project/vllm-omni/pull/2604) for the full investigation.
-To opt out of chunked streaming, pass `--no-async-chunk` — the pipeline auto-dispatches to the end-to-end codec processor.
+To opt out of chunked streaming, pass `--no-async-chunk` instead — the pipeline auto-dispatches to the end-to-end codec processor.
 ### Tuning stage 1 `max_num_seqs` per task type
 The bundled `qwen3_tts.yaml` ships stage 1 (Code2Wav) at `max_num_seqs: 10`, tuned for Base voice cloning: stage-1 lifetimes are long (~3 s/req), so admitting up to 10 concurrent codec sequences lets requests progress in parallel in the scheduler — ~2× TTFA p95 at c=4 / c=8 (1× H100, 1.7B-Base, seed-tts) at an 8–12 % audio-throughput cost.
 CustomVoice / VoiceDesign have much shorter stage-1 lifetimes (~50–200 ms) and are TTFA-optimal at `max_num_seqs: 1`. Override the default when serving those task types:
 ```bash
 vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base --omni \
    --stage-overrides '{"1": {"max_num_seqs": 1}}'
 ```
 ### Sending requests
 ```bash
@@ -419,11 +281,71 @@ python qwen3_tts/gradio_fastrtc_demo.py --api-base http://localhost:8000
 `qwen3_tts/batch_speech_client.py` issues many concurrent requests for throughput measurement.
 ### Notes
- Base voice cloning has per-request reference-audio cost; the uniproc default keeps IPC overhead off the critical path. See the executor-backend section above for background.
+- Base voice cloning has uniproc-vs-mp tradeoffs depending on per-request reference audio cost; see the executor-backend section above.
 - `vllm_omni/deploy/qwen3_tts.yaml` is the default deploy config (loaded by HF `model_type`); per-stage runtime overrides are available via `--stage-N-<field> <value>`.
 ---
 ## VoxCPM
 Split-stage TTS at 24 kHz.
 ### Prerequisites
 ```bash
 pip install voxcpm
 # or use a local source tree:
 export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
 ```
 If the native VoxCPM `config.json` lacks HF `model_type`, set up an HF-compatible config dir:
 ```bash
 export VOXCPM_MODEL=/path/to/voxcpm-model
 export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config
 mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"
 cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json"
 cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true
 python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)'
 ```
 ### Launch
 ```bash
 export VOXCPM_MODEL=/path/to/voxcpm-model
 ./voxcpm/run_server.sh                # async-chunk streaming (default)
 ./voxcpm/run_server.sh sync           # non-streaming
 ```
 Or directly:
 ```bash
 vllm serve "$VOXCPM_MODEL" \
    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
    --trust-remote-code --enforce-eager --omni --port 8091
 ```
 ### Sending requests
 ```bash
 # Basic TTS
 python voxcpm/openai_speech_client.py \
    --model "$VOXCPM_MODEL" \
    --text "This is a VoxCPM online text-to-speech example."
 # Voice cloning
 python voxcpm/openai_speech_client.py \
    --model "$VOXCPM_MODEL" \
    --text "This sentence is synthesized with a cloned voice." \
    --ref-audio /path/to/reference.wav \
    --ref-text "The exact transcript spoken in reference.wav."
 # Streaming PCM
 python voxcpm/openai_speech_client.py \
    --model "$VOXCPM_MODEL" \
    --text "This is a streaming VoxCPM request." \
    --stream --output voxcpm_stream.pcm
 ```
 ### Notes
 - `voxcpm.yaml` for one-shot decode; `voxcpm_async_chunk.yaml` for single-request streaming. Do not use the async-chunk config for concurrent requests or `/v1/audio/speech/batch`.
 - Generic TTS fields not supported by VoxCPM: `voice`, `instructions`, `language`, `speaker_embedding`, `x_vector_only_mode`.
 - For benchmark measurement, see [`benchmarks/voxcpm`](https://github.com/vllm-project/vllm-omni/tree/main/benchmarks/voxcpm/README.md).
 ---
 ## VoxCPM2
@@ -485,14 +407,6 @@ The demo handles voice-preset selection and reference-audio upload. `voxtral_tts
 ## Example materials
 ??? abstract "cosyvoice3/run_server.sh"
    --8<-- "examples/online_serving/text_to_speech/cosyvoice3/run_server.sh"
 ??? abstract "cosyvoice3/speech_client.py"
    --8<-- "examples/online_serving/text_to_speech/cosyvoice3/speech_client.py"
 ??? abstract "fish_speech/gradio_demo.py"
    ``````py
    --8<-- "examples/online_serving/text_to_speech/fish_speech/gradio_demo.py"
@@ -509,29 +423,6 @@ The demo handles voice-preset selection and reference-audio upload. `voxtral_tts
    ``````py
    --8<-- "examples/online_serving/text_to_speech/fish_speech/speech_client.py"
    ``````
 ??? abstract "higgs_audio_v2/batch_speech_client.py"
    ``````py
    --8<-- "examples/online_serving/text_to_speech/higgs_audio_v2/batch_speech_client.py"
    ``````
 ??? abstract "higgs_audio_v2/run_server.sh"
    ``````sh
    --8<-- "examples/online_serving/text_to_speech/higgs_audio_v2/run_server.sh"
 ??? abstract "glm_tts/gradio_demo.py"
    ``````py
    --8<-- "examples/online_serving/text_to_speech/glm_tts/gradio_demo.py"
    ``````
 ??? abstract "glm_tts/openai_speech_client.py"
    ``````py
    --8<-- "examples/online_serving/text_to_speech/glm_tts/openai_speech_client.py"
    ``````
 ??? abstract "glm_tts/run_gradio_demo.sh"
    ``````sh
    --8<-- "examples/online_serving/text_to_speech/glm_tts/run_gradio_demo.sh"
    ``````
 ??? abstract "glm_tts/run_server.sh"
    ``````sh
    --8<-- "examples/online_serving/text_to_speech/glm_tts/run_server.sh"
    ``````
 ??? abstract "omnivoice/run_server.sh"
    ``````sh
    --8<-- "examples/online_serving/text_to_speech/omnivoice/run_server.sh"
@@ -572,6 +463,14 @@ The demo handles voice-preset selection and reference-audio upload. `voxtral_tts
    ``````py
    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/tts_common.py"
    ``````
 ??? abstract "voxcpm/openai_speech_client.py"
    ``````py
    --8<-- "examples/online_serving/text_to_speech/voxcpm/openai_speech_client.py"
    ``````
 ??? abstract "voxcpm/run_server.sh"
    ``````sh
    --8<-- "examples/online_serving/text_to_speech/voxcpm/run_server.sh"
    ``````
 ??? abstract "voxcpm2/gradio_demo.py"
    ``````py
    --8<-- "examples/online_serving/text_to_speech/voxcpm2/gradio_demo.py"
@@ -32,9 +32,7 @@ guide. AutoRound is Intel-supported.
 |-------|------------|-------|--------|---------|
 | FLUX.1-dev | `vllm-project-org/FLUX.1-dev-AutoRound-w4a16` | Diffusion transformer | W4A16 | GPTQ-Marlin or Intel-supported AutoRound backend |
 | Qwen-Image | Not listed | Diffusion transformer | W4A16 | Not validated |
-| Wan2.2-I2V | `Intel/Wan2.2-I2V-A14B-Diffusers-int4-AutoRound` | Diffusion transformer | W4A16 | GPTQ-Marlin or Intel-supported AutoRound backend |
+| Wan2.2 | Not listed | Diffusion transformer | W4A16 | Not validated |
 | Wan2.2-T2V | `Intel/Wan2.2-T2V-A14B-Diffusers-int4-AutoRound` | Diffusion transformer | W4A16 | GPTQ-Marlin or Intel-supported AutoRound backend |
 | Wan2.2-TI2V | `Intel/Wan2.2-TI2V-5B-Diffusers-int4-AutoRound` | Diffusion transformer | W4A16 | GPTQ-Marlin or Intel-supported AutoRound backend |
 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
@@ -52,8 +50,8 @@ vLLM-Omni's runtime module names.
 | Model | Scope | Status | Notes |
 |-------|-------|--------|-------|
 | GLM-Image | Diffusion transformer  | ✅ | `Intel/GLM-Image-int4-AutoRound` |
 | BAGEL | Checkpoint-defined diffusion or transformer stage | Not validated | Requires a compatible AutoRound checkpoint |
 | GLM-Image | Checkpoint-defined diffusion or transformer stage | Not validated | Requires a compatible AutoRound checkpoint |
 ## Configuration
@@ -6,46 +6,34 @@ ModelOpt quantization loads checkpoints produced by NVIDIA ModelOpt. The
 quantized weights and scale tensors are generated before serving, so inference
 does not run online calibration or convert a BF16 checkpoint at startup.
-vLLM-Omni validates ModelOpt FP8, ModelOpt NVFP4, and ModelOpt mixed
+vLLM-Omni currently validates the ModelOpt FP8 checkpoint path for diffusion
-FP8/NVFP4 checkpoint loading for diffusion transformer stages. The loader
+transformers. The loader auto-detects supported ModelOpt FP8 checkpoint configs
-auto-detects supported ModelOpt checkpoint configs and keeps non-transformer
+and keeps non-transformer components, such as the tokenizer, scheduler, text
 components, such as the tokenizer, scheduler, text encoder, vision/audio
 encoder, and VAE, on the base checkpoint unless a model-specific guide says
 otherwise.
 !!! note
-    ModelOpt checkpoints are pre-quantized checkpoints. Do not pass
+    `--force-cutlass-fp8` is an explicit runtime override for diffusion
-    `--quantization fp8` for these checkpoints. The checkpoint
+    checkpoints that already carry a supported ModelOpt FP8 config. It does not
-    `quantization_config` selects the ModelOpt path.
+    quantize BF16 checkpoints and it does not apply to online `--quantization
-
+    fp8`. The flag only takes effect for ModelOpt FP8 diffusion stages on CUDA
-!!! note
+    SM89+ devices; other platforms and non-ModelOpt FP8 paths fall back to the
-    `--force-cutlass-fp8`, `--linear-backend cutlass`, and
+    normal vLLM kernel selection.
    `--moe-backend cutlass` are runtime backend selections for checkpoints that
    already carry supported ModelOpt quantized weights and scales. They do not
    quantize BF16 checkpoints at startup.
 ## Supported ModelOpt Checkpoint Formats
 vLLM-Omni treats ModelOpt checkpoints as pre-quantized checkpoints. The
 checkpoint config must identify ModelOpt as the quantization method or producer,
-and the quantization algorithm must be one of the validated algorithms below.
+and the quantization algorithm must be one of the validated FP8 algorithms.
 | Checkpoint field | Supported value |
 |------------------|-----------------|
-| `method` / `quant_method` | `modelopt`, `modelopt_fp4`, `modelopt_mixed` |
+| `method` / `quant_method` | `modelopt` |
 | `producer.name` | `modelopt` |
-| `quant_algo` | `FP8`, `FP8_PER_CHANNEL_PER_TOKEN`, `NVFP4`, `MIXED_PRECISION` |
+| `quant_algo` | `FP8`, `FP8_PER_CHANNEL_PER_TOKEN` |
-| `quant_algo` | Runtime method | Typical use |
+Other ModelOpt algorithms, such as NVFP4, are not enabled by this diffusion
-|--------------|----------------|-------------|
+FP8 path until they have separate model and quality validation.
 | `FP8`, `FP8_PER_CHANNEL_PER_TOKEN` | `modelopt` | FP8 diffusion transformer checkpoints |
 | `NVFP4` | `modelopt_fp4` | NVFP4 diffusion transformer checkpoints |
 | `MIXED_PRECISION` | `modelopt_mixed` | Mixed FP8/NVFP4 checkpoints with a ModelOpt per-layer policy |
 For multi-component diffusion or omni models, only the checkpoint component
 that contains ModelOpt quantized weights should use the ModelOpt quantization
 method. Encoders, decoders, tokenizers, schedulers, and other BF16 components
 stay unquantized unless the model-specific recipe validates otherwise.
 ## Hardware Support
@@ -59,10 +47,7 @@ stay unquantized unless the model-specific recipe validates otherwise.
 | Ascend NPU | ❌ |
 Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this guide.
-The optional CUTLASS FP8 runtime override requires CUDA SM89+. ModelOpt NVFP4
+The optional CUTLASS FP8 runtime override requires CUDA SM89+.
 and mixed FP8/NVFP4 diffusion checkpoints are currently validated on Blackwell
 CUDA systems in the recipes below; other CUDA generations require separate
 backend and quality validation.
 ## Model Type Support
@@ -71,19 +56,12 @@ backend and quality validation.
 | Model | HF checkpoint | Scope | Status |
 |-------|---------------|-------|--------|
 | Qwen-Image 2512 | `feizhai123/qwen-image-2512-modelopt-fp8-dynamic-all` | Diffusion transformer | Validated for ModelOpt FP8 checkpoints |
 | Qwen-Image 2512 | `feizhai123/qwen-image-2512-modelopt-mixed-fp8-sensitive-nvfp4-heavy` | Diffusion transformer | Validated for ModelOpt mixed FP8/NVFP4 checkpoints |
 | Z-Image | `feizhai123/z-image-modelopt-fp8-conservative` | Diffusion transformer | Validated for ModelOpt FP8 checkpoints |
 | FLUX.2-dev | `feizhai123/flux2-dev-modelopt-fp8` | Diffusion transformer | Validated for ModelOpt FP8 checkpoints |
 | FLUX.2-klein 4B | `feizhai123/flux2-klein-4b-modelopt-fp8` | Diffusion transformer | Validated for ModelOpt FP8 checkpoints |
 | HunyuanImage-3.0 | `feizhai123/hunyuan-image3-modelopt-fp8` | MoE diffusion transformer | Validated for ModelOpt FP8 checkpoints |
 | HunyuanImage-3.0 | `feizhai123/hunyuan-image3-modelopt-mixed-experts-nvfp4-dense-fp8` | MoE diffusion transformer | Validated for ModelOpt mixed FP8/NVFP4 checkpoints |
 | Wan2.2 | Not available | Diffusion transformer | Not validated |
 For full serving commands and benchmark context, see
 [`recipes/Qwen/Qwen-Image.md`](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen-Image.md)
 and
 [`recipes/Tencent/HunyuanImage-3.0-Instruct.md`](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/HunyuanImage-3.0-Instruct.md).
 ### Multi-Stage Omni/TTS Model
 | Model | Scope | Status |
@@ -102,28 +80,15 @@ ModelOpt targets yet.
 ## Configuration
-For pre-quantized ModelOpt checkpoints, no `--quantization fp8` flag is needed.
+For pre-quantized ModelOpt FP8 checkpoints, no `--quantization fp8` flag is
-The checkpoint config selects the ModelOpt path.
+needed. The checkpoint config selects the ModelOpt path.
 Online serving:
 ```bash
-vllm serve <modelopt-checkpoint> \
+vllm serve <modelopt-fp8-checkpoint> \
  --omni \
  --tensor-parallel-size <N> \
  --linear-backend cutlass \
  --force-cutlass-fp8
 ```
 For mixed FP8/NVFP4 MoE checkpoints, also select the validated MoE backend:
 ```bash
 vllm serve <modelopt-mixed-moe-checkpoint> \
  --omni \
  --tensor-parallel-size <N> \
  --enable-expert-parallel \
  --linear-backend cutlass \
  --moe-backend cutlass \
  --force-cutlass-fp8
 ```
@@ -131,14 +96,14 @@ Offline inference:
 ```bash
 python examples/offline_inference/text_to_image/text_to_image.py \
-  --model <modelopt-checkpoint> \
+  --model <modelopt-fp8-checkpoint> \
  --tensor-parallel-size <N> \
  --prompt "a red ceramic teapot on a wooden table" \
  --height 1024 \
  --width 1024 \
  --num-inference-steps 20 \
  --seed 42 \
-  --output outputs/modelopt.png
+  --output outputs/modelopt_fp8.png
 ```
 Python API:
@@ -147,7 +112,7 @@ Python API:
 from vllm_omni import Omni
 omni = Omni(
-    model="<modelopt-checkpoint>",
+    model="<modelopt-fp8-checkpoint>",
    tensor_parallel_size=2,
    force_cutlass_fp8=True,
 )
@@ -158,34 +123,16 @@ omni = Omni(
 | Parameter | Type | Default | Description |
 |-----------|------|---------|-------------|
 | `force_cutlass_fp8` / `--force-cutlass-fp8` | bool | `False` | Force CUTLASS FP8 linear kernels for supported ModelOpt FP8 diffusion stages on CUDA SM89+ |
 | `--linear-backend cutlass` | str | auto | Select the validated CUTLASS linear backend for supported ModelOpt NVFP4 or mixed FP8/NVFP4 diffusion stages |
 | `--moe-backend cutlass` | str | auto | Select the validated CUTLASS MoE backend for supported ModelOpt mixed MoE checkpoints |
 ## Validation and Notes
-1. Compare the ModelOpt checkpoint against the BF16 baseline with the same
+1. Compare the ModelOpt FP8 checkpoint against the BF16 baseline with the same
   prompt, resolution, seed, and inference steps.
 2. Use `tests/diffusion/quantization/test_quantization_quality.py` with
   `VLLM_OMNI_QUALITY_CONFIGS` to validate local baseline and quantized model
   paths.
-3. For HunyuanImage-3.0 quantized DiT checkpoints, the opt-in accuracy check is:
+3. Report LPIPS, PSNR, MAE, throughput, latency, and peak memory when adding a
-
+   new validated ModelOpt diffusion checkpoint.
-   ```bash
+4. Keep `--quantization fp8` for online FP8 from BF16 checkpoints; use this
-   CUDA_VISIBLE_DEVICES=2,3 \
+   ModelOpt path only when the checkpoint already contains ModelOpt FP8 weights
-   HUNYUAN_IMAGE3_RUN_QUANT_ACCURACY=1 \
+   and scales.
   HUNYUAN_IMAGE3_QUANT_DEVICES=0,1 \
   HUNYUAN_IMAGE3_QUANT_TP=2 \
   HUNYUAN_IMAGE3_BF16_MODEL=/path/to/hunyuan-image3-bf16 \
   HUNYUAN_IMAGE3_FP8_MODEL=/path/to/hunyuan-image3-modelopt-fp8 \
   HUNYUAN_IMAGE3_NVFP4_MODEL=/path/to/hunyuan-image3-modelopt-mixed-experts-nvfp4-dense-fp8 \
   PYTHONPATH=/path/to/vllm-omni:${PYTHONPATH:-} \
   python -m pytest -s -v \
     tests/e2e/accuracy/test_hunyuan_image3.py \
     -k quantized_dit_matches_bf16_accuracy
   ```
 4. Report CLIP score deltas, SSIM, PSNR, throughput, latency, and peak memory
   when adding a new validated ModelOpt diffusion checkpoint.
 5. Keep `--quantization fp8` for online FP8 from BF16 checkpoints; use this
   ModelOpt path only when the checkpoint already contains ModelOpt quantized
   weights and scales.
@@ -1,479 +0,0 @@
 # W4A4 MXFP4 Quantization
 ## Overview
 W4A4 MXFP4 (Microscaling FP4) quantizes both weights and activations to FP4
 (`float4_e2m1fn_x2`, packed 2 values per byte) using the OCP MX format: groups
 of 32 K-dimension elements share a single `float8_e8m0fnu` exponent scale.
 vLLM-Omni provides two quantization methods with different scale structures:
 | Method | Scale structure | Mode | Use case |
 |--------|----------------|------|----------|
 | `mxfp4` | Single-scale (per-32 fine only) | Online only | Quick accuracy baseline; no checkpoint prep needed |
 | `mxfp4_dualscale` | Dual-scale (fine per-32 + coarse per-512 + per-channel `mul_scale`) | Online + Offline | Production; better accuracy; offline recommended |
 !!! tip "Recommended: `mxfp4_dualscale` offline"
    For production deployments, use the `mxfp4_dualscale` offline mode with a
    pre-quantized checkpoint produced by msModelSlim. Offline checkpoints load
    calibrated `mul_scale` tensors from disk, providing measurably better accuracy
    than any online method. The one-time preprocessing cost amortises across all
    subsequent inference runs.
    Use `mxfp4` online only for quick experimentation where preprocessing time
    is not acceptable and accuracy loss is tolerable.
 !!! warning "Online single-scale ≠ Offline dual-scale"
    `mxfp4_dualscale` offline mode uses `NPUMxfp4DualScaleLinearMethod`:
    fine scale (per-32 K), coarse scale (per-512 K), and per-input-channel
    `mul_scale` from calibration — all loaded from the checkpoint.
    `mxfp4_dualscale` online mode uses `NPUMxfp4DualScaleOnlineLinearMethod`:
    dual-level scales computed on the fly from BF16 weights; no calibration
    `mul_scale` is available. Loading an offline checkpoint with the online
    method (or vice versa) will produce incorrect results or shape errors.
 ## Hardware Support
 | Device | Support |
 |--------|---------|
 | NVIDIA Blackwell GPU (SM 100+) | ⭕ |
 | NVIDIA Ada/Hopper GPU (SM 89+) | ⭕ |
 | NVIDIA Ampere GPU (SM 80+) | ⭕ |
 | AMD ROCm | ⭕ |
 | Intel XPU | ⭕ |
 | Ascend NPU (Atlas 950 A5) | ✅ |
 Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this guide.
 ## Model Type Support
 ### Diffusion Model (Wan2.2)
 | Model | Online | Offline | Notes |
 |-------|--------|---------|-------|
 | Wan2.2-T2V-A14B | `mxfp4` / `mxfp4_dualscale` | `mxfp4_dualscale` | MoE cascade (`transformer` + `transformer_2`); both transformers quantized with the same config |
 | Wan2.2-I2V-A14B | `mxfp4` / `mxfp4_dualscale` | `mxfp4_dualscale` | MoE cascade; same scheme as T2V-A14B |
 | Wan2.2-TI2V-5B | ❌ | ❌ | Parameter count too small; W4A4 causes unacceptable accuracy loss |
 The choice between `mxfp4` and `mxfp4_dualscale` in **online mode** is about
 quantization quality, not model compatibility — both work on cascade (A14B) and
 single-transformer models alike, the same as `mxfp8` online:
 - `mxfp4`: single-scale, lower overhead, simpler compute, online only
 - `mxfp4_dualscale`: dual-scale + optional BF16 fallback, better accuracy, online **and** offline
 **Offline** checkpoints for A14B are always in `mxfp4_dualscale` format (produced
 by the merge script); there is no offline `mxfp4` single-scale format.
 !!! note "Per-layer BF16 fallback in offline cascade models"
    The A14B offline checkpoint uses `quant_method: mxfp4_dualscale`. Most
    linear layers are stored as W4A4 MXFP4 DualScale; precision-sensitive layers
    retain their original BF16 weights and are listed in `ignored_layers` inside
    each transformer's `config.json`. The two transformers may have different
    `ignored_layers` sets — the pipeline reads each transformer's own `config.json`
    and rebuilds the config locally when they differ, so routing is always
    per-transformer-accurate.
 !!! warning "TI2V-5B not supported"
    Wan2.2-TI2V-5B is excluded from W4A4 quantization. Its smaller parameter
    count makes it significantly more sensitive to 4-bit quantization noise,
    resulting in unacceptable accuracy loss. Use [MXFP8](mxfp8.md) for TI2V-5B.
 ## Configuration
 ### `mxfp4` — Single-Scale Online Mode
 Online mode requires no pre-processing. vLLM-Omni quantizes BF16 weights to
 MXFP4 at load time using `npu_dynamic_mx_quant`. A single block scale
 (`float8_e8m0fnu`, one per 32 K elements) is computed on the fly; no
 calibration `mul_scale` is available. Applies equally to single-transformer
 and cascade (A14B) models — both transformers in a cascade receive the same
 quantization config automatically.
 ```python
 from vllm_omni import Omni
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 omni = Omni(model="<your-model>", quantization="mxfp4")
 outputs = omni.generate(
    "A cat sitting on a windowsill",
    OmniDiffusionSamplingParams(num_inference_steps=50),
 )
 ```
 ```bash
 # Single-transformer or cascade model — same command
 python text_to_video.py --model <your-model> --quantization mxfp4
 # Online serving
 vllm serve <your-model> --omni --quantization mxfp4
 ```
 ### `mxfp4_dualscale` — DualScale Online Mode
 Online DualScale mode computes both fine and coarse scales on the fly from BF16
 weights using `npu_dynamic_dual_level_mx_quant`. Applies equally to
 single-transformer and cascade (A14B) models. Compared to `mxfp4` online,
 DualScale provides better quantization accuracy at higher compute cost.
 The default configuration keeps the leading 5 transformer blocks in BF16
 (`num_bf16_fallback_layers=5`). Accuracy evaluation on Wan2.2-A14B shows this
 is sufficient to meet quality requirements and is the recommended setting.
 ```python
 omni = Omni(model="<your-model>", quantization="mxfp4_dualscale")
 ```
 ```bash
 python text_to_video.py --model <your-model> --quantization mxfp4_dualscale
 ```
 If accuracy debugging identifies additional precision-sensitive layers, they
 can be pinned to BF16 via the Python API:
 ```python
 omni = Omni(
    model="<your-model>",
    quantization_config={
        "method": "mxfp4_dualscale",
        "ignored_layers": ["blocks.10.attn1.to_q"],   # explicit per-layer override
    },
 )
 ```
 BF16 fallback routing in online mode applies two rules in priority order:
 1. **`ignored_layers`** (explicit per-layer override): any layer whose prefix
   matches is kept in BF16 regardless of block index.
 2. **`num_bf16_fallback_layers`** (coarse leading-block rule): the first N
   transformer blocks (`blocks.0` … `blocks.N-1`) fall back to BF16. Defaults
   to `5` (recommended). Layers outside `blocks.N.*`
   (e.g. `condition_embedder`) are always quantized.
 ### `mxfp4_dualscale` — DualScale Offline Mode (Recommended)
 Offline mode loads a pre-quantized DualScale checkpoint from msModelSlim. A
 preprocessing step converts the raw quantized output to the diffusers format
 expected by vLLM-Omni and injects the quantization config into each
 `transformer/config.json` so that vLLM-Omni auto-detects the offline path
 without a `--quantization` flag.
 BF16 fallback layers may be interleaved anywhere in the transformer — they are
 not restricted to leading blocks. The merge script detects them from
 `quant_model_description.json` and writes their prefixes into `ignored_layers`
 inside `config.json`. At runtime, each layer's prefix is matched against
 `ignored_layers` to decide BF16 vs. MXFP4 DualScale.
 #### Checkpoint tensor layout
 Each quantized linear layer stores four tensors:
 | Tensor | Shape | dtype | Description |
 |--------|-------|-------|-------------|
 | `weight` | `(N, K)` | float8_e4m3fn | FP4 packed (2 values per byte) |
 | `weight_scale` | `(N, K//32)` | uint8 | Fine block scale (`float8_e8m0fnu` bit pattern) |
 | `weight_dual_scale` | `(N, K//512, 1)` | float32 | Coarse block scale |
 | `mul_scale` | `(K,)` | float32 | Per-input-channel smooth pre-scale (from calibration) |
 BF16 fallback layers have no quantization tensors; only the original `weight`
 (and optional `bias`) are present, loaded directly from the base checkpoint.
 #### Step 1 — Quantize with msModelSlim
 ```bash
 msmodelslim quant \
  --model_path /path/to/Wan2.2-T2V-A14B-Diffusers \
  --save_path  /path/to/wan2_2_t2v_quantized_raw \
  --device npu \
  --model_type Wan2_2 \
  --config_path /path/to/wan2_2_w4a4_mxfp4_dualscale.yaml \
  --trust_remote_code True
 ```
 After this step, `--save_path` contains raw quantized safetensors files,
 scale files, and a metadata JSON (`quant_model_description*.json`).
 For cascade MoE models (T2V-A14B, I2V-A14B), msModelSlim outputs two
 subdirectories: `high_noise_model/` (transformer) and `low_noise_model/`
 (transformer_2).
 #### Step 2 — Preprocess with merge_mxfp4_dualscale_checkpoint.py
 The script (`vllm_omni/quantization/tools/merge_mxfp4_dualscale_checkpoint.py`):
 1. Copies the original diffusers model to `--output-path` (VAE, text encoder,
   scheduler, etc. are preserved).
 2. Remaps tensor names from msModelSlim convention to diffusers convention and
   strips `.linear.` / `.div.` wrappers added by the quantization tool.
 3. Overlays MXFP4 tensors (weight, fine/coarse scales, `mul_scale`) onto the
   BF16 base checkpoint. Non-quantized layers keep their original BF16 weights.
 4. Detects all linear layers that remain in BF16 and writes their prefixes into
   `ignored_layers` in `config.json`.
 5. Injects `quantization_config` so vLLM-Omni auto-detects offline MXFP4
   DualScale.
 For cascade MoE models, steps 2–5 run separately for each transformer.
 ```bash
 python vllm_omni/quantization/tools/merge_mxfp4_dualscale_checkpoint.py \
  --model-type     Wan2.2-T2V-A14B \
  --original-model /path/to/Wan2.2-T2V-A14B-Diffusers \
  --quant-path     /path/to/wan2_2_t2v_quantized_raw \
  --output-path    /path/to/Wan2.2-T2V-A14B-MXFP4-DualScale
 ```
 | Argument | Description |
 |----------|-------------|
 | `--model-type` | Model variant: `Wan2.2-T2V-A14B` or `Wan2.2-I2V-A14B` |
 | `--original-model` | Root directory of the original BF16 diffusers model |
 | `--quant-path` | Root directory of the msModelSlim quantized output |
 | `--output-path` | Output directory for the merged model (created by the script) |
 The script outputs a complete diffusers model directory at `--output-path`,
 with each transformer subfolder containing:
 - `diffusion_pytorch_model.safetensors` — MXFP4 weights + scale tensors, with BF16 fallback layers from the base checkpoint
 - `config.json` — original transformer config with `quantization_config` injected
 - `quant_model_description.json` — quantization metadata (reference only)
 The `quantization_config` injected into `config.json` for each transformer:
 ```json
 {
  "quant_method": "mxfp4_dualscale",
  "is_checkpoint_serialized": true,
  "ignored_layers": [
    "blocks.0.attn1.to_qkv",
    "blocks.0.attn1.to_out",
    "proj_out"
  ]
 }
 ```
 `ignored_layers` lists every linear layer that retains its original BF16 weight,
 using vllm-omni model parameter names (QKV-fused, FFN underscored, `to_out`
 unindexed). The exact entries are determined by the quantization tool (msModelSlim)
 and may differ between `transformer` and `transformer_2` in a cascade model.
 #### Step 3 — Serve
 ```bash
 python text_to_video.py --model /path/to/Wan2.2-T2V-A14B-MXFP4-DualScale
 # Online serving
 vllm serve /path/to/Wan2.2-T2V-A14B-MXFP4-DualScale --omni
 ```
 ```python
 omni = Omni(model="/path/to/Wan2.2-T2V-A14B-MXFP4-DualScale")
 ```
 !!! note
    No `--quantization` flag is needed for offline mode. The preprocessing
    script injects `quantization_config` into each `transformer/config.json`,
    which vLLM-Omni reads automatically to activate the correct offline path.
 ## Parameters
 ### `mxfp4` (single-scale, online only)
 | Parameter | Type | Default | Description |
 |-----------|------|---------|-------------|
 | `method` | str | — | `"mxfp4"` |
 | `ignored_layers` | list[str] | `[]` | Layer prefixes to keep in BF16 |
 ### `mxfp4_dualscale` (dual-scale, online + offline)
 | Parameter | Type | Default | Description |
 |-----------|------|---------|-------------|
 | `method` | str | — | `"mxfp4_dualscale"` |
 | `is_checkpoint_serialized` | bool | `False` | `True` for offline DualScale checkpoints; auto-set from `config.json` when using the preprocessing script |
 | `ignored_layers` | list[str] | `[]` | Layer prefixes to keep in BF16. **Works in both modes**: offline — populated by the merge script for interleaved sensitive layers; online — user-supplied for explicit per-layer precision override |
 | `num_bf16_fallback_layers` | int | `5` | **Online mode only**: leading N transformer blocks (`blocks.0` … `blocks.N-1`) kept in BF16. Applied after `ignored_layers`; ignored in offline mode. Default of `5` is the evaluated recommended value for Wan2.2-A14B |
 #### BF16 fallback priority (online mode)
 ```
 for each linear layer:
    if prefix in ignored_layers               → BF16  (explicit override, highest priority)
    elif block_idx < num_bf16_fallback_layers → BF16  (coarse leading-block rule)
    else                                      → MXFP4 DualScale online
 ```
 Layers outside `blocks.N.*` (e.g. `condition_embedder.*`) are always quantized
 unless they appear in `ignored_layers`.
 ## Validation and Notes
 1. **Online single-scale (`mxfp4`)** quantizes BF16 weights at load time using
   `npu_dynamic_mx_quant` (single-scale). No calibration `mul_scale` is
   available — all output partitions receive an identity pre-scale. No offline
   checkpoint format exists for this method.
 2. **Online dual-scale (`mxfp4_dualscale`, `is_checkpoint_serialized=False`)**
   quantizes BF16 weights using `npu_dynamic_dual_level_mx_quant` (fine + coarse
   scales computed on the fly). No calibration `mul_scale`; leading blocks or
   explicit `ignored_layers` stay in BF16 for accuracy.
 3. **Offline dual-scale (`mxfp4_dualscale`, `is_checkpoint_serialized=True`)** —
   **recommended for production** — loads four tensors per quantized layer: FP4
   weight, fine scale (`uint8` reinterpreted as `float8_e8m0fnu`), coarse scale
   (`float32`), and per-input-channel `mul_scale` (`float32`). BF16 fallback
   layers have no quantization tensors and are routed via `ignored_layers`.
 4. **Scale dtype**: fine scales are stored as `uint8` in safetensors (same bit
   layout as `float8_e8m0fnu`) and reinterpreted at load time without a lossy
   float32 round-trip.
 5. **Cascade model config propagation**: in a cascade model (transformer +
   transformer_2), vLLM-Omni reads each transformer's own `config.json` and
   rebuilds the quant config locally when `ignored_layers` differs between
   transformers, ensuring per-layer routing is accurate for each. The first
   transformer's config is propagated to `od_config` so the second transformer
   can reuse it as a starting point.
 6. **Self-attention QKV fusion**: Q, K, V projection weights are fused into a
   single `QKVParallelLinear` layer at runtime. `ignored_layers` entries use the
   fused name (`attn1.to_qkv`), written automatically by the merge script.
 7. W4A4 carries higher quantization noise than W8A8 (16 vs 256 levels). The
   DualScale offline method mitigates this with calibrated `mul_scale` smooth
   quantization. Use `ignored_layers` and `num_bf16_fallback_layers` to trade
   off compression vs. accuracy for precision-sensitive layers.
 ## Adapting MXFP4 for a New Model
 This section is aimed at developers who want to add MXFP4 support to a model
 other than Wan2.2. The three integration points are: (1) discovering the correct
 runtime layer names, (2) wiring `ignored_layers` into the model, and (3) writing
 a merge script for offline checkpoints.
 ### Step 1 — Discover runtime layer names
 `ignored_layers` entries must match the **runtime parameter names** used inside
 vllm-omni, which may differ from the names stored in the diffusers checkpoint.
 The canonical source of truth is the model's own `named_parameters()`.
 ```python
 from vllm_omni import Omni
 # Load the model without quantization to inspect parameter names.
 omni = Omni(model="/path/to/your-model")  # no --quantization flag
 for name, _ in omni.pipeline.transformer.named_parameters():
    if "weight" in name and "scale" not in name:
        print(name)
 ```
 Compare the printed names against the diffusers checkpoint keys
 (`safetensors.safe_open` or `torch.load`) to identify any renames your model
 applies. Common patterns that differ in Wan2.2 (and may appear in other
 models):
 | Diffusers checkpoint name | vllm-omni runtime name | Reason |
 |---------------------------|------------------------|--------|
 | `attn1.to_q`, `attn1.to_k`, `attn1.to_v` | `attn1.to_qkv` | Self-attention Q/K/V fused into `QKVParallelLinear` |
 | `ffn.net.0.proj` | `ffn.net_0.proj` | Dots in sub-module names replaced with underscores |
 | `ffn.net.2` | `ffn.net_2` | Same underscore rule |
 | `to_out.0` | `to_out` | Sequential index stripped |
 If your model has different fusion patterns, inspect `packed_modules_mapping`
 on the model class — this dict records how checkpoint keys are mapped to
 fused runtime parameters.
 !!! warning "Partial QKV fallback is not allowed"
    If your model fuses Q, K, V into a single layer, `ignored_layers` must
    include **all three or none**. A partial fallback (e.g. `to_q` in BF16 but
    `to_k`, `to_v` quantized) cannot be expressed at runtime because they share
    one `QKVParallelLinear`. The merge script enforces this and raises an error
    if only some of the trio appear as non-quantized.
 ### Step 2 — Add ignored_layers to the model
 #### Online mode
 Pass `ignored_layers` directly in the quantization config using the **runtime
 names** discovered in Step 1. No code changes to the model are required.
 ```python
 omni = Omni(
    model="/path/to/your-model",
    quantization={
        "method": "mxfp4_dualscale",
        "ignored_layers": [
            "blocks.0.attn1.to_qkv",   # runtime name, not diffusers name
            "blocks.0.attn1.to_out",
            "blocks.0.ffn.net_0.proj",
        ],
    },
 )
 ```
 ```bash
 # CLI does not support list-typed ignored_layers directly.
 # Use the Python API or set ignored_layers in config.json (offline).
 python your_script.py --model /path/to/your-model --quantization mxfp4_dualscale
 ```
 The `num_bf16_fallback_layers` coarse rule is an alternative to listing layers
 individually: set it to N to keep all linear layers in blocks 0 … N-1 in BF16.
 The right value depends on the model's sensitivity; evaluate on a validation
 set and pick the smallest N that meets your accuracy target.
 #### Offline mode
 For offline checkpoints, `ignored_layers` is written into each transformer's
 `config.json` by the merge script (see Step 3). No manual editing is needed if
 the merge script is correct. The injected block:
 ```json
 {
  "quant_method": "mxfp4_dualscale",
  "is_checkpoint_serialized": true,
  "ignored_layers": [
    "blocks.0.attn1.to_qkv",
    "blocks.0.attn1.to_out"
  ]
 }
 ```
 To add a layer manually (e.g. to pin an additional layer to BF16 without
 re-running the merge script), edit `config.json` inside the transformer
 subfolder. Use runtime names, not diffusers checkpoint names.
 ### Step 3 — Write a merge script for offline mode
 The merge script for a new model mirrors
 `vllm_omni/quantization/tools/merge_mxfp4_dualscale_checkpoint.py`. The four
 things it must do:
 1. **Remap tensor names** from the quantization tool convention to diffusers
   convention (strip wrappers like `.linear.`, `.div.`; fix any prefix
   differences).
 2. **Collect ignored_layers**: after loading, enumerate all `*.weight` keys that
   have no corresponding `*.weight_scale` (i.e. layers the tool left in BF16).
   Convert diffusers names to vllm-omni runtime names (fuse QKV, rename FFN
   sub-modules, etc.). Write the result to `config.json`.
 3. **Inject `quantization_config`** into `config.json`:
   ```python
   config["quantization_config"] = {
       "quant_method":              "mxfp4_dualscale",
       "is_checkpoint_serialized":  True,
       "ignored_layers":            ignored_layers,   # runtime names
   }
   ```
 4. **Save** the merged safetensors and the updated `config.json`.
 The key helper to implement is the diffusers-to-runtime name translator
 (equivalent to `_diffusers_to_vllm_ignored` in the Wan2.2 merge script).
 For each non-quantized diffusers weight key, apply your model's specific
 renaming rules and collect the results.
 !!! tip "Validate before serving"
    After producing the offline checkpoint, load it without a `--quantization`
    flag and verify that vLLM-Omni auto-detects the correct method. Check that
    the layer count reported in the startup log matches expectations: quantized
    layer count + `ignored_layers` count should equal total linear layer count.
    Any mismatch indicates a name-mapping bug in the merge script.
@@ -7,28 +7,26 @@ using the OCP MX format: groups of 32 K-dimension elements share a single
 `float8_e8m0fnu` exponent scale. This gives better accuracy than channel-wise
 FP8 while keeping the same 8-bit weight footprint.
-This method supports three modes:
+This method supports two modes:
 | Mode | Description |
 |------|-------------|
 | **Online** | BF16 weights are quantized to MXFP8 at load time — no pre-processing needed |
-| **Offline (Native)** | msModelSlim-exported MXFP8 weights converted to diffusers format via `merge_mxfp8_checkpoint.py` — weights and scales are loaded directly from the preprocessed checkpoint |
+| **Offline** | msModelSlim-exported MXFP8 weights converted to diffusers format via `merge_mxfp8_checkpoint.py` — weights and scales are loaded directly from the preprocessed checkpoint |
 | **Offline (AutoRound)** | AutoRound MXFP8 checkpoints with `data_type="mx_fp"` — auto-detected from `config.json` |
 ## Hardware Support
-| Device | Online | Offline (Native) | Offline (AutoRound) |
+| Device | Support |
-|--------|--------|------------------|---------------------|
+|--------|---------|
-| NVIDIA Blackwell GPU (SM 100+) | ⭕ | ⭕ | ⭕ |
+| NVIDIA Blackwell GPU (SM 100+) | ⭕ |
-| NVIDIA Ada/Hopper GPU (SM 89+) | ⭕ | ⭕ | ⭕ |
+| NVIDIA Ada/Hopper GPU (SM 89+) | ⭕ |
-| NVIDIA Ampere GPU (SM 80+) | ⭕ | ⭕ | ⭕ |
+| NVIDIA Ampere GPU (SM 80+) | ⭕ |
-| AMD ROCm | ⭕ | ⭕ | ⭕ |
+| AMD ROCm | ⭕ |
-| Intel XPU | ✅ | ❌ | ✅ |
+| Intel XPU | ⭕ |
-| Ascend NPU (Atlas 950 A5) | ✅ | ✅ | ⭕ |
+| Ascend NPU (Atlas 950 A5) | ✅ |
-Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this guide.
+Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
-
+guide.
 **Note**: Intel XPU only supports AutoRound MXFP8 for offline mode. Use AutoRound quantized checkpoints or online mode for XPU.
 ## Model Type Support
@@ -84,9 +82,9 @@ python text_to_video.py --model <your-model> --quantization mxfp8
 vllm serve <your-model> --omni --quantization mxfp8
 ```
-### Offline Mode (Native)
+### Offline Mode
-Native offline mode loads a pre-quantized checkpoint from msModelSlim. A preprocessing
+Offline mode loads a pre-quantized checkpoint from msModelSlim. A preprocessing
 step converts the raw quantized output to the diffusers format expected by
 vLLM-Omni and injects the quantization config into `transformer/config.json` so
 that vLLM-Omni auto-detects the offline path without a `--quantization` flag.
@@ -161,41 +159,10 @@ omni = Omni(model="/path/to/Wan2.2-TI2V-5B-MXFP8")
 ```
 !!! note
-    No `--quantization` flag is needed for native offline mode. The preprocessing
+    No `--quantization` flag is needed for offline mode. The preprocessing
    script injects `quantization_config` into each `transformer/config.json`,
    which vLLM-Omni reads automatically to activate the offline MXFP8 method.
 ### Offline Mode (AutoRound)
 AutoRound MXFP8 checkpoints declare `quant_method="auto-round"` with `data_type="mx_fp"` in their `config.json`. These are automatically detected and use the `IncMxfp8OfflineLinearMethod` backend.
 To use an AutoRound MXFP8 checkpoint:
 ```bash
 python text_to_video.py --model <autoround-mxfp8-model>
 # Online serving
 vllm serve <autoround-mxfp8-model> --omni
 ```
 Python API:
 ```python
 omni = Omni(model="<autoround-mxfp8-model>")
 ```
 !!! note
    AutoRound MXFP8 checkpoints are auto-detected from `config.json` and do not require a `--quantization` flag. The config must include:
    ```json
    {
      "quantization_config": {
        "quant_method": "auto-round",
        "data_type": "mx_fp",
        ...
      }
    }
    ```
 ## Parameters
 | Parameter | Type | Default | Description |
@@ -9,24 +9,20 @@ separate quantized checkpoint.
 This mode is different from pre-quantized checkpoint formats such as GGUF,
 AutoRound, msModelSlim, or serialized Int8 checkpoints. Those formats are
 prepared before serving and are documented in their method-specific guides.
 For MXFP8 and MXFP4, use this page for load-time quantization from BF16
 checkpoints, and use the method-specific pages for offline checkpoints produced
 by msModelSlim and the merge tools.
 ## Hardware Support
-| Device | FP8 W8A8 | Int8 W8A8 | MXFP8 W8A8 | MXFP4 W4A4 |
+| Device | FP8 W8A8 | Int8 W8A8 |
-|--------|----------|-----------|------------|------------|
+|--------|----------|-----------|
-| NVIDIA Blackwell GPU (SM 100+) | ✅ | ✅ | ⭕ | ⭕ |
+| NVIDIA Blackwell GPU (SM 100+) | ✅ | ✅ |
-| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ | ✅ | ⭕ | ⭕ |
+| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ | ✅ |
-| NVIDIA Ampere GPU (SM 80+) | ✅ | ✅ | ⭕ | ⭕ |
+| NVIDIA Ampere GPU (SM 80+) | ✅ | ✅ |
-| AMD ROCm | ⭕ | ⭕ | ⭕ | ⭕ |
+| AMD ROCm | ⭕ | ⭕ |
-| Intel XPU | ⭕ | ⭕ | ✅ | ⭕ |
+| Intel XPU | ⭕ | ⭕ |
-| Ascend NPU | ❌ | ✅ | ✅ | ✅ |
+| Ascend NPU | ❌ | ✅ |
 Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
-guide. FP8 on Ampere may use a weight-only path where available. MXFP8 and
+guide. FP8 on Ampere may use a weight-only path where available.
 MXFP4 are documented for the Ascend NPU path.
 ## Model Type Support
@@ -36,8 +32,6 @@ MXFP4 are documented for the Ascend NPU path.
 |--------|-------|----------------|--------|
 | FP8 W8A8 | [FP8](fp8.md) | Qwen-Image; Wan2.2 is not validated | Validated for Qwen-Image family and other DiT models |
 | Int8 W8A8 | [Int8](int8.md) | Qwen-Image; Wan2.2 is not validated | Validated for Qwen-Image and Z-Image |
 | MXFP8 W8A8 | [MXFP8](mxfp8.md) | Wan2.2-T2V-A14B, Wan2.2-I2V-A14B, Wan2.2-TI2V-5B | Validated on Ascend NPU and Intel XPU |
 | MXFP4 W4A4 | [MXFP4](mxfp4.md) | Wan2.2-T2V-A14B, Wan2.2-I2V-A14B | Ascend NPU only; TI2V-5B is not supported |
 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
@@ -59,9 +53,6 @@ from vllm_omni import Omni
 omni_fp8 = Omni(model="<your-model>", quantization="fp8")
 omni_int8 = Omni(model="<your-model>", quantization="int8")
 omni_mxfp8 = Omni(model="<your-model>", quantization="mxfp8")
 omni_mxfp4 = Omni(model="<your-model>", quantization="mxfp4")
 omni_mxfp4_dualscale = Omni(model="<your-model>", quantization="mxfp4_dualscale")
 ```
 CLI:
@@ -69,9 +60,6 @@ CLI:
 ```bash
 vllm serve <your-model> --omni --quantization fp8
 vllm serve <your-model> --omni --quantization int8
 vllm serve <your-model> --omni --quantization mxfp8
 vllm serve <your-model> --omni --quantization mxfp4
 vllm serve <your-model> --omni --quantization mxfp4_dualscale
 ```
 Per-component routing:
@@ -89,11 +77,10 @@ config = build_quant_config({
 | Parameter | Methods | Description |
 |-----------|---------|-------------|
-| `method` | FP8, Int8, MXFP8, MXFP4 | Quantization method: `"fp8"`, `"int8"`, `"mxfp8"`, `"mxfp4"`, or `"mxfp4_dualscale"` |
+| `method` | FP8, Int8 | Quantization method, usually `"fp8"` or `"int8"` |
-| `ignored_layers` | FP8, Int8, MXFP8, MXFP4 | Layer name patterns to keep in BF16/FP16 |
+| `ignored_layers` | FP8, Int8 | Layer name patterns to keep in BF16/FP16 |
 | `activation_scheme` | FP8, Int8 | The runtime value `"dynamic"` selects online activation scaling |
 | `weight_block_size` | FP8 | Optional block-wise FP8 weight quantization size |
 | `num_bf16_fallback_layers` | MXFP4 DualScale | Leading transformer blocks to keep in BF16 for online `mxfp4_dualscale`; defaults to `5` |
 ## Validation and Notes
@@ -104,5 +91,3 @@ config = build_quant_config({
   model as supported.
 4. If a model already ships quantized weights, use the matching pre-quantized
   method guide instead of online quantization.
 5. For Ascend MXFP4 deployments, prefer offline `mxfp4_dualscale` checkpoints
   when production quality is more important than avoiding preprocessing.
@@ -9,20 +9,20 @@ type has a different quantization scope.
 | Mode | Guide | Description | Methods |
 |------|-------|-------------|---------|
-| Online quantization | [Online Quantization](online.md) | vLLM-Omni computes quantized weights and scales while loading the model. | FP8 W8A8, Int8 W8A8, MXFP8 W8A8, MXFP4 W4A4 |
+| Online quantization | [Online Quantization](online.md) | vLLM-Omni computes quantized weights and scales while loading the model. | FP8 W8A8, Int8 W8A8, MXFP8 W8A8 |
 | Runtime attention quantization | [Quantized KV Cache](quantized_kvcache.md) | vLLM-Omni dynamically quantizes eligible diffusion Flash Attention tensors during inference. | FP8 FA |
-| Pre-quantized checkpoints | Method-specific guides | The checkpoint or an offline quantizer provides quantized weights and scales before serving. | ModelOpt, GGUF, AutoRound, msModelSlim, serialized Int8, offline MXFP8, offline MXFP4 DualScale |
+| Pre-quantized checkpoints | Method-specific guides | The checkpoint or an offline quantizer provides quantized weights and scales before serving. | ModelOpt, GGUF, AutoRound, msModelSlim, serialized Int8, offline MXFP8 |
 ## Hardware Support
-| Device | FP8 W8A8 | Int8 W8A8 | ModelOpt | MXFP8 W8A8 | MXFP4 W4A4 | GGUF | AutoRound | msModelSlim |
+| Device | FP8 W8A8 | Int8 W8A8 | ModelOpt | MXFP8 W8A8 | GGUF | AutoRound | msModelSlim |
-|--------|----------|-----------|----------|------------|------------|------|-----------|-------------|
+|--------|----------|-----------|----------|------------|------|-----------|-------------|
-| NVIDIA Blackwell GPU (SM 100+) | ✅ | ✅ | ✅ | ⭕ | ⭕ | ✅ | ✅ | ❌ |
+| NVIDIA Blackwell GPU (SM 100+) | ✅ | ✅ | ✅ | ⭕ | ✅ | ✅ | ❌ |
-| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ | ✅ | ✅ | ⭕ | ⭕ | ✅ | ✅ | ❌ |
+| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ | ✅ | ✅ | ⭕ | ✅ | ✅ | ❌ |
-| NVIDIA Ampere GPU (SM 80+) | ✅ | ✅ | ⭕ | ⭕ | ⭕ | ✅ | ✅ | ❌ |
+| NVIDIA Ampere GPU (SM 80+) | ✅ | ✅ | ⭕ | ⭕ | ✅ | ✅ | ❌ |
-| AMD ROCm | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ❌ |
+| AMD ROCm | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ❌ |
-| Intel XPU | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ✅ | ❌ |
+| Intel XPU | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ✅ | ❌ |
-| Ascend NPU | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ |
+| Ascend NPU | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ |
 Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
 guide. FP8 on Ampere may use a weight-only path where available.
@@ -42,7 +42,6 @@ otherwise.
 | Int8 W8A8 | [Int8](int8.md) | Online or serialized W8A8 | Qwen-Image; Wan2.2 is not validated | Validated for Qwen-Image and Z-Image |
 | ModelOpt | [ModelOpt](modelopt.md) | Pre-quantized FP8 checkpoints | Qwen-Image, Z-Image, FLUX.2, HunyuanImage-3.0 | Validated for ModelOpt FP8 diffusion checkpoints |
 | MXFP8 W8A8 | [MXFP8](mxfp8.md) | Online W8A8 or offline pre-quantized | Wan2.2-T2V-A14B, I2V-A14B, TI2V-5B | Ascend NPU only; validated for Wan2.2 |
 | MXFP4 W4A4 | [MXFP4](mxfp4.md) | `mxfp4`: online single-scale only; `mxfp4_dualscale`: online or offline dual-scale (offline recommended) | Wan2.2-T2V-A14B, I2V-A14B | Ascend NPU only; validated for Wan2.2 A14B cascade models; TI2V-5B not supported; offline `mxfp4_dualscale` uses calibrated `mul_scale` for best accuracy |
 | GGUF | [GGUF](gguf.md) | Pre-quantized transformer weights | Qwen-Image | Validated where a model-specific GGUF adapter exists |
 | AutoRound | [AutoRound](autoround.md) | Pre-quantized W4A16 checkpoints | FLUX.1-dev; Qwen-Image/Wan2.2 not validated | Checkpoint-driven |
 | msModelSlim | [msModelSlim](msmodelslim.md) | Pre-quantized Ascend checkpoints | Wan2.2 recipe; HunyuanImage-3.0 inference target | Ascend/NPU path |
@@ -59,7 +58,6 @@ in BF16 unless the model guide explicitly adds support.
 | ModelOpt | [ModelOpt](modelopt.md) | Thinker or language-model checkpoint config | Qwen3-Omni thinker | ModelOpt checkpoint path |
 | Int8 | [Int8](int8.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
 | MXFP8 | [MXFP8](mxfp8.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
 | MXFP4 | [MXFP4](mxfp4.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
 | GGUF | [GGUF](gguf.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
 | AutoRound | [AutoRound](autoround.md) | Thinker or language-model checkpoint config | Qwen2.5-Omni, Qwen3-Omni | Supported through AutoRound checkpoints |
 | msModelSlim | [msModelSlim](msmodelslim.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
@@ -75,7 +73,6 @@ attached to the intended stage rather than applied globally.
 | Int8 | [Int8](int8.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Requires model-specific validation |
 | ModelOpt | [ModelOpt](modelopt.md) | Checkpoint-defined diffusion stage | BAGEL, GLM-Image | Requires model-specific validation |
 | MXFP8 | [MXFP8](mxfp8.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Not validated |
 | MXFP4 | [MXFP4](mxfp4.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Not validated |
 | GGUF | [GGUF](gguf.md) | Stage-specific transformer weights | BAGEL, GLM-Image | No validated adapter listed |
 | AutoRound | [AutoRound](autoround.md) | Checkpoint-defined stage | BAGEL, GLM-Image | No validated checkpoint listed |
 | msModelSlim | [msModelSlim](msmodelslim.md) | Ascend-generated stage weights | GLM-Image | Requires model-specific adaptation |
@@ -103,7 +100,7 @@ config = build_quant_config({
 | Component | Default quantized? | Notes |
 |-----------|--------------------|-------|
-| Diffusion transformer | Yes | Primary target for FP8, Int8, ModelOpt, MXFP8, MXFP4, GGUF, AutoRound, and msModelSlim |
+| Diffusion transformer | Yes | Primary target for FP8, Int8, ModelOpt, GGUF, AutoRound, and msModelSlim |
 | Text encoder | No | Keep BF16 unless a method-specific guide documents support |
 | VAE | No | Keep BF16; storage-only paths are method-specific |
 | Scheduler/tokenizer | No | Loaded from the base model repository |
@@ -17,7 +17,7 @@ from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.inputs.data import OmniPromptType
 _REPO_ROOT = Path(__file__).resolve().parents[3]
-_DEFAULT_DEPLOY_CONFIG = str(_REPO_ROOT / "vllm_omni" / "deploy" / "hunyuan_image_3_moe.yaml")
+_DEFAULT_DEPLOY_CONFIG = str(_REPO_ROOT / "vllm_omni" / "deploy" / "hunyuan_image3.yaml")
 _DEFAULT_AR_DEPLOY_CONFIG = str(_REPO_ROOT / "vllm_omni" / "deploy" / "hunyuan_image3_ar.yaml")
 _MODALITY_TASK_MAP: dict[str, tuple[str, str | None]] = {
@@ -61,8 +61,8 @@ def parse_args():
    parser.add_argument("--steps", type=int, default=50, help="Number of inference steps.")
    parser.add_argument("--guidance-scale", type=float, default=5.0, help="Classifier-free guidance scale.")
    parser.add_argument("--seed", type=int, default=42, help="Random seed.")
-    parser.add_argument("--height", type=int, default=None, help="Output image height.")
+    parser.add_argument("--height", type=int, default=1024, help="Output image height.")
-    parser.add_argument("--width", type=int, default=None, help="Output image width.")
+    parser.add_argument("--width", type=int, default=1024, help="Output image width.")
    parser.add_argument("--vae-use-tiling", action="store_true", help="Enable VAE tiling.")
    parser.add_argument(
        "--bot-task",
@@ -229,21 +229,7 @@ def main():
    from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-    if (args.height is None) != (args.width is None):
+    ar_stop_token_ids = resolve_stop_token_ids(task=task, bot_task=bot_task, tokenizer=tokenizer)
        raise ValueError("--height and --width must both be specified or both omitted.")
    user_specified_size = args.height is not None and args.width is not None
    if args.modality in ("img2text", "text2text"):
        ar_image_size = "auto"
    elif user_specified_size:
        ar_image_size = f"{args.width}x{args.height}"
    else:
        ar_image_size = None
    ar_stop_token_ids = resolve_stop_token_ids(
        task=task, bot_task=bot_task, tokenizer=tokenizer, image_size=ar_image_size
    )
    print(
        f"[AR Config] task={task}, bot_task={bot_task}, image_size={ar_image_size}, stop_token_ids={ar_stop_token_ids}"
    )
    for sp in params_list:
        if isinstance(sp, OmniDiffusionSamplingParams):
            sp.num_inference_steps = args.steps
@@ -173,8 +173,8 @@ def parse_args() -> argparse.Namespace:
        "--quantization",
        type=str,
        default=None,
-        choices=["fp8", "mxfp8", "mxfp4", "mxfp4_dualscale", "int8", "gguf"],
+        choices=["fp8", "mxfp8", "int8", "gguf"],
-        help="Quantization method for the transformer. mxfp8: W8A8 MXFP8 (NPU). mxfp4: W4A4 MXFP4 (NPU). mxfp4_dualscale: W4A4 MXFP4 dual-scale + BF16 fallback mixed (NPU). fp8: online FP8 (GPU).",
+        help="Quantization method for the transformer. mxfp8: W8A8 MXFP8 online quant (NPU). fp8: online FP8 (GPU).",
    )
    # Distributed and parallel execution
@@ -127,12 +127,7 @@ def run_one_path(
            ),
            args.device,
        )
-        pred = run_pipeline_forward(
+        pred = run_pipeline_forward(pipeline, batch_inputs, noise)
            pipeline,
            batch_inputs,
            noise,
            request_id=f"internvla-a1-sample-{index}",
        )
        pred = pred[:, :, : dataset.physical_action_dim].to(torch.float32).cpu()
        results.append(
            {
@@ -155,7 +150,6 @@ def run_pipeline_forward(
    pipeline,
    batch_inputs: dict[str, torch.Tensor],
    noise: torch.Tensor,
    request_id: str,
 ) -> torch.Tensor:
    output = pipeline.forward(
        OmniDiffusionRequest(
@@ -167,7 +161,6 @@ def run_pipeline_forward(
                    "decode_image": False,
                }
            ),
            request_id=request_id,
        )
    )
    if output.error:
@@ -219,38 +212,23 @@ def benchmark_forward(
    _synchronize(args.device)
    cold_start_begin = time.perf_counter()
-    pred = run_pipeline_forward(
+    pred = run_pipeline_forward(pipeline, batch_inputs, noise)
        pipeline,
        batch_inputs,
        noise,
        request_id=f"internvla-a1-benchmark-{index}-cold",
    )
    _synchronize(args.device)
    cold_start_ms = (time.perf_counter() - cold_start_begin) * 1000.0
    warmup_ms: list[float] = []
-    for iter_idx in range(args.warmup_iters):
+    for _ in range(args.warmup_iters):
        _synchronize(args.device)
        begin = time.perf_counter()
-        _ = run_pipeline_forward(
+        _ = run_pipeline_forward(pipeline, batch_inputs, noise)
            pipeline,
            batch_inputs,
            noise,
            request_id=f"internvla-a1-benchmark-{index}-warmup-{iter_idx}",
        )
        _synchronize(args.device)
        warmup_ms.append((time.perf_counter() - begin) * 1000.0)
    benchmark_ms: list[float] = []
-    for iter_idx in range(args.benchmark_iters):
+    for _ in range(args.benchmark_iters):
        _synchronize(args.device)
        begin = time.perf_counter()
-        _ = run_pipeline_forward(
+        _ = run_pipeline_forward(pipeline, batch_inputs, noise)
            pipeline,
            batch_inputs,
            noise,
            request_id=f"internvla-a1-benchmark-{index}-iter-{iter_idx}",
        )
        _synchronize(args.device)
        benchmark_ms.append((time.perf_counter() - begin) * 1000.0)
@@ -317,12 +295,7 @@ def main() -> None:
            dataset=dataset,
            train_meta=train_meta,
            collate_samples=collate_open_loop_samples,
-            run_sample_actions=lambda policy, batch_inputs, noise: run_pipeline_forward(
+            run_sample_actions=lambda policy, batch_inputs, noise: run_pipeline_forward(policy, batch_inputs, noise),
                policy,
                batch_inputs,
                noise,
                request_id="internvla-a1-open-loop",
            ),
            num_episodes=args.num_episodes,
            seed=args.seed,
            device=args.device,
@@ -1,52 +0,0 @@
 # Lance (ByteDance) — offline inference
 [Lance](https://huggingface.co/bytedance-research/Lance) is a 3B unified
 autoregressive + diffusion multimodal model on a Qwen2.5-VL backbone. It is
 **BAGEL-lineage** (ByteDance Mixture-of-Transformers): the released `Lance_3B`
 checkpoint uses the same `*_moe_gen` MoT weight layout as BAGEL, so vLLM-Omni
 implements it by reusing the BAGEL transformer core and specializing only the
 ViT (Qwen2.5-VL vision), the VAE (Wan2.2) and the checkpoint layout.
 This example covers all six Lance modalities from the upstream HF model card:
 `t2i`, `t2v`, `image_edit`, `video_edit`, `x2t_image` (image understanding) and
 `x2t_video` (video understanding).
 ## Hardware
 Single NVIDIA GPU with 16 GB+ VRAM in BF16 (we test on B300 / A100). CUDA ≥ 12.4.
 ## Run
 ```bash
 # Text-to-image
 python examples/offline_inference/lance/end2end.py \
    --model bytedance-research/Lance \
    --prompts "a corgi astronaut on the moon, cinematic" \
    --steps 30 --cfg-text-scale 4.0 --timestep-shift 3.5 \
    --height 1024 --width 1024 \
    --output ./out
 # Text-to-video (uses the Lance_3B_Video subfolder; see ``--modality``
 # choices for all six task variants)
 python examples/offline_inference/lance/end2end.py \
    --model bytedance-research/Lance/Lance_3B_Video --modality text2video \
    --num-frames 25 --video-height 480 --video-width 768 \
    --prompts "a cat playing piano, cinematic" \
    --steps 30 --fps 8 --output ./out
 ```
 `video_edit` requires `--model bytedance-research/Lance/Lance_3B_Video` so
 the 3-D `latent_pos_embed` table is loaded; the other paths can point at
 the top-level `bytedance-research/Lance` repo and resolve the right
 sub-checkpoint automatically.
 The HF repo bundles everything (`Lance_3B/`, `Lance_3B_Video/`,
 `Qwen2.5-VL-ViT/`, `Wan2.2_VAE.pth`); no separate downloads are required.
 ## Defaults
 Matches upstream `inference_lance.sh`: 30 denoising steps, timestep-shift 3.5,
 text CFG 4.0, seed 42, 1024×1024 (override with `--height` / `--width`).  For
 the understanding paths (`img2text` / `video2text`), sampling is enabled by
 default at `--text-temperature 0.8` because Lance's greedy decoder emits an
 immediate EOS for many prompts.
@@ -1,333 +0,0 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 """Lance-3B offline inference (single-stage).
 Lance is BAGEL-lineage; this example covers the single-stage paths wired in
 this PR:
    text2img  — image generation from text
    img2text  — image understanding (caption / VQA)
 Examples:
    python examples/offline_inference/lance/end2end.py \
        --model bytedance-research/Lance \
        --prompts "a corgi astronaut on the moon, cinematic" \
        --output ./out
    python examples/offline_inference/lance/end2end.py \
        --model bytedance-research/Lance --modality img2text \
        --image-path /path/to/photo.jpg \
        --prompts "Describe this image in detail."
 """
 import argparse
 import os
 def parse_args():
    p = argparse.ArgumentParser()
    p.add_argument("--model", default="bytedance-research/Lance", help="HF repo or local path.")
    p.add_argument("--prompts", nargs="+", default=None, help="Input text prompts.")
    p.add_argument("--txt-prompts", type=str, default=None, help="File with one prompt per line.")
    p.add_argument(
        "--modality",
        default="text2img",
        choices=["text2img", "img2text", "text2video", "video2text", "img2img", "video2video", "image2video"],
        help="Lance single-stage modality.",
    )
    p.add_argument("--image-path", type=str, default=None, help="Input image for img2text.")
    p.add_argument("--video-path", type=str, default=None, help="Input video file for video2text.")
    p.add_argument("--num-frames", type=int, default=25, help="Number of RGB frames for text2video (max 121).")
    p.add_argument("--video-height", type=int, default=480, help="Video frame height.")
    p.add_argument("--video-width", type=int, default=768, help="Video frame width.")
    p.add_argument("--height", type=int, default=None, help="Image height (t2i). Default = max_hw (1024).")
    p.add_argument("--width", type=int, default=None, help="Image width (t2i). Default = max_hw (1024).")
    p.add_argument(
        "--fps", type=int, default=12, help="Output video FPS when saving MP4 (matches upstream Lance's save_fps=12)."
    )
    p.add_argument("--output", type=str, default=".", help="Output directory.")
    p.add_argument("--steps", type=int, default=30, help="Denoising steps (Lance default 30).")
    p.add_argument("--cfg-text-scale", type=float, default=4.0, help="Text CFG scale (Lance default 4.0).")
    p.add_argument("--timestep-shift", type=float, default=3.5, help="Flow-match timestep shift (Lance default 3.5).")
    p.add_argument("--negative-prompt", type=str, default=None)
    p.add_argument("--seed", type=int, default=42)
    p.add_argument("--max-text-tokens", type=int, default=512, help="Max tokens for img2text generation.")
    p.add_argument(
        "--do-sample",
        action="store_true",
        default=True,
        help="Sample (vs greedy) for x2t generation; greedy frequently emits an immediate EOS for Lance.",
    )
    p.add_argument("--no-sample", dest="do_sample", action="store_false", help="Disable sampling (greedy decoding).")
    p.add_argument(
        "--text-temperature",
        type=float,
        default=0.8,
        help="Sampling temperature for x2t generation. Lance frequently emits an immediate EOS below ~0.7; 0.8 is a good default.",
    )
    p.add_argument(
        "--system-prompt",
        type=str,
        default=None,
        help=(
            "Override the task system prompt (used by x2t_image / x2t_video for "
            'per-example QA instructions like "Look at the image carefully and '
            'answer the question."; without it x2t falls back to a '
            "caption-style default and the model describes instead of answering)."
        ),
    )
    # Lance is single-stage diffusion — no deploy YAML needed.  Required
    # engine knobs (``pipeline``, ``enforce_eager``, ``trust_remote_code``,
    # ``max_num_seqs=1`` …) are passed as flat kwargs to ``Omni`` below
    # and ``create_default_diffusion`` materializes the stage config.
    from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
    nullify_stage_engine_defaults(p)
    return p.parse_args()
 from vllm_omni.diffusion.models.lance.prompts import (  # noqa: E402
    VIDEO_PAD,
    VISION_END,
    VISION_START,
    render_lance_prompt,
 )
 # Visual block matching upstream Lance: <|vision_start|><|video_pad|><|vision_end|>
 # (Lance's renderer uses ``<|video_pad|>`` even for image inputs by default.)
 _VISION_BLOCK = f"{VISION_START}{VIDEO_PAD}{VISION_END}"
 def _format_text2img_prompts(prompts):
    return [{"prompt": render_lance_prompt("t2i", p), "modalities": ["image"]} for p in prompts]
 def _format_text2video_prompts(prompts, num_frames, video_h, video_w):
    extra = {"num_frames": num_frames, "video_height": video_h, "video_width": video_w}
    return [{"prompt": render_lance_prompt("t2v", p), "modalities": ["video"], "extra_args": extra} for p in prompts]
 def _format_image2video_prompts(prompts, image_path, num_frames, video_h, video_w):
    """Image-to-Video (no first-frame pin): image + text → long video.
    Passes the input image via ``multi_modal_data.first_frame``.  The pipeline
    treats it as a 1-frame reference (VAE+ViT prefill) and generates a fresh
    multi-frame video at the requested ``num_frames`` × ``video_h`` ×
    ``video_w`` shape.
    """
    import os as _os
    if not image_path or not _os.path.exists(image_path):
        raise ValueError(f"image2video requires --image-path pointing to an existing file, got: {image_path}")
    from PIL import Image as _Image
    img = _Image.open(image_path).convert("RGB")
    extra = {"num_frames": num_frames, "video_height": video_h, "video_width": video_w}
    return [
        {
            "prompt": render_lance_prompt("i2v", p, vision_token=_VISION_BLOCK),
            "multi_modal_data": {"first_frame": img},
            "modalities": ["video"],
            "extra_args": extra,
        }
        for p in prompts
    ]
 def _format_img2img_prompts(prompts, image_path):
    import os as _os
    if not image_path or not _os.path.exists(image_path):
        raise ValueError(f"img2img requires --image-path pointing to an existing file, got: {image_path}")
    from PIL import Image as _Image
    img = _Image.open(image_path).convert("RGB")
    return [
        {
            "prompt": render_lance_prompt("image_edit", p, vision_token=_VISION_BLOCK),
            "multi_modal_data": {"img2img": img},
            "modalities": ["image"],
        }
        for p in prompts
    ]
 def _format_video2video_prompts(prompts, video_path, num_frames, video_h, video_w):
    import os as _os
    if not video_path or not _os.path.exists(video_path):
        raise ValueError(f"video2video requires --video-path pointing to an existing file, got: {video_path}")
    extra = {"num_frames": num_frames, "video_height": video_h, "video_width": video_w}
    return [
        {
            "prompt": render_lance_prompt("video_edit", p, vision_token=_VISION_BLOCK),
            "multi_modal_data": {"video": video_path},
            "modalities": ["video"],
            "extra_args": extra,
        }
        for p in prompts
    ]
 def _format_video2text_prompts(prompts, video_path, system_prompt=None):
    import os as _os
    if not video_path or not _os.path.exists(video_path):
        raise ValueError(f"video2text requires --video-path pointing to an existing file, got: {video_path}")
    import imageio.v3 as iio
    video = iio.imread(video_path)
    return [
        {
            "prompt": render_lance_prompt("x2t_video", p, vision_token=_VISION_BLOCK, system_prompt=system_prompt),
            "multi_modal_data": {"video": video},
            "modalities": ["text"],
        }
        for p in prompts
    ]
 def _format_img2text_prompts(prompts, image_path, system_prompt=None):
    from PIL import Image
    if not image_path or not os.path.exists(image_path):
        raise ValueError(f"img2text requires --image-path pointing to an existing file, got: {image_path}")
    img = Image.open(image_path).convert("RGB")
    return [
        {
            "prompt": render_lance_prompt("x2t_image", p, vision_token=_VISION_BLOCK, system_prompt=system_prompt),
            "multi_modal_data": {"image": img},
            "modalities": ["text"],
        }
        for p in prompts
    ]
 def main():
    args = parse_args()
    os.makedirs(args.output, exist_ok=True)
    if args.txt_prompts:
        with open(args.txt_prompts, encoding="utf-8") as f:
            prompts = [ln.strip() for ln in f if ln.strip()]
    elif args.modality == "img2text":
        prompts = args.prompts or ["Describe this image in detail."]
    else:
        prompts = args.prompts or ["A cute cat, studio lighting, 4k"]
    from vllm_omni.entrypoints.omni import Omni
    # Video modalities require the Lance_3B_Video checkpoint (3D
    # latent_pos_embed shaped (126976, 2048)).  Without it, vllm-omni silently
    # loads Lance_3B (4096, 2048) — the image-only table — and any t_lat >= 1
    # immediately indexes out of bounds.  Rewrite the model path so
    # LancePipeline._select_video_variant picks the video subfolder.
    resolved_model = args.model
    if args.modality in ("text2video", "video2video", "video2text", "image2video"):
        if not str(resolved_model).rstrip("/").endswith("Lance_3B_Video"):
            candidate = os.path.join(resolved_model, "Lance_3B_Video")
            if os.path.isdir(candidate):
                resolved_model = candidate
    omni_kwargs = vars(args).copy()
    omni_kwargs["model"] = resolved_model
    # Lance single-stage defaults (formerly in vllm_omni/deploy/lance.yaml):
    omni_kwargs.setdefault("pipeline", "lance")
    omni_kwargs.setdefault("max_num_batched_tokens", 32768)
    omni_kwargs.setdefault("max_num_seqs", 1)
    omni_kwargs.setdefault("enforce_eager", True)
    omni_kwargs.setdefault("trust_remote_code", True)
    omni_kwargs.setdefault("enable_prefix_caching", False)
    omni_kwargs.setdefault("async_chunk", False)
    omni = Omni(**omni_kwargs)
    if args.modality == "img2text":
        formatted = _format_img2text_prompts(prompts, args.image_path, system_prompt=args.system_prompt)
    elif args.modality == "text2video":
        formatted = _format_text2video_prompts(prompts, args.num_frames, args.video_height, args.video_width)
    elif args.modality == "video2text":
        formatted = _format_video2text_prompts(prompts, args.video_path, system_prompt=args.system_prompt)
    elif args.modality == "img2img":
        formatted = _format_img2img_prompts(prompts, args.image_path)
    elif args.modality == "video2video":
        formatted = _format_video2video_prompts(
            prompts, args.video_path, args.num_frames, args.video_height, args.video_width
        )
    elif args.modality == "image2video":
        formatted = _format_image2video_prompts(
            prompts, args.image_path, args.num_frames, args.video_height, args.video_width
        )
    else:
        formatted = _format_text2img_prompts(prompts)
    params_list = omni.default_sampling_params_list
    diffusion_params = params_list[0]  # single-stage: one param set
    diffusion_params.num_inference_steps = args.steps  # type: ignore
    if args.seed is not None:
        diffusion_params.seed = args.seed  # type: ignore
    if args.height is not None:
        diffusion_params.height = args.height  # type: ignore
    if args.width is not None:
        diffusion_params.width = args.width  # type: ignore
    extra = getattr(diffusion_params, "extra_args", {}) or {}
    extra["cfg_text_scale"] = args.cfg_text_scale
    extra["timestep_shift"] = args.timestep_shift
    if args.modality in ("img2text", "video2text"):
        extra["max_think_tokens"] = args.max_text_tokens
        extra["do_sample"] = args.do_sample
        extra["text_temperature"] = args.text_temperature
    if args.negative_prompt is not None:
        extra["negative_prompt"] = args.negative_prompt
    diffusion_params.extra_args = extra  # type: ignore
    outputs = list(omni.generate(prompts=formatted, sampling_params_list=params_list))
    if args.modality in ("img2text", "video2text"):
        for i, req_output in enumerate(outputs):
            text = getattr(req_output, "output", None) or getattr(req_output, "text", None)
            if text is None and hasattr(req_output, "custom_output"):
                text = (req_output.custom_output or {}).get("text_output")
            print(f"[Output {i}] {text}")
        return
    if args.modality in ("text2video", "video2video", "image2video"):
        for i, req_output in enumerate(outputs):
            custom = getattr(req_output, "custom_output", None) or {}
            frames = custom.get("video_frames")
            if not frames:
                print(f"[Output {i}] no video frames returned")
                continue
            mp4_path = os.path.join(args.output, f"lance_{i}.mp4")
            try:
                import imageio.v3 as iio
                import numpy as np
                arr = np.stack([np.asarray(f) for f in frames], axis=0)
                iio.imwrite(mp4_path, arr, fps=args.fps, codec="libx264", quality=8)
                print(f"[Output {i}] Saved MP4 ({len(frames)} frames, {arr.shape[1]}x{arr.shape[2]}) to {mp4_path}")
            except Exception as exc:
                print(f"[Output {i}] MP4 encode failed ({exc!r}); writing per-frame ONGs instead")
                frame_dir = os.path.join(args.output, f"lance_{i}_frames")
                os.makedirs(frame_dir, exist_ok=True)
                for j, frame in enumerate(frames):
                    frame.save(os.path.join(frame_dir, f"{j:04d}.png"))
                print(f"  → {frame_dir} ({len(frames)} frames)")
        return
    idx = 0
    for req_output in outputs:
        images = getattr(req_output, "images", None)
        if not images:
            continue
        for j, img in enumerate(images):
            path = os.path.join(args.output, f"lance_{idx}_{j}.png")
            img.save(path)
            print(f"[Output] Saved image to {path}")
        idx += 1
 if __name__ == "__main__":
    main()
@@ -2,13 +2,12 @@
 [Ming-flash-omni-2.0](https://github.com/inclusionAI/Ming) is an omni-modal model supporting text, image, video, and audio understanding, with text and speech outputs.
-vLLM-Omni supports three deployment modes:
+vLLM-Omni supports two deployment modes:
 | Mode | Deploy config | Output |
 |------|--------------|--------|
 | Thinker + Talker (omni-speech, default) | `vllm_omni/deploy/ming_flash_omni.yaml` | Text + Audio |
 | Thinker only (multimodal understanding) | `vllm_omni/deploy/ming_flash_omni_thinker_only.yaml` | Text |
 | Thinker + Imagegen (text-to-image / img2img) | `vllm_omni/deploy/ming_flash_omni_image.yaml` | Image (online-serving only at the moment) |
 For standalone TTS (talker only), see the [Ming-flash-omni-TTS section in the Text-To-Speech hub](../text_to_speech/README.md#ming-flash-omni-tts).
@@ -88,17 +87,6 @@ The default deploy YAML allocates thinker on GPUs 0–3 and talker on GPU 3 for
 Pass `--deploy-config /path/to/your_deploy.yaml` to any of the commands above to override the bundled deploy config.
 ### Image generation (text-to-image / img2img)
 The diffusion-stage image-generation path is currently only wired through
 the online OpenAI-compatible chat endpoint (`/v1/chat/completions` with
 `"modalities": ["image"]`). For payloads, optional knobs
 (`steps`/`cfg`/`seed`/`byte5_text`/`negative_prompt`), and the img2img
 reference-image flow, see the
 [image-gen section in the online-serving README](../../online_serving/ming_flash_omni/README.md#image-generation-text-to-image).
 `end2end.py` does not yet exercise the imagegen stage; that is tracked as
 follow-up work.
 ## Online serving
 For online serving via the OpenAI-compatible API, see [examples/online_serving/ming_flash_omni/README.md](../../online_serving/ming_flash_omni/README.md).
@@ -37,10 +37,6 @@ python end2end.py --modality text2text \
 # Custom resolution (image generation)
 python end2end.py --prompt "A futuristic cityscape at sunset" \
                  --width 2048 --height 1024 --think
 # Text-to-image with Cache-DiT acceleration
 python end2end.py --prompt "A cute cat" \
                  --cache-backend cache_dit \
 ```
 > **Note**: Default configuration works on a single **NVIDIA A100 (80GB)** or **H100** GPU.
@@ -139,8 +135,6 @@ python end2end.py \
 | `--tensor-parallel-size` | 1 | Number of GPUs for tensor parallelism |
 | `--enforce-eager` | False | Disable torch.compile |
 | `--enable-cpu-offload` | False | Enable module-wise (sequential) CPU offload to reduce peak VRAM |
 | `--cache-backend` | None | Set to `cache_dit` for Cache-DiT acceleration |
 | `--enable-cache-dit-summary` | False | Print Cache-DiT cache statistics after generation |
 ## Reducing GPU Memory Usage
@@ -155,18 +155,6 @@ def parse_args():
            "of extra CPU<->GPU transfers."
        ),
    )
    parser.add_argument(
        "--cache-backend",
        type=str,
        default=None,
        choices=["cache_dit"],
        help="Cache backend to use for image generation acceleration.",
    )
    parser.add_argument(
        "--enable-cache-dit-summary",
        action="store_true",
        help="Enable cache-dit summary logging after diffusion forward passes.",
    )
    nullify_stage_engine_defaults(parser)
    return parser.parse_args()
@@ -194,8 +182,6 @@ def main():
        tensor_parallel_size=args.tensor_parallel_size,
        enforce_eager=args.enforce_eager,
        enable_cpu_offload=args.enable_cpu_offload,
        cache_backend=args.cache_backend,
        enable_cache_dit_summary=args.enable_cache_dit_summary,
    )
    extra_args = {
@@ -239,7 +225,6 @@ def main():
    print(f"  Seed           : {args.seed}")
    print(f"  Think mode     : {args.think}")
    print(f"  TP size        : {args.tensor_parallel_size}")
    print(f"  Cache backend  : {args.cache_backend or 'none'}")
    print(f"{'=' * 60}\n")
    # Build prompt dict
@@ -35,7 +35,6 @@ This folder provides several entrypoints for experimenting with text-to-image di
 | `black-forest-labs/FLUX.2-klein-9B` | 1024 x 1024 | 37.1 | 32.3 |
 | `black-forest-labs/FLUX.2-dev` | 1024 x 1024 | 65.7 | >80 (CPU offload required) |
 | `HunyuanImage-3.0` | 1024 x 1024 | 80.0 (TP≥3)  | 160 |
 | `HiDream-I1-Full` | 1024 x 1024 | 63.7  | 57.7 |
 !!! info
 *Peak VRAM:  based on basic single-card usage, batch size =1, without any acceleration/optimization features. FLUX.2-dev requires `--enable-cpu-offload` on a single 80 GiB GPU.
@@ -95,7 +94,6 @@ python text_to_image.py \
 | `--lora-scale` | float | `1.0` | Scale factor for LoRA weights |
 | `--use-system-prompt` | str | `None` | System prompt preset: `en_unified`, `en_vanilla`, `en_recaption`, `en_think_recaption`, `dynamic`, `None`, or custom text. Recommended: `en_unified`. Only for HunyuanImage-3.0.|
 | `--system-prompt` | str | `None` | Custom system prompt text. Only used when `--use-system-prompt` is set to `custom`. Only for HunyuanImage-3.0.|
 | `--auxiliary-text-encoder` | str | `None` | Supplementary auxiliary text encoder parameters model name or path (especially for Hidream-l1-full). |
 **NextStep-1.1 specific arguments:**
@@ -179,24 +177,6 @@ python examples/offline_inference/text_to_image/text_to_image.py \
  --output flux2-dev.png
 ```
 ### HiDream-I1-Full Models
 The `--auxiliary-text-encoder` parameter is required when running HiDream‑I1‑Full:
 ```bash
 python examples/offline_inference/text_to_image/text_to_image.py \
  --model HiDream-ai/HiDream-I1-Full \
  --prompt "The setting sun of late autumn dyes the riverside with a warm orange hue" \
  --seed 42 \
  --guidance-scale 5.0 \
  --tensor-parallel-size 1 \
  --num-images-per-prompt 1 \
  --num-inference-steps 50 \
  --auxiliary-text-encoder meta-llama/Meta-Llama-3.1-8B-Instruct \
  --output /output.png
 ```
 ### Batch Requests (Multiple Prompts)
 You can pass multiple prompts in a single `generate` call.
@@ -300,12 +300,6 @@ def parse_args() -> argparse.Namespace:
        default=None,
        help=("Custom system prompt. Used when --use-system-prompt is custom. "),
    )
    parser.add_argument(
        "--auxiliary-text-encoder",
        type=str,
        default=None,
        help="Supplementary auxiliary text encoder parameters model name or path (especially for Hidream-l1-full).",
    )
    current_omni_platform.pre_register_and_update(parser)
    from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
@@ -398,7 +392,6 @@ def main():
        "profiler_config": args.profiler_config,
        "init_timeout": args.init_timeout,
        "stage_init_timeout": args.stage_init_timeout,
        "auxiliary_text_encoder": args.auxiliary_text_encoder,
        **lora_args,
        **quant_kwargs,
    }
@@ -14,13 +14,13 @@ list of supported architectures across all modalities, see
 | Model | HuggingFace repo | Stages | Voice cloning | Streaming | Special modes | Sample rate |
 |---|---|---|---|---|---|---|
-| CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ | — | 24 kHz |
+| CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ | — | 22.05 kHz |
 | Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ | — | 44.1 kHz |
 | GLM-TTS | `zai-org/GLM-TTS` | 2 (AR + DiT) | ✓ (required) | ✓ | — | 24 kHz |
 | Ming-flash-omni-TTS | `Jonathan1909/Ming-flash-omni-2.0` | single (talker only) | — (caption-controlled) | — | style / IP / basic captions | 44.1 kHz |
 | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | single (AR + codec) | ✓ (required) | ✓ | voice_clone, continuation | 48 kHz |
 | OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design, language hint | 24 kHz |
 | Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | 2 (talker + code2wav) | ✓ (Base) | ✓ | 3 task variants | 24 kHz |
 | VoxCPM | `openbmb/VoxCPM-0.5B` | split | ✓ | ✓ | — | 24 kHz |
 | VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | ✓ (online) | continuation | 48 kHz |
 | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | varies | ✓ | ✓ | voice presets | 24 kHz |
@@ -41,7 +41,7 @@ python examples/offline_inference/text_to_speech/<model>/end2end.py \
 ## CosyVoice3
-2-stage TTS pipeline (`talker` + `code2wav`) at 24 kHz.
+2-stage TTS pipeline (`talker` + `code2wav`) at 22.05 kHz.
 ### Prerequisites
 ```bash
@@ -93,35 +93,6 @@ Streaming is enabled by default via `async_chunk: true` in `vllm_omni/deploy/cos
 ---
 ## GLM-TTS
 2-stage TTS pipeline (AR + DiT flow-matching) at 24 kHz. Every request requires reference audio and its transcript for zero-shot voice cloning.
 ### Quick start
 ```bash
 python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
    --model zai-org/GLM-TTS \
    --text "你好，这是语音合成测试。" \
    --ref-audio /path/to/reference.wav \
    --ref-text "这是参考音频的文本内容。" \
    --output-dir ./output
 ```
 ### Architecture
 ```
 Text → [Stage 0: AR] → Speech Tokens → [Stage 1: DiT + HiFT] → Audio (24 kHz)
        (Llama-based)    (32k vocab)      (Flow Matching)
 ```
 ### Notes
 - `--ref-audio` and `--ref-text` are **required** together; GLM-TTS does not support text-only synthesis.
 - Reference audio should be 3-10 seconds.
 - First run may be slow due to lazy loading of WhisperVQ tokenizer and CampPlus ONNX speaker embedder.
 - Default sampling: temperature=1.0, top_k=25, top_p=0.8 (RAS method).
 - The `--model` path should point to the repository root (not `llm/` subdirectory).
 ---
 ## Fish Speech S2 Pro
 4B dual-AR text-to-speech model from FishAudio with the DAC codec at 44.1 kHz.
@@ -271,14 +242,6 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
    --lang zh
 ```
 ### Seed for Reproducibility
 ```bash
 python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
    --model k2-fsa/OmniVoice \
    --text "Hello, this is a test." \
    --seed 42
 ```
 ### Notes
 - Stage 0 (Generator): Qwen3-0.6B with 32-step iterative unmasking.
 - Stage 1 (Decoder): HiggsAudioV2 RVQ + DAC at 24 kHz.
@@ -345,6 +308,56 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
 ---
 ## VoxCPM
 Split-stage TTS. The hub example covers single TTS, single voice cloning, and streaming. Use `benchmarks/voxcpm/` for warmup, batch JSONL prompts, profiler injection, and offline TTFP / RTF measurement.
 ### Prerequisites
 ```bash
 pip install voxcpm soundfile
 # or use a local source tree:
 export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
 ```
 ### Quick start
 ```bash
 python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
    --model openbmb/VoxCPM-0.5B \
    --text "This is a split-stage VoxCPM synthesis example running on vLLM Omni."
 ```
 ### Voice cloning
 ```bash
 python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
    --model openbmb/VoxCPM-0.5B \
    --text "This sentence is synthesized with a cloned voice." \
    --ref-audio /path/to/reference.wav \
    --ref-text  "The exact transcript spoken in reference.wav."
 ```
 ### Streaming
 Pass `--streaming` together with the legacy async-chunk stage config:
 ```bash
 python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
    --model openbmb/VoxCPM-0.5B \
    --streaming \
    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
    --text "This is a split-stage VoxCPM streaming example running on vLLM Omni."
 ```
 ### Persistent HF config (optional)
 The engine auto-patches a missing `model_type` in the checkpoint's `config.json` in-memory at startup (see `vllm_omni/engine/arg_utils.py`), so no manual setup is required.
 If you'd rather maintain a persistent patched config dir (e.g. to share across runs or to add fields the auto-patcher doesn't set), export `VLLM_OMNI_VOXCPM_HF_CONFIG_PATH` to its location — both the bundled deploy and async-chunk yamls read it via `${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,}`.
 ### Notes
 - Non-streaming auto-loads `vllm_omni/deploy/voxcpm.yaml` from the model directory name / HF `model_type`. Streaming uses the legacy `voxcpm_async_chunk.yaml` passed via `--stage-configs-path`.
 - Streaming is currently single-request oriented.
 - `--ref-text` must be the real transcript of `--ref-audio`; mismatched text degrades quality.
 - For online serving, see the [VoxCPM section in the online hub](../../online_serving/text_to_speech/README.md#voxcpm). For benchmark reporting, see [`benchmarks/voxcpm`](../../../benchmarks/voxcpm/README.md).
 ---
 ## VoxCPM2
 Single-stage native AR TTS at 48 kHz. Pipeline: `feat_encoder → MiniCPM4 → FSQ → residual_lm → LocDiT → AudioVAE`.
@@ -1,194 +0,0 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 """GLM-TTS End-to-End Offline Inference Example.
 GLM-TTS is a two-stage TTS system:
  - Stage 0 (AR): Llama-based model generates speech tokens from text
  - Stage 1 (DiT): Flow matching model converts speech tokens to audio
 Usage:
    # Sync two-stage (default)
    python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
        --model /path/to/GLM-TTS \
        --text "你好，这是一个语音合成测试。" \
        --ref-audio /path/to/reference.wav \
        --ref-text "参考音频的转录文本。" \
        --output-dir ./output
    # Async chunk mode (streaming DiT)
    python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
        --model /path/to/GLM-TTS --async-chunk \
        --text "你好，这是一个语音合成测试。" \
        --ref-audio /path/to/reference.wav \
        --ref-text "参考音频的转录文本。" \
        --output-dir ./output
 """
 import base64
 import io
 import logging
 import os
 import tempfile
 import time
 from typing import Any
 from urllib.request import urlopen
 import numpy as np
 import soundfile as sf
 import torch
 import yaml
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 from vllm_omni import Omni
 from vllm_omni.model_executor.models.glm_tts.glm_tts import build_glm_tts_prefill_metadata
 logger = logging.getLogger(__name__)
 DEFAULT_DEPLOY_CONFIG = os.path.join(
    os.path.dirname(__file__),
    "..",
    "..",
    "..",
    "..",
    "vllm_omni",
    "deploy",
    "glm_tts.yaml",
 )
 SAMPLE_RATE = 24000
 def _load_ref_audio(ref_audio: str) -> tuple[torch.Tensor, int]:
    """Load reference audio from file path, URL, or data URI."""
    if ref_audio.startswith(("http://", "https://")):
        with urlopen(ref_audio, timeout=60) as response:
            audio_obj: Any = io.BytesIO(response.read())
    elif ref_audio.startswith("data:"):
        _, _, encoded = ref_audio.partition(",")
        audio_obj = io.BytesIO(base64.b64decode(encoded))
    else:
        audio_obj = ref_audio
    wav_np, sr = sf.read(audio_obj, dtype="float32")
    if wav_np.ndim > 1:
        wav_np = wav_np.mean(axis=1)
    return torch.from_numpy(wav_np), int(sr)
 def _concat_audio(audio_val: Any) -> np.ndarray:
    """Concatenate audio tensors from multimodal output."""
    if isinstance(audio_val, list):
        tensors = [torch.as_tensor(t).float().reshape(-1) for t in audio_val if t is not None]
        if not tensors:
            return np.zeros((0,), dtype=np.float32)
        return torch.cat(tensors, dim=-1).cpu().numpy().astype(np.float32, copy=False)
    if isinstance(audio_val, torch.Tensor):
        return audio_val.float().cpu().numpy().reshape(-1)
    return np.asarray(audio_val, dtype=np.float32).reshape(-1)
 def _extract_sample_rate(audio_mm: dict) -> int:
    """Extract sample rate from multimodal output dict."""
    sr_raw = audio_mm.get("sr", SAMPLE_RATE)
    if isinstance(sr_raw, list):
        sr_raw = sr_raw[-1] if sr_raw else SAMPLE_RATE
    if hasattr(sr_raw, "item"):
        return int(sr_raw.item())
    return int(sr_raw)
 def _modify_deploy_config(base_path: str, async_chunk: bool) -> str:
    """Build deploy config with explicit sync/async mode and eager execution.
    Mirrors the logic in ``tests/e2e/offline_inference/test_glm_tts.py``
    (``_get_deploy_config``) so that example runs match CI behavior.
    """
    with open(base_path) as f:
        cfg = yaml.safe_load(f)
    cfg["async_chunk"] = async_chunk
    for stage in cfg.get("stages", []):
        stage["enforce_eager"] = True
        if stage.get("stage_id") == 0:
            stage["async_scheduling"] = bool(async_chunk)
    tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False, prefix="glm_tts_")
    yaml.dump(cfg, tmp)
    tmp.close()
    return tmp.name
 def main(args):
    """Run offline GLM-TTS inference."""
    os.makedirs(args.output_dir, exist_ok=True)
    base_deploy_config = args.deploy_config or DEFAULT_DEPLOY_CONFIG
    deploy_config_path = _modify_deploy_config(base_deploy_config, args.async_chunk)
    ref_audio_wav, ref_audio_sr = _load_ref_audio(args.ref_audio)
    if not args.ref_text:
        raise ValueError("GLM-TTS requires --ref-audio and --ref-text.")
    inputs = [
        {
            "prompt": args.text,
            "multi_modal_data": {
                "audio": (ref_audio_wav.float().cpu().numpy(), ref_audio_sr),
            },
            "modalities": ["audio"],
            "mm_processor_kwargs": {"prompt_text": args.ref_text},
            "additional_information": build_glm_tts_prefill_metadata(
                args.model,
                args.text,
                args.ref_text,
            ),
        }
    ]
    omni = Omni(
        model=args.model,
        stage_configs_path=deploy_config_path,
        log_stats=args.log_stats,
        stage_init_timeout=args.stage_init_timeout,
    )
    t_start = time.perf_counter()
    outputs = omni.generate(inputs)
    elapsed = (time.perf_counter() - t_start) * 1000
    assert outputs, "No outputs returned"
    audio_mm = outputs[0].multimodal_output
    assert "audio" in audio_mm, "No audio output found"
    audio = _concat_audio(audio_mm["audio"])
    sr = _extract_sample_rate(audio_mm)
    out_path = os.path.join(args.output_dir, "output.wav")
    sf.write(out_path, audio, samplerate=sr, format="WAV")
    logger.info("Saved %s (%.2fs @ %dHz)", out_path, len(audio) / sr, sr)
    logger.info("Total inference: %.1f ms", elapsed)
 def parse_args():
    parser = FlexibleArgumentParser(description="GLM-TTS Text-to-Speech Example")
    parser.add_argument("--model", type=str, required=True, help="Model path")
    parser.add_argument("--text", type=str, default="你好，这是一个语音合成测试。")
    parser.add_argument("--output-dir", type=str, default="./output")
    parser.add_argument("--ref-audio", type=str, required=True, help="Reference WAV path/URL")
    parser.add_argument("--ref-text", type=str, required=True, help="Transcript of ref audio")
    parser.add_argument("--deploy-config", type=str, default=None)
    parser.add_argument(
        "--async-chunk",
        action="store_true",
        default=False,
        help="Enable async_chunk mode (streaming DiT). Default: sync two-stage.",
    )
    parser.add_argument("--log-stats", action="store_true")
    parser.add_argument("--stage-init-timeout", type=int, default=600)
    return parser.parse_args()
 if __name__ == "__main__":
    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
    )
    main(parse_args())
@@ -1,38 +0,0 @@
 # higgs-audio v2 — offline example
 Drives Stage 0 (DualFFN talker) + Stage 1 (HiggsAudio codec) for `bosonai/higgs-audio-v2-generation-3B-base` end-to-end through the vLLM-Omni engine and writes a 24 kHz mono WAV per prompt.
 ## Prerequisites
 Voice clone needs `transformers>=5.3.0` — vllm-omni loads the audio codec via HF's `HiggsAudioV2TokenizerModel`, instantiated from the `k2-fsa/OmniVoice/audio_tokenizer/` subdirectory (only that ~806 MB subdir is downloaded). The boson-ai standalone tokenizer repo's `model.safetensors` is actually a copy of the 3B talker LM, so HF can't load it directly; the k2 bundle ships the same codec weights repackaged with HF-compatible key naming.
 ```bash
 pip install -U "transformers>=5.3.0"
 ```
 ## Quick start
 Plain TTS:
 ```bash
 python examples/offline_inference/text_to_speech/higgs_audio_v2/end2end.py \
    --texts "Hello world." "The quick brown fox jumps over the lazy dog." \
    --output-dir results/wavs
 ```
 ## Voice cloning
 Pass both `--ref-audio` and `--ref-text` together:
 ```bash
 python examples/offline_inference/text_to_speech/higgs_audio_v2/end2end.py \
    --texts "Hello, this is a cloned voice." \
    --ref-audio /path/to/reference.wav \
    --ref-text  "Exact transcript spoken in reference.wav." \
    --output-dir results/wavs
 ```
 ## Notes
 - Output: 24 kHz mono WAV.
 - Deploy config: `vllm_omni/deploy/higgs_audio_v2.yaml` (auto-loaded by HF `model_type`).
 - `--ref-text` must be the real transcript of `--ref-audio`; mismatched text degrades cloned-voice quality.
 - For online serving, see [`examples/online_serving/text_to_speech/higgs_audio_v2/`](../../../online_serving/text_to_speech/higgs_audio_v2/).
@@ -1,205 +0,0 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 """Offline higgs-audio v2 inference example.
 Runs Stage 0 (DualFFN talker) + Stage 1 (HiggsAudio codec) end-to-end
 through the vLLM-Omni engine without going through the HTTP server, and
 saves a 24 kHz mono WAV per prompt.
 Example:
    python examples/offline_inference/text_to_speech/higgs_audio_v2/end2end.py \\
        --texts "Hello world." \\
                "The quick brown fox jumps over the lazy dog." \\
        --output-dir results/wavs
 """
 from __future__ import annotations
 import os
 # DeepGEMM FP8 kernels require an optional backend that may not be installed.
 # Disable the warmup before importing vLLM so engine startup falls back to the
 # regular gemm path. Users with deep_gemm installed can override these.
 os.environ.setdefault("VLLM_USE_DEEP_GEMM", "0")
 os.environ.setdefault("VLLM_MOE_USE_DEEP_GEMM", "0")
 import time
 from pathlib import Path
 import numpy as np
 import soundfile as sf
 import torch
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 from vllm_omni import Omni
 from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 SAMPLE_RATE = 24_000
 DEFAULT_TEXTS = (
    "Hello world.",
    "The quick brown fox jumps over the lazy dog.",
 )
 def parse_args():
    parser = FlexibleArgumentParser(description="Offline higgs-audio v2 inference")
    parser.add_argument(
        "--model",
        type=str,
        default="bosonai/higgs-audio-v2-generation-3B-base",
        help="Stage-0 talker model id or path.",
    )
    parser.add_argument(
        "--texts",
        type=str,
        nargs="+",
        default=list(DEFAULT_TEXTS),
        help="One or more plain-text prompts to synthesize.",
    )
    parser.add_argument(
        "--output-dir",
        type=str,
        default="results/wavs",
        help="Directory to write per-prompt WAV files.",
    )
    parser.add_argument(
        "--deploy-config",
        type=str,
        default=None,
        help="Override the deploy config path. Auto-loads "
        "vllm_omni/deploy/higgs_audio_v2.yaml from the HF model_type by default.",
    )
    parser.add_argument(
        "--max-new-tokens",
        type=int,
        default=500,
        help="Cap on Stage-0 codec frames per request.",
    )
    parser.add_argument(
        "--ref-audio",
        type=str,
        default=None,
        help="Reference clip for voice clone (path to a WAV file). Paired with --ref-text.",
    )
    parser.add_argument(
        "--ref-text",
        type=str,
        default=None,
        help="Transcript of the reference clip. Required when --ref-audio is set.",
    )
    nullify_stage_engine_defaults(parser)
    return parser.parse_args()
 def _slugify(text: str) -> str:
    import re
    slug = re.sub(r"\s+", "_", text.strip().lower())
    slug = re.sub(r"[^a-z0-9_]+", "", slug)
    return slug[:48] or "prompt"
 def _extract_pcm(multimodal_output: dict) -> torch.Tensor:
    """Pull the final concatenated PCM tensor out of a request's multimodal_output."""
    audio = multimodal_output.get("model_outputs")
    if audio is None:
        audio = multimodal_output.get("audio")
    if audio is None:
        raise ValueError(f"no audio key in multimodal_output: {list(multimodal_output.keys())}")
    if isinstance(audio, list):
        valid = [torch.as_tensor(a).float().cpu().reshape(-1) for a in audio if a is not None]
        if not valid:
            raise ValueError("audio list is empty")
        return torch.cat(valid, dim=0) if len(valid) > 1 else valid[0]
    return torch.as_tensor(audio).float().cpu().reshape(-1)
 def _pcm_to_int16(pcm: torch.Tensor) -> np.ndarray:
    arr = pcm.numpy()
    if arr.dtype.kind == "f":
        arr = np.clip(arr, -1.0, 1.0)
        arr = (arr * 32767.0).astype(np.int16)
    else:
        arr = arr.astype(np.int16)
    return arr
 def main():
    args = parse_args()
    if (args.ref_audio is None) != (args.ref_text is None):
        raise SystemExit("--ref-audio and --ref-text must be supplied together")
    output_dir = Path(args.output_dir)
    output_dir.mkdir(parents=True, exist_ok=True)
    engine = Omni(model=args.model, deploy_config=args.deploy_config)
    # Build prompt_token_ids using the same path serving_speech.py takes online.
    from transformers import AutoProcessor
    from vllm_omni.model_executor.models.higgs_audio_v2.higgs_audio_v2_tokenizer import (
        build_plain_text_prompt,
        build_voice_clone_prompt,
        input_ids_to_python_list,
    )
    processor = AutoProcessor.from_pretrained(args.model, trust_remote_code=True)
    # Voice-clone path: load the reference clip once. The HF processor will
    # encode it via the bundled HiggsAudioV2TokenizerModel each time we build
    # a prompt below. This is cheap on CPU for a few-second clip.
    ref_wav: np.ndarray | None = None
    ref_sr: int | None = None
    if args.ref_audio is not None:
        ref_wav, ref_sr = sf.read(args.ref_audio, always_2d=False)
        if ref_wav.ndim == 2:
            ref_wav = ref_wav.mean(axis=1)
    print(f"Model       : {args.model}")
    print(f"Prompts     : {len(args.texts)}")
    print(f"Output dir  : {output_dir}")
    print(f"Voice clone : {'yes' if ref_wav is not None else 'no'}")
    # Run one prompt at a time. The Stage-0 talker's per-slot audio state is
    # request-scoped; submitting multiple prompts in the same engine.generate()
    # call would batch them in the AR runner and exercise a code path that is
    # not validated for this model yet.
    total_elapsed = 0.0
    total_dur = 0.0
    for text in args.texts:
        if ref_wav is not None:
            out = build_voice_clone_prompt(processor, text, ref_wav, int(ref_sr), args.ref_text)
            prompt = {
                "prompt_token_ids": out["prompt_token_ids"],
                # Bare tensors (NOT list-wrapped): the msgspec serializer in
                # vllm_omni.data_entry_keys routes torch.Tensor → tensor_data
                # and list[Tensor] → list_data (which silently strips tensors).
                "additional_information": {
                    "audio_input_ids": out["audio_input_ids"],
                    "audio_input_ids_mask": out["audio_input_ids_mask"],
                },
            }
        else:
            inputs = build_plain_text_prompt(processor, text)
            prompt = {"prompt_token_ids": input_ids_to_python_list(inputs)}
        t_start = time.perf_counter()
        outputs = engine.generate([prompt])
        elapsed = time.perf_counter() - t_start
        total_elapsed += elapsed
        mm = outputs[0].outputs[0].multimodal_output
        pcm = _extract_pcm(mm)
        slug = _slugify(text)
        out_path = output_dir / f"{slug}.wav"
        sf.write(str(out_path), _pcm_to_int16(pcm), SAMPLE_RATE, format="WAV", subtype="PCM_16")
        dur = pcm.numel() / SAMPLE_RATE
        total_dur += dur
        print(f"  {slug:<50} dur={dur:5.2f}s  -> {out_path}")
    rtf = total_elapsed / total_dur if total_dur > 0 else float("inf")
    print(f"Total infer : {total_elapsed:.2f}s  total audio: {total_dur:.2f}s  RTF: {rtf:.3f}")
 if __name__ == "__main__":
    os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
    main()
@@ -80,12 +80,6 @@ def run_e2e():
        default=600,
        help="Stage initialization timeout in seconds",
    )
    parser.add_argument(
        "--seed",
        type=int,
        default=None,
        help="Random seed for generation",
    )
    nullify_stage_engine_defaults(parser)
    args = parser.parse_args()
@@ -128,7 +122,7 @@ def run_e2e():
    if mm_processor_kwargs:
        prompts["mm_processor_kwargs"] = mm_processor_kwargs
-    sampling_params_list = [OmniDiffusionSamplingParams(extra_args={"seed": args.seed})]
+    sampling_params_list = [OmniDiffusionSamplingParams()]
    print(f"Generating speech for: {args.text}")
@@ -43,8 +43,8 @@ def _estimate_prompt_len(
    """
    try:
        from vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts import Qwen3TTSConfig
-        from vllm_omni.model_executor.models.qwen3_tts.prompt_embeds_builder import (
+        from vllm_omni.model_executor.models.qwen3_tts.qwen3_tts_talker import (
-            Qwen3TTSPromptEmbedsBuilder,
+            Qwen3TTSTalkerForConditionalGeneration,
        )
        if model_name not in _cache:
@@ -124,7 +124,7 @@ def _estimate_prompt_len(
            except Exception:
                return None
-        return Qwen3TTSPromptEmbedsBuilder.estimate_prompt_len_from_additional_information(
+        return Qwen3TTSTalkerForConditionalGeneration.estimate_prompt_len_from_additional_information(
            additional_information=additional_information,
            task_type=task_type,
            tokenize_prompt=lambda t: tok(t, padding=False)["input_ids"],
@@ -0,0 +1,220 @@
 """Minimal offline VoxCPM example for vLLM Omni."""
 from __future__ import annotations
 import asyncio
 import time
 from pathlib import Path
 from typing import Any
 import soundfile as sf
 import torch
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 from vllm_omni import AsyncOmni, Omni
 from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 def _build_prompt(args) -> dict[str, Any]:
    additional_information: dict[str, list[Any]] = {
        "text": [args.text],
        "cfg_value": [args.cfg_value],
        "inference_timesteps": [args.inference_timesteps],
        "min_len": [args.min_len],
        "max_new_tokens": [args.max_new_tokens],
    }
    if args.streaming_prefix_len is not None:
        additional_information["streaming_prefix_len"] = [args.streaming_prefix_len]
    if args.ref_audio is not None:
        additional_information["ref_audio"] = [args.ref_audio]
    if args.ref_text is not None:
        additional_information["ref_text"] = [args.ref_text]
    return {
        "prompt_token_ids": [1],
        "additional_information": additional_information,
    }
 def _extract_audio_tensor(mm: dict[str, Any]) -> torch.Tensor:
    audio = mm.get("audio", mm.get("model_outputs"))
    if audio is None:
        raise ValueError("No audio output found in multimodal output.")
    if isinstance(audio, list):
        parts = [torch.as_tensor(item).float().cpu().reshape(-1) for item in audio]
        audio = torch.cat(parts, dim=-1) if parts else torch.zeros(0)
    if not isinstance(audio, torch.Tensor):
        audio = torch.as_tensor(audio)
    return audio.float().cpu().reshape(-1)
 def _extract_sample_rate(mm: dict[str, Any]) -> int:
    sr_raw = mm.get("sr", 24000)
    if isinstance(sr_raw, list) and sr_raw:
        sr_raw = sr_raw[-1]
    if hasattr(sr_raw, "item"):
        return int(sr_raw.item())
    return int(sr_raw)
 def _save_audio(audio: torch.Tensor, sample_rate: int, output_dir: Path, request_id: str) -> Path:
    output_dir.mkdir(parents=True, exist_ok=True)
    output_path = output_dir / f"output_{request_id}.wav"
    sf.write(
        output_path,
        audio.float().cpu().clamp(-1.0, 1.0).numpy(),
        sample_rate,
        format="WAV",
        subtype="PCM_16",
    )
    return output_path
 async def _run_streaming(args) -> Path:
    prompt = _build_prompt(args)
    output_dir = Path(args.output_dir) if args.output_dir is not None else Path("output_audio_streaming")
    request_id = "streaming_example"
    sample_rate = 24000
    buffered_samples = 0
    chunks: list[torch.Tensor] = []
    started = time.perf_counter()
    omni = AsyncOmni(
        model=args.model,
        stage_configs_path=args.stage_configs_path,
        deploy_config=args.deploy_config,
        log_stats=args.log_stats,
        stage_init_timeout=args.stage_init_timeout,
    )
    try:
        async for stage_output in omni.generate(prompt, request_id=request_id):
            mm = getattr(stage_output, "multimodal_output", None)
            if not isinstance(mm, dict):
                request_output = getattr(stage_output, "request_output", None)
                if request_output is None:
                    continue
                mm = getattr(request_output, "multimodal_output", None)
                if not isinstance(mm, dict) and getattr(request_output, "outputs", None):
                    mm = getattr(request_output.outputs[0], "multimodal_output", None)
            if not isinstance(mm, dict):
                continue
            audio = _extract_audio_tensor(mm)
            if audio.numel() == 0:
                continue
            sample_rate = _extract_sample_rate(mm)
            if audio.numel() > buffered_samples:
                delta = audio[buffered_samples:]
                buffered_samples = int(audio.numel())
            else:
                delta = audio
                buffered_samples += int(delta.numel())
            if delta.numel() > 0:
                chunks.append(delta)
        if not chunks:
            raise RuntimeError("No streaming audio chunks received from VoxCPM.")
        output_audio = torch.cat(chunks, dim=0)
        output_path = _save_audio(output_audio, sample_rate, output_dir, request_id)
        print(f"Saved streaming audio to: {output_path} ({time.perf_counter() - started:.2f}s)")
        return output_path
    finally:
        omni.shutdown()
 def _run_sync(args) -> Path:
    prompt = _build_prompt(args)
    output_dir = Path(args.output_dir) if args.output_dir is not None else Path("output_audio")
    request_id = "sync_example"
    started = time.perf_counter()
    last_mm: dict[str, Any] | None = None
    omni = Omni(
        model=args.model,
        stage_configs_path=args.stage_configs_path,
        deploy_config=args.deploy_config,
        log_stats=args.log_stats,
        stage_init_timeout=args.stage_init_timeout,
    )
    for stage_outputs in omni.generate(prompt):
        request_output = getattr(stage_outputs, "request_output", None)
        if request_output is None:
            continue
        outputs = getattr(request_output, "outputs", None)
        if outputs:
            for output in outputs:
                mm = getattr(output, "multimodal_output", None)
                if isinstance(mm, dict):
                    last_mm = mm
        mm = getattr(request_output, "multimodal_output", None)
        if isinstance(mm, dict):
            last_mm = mm
    if last_mm is None:
        raise RuntimeError("No audio output received from VoxCPM.")
    output_path = _save_audio(
        _extract_audio_tensor(last_mm),
        _extract_sample_rate(last_mm),
        output_dir,
        request_id,
    )
    print(f"Saved audio to: {output_path} ({time.perf_counter() - started:.2f}s)")
    return output_path
 def parse_args():
    parser = FlexibleArgumentParser(description="Minimal offline VoxCPM example for vLLM Omni.")
    parser.add_argument("--model", type=str, required=True, help="Local VoxCPM model directory.")
    parser.add_argument(
        "--deploy-config",
        type=str,
        default=None,
        help=("Override the deploy config path."),
    )
    parser.add_argument(
        "--stage-configs-path",
        type=str,
        default=None,
        help=(
            "Legacy stage_args yaml path. Required for streaming "
            "(vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml); "
            "leave unset for non-streaming to use the auto-resolved deploy config."
        ),
    )
    parser.add_argument(
        "--streaming",
        action="store_true",
        default=False,
        help="Stream audio chunks as they arrive via AsyncOmni.",
    )
    parser.add_argument("--text", type=str, required=True, help="Input text for synthesis.")
    parser.add_argument("--ref-audio", type=str, default=None, help="Reference audio path for voice cloning.")
    parser.add_argument("--ref-text", type=str, default=None, help="Transcript of the reference audio.")
    parser.add_argument("--output-dir", type=str, default=None, help="Output directory for generated wav files.")
    parser.add_argument("--cfg-value", type=float, default=2.0, help="Guidance value passed to VoxCPM.")
    parser.add_argument("--inference-timesteps", type=int, default=10, help="Number of diffusion timesteps.")
    parser.add_argument("--min-len", type=int, default=2, help="Minimum latent length.")
    parser.add_argument("--max-new-tokens", type=int, default=4096, help="Maximum latent length.")
    parser.add_argument(
        "--streaming-prefix-len",
        type=int,
        default=3,
        help="Streaming prefix length used by voxcpm_async_chunk.yaml.",
    )
    parser.add_argument("--stage-init-timeout", type=int, default=600, help="Stage initialization timeout in seconds.")
    parser.add_argument("--log-stats", action="store_true", help="Enable vLLM Omni stats logging.")
    nullify_stage_engine_defaults(parser)
    args = parser.parse_args()
    if (args.ref_audio is None) != (args.ref_text is None):
        raise ValueError("Voice cloning requires --ref-audio and --ref-text together.")
    return args
 def main(args) -> None:
    route = "streaming" if args.streaming else "sync"
    print(f"Model: {args.model}")
    print(f"Deploy config: {args.deploy_config or '<auto from HF model_type>'}")
    print(f"Stage configs path: {args.stage_configs_path or '<unused>'}")
    print(f"Route: {route}")
    if route == "streaming":
        asyncio.run(_run_streaming(args))
    else:
        _run_sync(args)
 if __name__ == "__main__":
    main(parse_args())
@@ -163,8 +163,8 @@ def parse_args() -> argparse.Namespace:
        "--quantization",
        type=str,
        default=None,
-        choices=["fp8", "mxfp8", "mxfp4", "mxfp4_dualscale", "int8", "gguf"],
+        choices=["fp8", "mxfp8", "int8", "gguf"],
-        help="Quantization method for the transformer. mxfp8: W8A8 MXFP8 (NPU). mxfp4: W4A4 MXFP4 (NPU). mxfp4_dualscale: W4A4 MXFP4 dual-scale + BF16 fallback mixed (NPU). fp8: online FP8 (GPU).",
+        help="Quantization method for the transformer. mxfp8: W8A8 MXFP8 online quant (NPU). fp8: online FP8 (GPU).",
    )
    # Distributed and parallel execution
@@ -237,27 +237,6 @@ def parse_args() -> argparse.Namespace:
    return parser.parse_args()
 def _extract_peak_memory_mb(result: Any) -> float:
    """Pull worker-reported peak VRAM (MiB) generation result.
    Mirrors vllm_omni/entrypoints/openai/serving_video.py:_extract_peak_memory_mb.
    """
    if isinstance(result, list):
        result = result[0] if result else None
    if result is None:
        return 0.0
    val = getattr(result, "peak_memory_mb", 0.0)
    if not val:
        inner = getattr(result, "request_output", None)
        if isinstance(inner, list):
            inner = inner[0] if inner else None
        val = getattr(inner, "peak_memory_mb", 0.0)
    try:
        return float(val or 0.0)
    except (TypeError, ValueError):
        return 0.0
 def main():
    args = parse_args()
    model_class_name = args.model_class_name
@@ -369,10 +348,6 @@ def main():
    # Print profiling results
    print(f"Total generation time: {generation_time:.4f} seconds ({generation_time * 1000:.2f} ms)")
    peak_mb = _extract_peak_memory_mb(frames)
    if peak_mb:
        print(f"Worker peak GPU memory (reserved): {peak_mb:.2f} MiB ({peak_mb / 1024:.2f} GiB)")
    audio = None
    if isinstance(frames, list):
        frames = frames[0] if frames else None
@@ -1,105 +1,24 @@
 # VACE Video Generation
-Generate videos from text prompts, images, or video conditions using vLLM-Omni's VACE diffusion pipeline.
+[VACE](https://github.com/ali-vilab/VACE) (Video All-in-one Creation Engine) supports multiple video tasks through a single model.
- `vace_video_generation.py`: command-line script for multi-mode video generation with advanced options.
+| Model | Architecture | Model Weights (bf16) | HuggingFace |
 |-------|-------------|----------------------|-------------|
 | Wan2.1-VACE (1.3B) | Wan2.1 | ~10 GB | [Wan-AI/Wan2.1-VACE-1.3B-diffusers](https://huggingface.co/Wan-AI/Wan2.1-VACE-1.3B-diffusers) |
 | Wan2.1-VACE (14B) | Wan2.1 | ~38 GB | [Wan-AI/Wan2.1-VACE-14B-diffusers](https://huggingface.co/Wan-AI/Wan2.1-VACE-14B-diffusers) |
-## Table of Contents
+## Text-to-Video (T2V)
 - [Overview](#overview)
 - [Quick Start](#quick-start)
 - [Key Arguments](#key-arguments)
 - [More CLI Examples](#more-cli-examples)
 ## Overview
 [VACE](https://github.com/ali-vilab/VACE) (Video All-in-one Creation Engine) supports multiple video tasks through a single unified model, including text-to-video, image-to-video, first-last-frame interpolation, inpainting, and reference image-guided generation.
 ### Supported Models
 | Model | Architecture | Peak VRAM (GiB) * | Model Weights (GiB) | HuggingFace |
 |-------|-------------|-------------------|---------------------|-------------|
 | Wan2.1-VACE (1.3B) | Wan2.1 | TBD | ~10 | [Wan-AI/Wan2.1-VACE-1.3B-diffusers](https://huggingface.co/Wan-AI/Wan2.1-VACE-1.3B-diffusers) |
 | Wan2.1-VACE (14B) | Wan2.1 | TBD | ~38 | [Wan-AI/Wan2.1-VACE-14B-diffusers](https://huggingface.co/Wan-AI/Wan2.1-VACE-14B-diffusers) |
 !!! info
 *Peak VRAM: based on basic single-card usage, 480x832 resolution, 81 frames, 30 inference steps, without any acceleration/optimization features.
 Default model: `Wan-AI/Wan2.1-VACE-14B-diffusers`
 ## Quick Start
 ### Python API
 Text-to-video generation:
 ```python
 import torch
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 if __name__ == "__main__":
    omni = Omni(model="Wan-AI/Wan2.1-VACE-1.3B-diffusers")
    prompt = "A sleek robot stands in a vast warehouse filled with boxes"
    outputs = omni.generate(
        prompt,
        OmniDiffusionSamplingParams(
            height=480,
            width=832,
            num_frames=81,
            num_inference_steps=30,
            guidance_scale=5.0,
        ),
    )
    video = outputs[0].images
    from diffusers.utils import export_to_video
    export_to_video(list(video[0]), "t2v_output.mp4", fps=16)
    omni.close()
 ```
 ### Local CLI Usage
 ```bash
 python vace_video_generation.py \
  --mode t2v \
  --prompt "A sleek robot stands in a vast warehouse filled with boxes" \
  --height 480 --width 832 --num-frames 81 \
  --num-inference-steps 30 --guidance-scale 5.0 --flow-shift 5.0 \
  --output t2v_output.mp4
 ```
-## Key Arguments
+## Image-to-Video (I2V)
 | Argument | Type | Default | Description |
 | -------- | ---- | ------- | ----------- |
 | `--mode` | str | `"t2v"` | VACE task mode: `t2v`, `i2v`, `v2lf`, `flf2v`, `inpaint`, `r2v` |
 | `--model` | str | `"Wan-AI/Wan2.1-VACE-14B-diffusers"` | Model ID or local path |
 | `--prompt` | str | `"A cat walking in a garden"` | Text description of desired video |
 | `--negative-prompt` | str | `""` | Negative prompt for classifier-free guidance |
 | `--image` | str | `None` | Input image path (for I2V, R2V, FLF2V, inpaint modes) |
 | `--last-image` | str | `None` | Last frame image path (for FLF2V mode) |
 | `--height` | int | `480` | Output video height in pixels (should be a multiple of 16) |
 | `--width` | int | `832` | Output video width in pixels (should be a multiple of 16) |
 | `--num-frames` | int | `81` | Number of video frames to generate |
 | `--num-inference-steps` | int | `30` | Number of denoising steps (more steps = higher quality, slower) |
 | `--guidance-scale` | float | `5.0` | Classifier-free guidance scale |
 | `--flow-shift` | float | `5.0` | Scheduler flow shift parameter |
 | `--seed` | int | `42` | Random seed for deterministic sampling |
 | `--fps` | int | `16` | Frames per second for the saved MP4 |
 | `--output` | str | `"vace_output.mp4"` | Path to save the generated video |
 | `--vae-use-tiling` | flag | on | Enable VAE tiling for memory optimization |
 | `--cfg-parallel-size` | int | `1` | Set to `2` to enable CFG Parallel |
 | `--ulysses-degree` | int | `1` | Ulysses sequence parallel degree for multi-GPU inference |
 | `--ring-degree` | int | `1` | Ring sequence parallel degree for hybrid Ulysses + Ring inference |
 | `--tensor-parallel-size` | int | — | Tensor parallel size |
 | `--enforce-eager` | flag | off | Disable torch.compile |
 > If you encounter OOM errors, try `--vae-use-tiling` or multi-GPU parallelism options (`--ulysses-degree`, `--cfg-parallel-size`).
 ## More CLI Examples
 ### Image-to-Video (I2V)
 First frame is kept, remaining frames are generated:
@@ -112,7 +31,7 @@ python vace_video_generation.py \
  --output i2v_output.mp4
 ```
-### First-Last-Frame Interpolation (FLF2V)
+## First-Last-Frame Interpolation (FLF2V)
 ```bash
 python vace_video_generation.py \
@@ -123,7 +42,7 @@ python vace_video_generation.py \
  --output flf2v_output.mp4
 ```
-### Inpainting
+## Inpainting
 Center vertical stripe is masked and regenerated:
@@ -136,7 +55,7 @@ python vace_video_generation.py \
  --output inpaint_output.mp4
 ```
-### Reference Image-guided (R2V)
+## Reference Image-guided (R2V)
 ```bash
 python vace_video_generation.py \
@@ -146,3 +65,24 @@ python vace_video_generation.py \
  --height 480 --width 832 --num-frames 81 \
  --output r2v_output.mp4
 ```
 ## Key Arguments
 - `--mode`: VACE task mode (`t2v`, `i2v`, `flf2v`, `inpaint`, `r2v`).
 - `--model`: Model ID (default: `Wan-AI/Wan2.1-VACE-1.3B-diffusers`).
 - `--image`: Input image for I2V, inpainting, and R2V modes.
 - `--last-image`: Last frame image for FLF2V mode.
 - `--prompt`: Text description of desired video.
 - `--height/--width`: Output resolution (default 480x832). Dimensions should be multiples of 16.
 - `--num-frames`: Number of frames (default 81).
 - `--guidance-scale`: CFG scale (default 5.0).
 - `--flow-shift`: Scheduler flow shift (default 5.0).
 - `--num-inference-steps`: Number of denoising steps (default 30).
 - `--fps`: Frames per second for the saved MP4 (default 16).
 - `--output`: Path to save the generated video.
 - `--vae-use-tiling`: Enable VAE tiling for memory optimization.
 - `--ulysses-degree`: Ulysses sequence parallelism degree for multi-GPU.
 - `--cfg-parallel-size`: CFG parallel size for multi-GPU.
 - `--tensor-parallel-size`: Tensor parallel size.
 > If you encounter OOM errors, try `--vae-use-tiling` or multi-GPU parallelism options.
@@ -101,6 +101,3 @@ Key arguments:
 - `--video-negative-prompt`: negative prompt for video generation.
 - `--audio-negative-prompt`: negative prompt for audio generation.
 - `--enable-cpu-offload`: enable CPU offload (defaults False).
 - `--cache-backend`: enable `cache_dit` for acceleration.
 - `--quantization`: online (dynamic) quantization method — `fp8` or `int8`.
  (VAEs, the T5 text encoder, norms and modulation stay bf16.)
@@ -4,7 +4,6 @@
 import argparse
 import re
 import time
 from typing import Any
 import numpy as np
 from PIL import Image
@@ -42,12 +41,6 @@ def parse_args() -> argparse.Namespace:
        choices=[1, 2, 3, 4],
        help="Number of GPUs used for classifier free guidance parallel size (max 4 branches).",
    )
    parser.add_argument(
        "--tensor-parallel-size",
        type=int,
        default=1,
        help=("Number of GPUs used for tensor parallelism (TP) inside the DiT."),
    )
    parser.add_argument(
        "--video-negative-prompt",
        default="jitter, bad hands, blur, distortion",
@@ -59,13 +52,6 @@ def parse_args() -> argparse.Namespace:
        help="Negative prompt for audio.",
    )
    parser.add_argument("--output", default="dreamid_output.mp4", help="Output video path.")
    parser.add_argument(
        "--quantization",
        type=str,
        default=None,
        choices=["fp8", "int8"],
        help=("Online (dynamic) quantization method for the DreamID-Omni transformer. "),
    )
    parser.add_argument(
        "--enable-cpu-offload",
        action="store_true",
@@ -77,7 +63,6 @@ def parse_args() -> argparse.Namespace:
        action="store_true",
        help="Enable layerwise (blockwise) offloading on DiT modules.",
    )
    parser.add_argument("--cache-backend", type=str, default=None, choices=["cache_dit"], help="Cache backend.")
    parser.add_argument(
        "--use-hsdp",
        action="store_true",
@@ -114,27 +99,6 @@ def load_image_and_audio(image_paths, audio_paths):
    return image, audio
 def _extract_peak_memory_mb(result: Any) -> float:
    """Pull worker-reported peak VRAM (MiB) generation result.
    Mirrors vllm_omni/entrypoints/openai/serving_video.py:_extract_peak_memory_mb.
    """
    if isinstance(result, list):
        result = result[0] if result else None
    if result is None:
        return 0.0
    val = getattr(result, "peak_memory_mb", 0.0)
    if not val:
        inner = getattr(result, "request_output", None)
        if isinstance(inner, list):
            inner = inner[0] if inner else None
        val = getattr(inner, "peak_memory_mb", 0.0)
    try:
        return float(val or 0.0)
    except (TypeError, ValueError):
        return 0.0
 def main() -> None:
    args = parse_args()
    if args.prompt is None and args.prompt_file is None:
@@ -177,39 +141,18 @@ def main() -> None:
    parallel_config = DiffusionParallelConfig(
        cfg_parallel_size=args.cfg_parallel_size,
        tensor_parallel_size=args.tensor_parallel_size,
        use_hsdp=args.use_hsdp,
        hsdp_shard_size=args.hsdp_shard_size,
        hsdp_replicate_size=args.hsdp_replicate_size,
    )
-    cache_config = None
+    omni = Omni(
    if args.cache_backend == "cache_dit":
        cache_config = {
            "Fn_compute_blocks": 1,
            "Bn_compute_blocks": 0,
            "max_warmup_steps": 4,
            "max_cached_steps": 20,
            "residual_diff_threshold": 0.24,
            "max_continuous_cached_steps": 3,
            "enable_taylorseer": False,
            "taylorseer_order": 1,
            "scm_steps_mask_policy": None,
            "scm_steps_policy": "dynamic",
        }
    omni_kwargs = dict(
        model=args.model,
        parallel_config=parallel_config,
        model_type=args.model_type,
        enable_cpu_offload=args.enable_cpu_offload,
        enable_layerwise_offload=args.enable_layerwise_offload,
        cache_backend=args.cache_backend,
        cache_config=cache_config,
    )
    if args.quantization is not None:
        omni_kwargs["quantization"] = args.quantization
    omni = Omni(**omni_kwargs)
    start = time.perf_counter()
    outputs = omni.generate(prompt, sampling_params)
    elapsed = time.perf_counter() - start
@@ -247,9 +190,6 @@ def main() -> None:
        f.write(video_bytes)
    print(f"Saved generated video to {output_path}")
    print(f"Total time: {elapsed:.2f}s")
    peak_mb = _extract_peak_memory_mb(outputs)
    if peak_mb:
        print(f"Worker peak GPU memory (reserved): {peak_mb:.2f} MiB ({peak_mb / 1024:.2f} GiB)")
 if __name__ == "__main__":
@@ -1,40 +0,0 @@
 # Lance — online serving
 OpenAI-compatible chat completions API for Lance, served by `vllm-omni`.
 ## Start the server
 ```bash
 bash examples/online_serving/lance/run_server.sh
 # or, with overrides
 MODEL=bytedance-research/Lance \
 DEPLOY_CONFIG=vllm_omni/deploy/lance.yaml \
 PORT=8091 \
    bash examples/online_serving/lance/run_server.sh
 ```
 The default deploy config is the single-stage `lance.yaml`; for `video_edit`
 or `text2video` workloads, point ``DEPLOY_CONFIG`` at a video-checkpoint
 config (or pass ``--model bytedance-research/Lance/Lance_3B_Video`` to
 `run_server.sh`).
 ## Send a request
 ```bash
 # text-to-image
 python examples/online_serving/lance/openai_chat_client.py \
    --prompt "A cute corgi astronaut on the moon, cinematic" \
    --modality text2img \
    --output corgi.png
 # image edit
 python examples/online_serving/lance/openai_chat_client.py \
    --prompt "Convert this into a vibrant cartoon-style illustration" \
    --modality img2img \
    --image-url path/to/photo.png \
    --output edited.png
 ```
 The client is shared with BAGEL — same OpenAI message format, same
 ``modalities`` and ``num_inference_steps`` / ``seed`` / ``height`` /
 ``width`` knobs.
@@ -1,184 +0,0 @@
 #!/usr/bin/env python3
 """
 Lance OpenAI-compatible chat client for image generation and multimodal tasks.
 Usage:
    python examples/online_serving/lance/openai_chat_client.py --prompt "A cute cat" --output output.png
    python examples/online_serving/lance/openai_chat_client.py --prompt "Describe this image" --image-url https://example.com/image.png
 """
 import argparse
 import base64
 from pathlib import Path
 import requests
 def generate_image(
    prompt: str,
    server_url: str = "http://localhost:8091",
    image_url: str | None = None,
    height: int | None = None,
    width: int | None = None,
    steps: int | None = None,
    seed: int | None = None,
    negative_prompt: str | None = None,
    modality: str = "text2img",  # "text2img" (default), "img2img", "img2text", "text2text"
 ) -> bytes | str | None:
    """Generate an image or text using the chat completions API.
    Args:
        prompt: Text description or prompt
        server_url: Server URL
        image_url: URL or path to input image (for img2img/img2text)
        height: Image height in pixels
        width: Image width in pixels
        steps: Number of inference steps
        seed: Random seed
        negative_prompt: Negative prompt
        modality: Task modality hint
    Returns:
        Image bytes (for image outputs) or Text string (for text outputs) or None if failed
    """
    # Construct Message Content
    content = [{"type": "text", "text": f"<|im_start|>{prompt}<|im_end|>"}]
    if image_url:
        # Check if local file
        if Path(image_url).exists():
            with open(image_url, "rb") as f:
                b64_data = base64.b64encode(f.read()).decode("utf-8")
                final_image_url = f"data:image/jpeg;base64,{b64_data}"
        else:
            final_image_url = image_url
        content.append({"type": "image_url", "image_url": {"url": final_image_url}})
    messages = [{"role": "user", "content": content}]
    # Build request payload with all parameters at top level
    # Note: vLLM ignores "extra_body", so we put parameters directly in the payload
    payload = {"messages": messages}
    # Set output modalities at top level
    if modality == "text2img" or modality == "img2img":
        payload["modalities"] = ["image"]
    elif modality == "img2text" or modality == "text2text":
        payload["modalities"] = ["text"]
    # Add generation parameters directly to payload
    if height is not None:
        payload["height"] = height
    if width is not None:
        payload["width"] = width
    if steps is not None:
        payload["num_inference_steps"] = steps
    if seed is not None:
        payload["seed"] = seed
    if negative_prompt:
        payload["negative_prompt"] = negative_prompt
    # Send request
    try:
        print(f"Sending request to {server_url} with modality {modality}...")
        response = requests.post(
            f"{server_url}/v1/chat/completions",
            headers={"Content-Type": "application/json"},
            json=payload,
            timeout=300,
        )
        response.raise_for_status()
        data = response.json()
        # Extract content - check ALL choices since server may return multiple
        # (e.g., text in choices[0], image in choices[1])
        choices = data.get("choices", [])
        # First pass: look for image output in any choice
        for choice in choices:
            choice_content = choice.get("message", {}).get("content")
            # Handle Image Output
            if isinstance(choice_content, list) and len(choice_content) > 0:
                first_item = choice_content[0]
                if isinstance(first_item, dict) and "image_url" in first_item:
                    img_url_str = first_item["image_url"].get("url", "")
                    if img_url_str.startswith("data:image"):
                        _, b64_data = img_url_str.split(",", 1)
                        return base64.b64decode(b64_data)
        # Second pass: look for text output if no image found
        for choice in choices:
            choice_content = choice.get("message", {}).get("content")
            if isinstance(choice_content, str) and choice_content:
                return choice_content
        print(f"Unexpected response format: {choices}")
        return None
    except Exception as e:
        print(f"Error: {e}")
        return None
 def main():
    parser = argparse.ArgumentParser(description="Bagel multimodal chat client")
    parser.add_argument("--prompt", "-p", default="A cute cat", help="Text prompt")
    parser.add_argument("--output", "-o", default="bagel_output.png", help="Output file (for image results)")
    parser.add_argument("--server", "-s", default="http://localhost:8091", help="Server URL")
    # Modality Control
    parser.add_argument("--image-url", "-i", type=str, help="Input image URL or local path")
    parser.add_argument(
        "--modality",
        "-m",
        default="text2img",
        choices=["text2img", "img2img", "img2text", "text2text"],
        help="Task modality",
    )
    # Generation Params
    parser.add_argument("--height", type=int, default=512, help="Image height")
    parser.add_argument("--width", type=int, default=512, help="Image width")
    parser.add_argument("--steps", type=int, default=25, help="Inference steps")
    parser.add_argument("--seed", type=int, default=42, help="Random seed")
    parser.add_argument("--negative", help="Negative prompt")
    args = parser.parse_args()
    print(f"Mode: {args.modality}")
    if args.image_url:
        print(f"Input Image: {args.image_url}")
    result = generate_image(
        prompt=args.prompt,
        server_url=args.server,
        image_url=args.image_url,
        height=args.height,
        width=args.width,
        steps=args.steps,
        seed=args.seed,
        negative_prompt=args.negative,
        modality=args.modality,
    )
    if result:
        if isinstance(result, bytes):
            # It's an image
            output_path = Path(args.output)
            output_path.write_bytes(result)
            print(f"Image saved to: {output_path}")
            print(f"Size: {len(result) / 1024:.1f} KB")
        elif isinstance(result, str):
            # It's text
            print("Response:")
            print(result)
    else:
        print("Failed to generate response")
        exit(1)
 if __name__ == "__main__":
    main()
@@ -1,15 +0,0 @@
 #!/bin/bash
 # Lance online serving startup script
 MODEL="${MODEL:-bytedance-research/Lance}"
 PORT="${PORT:-8091}"
 DEPLOY_CONFIG="${DEPLOY_CONFIG:-vllm_omni/deploy/lance.yaml}"
 echo "Starting Lance server..."
 echo "Model:         $MODEL"
 echo "Deploy config: $DEPLOY_CONFIG"
 echo "Port:          $PORT"
 vllm serve "$MODEL" --omni \
    --deploy-config "$DEPLOY_CONFIG" \
    --port "$PORT"
@@ -10,7 +10,6 @@ Please refer to [README.md](../../../README.md)
 |------|---------------|--------|
 | Thinker + Talker (omni-speech, default) | `vllm serve ... --omni` | Text + Audio |
 | Thinker only (multimodal understanding) | `vllm serve ... --omni --deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml` | Text |
 | Thinker + Imagegen (text-to-image / img2img) | `vllm serve ... --omni --deploy-config vllm_omni/deploy/ming_flash_omni_image.yaml` | Image |
 For standalone TTS (talker only), see the [Ming-flash-omni-TTS section in the Text-To-Speech hub](../text_to_speech/README.md#ming-flash-omni-tts).
@@ -57,91 +56,8 @@ bash run_curl_multimodal_generation.sh use_image
 bash run_curl_multimodal_generation.sh use_audio
 bash run_curl_multimodal_generation.sh use_video
 bash run_curl_multimodal_generation.sh use_mixed_modalities
 bash run_curl_multimodal_generation.sh use_image_gen
 ```
 ## Image generation (text-to-image)
 Ming-flash-omni-2.0 also exposes an image-generation (diffusion) stage.
 Launch with the image deploy YAML, which adds an image-gen stage behind
 the thinker:
 ```bash
 vllm serve Jonathan1909/Ming-flash-omni-2.0 --omni \
    --deploy-config vllm_omni/deploy/ming_flash_omni_image.yaml \
    --stage-init-timeout 1800 \
    --init-timeout 1800 \
    --port 8091
 ```
 Then request image output by passing `"modalities": ["image"]`:
 ```bash
 curl -s http://127.0.0.1:8091/v1/chat/completions \
    -H "Content-Type: application/json" \
    -d '{
      "model": "Jonathan1909/Ming-flash-omni-2.0",
      "messages": [{"role": "user", "content": "Please draw a cute cat."}],
      "modalities": ["image"]
    }' | jq -r '.choices[0].message.content[0].image_url.url | split(",")[1]' | base64 -d > ming_imagegen.png
 ```
 ### Optional knobs
 Pass image-gen overrides via the diffusion-stage `sampling_params_list[1].extra_args.image_gen`:
 | Key | Default | Description |
 | --- | --- | --- |
 | `height` / `width` | from config (1024) | Output resolution (multiples of `vae_scale_factor * 2`, currently 16). |
 | `steps` | 30 | Number of FlowMatchEuler denoise steps. |
 | `cfg` | 2.0 | Classifier-free guidance scale. |
 | `seed` | 42 | Per-request RNG seed (deterministic when ByT5 is also seed-stable). |
 | `byte5_text` | (auto) | Override the glyph text for ByT5 enhancement; raw strings are auto-wrapped to Ming's `Text "...". ` format. |
 | `negative_prompt` | empty | Real CFG negative conditioning (must be set on **stage-0 thinker** `sampling_params` so `expand_cfg_prompts` spawns the companion). |
 Example with all the knobs:
 ```bash
 curl http://127.0.0.1:8091/v1/chat/completions \
    -H "Content-Type: application/json" \
    -d '{
      "model": "Jonathan1909/Ming-flash-omni-2.0",
      "modalities": ["image"],
      "sampling_params_list": [
        {"temperature":0.4,"top_p":0.9,"top_k":1,"max_tokens":1,"seed":42,
         "extra_args":{"image_gen":{"negative_prompt":"ugly, blurry, distorted"}}},
        {"seed":42,"extra_args":{"image_gen":{
            "steps":6,"cfg":1.5,"height":512,"width":512,"seed":123,
            "byte5_text":["理解与生成统一"]}}}
      ],
      "messages": [{"role": "user", "content": "Draw a poster."}]
    }' | jq -r '.choices[0].message.content[0].image_url.url | split(",")[1]' | base64 -d > ming_imagegen_knobs.png
 ```
 ### img2img (reference image)
 Add an `image_url` content part to the user message; the parser routes it
 into the diffusion stage as `extra[reference_image]`:
 ```jsonc
 "messages": [{
  "role": "user",
  "content": [
    {"type": "text", "text": "Change the background to a sandy beach at sunset."},
    {"type": "image_url", "image_url": {"url": "data:image/png;base64,<base64>"}}
  ]
 }]
 ```
 ### GPU layout
 The shipped `ming_flash_omni_image.yaml` allocates the thinker on GPUs 0–3
 (TP=4) and the diffusion stage on GPU 4 (TP=1). Copy the YAML and edit
 `devices` per stage to relocate; with fewer GPUs available, drop
 the thinker TP to 2 and run the diffusion stage on a free card. Image-gen
 warmup takes roughly an extra 30–60 s on top of the thinker — set
 `--stage-init-timeout 1800` if the default 300 s is too tight.
 ## Modality control
 | `modalities` | Server config | Output |
@@ -149,7 +65,6 @@ warmup takes roughly an extra 30–60 s on top of the thinker — set
 | `["text"]` or omitted | Thinker only | Text |
 | `["audio"]` | Thinker + Talker | Audio (speech) |
 | `["text", "audio"]` | Thinker + Talker | Text + Audio |
 | `["image"]` | Thinker + Imagegen (image deploy YAML) | Image (PNG, base64 in `choices[0].message.content`) |
 For ready-to-copy curl examples (text / audio / multimodal input, SSE
 streaming, reasoning mode), see the recipe at
@@ -7,15 +7,14 @@ PORT="${PORT:-8091}"
 QUERY_TYPE="${1:-text}"
 # Validate query type
-if [[ ! "$QUERY_TYPE" =~ ^(text|use_audio|use_image|use_video|use_mixed_modalities|use_image_gen)$ ]]; then
+if [[ ! "$QUERY_TYPE" =~ ^(text|use_audio|use_image|use_video|use_mixed_modalities)$ ]]; then
    echo "Error: Invalid query type '$QUERY_TYPE'"
-    echo "Usage: $0 [text|use_audio|use_image|use_video|use_mixed_modalities|use_image_gen]"
+    echo "Usage: $0 [text|use_audio|use_image|use_video|use_mixed_modalities]"
    echo "  text: Text-only query"
    echo "  use_audio: Audio + Text query"
    echo "  use_image: Image + Text query"
    echo "  use_video: Video + Text query"
    echo "  use_mixed_modalities: Audio + Image + Video + Text query"
    echo "  use_image_gen: Text-to-image (diffusion stage) — saves the generated PNG locally"
    exit 1
 fi
@@ -101,43 +100,6 @@ esac
 echo "Running query type: $QUERY_TYPE"
 echo ""
 if [[ "$QUERY_TYPE" == "use_image_gen" ]]; then
    # Image-gen branch: dual-stage diffusion endpoint, modalities=["image"].
    # The shipped ming_flash_omni_image.yaml runs the thinker on cards 0-3
    # (TP=4) and the diffusion stage on card 4 (TP=1). See the image-gen
    # section in README.md for knobs and img2img.
    out_path="${OUT_PATH:-/tmp/ming_imagegen.png}"
    request_body=$(cat <<'EOF'
 {
  "model": "Jonathan1909/Ming-flash-omni-2.0",
  "modalities": ["image"],
  "messages": [
    {"role": "user", "content": "Draw a beautiful girl with short black hair and red dress."}
  ]
 }
 EOF
 )
    output=$(curl -sS --retry 3 --retry-delay 3 --retry-connrefused \
        -X POST "http://localhost:${PORT}/v1/chat/completions" \
        -H "Content-Type: application/json" \
        -d "$request_body")
    # Server returns base64-encoded PNG either as a data URL in
    # ``choices[0].message.content[0].image_url.url`` or as a raw string.
    b64=$(echo "$output" | jq -r '.choices[0].message.content
        | if type == "array"
            then (map(select(.image_url? != null)) | .[0].image_url.url // "")
            elif type == "string" then . else "" end')
    if [[ -z "$b64" ]]; then
        echo "Error: no image returned. Raw response:"
        echo "$output" | jq '.'
        exit 1
    fi
    if [[ "$b64" == data:image* ]]; then b64="${b64#*,}"; fi
    printf '%s' "$b64" | base64 --decode > "$out_path"
    echo "Saved generated image: $out_path ($(stat -c%s "$out_path" 2>/dev/null || stat -f%z "$out_path") bytes)"
    exit 0
 fi
 request_body=$(cat <<EOF
 {
  "model": "Jonathan1909/Ming-flash-omni-2.0",
@@ -18,6 +18,10 @@ Asynchronous chunk streaming operates as **enabled by default** within this bund
 Additionally, NPU, ROCm, and XPU per-platform configuration deltas are deterministically merged from the
 `platforms`: section of the corresponding YAML.
 **Note:** The OpenAI-style **`/v1/realtime`** WebSocket interface (facilitating streaming PCM audio input alongside audio and transcription output)
 is currently **unsupported** while the `async_chunk` configuration attribute is enabled.
 It is requisite to instantiate the default omni architecture or utilize a deployment configuration specifying `async_chunk: false` to facilitate real-time streaming sessions.
 To explicitly utilize a custom deployment YAML, mandate the configuration path accordingly:
 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
@@ -285,7 +289,7 @@ python openai_realtime_client.py \
 | `--num-requests` | `1` | Number of sequential sessions (see `--concurrency`) |
 | `--concurrency` | `1` | Max concurrent WebSocket sessions when `--num-requests` > 1 |
-Ensure the server is running, for example:
+Ensure the server is running **without** `async_chunk` if you use `/v1/realtime`, for example:
 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
@@ -6,21 +6,11 @@
 vllm serve SenseNova/SenseNova-U1-8B-MoT --omni --port 8091
 ```
 With Cache-DiT acceleration:
 ```bash
 vllm serve SenseNova/SenseNova-U1-8B-MoT --omni --port 8091 \
    --cache-backend cache_dit
 ```
 Or use the convenience script:
 ```bash
 cd examples/online_serving/sensenova_u1
 bash run_server.sh
 # Cache-DiT
 CACHE_BACKEND=cache_dit bash run_server.sh
 ```
 ### Tensor Parallelism (TP)
@@ -3,18 +3,10 @@
 MODEL="${MODEL:-SenseNova/SenseNova-U1-8B-MoT}"
 PORT="${PORT:-8091}"
 CACHE_BACKEND="${CACHE_BACKEND:-none}"
 echo "Starting SenseNova-U1 server..."
 echo "Model: $MODEL"
 echo "Port: $PORT"
 echo "Cache backend: $CACHE_BACKEND"
 EXTRA_ARGS=()
 if [ "$CACHE_BACKEND" != "none" ]; then
    EXTRA_ARGS+=(--cache-backend "$CACHE_BACKEND")
 fi
 vllm serve "$MODEL" --omni \
-    --port "$PORT" \
+    --port "$PORT"
    "${EXTRA_ARGS[@]}"
@@ -15,11 +15,11 @@ For the full list of supported architectures across all modalities, see
 | Model | HuggingFace repo | Voice cloning | Streaming | Voice presets / upload | Gradio demo |
 |---|---|---|---|---|---|
 | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
 | GLM-TTS | `zai-org/GLM-TTS` | ✓ (`ref_audio`+`ref_text`, required) | ✓ (PCM stream) | — | ✓ |
 | Ming-flash-omni-TTS | `Jonathan1909/Ming-flash-omni-2.0` | — (caption-controlled) | — | caption fields (`instructions`) | — |
 | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✓ (`ref_audio` required) | ✓ (PCM stream) | — | ✓ |
-| OmniVoice | `k2-fsa/OmniVoice` | ✓ | — | — | — |
+| OmniVoice | `k2-fsa/OmniVoice` | (offline only) | — | — | — |
 | Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | ✓ (Base) | ✓ (PCM + WebSocket) | ✓ (presets + `/v1/audio/voices` upload) | ✓ (standard + FastRTC) |
 | VoxCPM | `openbmb/VoxCPM-0.5B` | ✓ | ✓ (PCM stream) | — | — |
 | VoxCPM2 | `openbmb/VoxCPM2` | ✓ | ✓ (AudioWorklet via gradio) | — | ✓ |
 | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✓ (gated upstream) | ✓ | ✓ (presets) | ✓ |
@@ -95,45 +95,6 @@ For full request-shape documentation (all parameters, response formats, error co
 ---
 ## GLM-TTS
 2-stage TTS (AR + DiT flow-matching) at 24 kHz. Every request requires `ref_audio` + `ref_text`.
 ### Launch
 ```bash
 vllm serve zai-org/GLM-TTS --omni --trust-remote-code --port 8091
 # or:
 bash examples/online_serving/text_to_speech/glm_tts/run_server.sh /path/to/GLM-TTS
 ```
 ### Sending requests
 ```bash
 # Voice cloning (required)
 python examples/online_serving/text_to_speech/glm_tts/openai_speech_client.py \
    --text "你好，这是语音克隆测试。" \
    --ref-audio file:///path/to/ref.wav \
    --ref-text "这是参考音频的文本内容。"
 # Custom format
 python examples/online_serving/text_to_speech/glm_tts/openai_speech_client.py \
    --text "Hello, this is a voice cloning test." \
    --ref-audio file:///path/to/ref.wav \
    --ref-text "Transcript of the reference audio." \
    --response-format mp3 -o output.mp3
 ```
 ### Gradio demo
 ```bash
 bash examples/online_serving/text_to_speech/glm_tts/run_gradio_demo.sh
 ```
 ### Notes
 - Output: 24 kHz mono WAV via HiFT vocoder.
 - `ref_audio` + `ref_text` are **required** together on every request. Reference audio should be 3-10 seconds.
 - Voice cloning feature extraction (WhisperVQ, CampPlus, mel) runs on the model side — no external dependency on the serving layer.
 ---
 ## Fish Speech S2 Pro
 4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec.
@@ -143,26 +104,6 @@ bash examples/online_serving/text_to_speech/glm_tts/run_gradio_demo.sh
 pip install fish-speech
 ```
 ### Kvcache attention fast path
 Fish Speech S2 Pro uses a Triton decode-only kvcache attention fast path by
 default on CUDA builds. Set `VLLM_OMNI_FISH_KVCACHE_ATTN=0` to disable it, or
 `VLLM_OMNI_FISH_KVCACHE_ATTN=required` to fail fast if the fast path cannot be
 installed.
 ```bash
 # Verify fast path availability.
 python - <<'PY'
 from vllm_omni.attention import fish_kvcache_attn
 print(fish_kvcache_attn.is_available())
 print(fish_kvcache_attn.load_error())
 PY
 # Optional: disable the runtime fast path.
 export VLLM_OMNI_FISH_KVCACHE_ATTN=0
 ```
 ### Launch
 ```bash
 vllm serve fishaudio/s2-pro --omni --port 8091
@@ -321,29 +262,16 @@ vllm serve k2-fsa/OmniVoice --omni --port 8091 --trust-remote-code
 ### CLI client
 ```bash
 cd examples/online_serving/text_to_speech/omnivoice
 # Text-only (auto voice)
 python speech_client.py --text "Hello, how are you?"
 # Language hint
 python speech_client.py --text "Bonjour, comment allez-vous?" --language French
 # Voice cloning (reference audio + optional ref_text)
 python speech_client.py \
 --text "Bonjour, comment allez-vous?" \
 --ref-audio /path/to/ref_audio.wav \
 --ref-text "Bonjour, comment allez-vous?"
 # Style instruction (voice design-style control)
 python speech_client.py \
 --text "Bonjour, comment allez-vous?" \
 --language French \
 --instructions "loud voice"
 # Deterministic output with seed parameter
 python speech_client.py --text "Hello, how are you?" --seed 42
 ```
-The client supports `--api-base`, `--model`, `--text`, `--response-format`, `--language`, `--voice`, `--ref-audio`, `--ref-text`, `--instructions`, `--seed`, and `--output`.
+The client supports `--api-base`, `--model`, `--text`, `--response-format`, `--language`, `--output`.
 ### Notes
 - Voice cloning and voice design require offline inference; see the [offline OmniVoice section](../../offline_inference/text_to_speech/README.md#omnivoice).
 ---
 ## Qwen3-TTS
@@ -366,10 +294,10 @@ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni --port 8091
 ./qwen3_tts/run_server.sh Base
 ```
-### Executor backend
+### Choosing an executor backend (uniproc vs mp)
-Single-GPU serves now default to the uniproc executor (lower IPC overhead, the Base cloning use case from [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) / [#2604](https://github.com/vllm-project/vllm-omni/pull/2604)). `vllm_omni/deploy/qwen3_tts.yaml` is the only Qwen3-TTS deploy config; pass `--deploy-config <path>` to override.
+Stage configs ship with the chunked-streaming default. To use the uniproc executor (lower IPC overhead for the Base cloning task), pass `--stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts_uniproc.yaml`. See [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) and [#2604](https://github.com/vllm-project/vllm-omni/pull/2604) for the full investigation.
-To opt out of chunked streaming, pass `--no-async-chunk` — the pipeline auto-dispatches to the end-to-end codec processor.
+To opt out of chunked streaming, pass `--no-async-chunk` instead — the pipeline auto-dispatches to the end-to-end codec processor.
 ### Sending requests
 ```bash
@@ -447,11 +375,73 @@ python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-de
 `qwen3_tts/batch_speech_client.py` issues many concurrent requests for throughput measurement.
 ### Notes
- Base voice cloning has per-request reference-audio cost; the uniproc default keeps IPC overhead off the critical path. See the executor-backend section above for background.
+- Base voice cloning has uniproc-vs-mp tradeoffs depending on per-request reference audio cost; see the executor-backend section above.
 - `vllm_omni/deploy/qwen3_tts.yaml` is the default deploy config (loaded by HF `model_type`); per-stage runtime overrides are available via `--stage-N-<field> <value>`.
 ---
 ## VoxCPM
 Split-stage TTS at 24 kHz.
 ### Prerequisites
 ```bash
 pip install voxcpm
 # or use a local source tree:
 export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
 ```
 If the native VoxCPM `config.json` lacks HF `model_type`, set up an HF-compatible config dir:
 ```bash
 export VOXCPM_MODEL=/path/to/voxcpm-model
 export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config
 mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"
 cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json"
 cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true
 python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)'
 ```
 ### Launch
 ```bash
 export VOXCPM_MODEL=/path/to/voxcpm-model
 ./voxcpm/run_server.sh                # async-chunk streaming (default)
 ./voxcpm/run_server.sh sync           # non-streaming
 ```
 Or directly:
 ```bash
 vllm serve "$VOXCPM_MODEL" \
    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
    --trust-remote-code --enforce-eager --omni --port 8091
 ```
 ### Sending requests
 ```bash
 # Basic TTS
 python voxcpm/openai_speech_client.py \
    --model "$VOXCPM_MODEL" \
    --text "This is a VoxCPM online text-to-speech example."
 # Voice cloning
 python voxcpm/openai_speech_client.py \
    --model "$VOXCPM_MODEL" \
    --text "This sentence is synthesized with a cloned voice." \
    --ref-audio /path/to/reference.wav \
    --ref-text "The exact transcript spoken in reference.wav."
 # Streaming PCM
 python voxcpm/openai_speech_client.py \
    --model "$VOXCPM_MODEL" \
    --text "This is a streaming VoxCPM request." \
    --stream --output voxcpm_stream.pcm
 ```
 ### Notes
 - `voxcpm.yaml` for one-shot decode; `voxcpm_async_chunk.yaml` for single-request streaming. Do not use the async-chunk config for concurrent requests or `/v1/audio/speech/batch`.
 - Generic TTS fields not supported by VoxCPM: `voice`, `instructions`, `language`, `speaker_embedding`, `x_vector_only_mode`.
 - For benchmark measurement, see [`benchmarks/voxcpm`](../../../benchmarks/voxcpm/README.md).
 ---
 ## VoxCPM2
 Single-stage native AR TTS at 48 kHz.
@@ -1,28 +0,0 @@
 #!/bin/bash
 # Launch vLLM-Omni server for CosyVoice3 TTS
 #
 # Usage:
 #   ./run_server.sh
 #   CUDA_VISIBLE_DEVICES=0 ./run_server.sh
 #
 # Streaming (async-chunk) is on by default via vllm_omni/deploy/cosyvoice3.yaml.
 # Set NO_ASYNC_CHUNK=1 to use the legacy synchronous path.
 set -e
 MODEL="${MODEL:-FunAudioLLM/Fun-CosyVoice3-0.5B-2512}"
 PORT="${PORT:-8091}"
 EXTRA_ARGS=()
 if [[ -n "${NO_ASYNC_CHUNK:-}" ]]; then
    EXTRA_ARGS+=(--no-async-chunk)
 fi
 echo "Starting CosyVoice3 server with model: $MODEL"
 vllm serve "$MODEL" \
    --host 0.0.0.0 \
    --port "$PORT" \
    --trust-remote-code \
    --omni \
    "${EXTRA_ARGS[@]}"
@@ -1,142 +0,0 @@
 """Client for CosyVoice3 TTS via /v1/audio/speech endpoint.
 CosyVoice3 has no built-in voice presets: every request is voice cloning
 driven by ``ref_audio`` + ``ref_text``. The defaults below point at the
 official upstream zero-shot prompt so the script runs out of the box.
 Examples:
    # Voice cloning with the default upstream prompt
    python speech_client.py --text "收到好友从远方寄来的生日礼物。"
    # Custom reference clip + transcript
    python speech_client.py --text "Hello, this is a cloned voice." \
        --ref-audio /path/to/reference.wav \
        --ref-text "Transcript of the reference audio."
    # Streaming PCM output
    python speech_client.py --text "Hello world" --stream --output output.pcm
 """
 import argparse
 import base64
 import os
 import httpx
 DEFAULT_API_BASE = "http://localhost:8091"
 DEFAULT_API_KEY = "EMPTY"
 DEFAULT_MODEL = "FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
 # Official CosyVoice zero-shot prompt and its transcript.
 DEFAULT_REF_AUDIO = "https://raw.githubusercontent.com/FunAudioLLM/CosyVoice/main/asset/zero_shot_prompt.wav"
 DEFAULT_REF_TEXT = "希望你以后能够做的比我还好呦。"
 def encode_audio_to_base64(audio_path: str) -> str:
    """Encode a local audio file to a base64 data URL."""
    if not os.path.exists(audio_path):
        raise FileNotFoundError(f"Audio file not found: {audio_path}")
    ext = audio_path.lower().rsplit(".", 1)[-1]
    mime_map = {"wav": "audio/wav", "mp3": "audio/mpeg", "flac": "audio/flac", "ogg": "audio/ogg"}
    mime_type = mime_map.get(ext, "audio/wav")
    with open(audio_path, "rb") as f:
        audio_b64 = base64.b64encode(f.read()).decode("utf-8")
    return f"data:{mime_type};base64,{audio_b64}"
 def run_tts(args) -> None:
    """Generate speech via the /v1/audio/speech API."""
    payload = {
        "model": args.model,
        "input": args.text,
        "response_format": args.response_format,
    }
    if args.ref_audio.startswith(("http://", "https://")):
        payload["ref_audio"] = args.ref_audio
    else:
        payload["ref_audio"] = encode_audio_to_base64(args.ref_audio)
    payload["ref_text"] = args.ref_text
    if args.stream:
        payload["stream"] = True
        payload["response_format"] = "pcm"
    print(f"Model: {args.model}")
    print(f"Text: {args.text}")
    print(f"Voice cloning: ref_audio={args.ref_audio}, ref_text={args.ref_text}")
    print("Generating audio...")
    api_url = f"{args.api_base}/v1/audio/speech"
    headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {args.api_key}",
    }
    if args.stream:
        output_path = args.output or "output.pcm"
        with httpx.Client(timeout=300.0) as client:
            with client.stream("POST", api_url, json=payload, headers=headers) as resp:
                if resp.status_code != 200:
                    print(f"Error: {resp.status_code}")
                    print(resp.read().decode())
                    return
                total_bytes = 0
                with open(output_path, "wb") as f:
                    for chunk in resp.iter_bytes():
                        f.write(chunk)
                        total_bytes += len(chunk)
                print(f"Streamed {total_bytes} bytes to: {output_path}")
    else:
        with httpx.Client(timeout=300.0) as client:
            response = client.post(api_url, json=payload, headers=headers)
        if response.status_code != 200:
            print(f"Error: {response.status_code}")
            print(response.text)
            return
        try:
            text = response.content.decode("utf-8")
            if text.startswith('{"error"'):
                print(f"Error: {text}")
                return
        except UnicodeDecodeError:
            pass
        output_path = args.output or "output.wav"
        with open(output_path, "wb") as f:
            f.write(response.content)
        print(f"Audio saved to: {output_path}")
 def main():
    parser = argparse.ArgumentParser(description="CosyVoice3 TTS client")
    parser.add_argument("--api-base", default=DEFAULT_API_BASE, help="API base URL")
    parser.add_argument("--api-key", default=DEFAULT_API_KEY, help="API key")
    parser.add_argument("--model", "-m", default=DEFAULT_MODEL, help="Model name")
    parser.add_argument("--text", required=True, help="Text to synthesize")
    parser.add_argument(
        "--ref-audio",
        default=DEFAULT_REF_AUDIO,
        help="Reference audio for voice cloning (path or URL)",
    )
    parser.add_argument(
        "--ref-text",
        default=DEFAULT_REF_TEXT,
        help="Transcript of the reference audio",
    )
    parser.add_argument("--stream", action="store_true", help="Enable streaming (PCM output)")
    parser.add_argument(
        "--response-format",
        default="wav",
        choices=["wav", "mp3", "flac", "pcm", "aac", "opus"],
        help="Audio format (default: wav)",
    )
    parser.add_argument("--output", "-o", default=None, help="Output file path")
    args = parser.parse_args()
    run_tts(args)
 if __name__ == "__main__":
    main()
@@ -1,291 +0,0 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 """Gradio demo for GLM-TTS online serving via /v1/audio/speech API.
 Supports:
  - Voice-cloned text-to-speech synthesis from reference audio (upload or URL)
  - Streaming (progressive PCM chunks) and non-streaming modes
 Usage:
    # Start the server first (see run_server.sh), then:
    python gradio_demo.py --api-base http://localhost:8091
    # Or use run_gradio_demo.sh to start both server and demo together.
 """
 import argparse
 import base64
 import io
 try:
    import gradio as gr
 except ImportError:
    raise ImportError("gradio is required to run this demo. Install it with: pip install 'vllm-omni[demo]'") from None
 import httpx
 import numpy as np
 import soundfile as sf
 PCM_SAMPLE_RATE = 24000
 DEFAULT_API_BASE = "http://localhost:8091"
 def encode_audio_to_base64(audio_data: tuple) -> str:
    """Encode Gradio audio input (sample_rate, numpy_array) to base64 data URL."""
    sample_rate, audio_np = audio_data
    if audio_np.dtype != np.int16:
        if audio_np.dtype in (np.float32, np.float64):
            audio_np = np.clip(audio_np, -1.0, 1.0)
            audio_np = (audio_np * 32767).astype(np.int16)
        else:
            audio_np = audio_np.astype(np.int16)
    buf = io.BytesIO()
    sf.write(buf, audio_np, sample_rate, format="WAV")
    wav_b64 = base64.b64encode(buf.getvalue()).decode("utf-8")
    return f"data:audio/wav;base64,{wav_b64}"
 def build_payload(
    text: str,
    ref_audio: tuple | None,
    ref_audio_url: str,
    ref_text: str,
    response_format: str = "wav",
    stream: bool = False,
    max_new_tokens: int = 4096,
 ) -> dict:
    """Build the /v1/audio/speech request payload."""
    if not text or not text.strip():
        raise gr.Error("Please enter text to synthesize.")
    payload: dict = {
        "input": text.strip(),
        "voice": "default",
        "response_format": "pcm" if stream else response_format,
        "stream": stream,
        "max_new_tokens": max_new_tokens,
    }
    # Voice cloning: ref_audio takes priority over URL.
    ref_url_stripped = ref_audio_url.strip() if ref_audio_url else ""
    if ref_audio is not None:
        payload["ref_audio"] = encode_audio_to_base64(ref_audio)
    elif ref_url_stripped:
        payload["ref_audio"] = ref_url_stripped
    else:
        raise gr.Error("GLM-TTS requires a reference audio file or URL.")
    if not ref_text or not ref_text.strip():
        raise gr.Error("GLM-TTS requires a transcript of the reference audio.")
    payload["ref_text"] = ref_text.strip()
    return payload
 def generate_speech(api_base: str, text: str, ref_audio, ref_audio_url, ref_text, response_format, max_new_tokens):
    """Non-streaming: call /v1/audio/speech and return full audio."""
    payload = build_payload(
        text, ref_audio, ref_audio_url, ref_text, response_format, stream=False, max_new_tokens=max_new_tokens
    )
    try:
        with httpx.Client(timeout=300.0) as client:
            resp = client.post(
                f"{api_base}/v1/audio/speech",
                json=payload,
                headers={"Content-Type": "application/json", "Authorization": "Bearer EMPTY"},
            )
    except httpx.TimeoutException:
        raise gr.Error("Request timed out. The server may be busy.")
    except httpx.ConnectError:
        raise gr.Error(f"Cannot connect to server at {api_base}. Is the server running?")
    if resp.status_code != 200:
        raise gr.Error(f"Server error ({resp.status_code}): {resp.text}")
    content_type = resp.headers.get("content-type", "")
    if "application/json" in content_type:
        try:
            raise gr.Error(f"Server error: {resp.json()}")
        except ValueError:
            pass
    try:
        if response_format == "pcm":
            audio_np = np.frombuffer(resp.content, dtype=np.int16).astype(np.float32) / 32767.0
            return (PCM_SAMPLE_RATE, audio_np)
        audio_np, sample_rate = sf.read(io.BytesIO(resp.content))
        if audio_np.ndim > 1:
            audio_np = audio_np[:, 0]
        return (sample_rate, audio_np.astype(np.float32))
    except Exception as e:
        raise gr.Error(f"Failed to decode audio: {e}")
 def generate_speech_stream(
    api_base, text, ref_audio, ref_audio_url, ref_text, response_format, max_new_tokens, chunk_seconds=0.5
 ):
    """Streaming: yield PCM chunks for progressive playback."""
    payload = build_payload(
        text, ref_audio, ref_audio_url, ref_text, response_format, stream=True, max_new_tokens=max_new_tokens
    )
    min_chunk_samples = int(PCM_SAMPLE_RATE * chunk_seconds)
    prebuffer_chunks = 2
    pending: list[np.ndarray] = []
    pending_len = 0
    chunks_yielded = 0
    prebuffer: list[np.ndarray] = []
    try:
        leftover = b""
        with httpx.Client(timeout=300.0) as client:
            with client.stream(
                "POST",
                f"{api_base}/v1/audio/speech",
                json=payload,
                headers={"Content-Type": "application/json", "Authorization": "Bearer EMPTY"},
            ) as resp:
                if resp.status_code != 200:
                    resp.read()
                    raise gr.Error(f"Server error ({resp.status_code}): {resp.text}")
                for chunk in resp.iter_bytes():
                    if not chunk:
                        continue
                    raw = leftover + chunk
                    usable = len(raw) - (len(raw) % 2)
                    leftover = raw[usable:]
                    if usable == 0:
                        continue
                    float_samples = np.frombuffer(raw[:usable], dtype=np.int16).copy().astype(np.float32) / 32767.0
                    pending.append(float_samples)
                    pending_len += len(float_samples)
                    if pending_len >= min_chunk_samples:
                        audio_chunk = np.concatenate(pending)
                        pending.clear()
                        pending_len = 0
                        if chunks_yielded < prebuffer_chunks:
                            prebuffer.append(audio_chunk)
                            chunks_yielded += 1
                            if chunks_yielded == prebuffer_chunks:
                                yield (PCM_SAMPLE_RATE, np.concatenate(prebuffer))
                                prebuffer.clear()
                        else:
                            yield (PCM_SAMPLE_RATE, audio_chunk)
        remaining = prebuffer + pending
        if remaining:
            yield (PCM_SAMPLE_RATE, np.concatenate(remaining))
    except httpx.TimeoutException:
        raise gr.Error("Request timed out. The server may be busy.")
    except httpx.ConnectError:
        raise gr.Error(f"Cannot connect to server at {api_base}. Is the server running?")
 def on_stream_change(stream: bool):
    """Lock format to PCM when streaming."""
    if stream:
        return gr.update(value="pcm", interactive=False)
    return gr.update(value="wav", interactive=True)
 def build_interface(api_base: str, stream_chunk_seconds: float = 0.5):
    """Build the Gradio interface."""
    with gr.Blocks(title="GLM-TTS Demo") as demo:
        gr.Markdown("# GLM-TTS - Voice Cloning")
        gr.Markdown(f"**Server:** `{api_base}` | **Model:** GLM-TTS | **Output:** 24kHz")
        with gr.Row():
            with gr.Column(scale=3):
                text_input = gr.Textbox(
                    label="Text to Synthesize",
                    placeholder="Enter text here (supports Chinese and English)...",
                    lines=4,
                )
                with gr.Accordion("Reference Voice (required)", open=True):
                    gr.Markdown("Upload or link a short reference audio (3-10s) and provide its transcript.")
                    ref_audio = gr.Audio(
                        label="Reference Audio",
                        type="numpy",
                        sources=["upload", "microphone"],
                    )
                    ref_audio_url = gr.Textbox(
                        label="Reference Audio URL (alternative to upload)",
                        placeholder="https://example.com/reference.wav",
                        lines=1,
                    )
                    ref_text = gr.Textbox(
                        label="Reference Audio Transcript (required for cloning)",
                        placeholder="Exact transcript of the reference audio...",
                        lines=2,
                    )
                with gr.Row():
                    response_format = gr.Dropdown(
                        choices=["wav", "mp3", "flac", "pcm"],
                        value="wav",
                        label="Audio Format",
                        scale=1,
                    )
                    stream_checkbox = gr.Checkbox(
                        label="Stream output",
                        value=False,
                        info="Progressive PCM streaming",
                        scale=1,
                    )
                    max_new_tokens = gr.Slider(
                        minimum=256,
                        maximum=8192,
                        value=4096,
                        step=256,
                        label="Max New Tokens",
                        scale=1,
                    )
                generate_btn = gr.Button("Generate Speech", variant="primary", size="lg")
            with gr.Column(scale=2):
                audio_output = gr.Audio(
                    label="Generated Audio",
                    interactive=False,
                    streaming=True,
                    autoplay=True,
                )
                gr.Markdown(
                    "### About\n"
                    "- **GLM-TTS** by Zhipu AI: Llama-based AR + Flow Matching DiT\n"
                    "- **Voice cloning**: upload 3-10s reference + transcript\n"
                    "- **Streaming**: real-time PCM output\n"
                    "- **24kHz** output via Vocos vocoder"
                )
        stream_checkbox.change(fn=on_stream_change, inputs=[stream_checkbox], outputs=[response_format])
        all_inputs = [text_input, ref_audio, ref_audio_url, ref_text, response_format, max_new_tokens]
        def dispatch(stream_enabled, *args):
            if stream_enabled:
                yield from generate_speech_stream(api_base, *args, chunk_seconds=stream_chunk_seconds)
            else:
                yield generate_speech(api_base, *args)
        generate_btn.click(fn=dispatch, inputs=[stream_checkbox] + all_inputs, outputs=[audio_output])
        demo.queue()
    return demo
 def main():
    parser = argparse.ArgumentParser(description="Gradio demo for GLM-TTS")
    parser.add_argument("--api-base", default=DEFAULT_API_BASE, help=f"API base URL (default: {DEFAULT_API_BASE})")
    parser.add_argument("--host", default="0.0.0.0", help="Gradio host (default: 0.0.0.0)")
    parser.add_argument("--port", type=int, default=7860, help="Gradio port (default: 7860)")
    parser.add_argument("--share", action="store_true", help="Share publicly")
    parser.add_argument("--stream-chunk-seconds", type=float, default=0.5, help="Seconds per streaming chunk")
    args = parser.parse_args()
    print(f"Connecting to vLLM server at: {args.api_base}")
    demo = build_interface(args.api_base, stream_chunk_seconds=args.stream_chunk_seconds)
    demo.launch(server_name=args.host, server_port=args.port, share=args.share)
 if __name__ == "__main__":
    main()
@@ -1,173 +0,0 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 """OpenAI-compatible client for GLM-TTS via /v1/audio/speech endpoint.
 GLM-TTS is a two-stage TTS system (AR + DiT) that generates audio from text
 conditioned on reference speech. Each request requires ref_audio + ref_text.
 Usage:
    # Voice cloning
    python openai_speech_client.py --text "你好" --ref-audio file:///path/to/ref.wav --ref-text "参考文本"
    # Streaming response, for async_chunk server mode
    python openai_speech_client.py --text "你好" --stream --ref-audio file:///path/to/ref.wav --ref-text "参考文本"
    # Specify output format
    python openai_speech_client.py --text "你好" --ref-audio file:///path/to/ref.wav \
        --ref-text "参考文本" --response-format mp3 -o output.mp3
 """
 import argparse
 import httpx
 # Default server configuration
 DEFAULT_API_BASE = "http://localhost:8091"
 DEFAULT_API_KEY = "EMPTY"
 def run_tts_generation(args) -> None:
    """Run TTS generation via OpenAI-compatible /v1/audio/speech API."""
    if not args.ref_audio or not args.ref_text:
        raise ValueError("GLM-TTS requires --ref-audio and --ref-text for voice cloning.")
    payload = {
        "model": args.model,
        "voice": "default",
        "input": args.text,
        "response_format": args.response_format,
        "stream": bool(args.stream),
        "ref_audio": args.ref_audio,
        "ref_text": args.ref_text,
    }
    if args.max_new_tokens:
        payload["max_new_tokens"] = args.max_new_tokens
    print(f"Model: {args.model}")
    print(f"Text: {args.text}")
    print(f"Voice cloning: ref_audio={args.ref_audio}, ref_text={args.ref_text}")
    print(f"Stream: {args.stream}")
    print("Generating audio...")
    api_url = f"{args.api_base}/v1/audio/speech"
    headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {args.api_key}",
    }
    if args.stream:
        output_path = args.output or "tts_output.pcm"
        with httpx.Client(timeout=300.0) as client, open(output_path, "wb") as f:
            with client.stream("POST", api_url, json=payload, headers=headers) as response:
                if response.status_code != 200:
                    print(f"Error: {response.status_code}")
                    response.read()
                    print(response.text)
                    return
                for chunk in response.iter_bytes():
                    f.write(chunk)
        print(f"Streaming audio saved to: {output_path}")
    else:
        with httpx.Client(timeout=300.0) as client:
            response = client.post(api_url, json=payload, headers=headers)
        if response.status_code != 200:
            print(f"Error: {response.status_code}")
            print(response.text)
            return
        try:
            text = response.content.decode("utf-8")
            if text.startswith('{"error"'):
                print(f"Error: {text}")
                return
        except UnicodeDecodeError:
            pass
        output_path = args.output or f"tts_output.{args.response_format}"
        with open(output_path, "wb") as f:
            f.write(response.content)
        print(f"Audio saved to: {output_path}")
 def parse_args():
    """Parse command line arguments."""
    parser = argparse.ArgumentParser(
        description="OpenAI-compatible client for GLM-TTS via /v1/audio/speech",
    )
    # Server configuration
    parser.add_argument(
        "--api-base",
        type=str,
        default=DEFAULT_API_BASE,
        help=f"API base URL (default: {DEFAULT_API_BASE})",
    )
    parser.add_argument(
        "--api-key",
        type=str,
        default=DEFAULT_API_KEY,
        help="API key (default: EMPTY)",
    )
    parser.add_argument(
        "--model",
        "-m",
        type=str,
        default="glm-tts",
        help="Model name/path",
    )
    # Input text
    parser.add_argument(
        "--text",
        type=str,
        required=True,
        help="Text to synthesize",
    )
    # Generation parameters
    parser.add_argument(
        "--max-new-tokens",
        type=int,
        default=None,
        help="Maximum new tokens to generate (default: model default)",
    )
    # Output
    parser.add_argument(
        "--response-format",
        type=str,
        default="wav",
        choices=["wav", "mp3", "flac", "pcm", "aac", "opus"],
        help="Audio output format (default: wav)",
    )
    parser.add_argument(
        "--stream",
        action="store_true",
        help="Request a streaming audio response (use with async_chunk server mode).",
    )
    parser.add_argument(
        "--output",
        "-o",
        type=str,
        default=None,
        help="Output audio file path (default: tts_output.<format>)",
    )
    # Voice cloning parameters
    parser.add_argument(
        "--ref-audio",
        type=str,
        default=None,
        help="Reference audio URL, file:// URI, or base64 data URL for voice cloning",
    )
    parser.add_argument(
        "--ref-text",
        type=str,
        default=None,
        help="Transcript of the reference audio (required with --ref-audio)",
    )
    return parser.parse_args()
 if __name__ == "__main__":
    args = parse_args()
    run_tts_generation(args)
@@ -1,48 +0,0 @@
 #!/bin/bash
 # Launch GLM-TTS server + Gradio demo together.
 #
 # Usage:
 #   ./run_gradio_demo.sh
 #   CUDA_VISIBLE_DEVICES=0 PORT=8091 GRADIO_PORT=7860 ./run_gradio_demo.sh
 set -e
 MODEL="${MODEL:-zai-org/GLM-TTS}"
 PORT="${PORT:-8091}"
 GRADIO_PORT="${GRADIO_PORT:-7860}"
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 REPO_ROOT="$(cd "$SCRIPT_DIR/../../../.." && pwd)"
 echo "Starting GLM-TTS server (port $PORT)..."
 FLASHINFER_DISABLE_VERSION_CHECK=1 \
 vllm-omni serve "$MODEL" \
    --deploy-config "$REPO_ROOT/vllm_omni/deploy/glm_tts.yaml" \
    --host 0.0.0.0 \
    --port "$PORT" \
    --gpu-memory-utilization 0.9 \
    --trust-remote-code \
    --enforce-eager \
    --omni &
 SERVER_PID=$!
 cleanup() {
    echo "Stopping server (PID $SERVER_PID)..."
    kill $SERVER_PID 2>/dev/null
    wait $SERVER_PID 2>/dev/null
 }
 trap cleanup EXIT
 # Wait for server to be ready.
 echo "Waiting for server to start..."
 for i in $(seq 1 120); do
    if curl -s "http://localhost:$PORT/health" > /dev/null 2>&1; then
        echo "Server ready."
        break
    fi
    sleep 2
 done
 echo "Starting Gradio demo (port $GRADIO_PORT)..."
 python "$SCRIPT_DIR/gradio_demo.py" \
    --api-base "http://localhost:$PORT" \
    --port "$GRADIO_PORT"
@@ -1,38 +0,0 @@
 #!/bin/bash
 # Launch vLLM-Omni server for GLM-TTS models
 #
 # Usage:
 #   ./run_server.sh                           # Default model path, async_chunk mode
 #   ./run_server.sh /path/to/GLM-TTS          # Custom model path, async_chunk mode
 #   ./run_server.sh /path/to/GLM-TTS sync     # Sync two-stage mode
 #
 # NOTE: The model path should point to the repo ROOT (not llm/ subdirectory).
 # model_subdir/tokenizer_subdir in the pipeline config resolve subdirectories.
 set -e
 MODEL="${1:-zai-org/GLM-TTS}"
 MODE="${2:-async}"
 EXTRA_ARGS=()
 case "$MODE" in
    async|async_chunk)
        ;;
    sync|no_async_chunk)
        EXTRA_ARGS+=("--no-async-chunk")
        ;;
    *)
        echo "Unknown mode: $MODE (expected async or sync)" >&2
        exit 1
        ;;
 esac
 echo "Starting GLM-TTS server with model: $MODEL (mode: $MODE)"
 vllm-omni serve "$MODEL" \
    --deploy-config vllm_omni/deploy/glm_tts.yaml \
    --host 0.0.0.0 \
    --port 8091 \
    --trust-remote-code \
    --omni \
    "${EXTRA_ARGS[@]}"
@@ -1,63 +0,0 @@
 # higgs-audio v2 online example
 This directory contains the online-serving entry points for boson-ai's higgs-audio v2 as integrated by vllm-omni: a 2-stage TTS pipeline (Llama-3.2-3B talker with DualFFN audio expert + HiggsAudio codec decoder) emitting 24 kHz mono speech.
 ## Prerequisites
 Voice clone uses HF's `HiggsAudioV2TokenizerModel` loaded from `k2-fsa/OmniVoice/audio_tokenizer/` (the boson-ai standalone tokenizer Hub repo's `model.safetensors` is the 3B talker LM, not the codec). Only that ~806 MB subdir is downloaded.
 ```bash
 pip install -U "transformers>=5.3.0"
 ```
 ## Files
 - `run_server.sh` — launch the vllm-omni server with the bundled `vllm_omni/deploy/higgs_audio_v2.yaml` deploy config.
 - `batch_speech_client.py` — send a list of prompts to `/v1/audio/speech` and save the returned WAV / PCM bytes to a directory; optionally passes `--ref-audio` + `--ref-text` for shallow voice clone.
 ## Launching the server
 ```bash
 GPUS=6,7 PORT=8094 bash examples/online_serving/text_to_speech/higgs_audio_v2/run_server.sh
 ```
 Environment overrides:
 - `MODEL` — HF id of the talker (default `bosonai/higgs-audio-v2-generation-3B-base`).
 - `PORT` — server port (default `8094`).
 - `GPUS` — `CUDA_VISIBLE_DEVICES` value (default `6,7`).
 - `GPU_UTIL` — `--gpu-memory-utilization` (default `0.4`).
 The script also exports `VLLM_USE_DEEP_GEMM=0` / `VLLM_MOE_USE_DEEP_GEMM=0` so the example works on images without the optional `deep_gemm` backend.
 The deploy YAML ships with `async_chunk: false` and `codec_streaming: true`, i.e. Stage 0 finishes its codec frames before Stage 1 starts decoding, and Stage 1 streams WAV/PCM bytes to the client chunk-by-chunk.
 ## Driving the server
 Plain TTS:
 ```bash
 python examples/online_serving/text_to_speech/higgs_audio_v2/batch_speech_client.py \
    --base-url http://localhost:8094 \
    --model bosonai/higgs-audio-v2-generation-3B-base \
    --output-dir /tmp/higgs_audio_v2_batch \
    --prompts "Hello world." \
              "The quick brown fox jumps over the lazy dog."
 ```
 Voice clone — pass a reference clip and its transcript (both required together):
 ```bash
 python examples/online_serving/text_to_speech/higgs_audio_v2/batch_speech_client.py \
    --base-url http://localhost:8094 \
    --model bosonai/higgs-audio-v2-generation-3B-base \
    --output-dir /tmp/higgs_audio_v2_clone \
    --ref-audio /path/to/reference.wav \
    --ref-text  "Exact transcript spoken in reference.wav." \
    --prompts "Hello, this is a cloned voice."
 ```
 ## Notes
 - `--ref-text` must be the real transcript of `--ref-audio`; mismatched text degrades cloned-voice quality.
 - Out of scope (rejected with explicit 4xx by the request validator): multi-speaker `[SPEAKERn]` tags inside `input`, `profile:` text-only speaker descriptions, the `ref_audio_in_system_message` system-block variant, chunked long-form generation, and per-request `voice` / `instructions` / `task_type` / `language` / `speed != 1.0` / `x_vector_only_mode` / `speaker_embedding`.
@@ -1,122 +0,0 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 """Batch client for the higgs-audio v2 online server.
 Sends a fixed list of prompts to ``/v1/audio/speech`` and saves the returned
 WAV files (or raw PCM bytes when ``--format pcm``) into ``--output-dir``.
 Usage (plain text -> speech):
  python examples/online_serving/text_to_speech/higgs_audio_v2/batch_speech_client.py \
      --base-url http://localhost:8094 \
      --output-dir /tmp/higgs_audio_v2_batch \
      --prompts "Hello world." "The quick brown fox jumps over the lazy dog."
 Usage (shallow voice clone — pass a reference clip + its transcript):
  python examples/online_serving/text_to_speech/higgs_audio_v2/batch_speech_client.py \
      --base-url http://localhost:8094 \
      --output-dir /tmp/higgs_audio_v2_clone \
      --ref-audio path/to/reference.wav \
      --ref-text "the transcript of the reference clip" \
      --prompts "Hello world."
 """
 from __future__ import annotations
 import argparse
 import base64
 import sys
 from pathlib import Path
 DEFAULT_PROMPTS = (
    "Hello world.",
    "The quick brown fox jumps over the lazy dog.",
    "It was the night before my birthday.",
    "Innovation distinguishes between a leader and a follower.",
 )
 def _slug(text: str) -> str:
    import re
    s = re.sub(r"\s+", "_", text.strip().lower())
    return re.sub(r"[^a-z0-9_]+", "", s)[:32] or "prompt"
 def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("--base-url", default="http://localhost:8094")
    parser.add_argument("--model", default="higgs_audio_v2")
    parser.add_argument("--prompts", nargs="+", default=list(DEFAULT_PROMPTS))
    parser.add_argument("--output-dir", type=Path, default=Path("/tmp/higgs_audio_v2_batch"))
    parser.add_argument("--format", choices=("wav", "pcm"), default="wav")
    parser.add_argument("--max-new-tokens", type=int, default=300)
    parser.add_argument("--seed", type=int, default=42)
    parser.add_argument("--timeout-s", type=float, default=120.0)
    parser.add_argument(
        "--ref-audio",
        type=Path,
        default=None,
        help="Reference clip for voice clone (path to a WAV file). Must be paired with --ref-text.",
    )
    parser.add_argument(
        "--ref-text",
        type=str,
        default=None,
        help="Transcript of the reference clip. Required when --ref-audio is set.",
    )
    args = parser.parse_args()
    if (args.ref_audio is None) != (args.ref_text is None):
        print("--ref-audio and --ref-text must be supplied together", file=sys.stderr)
        return 2
    ref_audio_data_url: str | None = None
    if args.ref_audio is not None:
        if not args.ref_audio.exists():
            print(f"ref-audio file not found: {args.ref_audio}", file=sys.stderr)
            return 2
        mime = "audio/wav" if args.ref_audio.suffix.lower() == ".wav" else "audio/mpeg"
        ref_b64 = base64.b64encode(args.ref_audio.read_bytes()).decode("ascii")
        ref_audio_data_url = f"data:{mime};base64,{ref_b64}"
    try:
        import httpx
    except ImportError:
        print(
            "this client needs `httpx`. Install with `pip install httpx`.",
            file=sys.stderr,
        )
        return 2
    args.output_dir.mkdir(parents=True, exist_ok=True)
    url = args.base_url.rstrip("/") + "/v1/audio/speech"
    failures = 0
    with httpx.Client(timeout=args.timeout_s) as client:
        for prompt in args.prompts:
            payload = {
                "model": args.model,
                "input": prompt,
                "response_format": args.format,
                "max_new_tokens": args.max_new_tokens,
                "seed": args.seed,
            }
            if ref_audio_data_url is not None:
                payload["ref_audio"] = ref_audio_data_url
                payload["ref_text"] = args.ref_text
            resp = client.post(url, json=payload)
            if resp.status_code != 200:
                print(f"[FAIL] {prompt!r} -> {resp.status_code}: {resp.text[:200]}", file=sys.stderr)
                failures += 1
                continue
            suffix = ".wav" if args.format == "wav" else ".pcm"
            out = args.output_dir / f"{_slug(prompt)}{suffix}"
            out.write_bytes(resp.content)
            print(f"[ ok ] {prompt!r} -> {out} ({len(resp.content)} bytes)")
    return 1 if failures else 0
 if __name__ == "__main__":
    sys.exit(main())
@@ -1,37 +0,0 @@
 #!/bin/bash
 # Launch vLLM-Omni server for higgs-audio v2.
 #
 # v1 scope: plain text -> 24 kHz speech only. Voice cloning, multi-speaker,
 # ChatML rich content, and language overrides are rejected by the validator
 # with explicit 4xx (see vllm_omni/entrypoints/openai/serving_speech.py).
 #
 # Usage:
 #   ./run_server.sh                 # default port 8094, GPUs 6 and 7
 #   PORT=8095 GPUS=6,7 ./run_server.sh
 #   MODEL=bosonai/higgs-audio-v2-generation-3B-base ./run_server.sh
 set -e
 MODEL="${MODEL:-bosonai/higgs-audio-v2-generation-3B-base}"
 PORT="${PORT:-8094}"
 GPUS="${GPUS:-6,7}"
 GPU_UTIL="${GPU_UTIL:-0.4}"
 echo "Starting higgs-audio v2 server"
 echo "  MODEL=$MODEL"
 echo "  PORT=$PORT"
 echo "  CUDA_VISIBLE_DEVICES=$GPUS"
 # DeepGEMM FP8 kernels are optional and trip warmup on builds without
 # the deep_gemm backend; disable them so the example works out of the box.
 # Users with deep_gemm installed can re-enable via the same env vars.
 CUDA_VISIBLE_DEVICES="$GPUS" \
 VLLM_USE_DEEP_GEMM=0 \
 VLLM_MOE_USE_DEEP_GEMM=0 \
 vllm-omni serve "$MODEL" \
    --deploy-config vllm_omni/deploy/higgs_audio_v2.yaml \
    --host 0.0.0.0 \
    --port "$PORT" \
    --gpu-memory-utilization "$GPU_UTIL" \
    --trust-remote-code \
    --omni
@@ -6,14 +6,9 @@ Examples:
    # Specify language
    python speech_client.py --text "Bonjour, comment allez-vous?" --language French
    # Use a specific uploaded/supported voice
    python speech_client.py --text "Hello" --voice my_uploaded_voice
 """
 import argparse
 import base64
 import os
 import httpx
@@ -21,61 +16,20 @@ DEFAULT_API_BASE = "http://localhost:8091"
 DEFAULT_API_KEY = "EMPTY"
 def encode_audio_to_base64(audio_path: str) -> str:
    """Encode a local audio file to a base64 data URL."""
    if not os.path.exists(audio_path):
        raise FileNotFoundError(f"Audio file not found: {audio_path}")
    ext = audio_path.lower().rsplit(".", 1)[-1]
    mime = {
        "wav": "audio/wav",
        "mp3": "audio/mpeg",
        "flac": "audio/flac",
        "ogg": "audio/ogg",
    }.get(ext, "audio/wav")
    with open(audio_path, "rb") as f:
        b64 = base64.b64encode(f.read()).decode("utf-8")
    return f"data:{mime};base64,{b64}"
 def run_tts(args) -> None:
    """Generate speech via /v1/audio/speech API."""
    payload = {
        "model": args.model,
        "input": args.text,
        "voice": "default",
        "response_format": args.response_format,
    }
    if args.seed is not None:
        payload["extra_params"] = {}
        payload["extra_params"]["seed"] = args.seed
    if args.voice:
        payload["voice"] = args.voice
    if args.language:
        payload["language"] = args.language
    if args.ref_audio:
        ref = args.ref_audio
        if ref.startswith(("http://", "https://", "data:")):
            payload["ref_audio"] = ref
        else:
            payload["ref_audio"] = encode_audio_to_base64(ref)
    if args.ref_text:
        payload["ref_text"] = args.ref_text
    if args.instructions:
        payload["instructions"] = args.instructions
    print(f"Model: {args.model}")
    print(f"Text: {args.text}")
    if args.seed:
        print(f"Seed: {args.seed}")
    if args.voice:
        print(f"Voice: {args.voice}")
    if args.language:
        print(f"Language: {args.language}")
    print("Generating audio...")
@@ -85,6 +39,7 @@ def run_tts(args) -> None:
        "Content-Type": "application/json",
        "Authorization": f"Bearer {args.api_key}",
    }
    with httpx.Client(timeout=300.0) as client:
        response = client.post(api_url, json=payload, headers=headers)
@@ -113,11 +68,6 @@ def main():
    parser.add_argument("--api-key", default=DEFAULT_API_KEY, help="API key")
    parser.add_argument("--model", "-m", default="k2-fsa/OmniVoice", help="Model name")
    parser.add_argument("--text", required=True, help="Text to synthesize")
    parser.add_argument(
        "--voice",
        default=None,
        help="Voice name (omit for auto voice; must match a supported or uploaded speaker if set)",
    )
    parser.add_argument("--language", default=None, help="Language hint (e.g., English, Chinese, French)")
    parser.add_argument(
        "--response-format",
@@ -125,30 +75,6 @@ def main():
        choices=["wav", "mp3", "flac", "pcm", "aac", "opus"],
        help="Audio format (default: wav)",
    )
    parser.add_argument(
        "--ref-audio",
        type=str,
        default=None,
        help="Reference audio for voice cloning (local path, URL, or data: URI)",
    )
    parser.add_argument(
        "--ref-text",
        type=str,
        default=None,
        help="Reference text for voice cloning",
    )
    parser.add_argument(
        "--instructions",
        type=str,
        default=None,
        help="Voice style/emotion instructions",
    )
    parser.add_argument(
        "--seed",
        type=int,
        default=None,
        help="Random seed for generation, default: None for stochastic output)",
    )
    parser.add_argument("--output", "-o", default=None, help="Output file path")
    args = parser.parse_args()
    run_tts(args)
@@ -0,0 +1,155 @@
 """OpenAI-compatible client for VoxCPM via /v1/audio/speech.
 Examples:
    # Basic text-to-speech
    python openai_speech_client.py --text "Hello from VoxCPM"
    # Voice cloning
    python openai_speech_client.py \
        --text "This sentence uses the cloned voice." \
        --ref-audio /path/to/reference.wav \
        --ref-text "The exact transcript spoken in the reference audio."
    # Streaming PCM output
    python openai_speech_client.py \
        --text "This is a streaming VoxCPM request." \
        --stream \
        --output output.pcm
 """
 import argparse
 import base64
 import os
 import httpx
 DEFAULT_API_BASE = "http://localhost:8091"
 DEFAULT_API_KEY = "EMPTY"
 DEFAULT_MODEL = "OpenBMB/VoxCPM1.5"
 def encode_audio_to_base64(audio_path: str) -> str:
    """Encode a local audio file to base64 data URL."""
    if not os.path.exists(audio_path):
        raise FileNotFoundError(f"Audio file not found: {audio_path}")
    ext = audio_path.lower().rsplit(".", 1)[-1]
    mime_map = {
        "wav": "audio/wav",
        "mp3": "audio/mpeg",
        "flac": "audio/flac",
        "ogg": "audio/ogg",
    }
    mime_type = mime_map.get(ext, "audio/wav")
    with open(audio_path, "rb") as f:
        audio_b64 = base64.b64encode(f.read()).decode("utf-8")
    return f"data:{mime_type};base64,{audio_b64}"
 def build_payload(args) -> dict[str, object]:
    payload: dict[str, object] = {
        "model": args.model,
        "input": args.text,
        "response_format": "pcm" if args.stream else args.response_format,
    }
    if args.ref_audio:
        if args.ref_audio.startswith(("http://", "https://", "data:")):
            payload["ref_audio"] = args.ref_audio
        else:
            payload["ref_audio"] = encode_audio_to_base64(args.ref_audio)
    if args.ref_text:
        payload["ref_text"] = args.ref_text
    if args.max_new_tokens is not None:
        payload["max_new_tokens"] = args.max_new_tokens
    if args.stream:
        payload["stream"] = True
    return payload
 def run_tts(args) -> None:
    payload = build_payload(args)
    api_url = f"{args.api_base}/v1/audio/speech"
    headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {args.api_key}",
    }
    print(f"Model: {args.model}")
    print(f"Text: {args.text}")
    if args.ref_audio:
        print("Mode: voice cloning")
        print(f"Reference audio: {args.ref_audio}")
    else:
        print("Mode: text-to-speech")
    if args.stream:
        output_path = args.output or "voxcpm_output.pcm"
        with httpx.Client(timeout=300.0) as client:
            with client.stream("POST", api_url, json=payload, headers=headers) as response:
                if response.status_code != 200:
                    print(f"Error: {response.status_code}")
                    print(response.read().decode("utf-8", errors="ignore"))
                    return
                total_bytes = 0
                with open(output_path, "wb") as f:
                    for chunk in response.iter_bytes():
                        if not chunk:
                            continue
                        f.write(chunk)
                        total_bytes += len(chunk)
        print(f"Streamed {total_bytes} bytes to: {output_path}")
        return
    with httpx.Client(timeout=300.0) as client:
        response = client.post(api_url, json=payload, headers=headers)
    if response.status_code != 200:
        print(f"Error: {response.status_code}")
        print(response.text)
        return
    try:
        text = response.content.decode("utf-8")
        if text.startswith('{"error"'):
            print(f"Error: {text}")
            return
    except UnicodeDecodeError:
        pass
    output_path = args.output or "voxcpm_output.wav"
    with open(output_path, "wb") as f:
        f.write(response.content)
    print(f"Audio saved to: {output_path}")
 def main():
    parser = argparse.ArgumentParser(description="VoxCPM OpenAI-compatible speech client")
    parser.add_argument("--api-base", default=DEFAULT_API_BASE, help="API base URL")
    parser.add_argument("--api-key", default=DEFAULT_API_KEY, help="API key")
    parser.add_argument("--model", "-m", default=DEFAULT_MODEL, help="Model name or path")
    parser.add_argument("--text", required=True, help="Text to synthesize")
    parser.add_argument("--ref-audio", default=None, help="Reference audio path, URL, or data URL")
    parser.add_argument(
        "--ref-text",
        default=None,
        help="The exact transcript spoken in the reference audio",
    )
    parser.add_argument("--stream", action="store_true", help="Enable streaming PCM output")
    parser.add_argument(
        "--response-format",
        default="wav",
        choices=["wav", "pcm", "flac", "mp3", "aac", "opus"],
        help="Audio format for non-streaming mode (default: wav)",
    )
    parser.add_argument("--max-new-tokens", type=int, default=None, help="Maximum tokens to generate")
    parser.add_argument("--output", "-o", default=None, help="Output file path")
    args = parser.parse_args()
    run_tts(args)
 if __name__ == "__main__":
    main()
@@ -0,0 +1,38 @@
 #!/bin/bash
 # Launch vLLM-Omni server for VoxCPM online speech serving.
 #
 # Usage:
 #   ./run_server.sh                 # default: async_chunk stage config
 #   ./run_server.sh async           # async_chunk stage config
 #   ./run_server.sh sync            # no-async-chunk stage config
 #   VOXCPM_MODEL=/path/to/model ./run_server.sh
 set -e
 MODE="${1:-async}"
 MODEL="${VOXCPM_MODEL:-OpenBMB/VoxCPM1.5}"
 case "$MODE" in
    async)
        STAGE_CONFIG="vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml"
        ;;
    sync)
        STAGE_CONFIG="vllm_omni/deploy/voxcpm.yaml"
        ;;
    *)
        echo "Unknown mode: $MODE"
        echo "Supported: async, sync"
        exit 1
        ;;
 esac
 echo "Starting VoxCPM server with model: $MODEL"
 echo "Stage config: $STAGE_CONFIG"
 vllm serve "$MODEL" \
    --stage-configs-path "$STAGE_CONFIG" \
    --host 0.0.0.0 \
    --port 8091 \
    --trust-remote-code \
    --enforce-eager \
    --omni
@@ -38,12 +38,13 @@ theme:
  features:
    - content.action.edit
    - content.code.copy
    - navigation.instant
    - navigation.instant.progress
    - navigation.tracking
    - navigation.tabs
    - navigation.tabs.sticky
    - navigation.sections
    - navigation.indexes
    - navigation.prune
    - navigation.top
    - search.suggest
    - search.highlight
@@ -55,7 +56,6 @@ theme:
  custom_dir: docs/mkdocs/overrides
 hooks:
  - docs/mkdocs/hooks/search_index.py
  - docs/mkdocs/hooks/generate_api_readme.py
  - docs/mkdocs/hooks/url_schemes.py
  - docs/mkdocs/hooks/generate_examples.py
@@ -117,13 +117,10 @@ plugins:
            separate_signature: true
            show_overloads: true
            signature_crossrefs: true
            show_source: false
          inventories:
          - https://docs.python.org/3/objects.inv
          - https://typing-extensions.readthedocs.io/en/latest/objects.inv
-          # Disabled because Read the Docs strict builds fail if this
+          - https://docs.aiohttp.org/en/stable/objects.inv
          # external inventory intermittently resets the connection.
          # - https://docs.aiohttp.org/en/stable/objects.inv
          - https://pillow.readthedocs.io/en/stable/objects.inv
          - https://numpy.org/doc/stable/objects.inv
          # Temporarily disabled due to decompression errors
@@ -168,8 +165,9 @@ extra_css:
  - mkdocs/stylesheets/extra.css
 extra_javascript:
  - mkdocs/javascript/memory.js
  - mkdocs/javascript/mathjax.js
  - https://unpkg.com/mathjax@3.2.2/es5/tex-mml-chtml.js
  - https://unpkg.com/mermaid@10/dist/mermaid.min.js
  - mkdocs/javascript/mermaid.js
  - mkdocs/javascript/edit_and_feedback.js
  - mkdocs/javascript/slack_and_forum.js
@@ -63,7 +63,7 @@ dev = [
    "funasr>=1.0.0",
    "lpips>=0.1.4", # for quantization fp8
    "orjson>=3.11.7", # to make v1/embeddings API fast
-    "voxcpm>=2.0.2", # for VoxCPM2 path-locating in tests
+    "voxcpm>=2.0.2", # for voxcpm model Test
    "torchdiffeq>=0.2.5" # for covo_audio Test
 ]
@@ -71,11 +71,6 @@ demo = [
    "gradio>=6.7.0",
 ]
 # Local-test-only extras
 local = [
    "FlagEmbedding",
 ]
 docs = [
    "mkdocs>=1.5.0",
    "mkdocs-api-autonav",
@@ -189,7 +184,6 @@ markers = [
    "core_model: L1&L2 tests (run in each PR)",
    "advanced_model: L3 level tests (run on each merge)",
    "full_model: L4 level tests (run nightly)",
    "local_model: tests (run locally)",
    # function module markers
    "diffusion: Diffusion model tests",
    "omni: Omni model tests",
@@ -241,15 +235,12 @@ extend-ignore-identifiers-re = [
    ".*MoBA",
    ".*temperal_downsample",
    ".*_nd",
    ".*nd$",
    ".*EmbedND$",
    ".*nothink.*",
    ".*NOTHINK.*",
    ".*nin.*",
    ".*[Oo]no_[Aa]nna.*",
    ".*cann.*",
    ".*[Nn][Dd]3.*",
    "byt5.*",
 ]
 extend-ignore-re = [
    "\\.\\.\\.n,d->\\.\\.\\.nd",
@@ -262,4 +253,3 @@ Vai = "Vai"
 tockens = "tockens"
 CANN = "CANN"
 ND = "ND"
 byt5 = "byt5"
@@ -1,177 +0,0 @@
 # Lance
 > Unified autoregressive + diffusion multimodal (text / image / video)
 ## Summary
 - Vendor: ByteDance
 - Model: [`bytedance-research/Lance`](https://huggingface.co/bytedance-research/Lance) (`Lance_3B`, `Lance_3B_Video`)
 - Task: text2img, text2video, img2img (image edit), video2video (video edit),
  img2text (image understanding), video2text (video understanding)
 - Mode: Offline inference, Online serving (OpenAI-compatible API)
 - Maintainer: Community
 ## When to use this recipe
 Use this recipe to run Lance — a 3B unified AR + diffusion multimodal model
 on a Qwen2.5-VL backbone — via vLLM-Omni. Lance is **BAGEL-lineage**: the
 released checkpoint uses the same `*_moe_gen` Mixture-of-Transformers weight
 layout as BAGEL, so vLLM-Omni reuses BAGEL's transformer core and specializes
 only the ViT (Qwen2.5-VL vision), the VAE (Wan2.2) and the checkpoint layout.
 All six single-stage modalities are supported on the `Lance_3B` checkpoint;
 `text2video` and `video2video` additionally need `Lance_3B_Video` for the 3-D
 `latent_pos_embed` table. The two-stage AR + DiT topology is deferred and
 needs `LanceConfig` / `LanceProcessor` registered in the `vllm` package (a
 separate upstream PR).
 ## References
 - Offline example:
  [`examples/offline_inference/lance/end2end.py`](../../examples/offline_inference/lance/end2end.py)
 - Online serving:
  [`examples/online_serving/lance/`](../../examples/online_serving/lance/)
 - Deploy config:
  [`vllm_omni/deploy/lance.yaml`](../../vllm_omni/deploy/lance.yaml)
 - E2E tests:
  [`tests/e2e/online_serving/test_lance.py`](../../tests/e2e/online_serving/test_lance.py)
 - HuggingFace model page:
  [bytedance-research/Lance](https://huggingface.co/bytedance-research/Lance)
 - PR: [vllm-project/vllm-omni#3710](https://github.com/vllm-project/vllm-omni/pull/3710)
  (closes [#3697](https://github.com/vllm-project/vllm-omni/issues/3697))
 ## Hardware Support
 ## GPU
 ### 1x NVIDIA B300 / A100 80GB
 #### Environment
 - OS: Linux
 - Python: 3.12
 - Driver / runtime: CUDA ≥ 12.4
 - vLLM-Omni version: 0.18.x.dev
 #### Command — text-to-image (default)
 ```bash
 python examples/offline_inference/lance/end2end.py \
    --model bytedance-research/Lance \
    --prompts "a corgi astronaut on the moon, cinematic" \
    --steps 30 --cfg-text-scale 4.0 --timestep-shift 3.5 \
    --height 1024 --width 1024 \
    --seed 42 --output ./out
 ```
 Defaults match upstream `inference_lance.sh`: 30 denoising steps,
 `timestep-shift 3.5`, text CFG 4.0, seed 42, 1024×1024.
 #### Command — image edit (img2img)
 ```bash
 python examples/offline_inference/lance/end2end.py \
    --model bytedance-research/Lance --modality img2img \
    --image-path /path/to/input.png \
    --prompts "Convert this into a vibrant cartoon-style illustration" \
    --steps 30 --cfg-text-scale 4.0 --timestep-shift 3.5 \
    --output ./out
 ```
 The Lance-native VAE prefill scatters Wan2.2 latents into the LLM query
 sequence; no separate image encoder is needed.
 #### Command — text-to-video
 ```bash
 python examples/offline_inference/lance/end2end.py \
    --model bytedance-research/Lance/Lance_3B_Video --modality text2video \
    --num-frames 25 --video-height 480 --video-width 768 \
    --prompts "a cat playing piano, cinematic" \
    --steps 30 --fps 8 --output ./out
 ```
 Use the `Lance_3B_Video` subfolder for any video path so the 3-D
 `latent_pos_embed` table is loaded; image / understanding paths can point at
 the top-level repo and resolve the right sub-checkpoint automatically.
 #### Command — image / video understanding
 ```bash
 # Image → text (caption / VQA)
 python examples/offline_inference/lance/end2end.py \
    --model bytedance-research/Lance --modality img2text \
    --image-path /path/to/photo.jpg \
    --prompts "Describe this image in detail." \
    --do-sample --text-temperature 0.8
 # Video → text
 python examples/offline_inference/lance/end2end.py \
    --model bytedance-research/Lance --modality video2text \
    --video-path /path/to/clip.mp4 \
    --prompts "What is happening in this video?"
 ```
 Sampling is enabled by default at `--text-temperature 0.8` for the
 understanding paths because Lance's greedy decoder emits an immediate EOS for
 many prompts.
 #### Verification
 ```bash
 pytest -s -v tests/e2e/online_serving/test_lance.py
 ```
 #### Notes
 - BF16 footprint: ~7 GB LLM + Qwen2.5-VL ViT + Wan2.2 VAE; comfortably fits
  on a single 16 GB+ GPU for `Lance_3B`.
 - `rope_scaling = {"type": "mrope", "mrope_section": [16, 24, 24]}` is wired
  through `BagelRotaryEmbedding`. `text2img` uses scalar position ids
  (BAGEL-equivalent); `img2text` / `video` / edit paths thread per-axis 3-D
  position ids.
 - `video_edit` quality is more abstract than `text2video` at the same
  resolution (known position-id offset between VAE-prefill and gen-latent
  blocks). Functionally correct end-to-end.
 ## Online Serving
 Lance supports all single-stage modalities via the OpenAI-compatible
 `/v1/chat/completions` API.
 ### Launch
 ```bash
 bash examples/online_serving/lance/run_server.sh
 # or, with overrides
 MODEL=bytedance-research/Lance \
 DEPLOY_CONFIG=vllm_omni/deploy/lance.yaml \
 PORT=8091 \
    bash examples/online_serving/lance/run_server.sh
 ```
 For `text2video` / `video2video`, set
 `MODEL=bytedance-research/Lance/Lance_3B_Video`.
 ### Send requests
 ```bash
 # Text-to-image
 python examples/online_serving/lance/openai_chat_client.py \
    --prompt "A cute corgi astronaut on the moon, cinematic" \
    --modality text2img --output corgi.png
 # Image edit
 python examples/online_serving/lance/openai_chat_client.py \
    --prompt "Convert this into a vibrant cartoon-style illustration" \
    --modality img2img --image-url path/to/photo.png \
    --output edited.png
 # Image understanding
 python examples/online_serving/lance/openai_chat_client.py \
    --prompt "Describe this image" \
    --modality img2text --image-url photo.jpg
 ```
 The client is shared with BAGEL — same OpenAI message format, same
 `modalities` / `num_inference_steps` / `seed` / `height` / `width` knobs.
@@ -1,112 +0,0 @@
 # GLM-Image for text-to-image and image editing
 ## Summary
 - Vendor: Z.ai
 - Model: `GLM/GLM-Image`
 - Task: Text-to-image (T2I) and image-to-image
 - Mode: Online serving with the OpenAI-compatible API
 - Maintainer: Community
 ## When to use this recipe
 Use this recipe when you want a known-good starting point for serving
 `GLM/GLM-Image` with vLLM-Omni on **two 80 GB NVIDIA A800** GPUs (Ampere-class,
 same default layout as the upstream **2×A100 80GB** example: Stage 0 AR on GPU 0,
 Stage 1 diffusion on GPU 1) and validate the deployment with the existing
 `examples/online_serving/glm_image` clients.
 ## References
 - Upstream or canonical docs:
  [`docs/user_guide/examples/online_serving/glm_image.md`](../../docs/user_guide/examples/online_serving/glm_image.md)
 - Related issue or discussion:
  [#2888](https://github.com/vllm-project/vllm-omni/pull/2888)
 ## Hardware Support
 This recipe documents **dual-GPU** CUDA layouts on A800 80 GB
 for the same software stack. Add more platforms (for example ROCm / NPU) as
 community validation lands.
 ## GPU
 ### 2× A800 80GB
 #### Environment
 These versions were taken from a working **editable** install: activate `vllm-omni/.venv` (or your equivalent), then align `pip` / Git with the rows below when reproducing this recipe.
 - OS: Linux
 - Python: 3.12
 - Driver / runtime: NVIDIA CUDA stack with **two** A800 80 GB GPUs visible (set `CUDA_VISIBLE_DEVICES` on your host if needed)
 - vLLM: **0.19.0**
 - vLLM-Omni: **0.19.0rc2.dev138+g38d5f2d53** (editable install from this repo; Git **`38d5f2d5`**, `git describe` ≈ **`v0.19.0rc1-138-g38d5f2d5`**)
 - Transformers: **5.5.4** (same `.venv` as above; required so `glm_image` configs load for Stage 0)
 #### Command
 Start the server from the repository root:
 ```bash
 vllm serve zai-org/GLM-Image --omni --port 8091
 ```
 To use the bundled stage config explicitly (same default as above):
 ```bash
 vllm serve zai-org/GLM-Image \
  --omni \
  --port 8091 \
  --deploy-config vllm_omni/deploy/glm_image.yaml
 ```
 #### Verification
 Run one of the existing example clients after the server is ready:
 ```bash
 curl -s http://172.18.69.133:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "messages": [
      {"role": "user", "content": "A beautiful landscape painting"}
    ],
    "extra_body": {
      "height": 1920,
      "width": 1920,
      "num_inference_steps": 50,
      "true_cfg_scale": 1.5,
      "seed": 42
    }
  }' | jq -r '.choices[0].message.content[0].image_url.url' | cut -d',' -f2- | base64 -d > land.png
 ```
 After the command finishes, check for the output files:
 ```bash
 ls output.png
 ```
 #### Sample end-to-end metrics
 One representative **offline** GLM-Image E2E run on this recipe’s **2× A800 80GB**.
 Overall summary from the run’s metrics. Rough wall-time split: **Stage 0 (AR)** ~**25 s**,
 **Stage 1 (diffusion)** ~**34 s** (see `e2e_stage_*_wall_time_ms` below).
 | Field | Value |
 | --- | ---: |
 | e2e_requests | 1 |
 | e2e_wall_time_ms | 61,148.679 |
 | e2e_total_tokens | 1,300 |
 | e2e_avg_time_per_request_ms | 61,148.679 |
 | e2e_avg_tokens_per_s | 21.260 |
 | e2e_stage_0_wall_time_ms | 24,708.760 |
 | e2e_stage_1_wall_time_ms | 33,787.442 |
 #### Notes
 - Memory usage: Roughly **~38 GiB + KV** on Stage 0 (AR) and **~20 GiB** on Stage 1 (DiT+VAE) per the user guide; two 80 GB cards match the default split.
 - Key flags: `--omni` is required; `--stage-configs-path` is optional unless you use a custom YAML (for example single-GPU).
 - Keep **Transformers ≥ 5.5.1** (this recipe used **5.5.4**) so `glm_image` configs resolve; otherwise Stage 0 can fail at `ModelConfig` validation.
 - Known limitations: This starter recipe follows the dual-GPU online path documented under `examples/online_serving/glm_image`. The first request may be slower due to warmup.
 - Generation time: about **61 s** wall time end-to-end for the sample above (50 inference steps, 1024×1024).
@@ -1,109 +0,0 @@
 # Helios for video generation
 ## Summary
 - Vendor: Helios
 - Model: `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled`
 - Task: Text-to-video, image-to-video, and video-to-video generation
 - Mode: Offline inference with the bundled Helios example
 - Maintainer: Community
 ## When to use this recipe
 Use this recipe when you want a known-good starting point for running Helios
 video generation with vLLM-Omni. The concrete command below focuses on
 `BestWishYsh/Helios-Base` text-to-video generation on one NVIDIA H20 GPU. The
 same example script also supports Helios-Mid, Helios-Distilled, image-to-video,
 and video-to-video; see the linked example docs for those model-specific flags.
 ## References
 - Upstream repository: <https://github.com/PKU-YuanGroup/Helios>
 - Model weights:
  <https://huggingface.co/BestWishYsh/Helios-Base>
 - Upstream or canonical docs:
  [`docs/user_guide/examples/offline_inference/helios.md`](../../docs/user_guide/examples/offline_inference/helios.md)
 - Related example under `examples/`:
  [`examples/offline_inference/helios/README.md`](../../examples/offline_inference/helios/README.md)
 ## Hardware Support
 ## GPU
 ### 1x NVIDIA H20
 #### Environment
 - OS: Ubuntu Linux x86_64
 - Python: 3.12.12
 - Driver / runtime: NVIDIA driver `580.126.20`, CUDA `13.0`
 - Hardware: 1x NVIDIA H20 GPU from an 8x H20 host
 - vLLM version: `0.19.0`
 - vLLM-Omni version or commit: `a3903810`
 #### Command
 Run the baseline Helios-Base text-to-video example from the repository root:
 ```bash
 cd examples/offline_inference/helios
 python end2end.py \
  --model BestWishYsh/Helios-Base \
  --sample-type t2v \
  --prompt "A dynamic time-lapse video showing the rapidly moving scenery from the window of a speeding train." \
  --guidance-scale 5.0 \
  --output helios_t2v_base.mp4
 ```
 To use cache-dit acceleration, run on a vLLM-Omni checkout that includes Helios cache-dit support and enable the cache backend:
 ```bash
 cd examples/offline_inference/helios
 python end2end.py \
  --cache-backend cache_dit \
  --enable-cache-dit-summary \
  --model BestWishYsh/Helios-Base \
  --sample-type t2v \
  --prompt "A dynamic time-lapse video showing the rapidly moving scenery from the window of a speeding train." \
  --guidance-scale 5.0 \
  --output helios_t2v_base.mp4
 ```
 #### Verification
 The script should print the resolved generation configuration, total generation
 time, and output path. A successful run ends with output similar to:
 ```text
 Total generation time: <seconds> seconds (<milliseconds> ms)
 Saved generated video to helios_t2v_base.mp4
 ```
 #### Important flags and stage configs
 - `--model BestWishYsh/Helios-Base` selects the base Helios checkpoint used by
  this recipe.
 - `--sample-type t2v` selects text-to-video generation. Use `i2v` with
  `--image-path` for image-to-video, or `v2v` with `--video-path` for
  video-to-video.
 - `--guidance-scale 5.0` matches the Helios-Base recommendation. Use
  `--guidance-scale 1.0` for Helios-Distilled.
 - `--cache-backend cache_dit` enables the cache-dit acceleration path.
 - `--enable-cache-dit-summary` prints cache-dit summary information after
  diffusion forward passes.
 - No separate stage config is required for this offline recipe; the bundled
  Helios example configures the pipeline through `end2end.py` arguments.
 - For Helios-Mid, add `--is-enable-stage2`,
  `--pyramid-num-inference-steps-list 20 20 20`, `--use-cfg-zero-star`,
  `--use-zero-init`, and `--zero-steps 1`.
 - For Helios-Distilled, add `--is-enable-stage2`,
  `--pyramid-num-inference-steps-list 2 2 2`, and
  `--is-amplify-first-chunk`.
 #### Known limitations
 - Helios generates video in 33-frame chunks. For best performance, set
  `--num-frames` to a multiple of `33`; non-multiple values are rounded up to
  the nearest multiple of `33`.
@@ -1,106 +0,0 @@
 # HiDream-I1-Full
 > Text-to-image serving and benchmark
 ## Summary
 - Vendor: HiDream.ai
 - Model: `HiDream-ai/HiDream-I1-Full`
 - Task: Text-to-image generation
 - Mode: Online serving with the OpenAI-compatible API
 - Maintainer: Community
 ## When to use this recipe
 Use this recipe when you want a known-good starting point for serving
 `HiDream-ai/HiDream-I1-Full` on a single 80 GB A800, validate the normal online-serving path.
 ## References
 - Model: <https://huggingface.co/HiDream-ai/HiDream-I1-Full>
 - Upstream or canonical docs:
  [`docs/user_guide/examples/online_serving/text_to_image.md`](../../docs/user_guide/examples/online_serving/text_to_image.md)
 - Related example under `examples/`:
  [`examples/online_serving/text_to_image/README.md`](../../examples/online_serving/text_to_image/README.md)
 ## Hardware Support
 This recipe currently documents one CUDA GPU serving configuration.
 ## GPU
 ### 1x A800 80GB
 #### Environment
 - OS: Linux
 - Python: 3.10+
 - Driver / runtime: NVIDIA CUDA environment with an A100 80 GB GPU
 - vLLM version: Match the repository requirements for your checkout
 - vLLM-Omni version or commit: Use the commit you are deploying from
 #### Command
 Start the baseline server:
 ```bash
 export MODEL_NAME_OR_PATH=HiDream-ai/HiDream-I1-Full
 vllm serve ${MODEL_NAME_OR_PATH} \
   --omni \
   --port 8092 \
   --auxiliary-text-encoder meta-llama/Llama-3.1-8B-Instruct \
   --tensor-parallel-size 1 \
   --vae_use_slicing \
   --vae_use_tiling
 ```
 You can also use the example launcher and pass the extra flags through:
 ```bash
 python examples/offline_inference/text_to_image/text_to_image.py \
  --model HiDream-ai/HiDream-I1-Full \
  --prompt "The setting sun of late autumn dyes the riverside with a warm orange hue" \
  --seed 42 \
  --guidance-scale 5.0 \
  --tensor-parallel-size 1 \
  --num-images-per-prompt 1 \
  --num-inference-steps 50 \
  --auxiliary-text-encoder meta-llama/Llama-3.1-8B-Instruct \
  --output /workspace/mnt/cmss-wmd/result/output.png
 ```
 #### Verification
 Run the existing client example after the server is ready:
 ```bash
 python examples/online_serving/text_to_image/openai_chat_client.py \
  --server http://localhost:8092 \
  --prompt "The setting sun of late autumn dyes the riverside with a warm orange hue" \
  --output /tmp/hidream_i1_recipe.png
 ```
 For a direct API smoke test:
 ```bash
 curl -s http://localhost:8092/v1/chat/completions   -H "Content-Type: application/json"   -d '{
    "messages": [
      {"role": "user", "content": "The setting sun of late autumn dyes the riverside with a warm orange hue"}
    ],
    "extra_body": {
      "num_inference_steps": 50,
      "seed": 42,
      "height": 1024,
      "width": 1024
    }
  }' | jq -r '.choices[0].message.content[0].image_url.url' | cut -d',' -f2- | base64 -d > output_online.png
 ```
 #### Notes
 - Key flags: `--auxiliary-text-encoder` designates the path for the auxiliary text model meta-llama/Llama-3.1-8B-Instruct. For HiDream-I1-Full, unspecified use defaults meta-llama/Llama-3.1-8B-Instruct (downloaded from official Hugging Face), and custom paths are supported.
@@ -86,6 +86,7 @@ curl -X POST http://localhost:8000/v1/videos \
 - Duration: Controlled by `num_frames` (frames = duration_seconds * 24 + 1)
 - Known limitations:
  - No image-to-video support yet (LTX23ImageToVideoPipeline is a placeholder)
  - No CFG-parallel support (single-GPU only)
  - Requires `diffusers >= 0.38.0` (not yet on PyPI, install from git)
 ## Hardware Support
@@ -130,108 +130,3 @@ and compare the output JSON or terminal metrics.
 - Key flags: `--step-execution` enables the step-wise runtime; `--max-num-seqs` controls how many compatible requests may stay active together.
 - Keep `--max-num-seqs 1` when you want the more conservative path, when traffic is mostly single-request, or when you are debugging correctness before measuring throughput.
 - Current batching is still shape-sensitive: different step progress can co-batch, but different resolutions do not yet co-batch.
 ### 2x B200 ModelOpt mixed FP8/NVFP4
 #### Environment
 - OS: Linux
 - Python: 3.12
 - Driver / runtime: NVIDIA CUDA environment with two B200 GPUs
 - vLLM version: Match the vLLM-Omni checkout used for deployment
 - vLLM-Omni version or commit: Use the commit that contains ModelOpt mixed
  FP8/NVFP4 checkpoint loading for diffusion models
 #### Command
 This configuration uses a pre-quantized ModelOpt mixed checkpoint for
 Qwen-Image-2512. The validated checkpoint keeps boundary, modulation,
 normalization, and output layers in BF16, uses FP8 for sensitive attention
 projections, and uses NVFP4 for heavier MLP / attention-output linear layers.
 Validated checkpoint:
 ```text
 feizhai123/qwen-image-2512-modelopt-mixed-fp8-sensitive-nvfp4-heavy
 ```
 Start the server with the CUTLASS quantized linear backend:
 ```bash
 CUDA_VISIBLE_DEVICES=2,3 \
 vllm serve feizhai123/qwen-image-2512-modelopt-mixed-fp8-sensitive-nvfp4-heavy \
  --omni \
  --host 0.0.0.0 \
  --port 8091 \
  --trust-remote-code \
  --tensor-parallel-size 2 \
  --linear-backend cutlass \
  --force-cutlass-fp8
 ```
 #### Verification
 Run a direct image generation request after the server is ready:
 ```bash
 curl http://localhost:8091/v1/images/generations \
  -H "Content-Type: application/json" \
  -d '{
    "model": "feizhai123/qwen-image-2512-modelopt-mixed-fp8-sensitive-nvfp4-heavy",
    "prompt": "A cinematic Warhammer 40,000 Ultramarine warrior in blue power armor, armor stained with alien ichor, swinging a chainsword at a Tyranid creature, gothic battlefield, dramatic lighting, high detail, no text, no watermark",
    "negative_prompt": "blurry, low quality, text, watermark, logo, malformed armor, bad hands",
    "size": "1024x1024",
    "num_inference_steps": 20,
    "true_cfg_scale": 4.0,
    "guidance_scale": 4.0,
    "seed": 48
  }'
 ```
 Check that:
 - The server responds on `http://localhost:8091/health`.
 - The response contains one generated image.
 - Logs show CUTLASS selection for ModelOpt FP8 / mixed-precision linear
  layers instead of falling back to the default automatic backend choice.
 #### Benchmark
 The numbers below are loaded-once request-level measurements, not an HTTP
 concurrency benchmark. Each case loads the model once, runs warmup requests,
 then measures 100 sequential requests.
 ```text
 Hardware: 2x B200
 Tensor parallel size: 2
 Requests: 100
 Concurrency: 1
 Resolution: 1024x1024
 Denoising steps: 20
 ```
 | Config | Mean | P90 | P99 | Peak VRAM |
 | --- | ---: | ---: | ---: | ---: |
 | BF16 | 2.696s | 2.700s | 2.707s | 99,000 MiB |
 | ModelOpt FP8 CUTLASS | 2.663s | 2.671s | 2.676s | 85,760 MiB |
 | Mixed FP8/NVFP4 CUTLASS | 2.584s | 2.590s | 2.594s | 84,412 MiB |
 The same 2-GPU setup was also used to sweep NVFP4 linear backends. Direct
 CUTLASS was the fastest validated backend for this checkpoint.
 | Backend | Mean | P90 | P99 | Peak VRAM |
 | --- | ---: | ---: | ---: | ---: |
 | auto | 2.927s | 2.948s | 3.535s | 85,340 MiB |
 | flashinfer-cudnn | 3.523s | 3.528s | 3.766s | 84,500 MiB |
 | flashinfer-trtllm | 2.711s | 2.727s | 3.010s | 84,996 MiB |
 | flashinfer-cutlass | 2.935s | 2.950s | 3.011s | 85,340 MiB |
 | cutlass | 2.578s | 2.584s | 2.592s | 84,412 MiB |
 #### Notes
 - Use `--linear-backend cutlass` to select the validated quantized linear
  backend from the CLI.
 - `--force-cutlass-fp8` keeps ModelOpt FP8 layers on the CUTLASS FP8 path.
 - The benchmark table intentionally reports Qwen-Image-2512 as
  concurrency-1 request-level data; do not compare it directly with an online
  HTTP concurrency benchmark.
@@ -29,25 +29,19 @@ recipes/
 |--------|------|----------|
 | [`audiox/AudioX.md`](./audiox/AudioX.md) | Offline + online unified text/video→audio diffusion | 1x L4 24GB |
 | [`Baidu/ERNIE-Image.md`](./Baidu/ERNIE-Image.md) | Text-to-image online serving (ERNIE-Image 8B) | 1x or 2x RTX 4090 24GB |
 | [`ByteDance/Lance.md`](./ByteDance/Lance.md) | Unified AR+diffusion: text/img/video gen + understanding (Lance 3B) | 1x B300 / A100 80GB |
 | [`fishaudio/Fish-Speech-S2-Pro.md`](./fishaudio/Fish-Speech-S2-Pro.md) | Online serving for TTS | 1x A800 80GB |
 | [`Helios/Helios.md`](./Helios/Helios.md) | Text-to-video, image-to-video, and video-to-video generation | 1x NVIDIA H20 |
 | [`inclusionAI/Ming-flash-omni-2.0.md`](./inclusionAI/Ming-flash-omni-2.0.md) | Online serving for multimodal chat + standalone TTS | 4x H100 / 1x H100 80GB |
 | [`LTX/LTX-2.md`](./LTX/LTX-2.md) | Text-to-video and image-to-video serving | 1x H200 141GB |
 | [`LTX/LTX-2.3.md`](./LTX/LTX-2.3.md) | Text-to-video with audio generation (22B) | 1x GPU (96GB VRAM) |
 | [`mistralai/Voxtral-TTS.md`](./mistralai/Voxtral-TTS.md) | Online serving for TTS | 1x RTX 4090 24GB |
 | [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt mixed FP8/NVFP4 | 1x A100 80GB / 2x B200 |
 | [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md) | Text-to-image serving with step-wise continuous batching replay | 1x A100 80GB |
 | [`Qwen/Qwen-Image-Edit.md`](./Qwen/Qwen-Image-Edit.md) | Text-guided single-image editing | 1x or 2x H200 141GB |
 | [`Qwen/Qwen3-Omni.md`](./Qwen/Qwen3-Omni.md) | Online serving for multimodal chat | 1x A100 80GB |
 | [`Qwen/Qwen3-TTS.md`](./Qwen/Qwen3-TTS.md) | Text-to-speech serving (CustomVoice / VoiceDesign / Base) | 1x H100/A100 80GB |
 | [`SenseNova/SenseNova-U1.md`](./SenseNova/SenseNova-U1.md) | Unified image generation and understanding | 1x H200 (144GB) |
 | [`Tencent/Covo-Audio-Chat.md`](./Tencent/Covo-Audio-Chat.md) | Online serving for audio chat | 1x A100 80GB |
-| [`Tencent/HunyuanImage-3.0-Instruct.md`](./Tencent/HunyuanImage-3.0-Instruct.md) | DiT-only text-to-image serving and benchmark, including ModelOpt mixed FP8/NVFP4 | 4x H100/H800 80GB / 2x B200 |
+| [`Tencent/HunyuanImage-3.0-Instruct.md`](./Tencent/HunyuanImage-3.0-Instruct.md) | DiT-only text-to-image serving and benchmark | 4x H100/H800 80GB |
 | [`Wan-AI/Wan2.2-I2V.md`](./Wan-AI/Wan2.2-I2V.md) | Image-to-video serving (Wan2.2 14B) | 8x Ascend NPU (A2/A3) |
 | [`Wan-AI/Wan2.2-S2V.md`](./Wan-AI/Wan2.2-S2V.md) | Speech-to-video serving (Wan2.2 14B) | 2x A100/H100 80GB |
 | [`zai-org/GLM-TTS.md`](./zai-org/GLM-TTS.md) | Online serving for Chinese/English zero-shot voice-cloned TTS | 1x A40 48GB |
 | [`GLM/GLM-Image.md`](./GLM/GLM-Image.md) | Online serving for image generation | 1x A800 80GB / 2x A800 80GB |
 Within a single recipe file, include different hardware support sections such
 as `GPU`, `ROCm`, and `NPU`, and add concrete tested configurations like
		`@@ -1,2 +0,0 @@`
			`// Opt out of bfcache so large generated API pages are released after navigation.`
			`window.addEventListener("unload", () => {});`