[BugFix] Probe __dict__ instead of hasattr when patching WanRMS_norm (#3400)

Signed-off-by: gcanlin <canlinguosdu@gmail.com>

.buildkite/pipeline-intel.yaml

@@ -10,7 +10,7 @@ steps:
           DOCKER_BUILDKIT: "1"
           # Buildkite will automatically replace this with the actual commit hash
           VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.19.0"
+          VLLM_VERSION: "v0.20.0"
         priority: 100
         timeout_in_minutes: 60
-        soft_fail: true
+        soft_fail: false

.buildkite/test-amd-merge.yml

@@ -7,7 +7,9 @@ steps:
   grade: Blocking
   commands:
     - export VLLM_ROCM_USE_AITER=0
-    - "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
+    # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
+    # that is not supported on CPU
+    - "timeout 20m pytest -v -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
 
 - label: "Diffusion Model Test"
   agent_pool: mi325_2
@@ -103,16 +105,17 @@ steps:
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
     - timeout 20m pytest -s -v tests/engine/test_async_omni_engine_abort.py
 
+# This is from test-nightly.yml.
+# This model seems to be important, so I am trying to run it here.
+# Since this is from test-nightly.yml,
+# I keep this non-blocking.
 - label: "Omni Model Test Qwen2-5-Omni"
-  agent_pool: mi325_2
+  agent_pool: mi325_4
   depends_on: amd-build
   mirror_hardwares: [amdproduction]
-  grade: Blocking
   commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
-    - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni_expansion.py -m "advanced_model" --run-level "core_model"
+    - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+    - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
 
 - label: "Omni Model Test Qwen3-Omni"
   agent_pool: mi325_2
@@ -120,10 +123,9 @@ steps:
   mirror_hardwares: [amdproduction]
   grade: Blocking
   commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
-    - timeout 30m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_mimo_audio.py -m "advanced_model" --run-level "advanced_model"
+    - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_mimo_audio.py -m "advanced_model" --run-level "advanced_model"
 
 - label: "Qwen3-TTS CustomVoice E2E Test"
   agent_pool: mi325_1

.buildkite/test-amd-ready.yaml

@@ -7,7 +7,9 @@ steps:
   grade: Blocking
   commands:
     - export VLLM_ROCM_USE_AITER=0
-    - "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
+    # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
+    # that is not supported on CPU
+    - "timeout 20m pytest -vvvv -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
 
 - label: "Voxtral TTS CUDA Unit Test"
   agent_pool: mi325_1
@@ -71,6 +73,17 @@ steps:
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
 
+- label: "AudioX Online Test"
+  agent_pool: mi325_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: Blocking
+  commands:
+    - export GPU_ARCHS=gfx942
+    - export VLLM_LOGGING_LEVEL=DEBUG
+    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+    - timeout 20m pytest -s -v tests/e2e/online_serving/test_audiox_online.py
+
 - label: "Diffusion Cache Backend Test"
   agent_pool: mi325_1
   depends_on: amd-build

.buildkite/test-merge.yml

@@ -216,27 +216,27 @@ steps:
           volumes:
             - "/fsx/hf_cache:/fsx/hf_cache"
 
-  # - label: "MOSS-TTS-Nano E2E Test"
-  #   depends_on: upload-merge-pipeline
-  #   commands:
-  #     - |
-  #       timeout 20m bash -c '
-  #         export VLLM_LOGGING_LEVEL=DEBUG
-  #         export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-  #         pytest -s -v tests/e2e/online_serving/test_moss_tts_nano.py tests/e2e/offline_inference/test_moss_tts_nano.py -m "advanced_model" --run-level "advanced_model"
-  #       '
-  #   agents:
-  #     queue: "gpu_1_queue"
-  #   plugins:
-  #     - docker#v5.2.0:
-  #         image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-  #         always-pull: true
-  #         propagate-environment: true
-  #         shm-size: "8gb"
-  #         environment:
-  #           - "HF_HOME=/fsx/hf_cache"
-  #         volumes:
-  #           - "/fsx/hf_cache:/fsx/hf_cache"
+  - label: "MOSS-TTS-Nano E2E Test"
+    depends_on: upload-merge-pipeline
+    commands:
+      - |
+        timeout 20m bash -c '
+          export VLLM_LOGGING_LEVEL=DEBUG
+          export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          pytest -s -v tests/e2e/online_serving/test_moss_tts_nano.py tests/e2e/offline_inference/test_moss_tts_nano.py -m "advanced_model" --run-level "advanced_model"
+        '
+    agents:
+      queue: "gpu_1_queue"
+    plugins:
+      - docker#v5.2.0:
+          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+          always-pull: true
+          propagate-environment: true
+          shm-size: "8gb"
+          environment:
+            - "HF_HOME=/fsx/hf_cache"
+          volumes:
+            - "/fsx/hf_cache:/fsx/hf_cache"
 
   - label: "Omni Model Test with H100"
     timeout_in_minutes: 30

.buildkite/test-nightly.yml

@@ -215,6 +215,44 @@ steps:
                       type: DirectoryOrCreate
 
 
+      - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100"
+        timeout_in_minutes: 45
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "core_model" --run-level "core_model"
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 4
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
   - group: ":card_index_dividers: TTS Model Test"
     key: nightly-tts-test-group
     depends_on: upload-nightly-pipeline
@@ -298,6 +336,9 @@ steps:
         timeout_in_minutes: 120
         commands:
           - pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and H100" --run-level "full_model" --ignore=tests/e2e/accuracy
+          # revoke after bagel optimization finished
+          # Keep Bagel expansion and multi-replica tests in their dedicated H100 jobs below.
+          # - pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and H100" --run-level "full_model" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_bagel_multi_replicas.py
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -370,6 +411,45 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
+      # revoke after bagel optimization finished
+      # - label: ":full_moon: Diffusion X2I(&A&T) · BAGEL Multi-Replica with H100"
+      #   timeout_in_minutes: 120
+      #   commands:
+      #     - pytest -s -v tests/e2e/online_serving/test_bagel_multi_replicas.py -m "full_model and diffusion and H100" --run-level "full_model"
+      #   agents:
+      #     queue: "mithril-h100-pool"
+      #   plugins:
+      #     - kubernetes:
+      #         podSpec:
+      #           containers:
+      #             - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+      #               resources:
+      #                 limits:
+      #                   nvidia.com/gpu: 4
+      #               volumeMounts:
+      #                 - name: devshm
+      #                   mountPath: /dev/shm
+      #                 - name: hf-cache
+      #                   mountPath: /root/.cache/huggingface
+      #               env:
+      #                 - name: HF_HOME
+      #                   value: /root/.cache/huggingface
+      #                 - name: HF_TOKEN
+      #                   valueFrom:
+      #                     secretKeyRef:
+      #                       name: hf-token-secret
+      #                       key: token
+      #           nodeSelector:
+      #             node.kubernetes.io/instance-type: gpu-h100-sxm
+      #           volumes:
+      #             - name: devshm
+      #               emptyDir:
+      #                 medium: Memory
+      #             - name: hf-cache
+      #               hostPath:
+      #                 path: /mnt/hf-cache
+      #                 type: DirectoryOrCreate
+
       - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
         timeout_in_minutes: 60
         commands:
@@ -512,6 +592,7 @@ steps:
         timeout_in_minutes: 180
         commands:
           - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -550,7 +631,7 @@ steps:
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test"
         key: nightly-diffusion-x2iat-performance
-        timeout_in_minutes: 180
+        timeout_in_minutes: 360
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
           - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
@@ -604,6 +685,60 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
+      - label: ":full_moon: Diffusion X2I(&A&T) · HunyuanImage3 · DiT Perf Test"
+        key: nightly-hunyuan-image3-performance
+        soft_fail: true
+        timeout_in_minutes: 120
+        if: build.env("RUN_HUNYUAN_IMAGE3_PERF") == "1"
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - export CACHE_DIT_VERSION=1.3.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp4_fp8.json
+            EXIT1=$$?
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_sp2.json
+            EXIT2=$$?
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_cfgp2.json
+            EXIT3=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$((EXIT1 | EXIT2 | EXIT3))
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 4
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
   # Diffusion x2v only (Wan, HunyuanVideo, …). x2i/x2a/x2t live in the X2I group above, not here.
   - group: ":card_index_dividers: Diffusion X2V Model Test"
     key: nightly-diffusion-x2v-group
@@ -656,6 +791,7 @@ steps:
         commands:
           - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model
           - pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m advanced_model --run-level advanced_model
+          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
         agents:
           queue: "mithril-h100-pool"
         plugins:

.buildkite/test-ready.yml

@@ -120,6 +120,23 @@ steps:
           volumes:
             - "/fsx/hf_cache:/fsx/hf_cache"
 
+  - label: "AudioX Online Test"
+    depends_on: upload-ready-pipeline
+    commands:
+      - timeout 20m pytest -s -v tests/e2e/online_serving/test_audiox_online.py -m "core_model and diffusion" --run-level core_model
+    agents:
+      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
+    plugins:
+      - docker#v5.2.0:
+          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+          always-pull: true
+          propagate-environment: true
+          environment:
+            - "HF_HOME=/fsx/hf_cache"
+            - "HF_TOKEN"
+          volumes:
+            - "/fsx/hf_cache:/fsx/hf_cache"
+
   - label: "Diffusion Cache Backend Test"
     depends_on: upload-ready-pipeline
     commands:

.claude/skills/add-tts-model/SKILL.md

@@ -211,8 +211,8 @@ See `plan/voxcpm2_native_ar_design.md`.
 
 - Model files in `vllm_omni/model_executor/models/<model_name>/`
 - Stage config YAML
-- Working `end2end.py` with correct audio output
-- README.md in the example directory
+- Working `end2end.py` at `examples/offline_inference/text_to_speech/<model>/end2end.py`
+- New section in `examples/offline_inference/text_to_speech/README.md` (table row + per-model section). Do **not** create a top-level `examples/offline_inference/<model>/` dir or a per-model `README.md` inside `text_to_speech/<model>/` — the hub README is the documented surface and the mkdocs `generate_examples` hook only descends one level into `examples/<category>/`.
 
 ## Phase 3: Online Serving
 
@@ -308,11 +308,11 @@ def build_voice_clone_prompt(ref_audio_path: str, text: str, codec) -> list:
 ### Deliverables
 
 - Updated `serving_speech.py` with all 5 integration points (single commit)
-- Client scripts and server launcher
-- Gradio demo with streaming and voice cloning UI
+- Client scripts and server launcher under `examples/online_serving/text_to_speech/<model>/`
+- Gradio demo with streaming and voice cloning UI in the same dir
 - E2E online serving test (`tests/e2e/online_serving/test_<model>.py`)
 - Buildkite CI entry in `.buildkite/test-merge.yml`
-- Documentation (offline + online serving docs)
+- New section in `examples/online_serving/text_to_speech/README.md` (table row + per-model section). Do **not** create a top-level `examples/online_serving/<model>/` dir or a per-model `README.md` inside `text_to_speech/<model>/`.
 
 ### E2E test pitfalls to avoid
 

.github/mergify.yml

@@ -0,0 +1,223 @@
+pull_request_rules:
+- name: label-documentation
+  description: Automatically apply documentation label
+  conditions:
+    - label != stale
+    - or:
+      - files~=^[^/]+\.md$
+      - files~=^docs/
+      - files~=^examples/
+  actions:
+    label:
+      add:
+        - documentation
+    comment:
+      message: "Documentation preview: https://vllm--{{number}}.org.readthedocs.build/en/{{number}}/"
+
+- name: comment-pre-commit-failure
+  description: Comment on PR when pre-commit check fails
+  conditions:
+    - status-failure=pre-commit
+    - -closed
+    - -draft
+  actions:
+    comment:
+      message: |
+        Hi @{{author}}, the pre-commit checks have failed. Please run:
+
+        ```bash
+        uv pip install pre-commit
+        pre-commit install
+        pre-commit run --all-files
+        ```
+
+        Then, commit the changes and push to your branch.
+
+        For future commits, `pre-commit` will run automatically on changed files before each commit.
+
+        > [!TIP]
+        > <details>
+        > <summary>Is <code>mypy</code> or <code>markdownlint</code> failing?</summary>
+        > <br/>
+        > <code>mypy</code> and <code>markdownlint</code> are run differently in CI. If the failure is related to either of these checks, please use the following commands to run them locally:
+        >
+        > ```bash
+        > # For mypy (substitute "3.10" with the failing version if needed)
+        > pre-commit run --hook-stage manual mypy-3.10
+        > # For markdownlint
+        > pre-commit run --hook-stage manual markdownlint
+        > ```
+        > </details>
+
+- name: comment-dco-failure
+  description: Comment on PR when DCO check fails
+  conditions:
+    - status-failure=dco
+    - -closed
+    - -draft
+  actions:
+    comment:
+      message: |
+        Hi @{{author}}, the DCO check has failed. Please click on DCO in the Checks section for instructions on how to resolve this.
+
+- name: label-ci-build
+  description: Automatically apply ci/build label
+  conditions:
+    - label != stale
+    - or:
+      - files~=^\.github/
+      - files~=\.buildkite/
+      - files~=^cmake/
+      - files=CMakeLists.txt
+      - files~=^docker/Dockerfile
+      - files~=^requirements.*\.txt
+      - files=setup.py
+  actions:
+    label:
+      add:
+        - ci/build
+
+- name: label-frontend
+  description: Automatically apply frontend label
+  conditions:
+    - label != stale
+    - files~=^vllm_omni/entrypoints/
+  actions:
+    label:
+      add:
+        - frontend
+
+- name: label-diffusion
+  description: Automatically apply diffusion label
+  conditions:
+    - label != stale
+    - files~=^vllm_omni/diffusion/
+  actions:
+    label:
+      add:
+        - diffusion
+
+- name: label-new-model
+  description: Automatically apply new-model label
+  conditions:
+    - label != stale
+    - or:
+        - files=vllm_omni/model_executor/models/registry.py
+        - files=vllm_omni/diffusion/registry.py
+  actions:
+    label:
+      add:
+        - new-model
+
+- name: label-npu
+  description: Automatically apply npu label
+  conditions:
+    - label != stale
+    - or:
+      - files~=^vllm_omni/platforms/npu/
+      - title~=(?i)NPU
+      - title~=(?i)Ascend
+  actions:
+    label:
+      add:
+        - npu
+    assign:
+      users:
+        - "gcanlin"
+
+- name: label-rocm
+  description: Automatically apply rocm label
+  conditions:
+    - label != stale
+    - or:
+      - files~=^vllm_omni/platforms/rocm/
+      - title~=(?i)AMD
+      - title~=(?i)ROCm
+  actions:
+    label:
+      add:
+        - rocm
+    assign:
+      users:
+        - "tjtanaa"
+
+- name: label-xpu
+  description: Automatically apply xpu label
+  conditions:
+    - label != stale
+    - or:
+      - files~=^vllm_omni/platforms/xpu/
+      - title~=(?i)Intel
+      - title~=(?i)XPU
+  actions:
+    label:
+      add:
+        - xpu
+    assign:
+      users:
+        - "yma11"
+
+- name: label-modelrunner
+  description: Automatically apply modelrunner label
+  conditions:
+    - label != stale
+    - or:
+      - files~=^vllm_omni/worker/
+      - title~=(?i)ModelRunner
+      - title~=(?i)worker
+  actions:
+    label:
+      add:
+        - ModelRunner
+    assign:
+      users:
+        - "gcanlin" # To check whether need to sync to NPU
+
+- name: auto-rebase if approved, ready, and 30 commits behind main
+  conditions:
+    - base = main
+    - label=ready
+    - "#approved-reviews-by >= 1"
+    - "#commits-behind >= 30"
+    - -closed
+    - -draft
+    - -conflict
+  actions:
+    rebase: {}
+
+- name: ping author on conflicts and add 'needs-rebase' label
+  conditions:
+    - label != stale
+    - conflict
+    - -closed
+  actions:
+    label:
+      add:
+        - needs-rebase
+    comment:
+      message: |
+       This pull request has merge conflicts that must be resolved before it can be
+       merged. Please rebase the PR, @{{author}}.
+
+       https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork
+
+- name: remove 'needs-rebase' label when conflict is resolved
+  conditions:
+    - -conflict
+    - -closed
+  actions:
+    label:
+      remove:
+        - needs-rebase
+
+- name: label-bug
+  description: Automatically apply bug label
+  conditions:
+    - label != stale
+    - or:
+      - title~=(?i)\bbug\b
+      - title~=(?i)\bbugfix\b
+  actions:
+    label:
+      add:
+        - bug

benchmarks/diffusion/README.md

@@ -32,6 +32,8 @@ python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 - The benchmark talks to `http://<host>:<port>/v1/chat/completions`.
 - If you run the server on another host or port, pass `--base-url` accordingly.
+- To compare diffusion multi-replica serving, restart the server with different
+  `runtime.num_replicas` values and keep the benchmark parameters unchanged.
 
 ## 2. Supported Datasets
 
@@ -93,6 +95,35 @@ You can point to your own trace using `--dataset-path`.
 
 ## 3. Benchmark Parameters
 
+### Multi-replica sweep
+
+For replica scaling, use the same model, prompt count, image/video size, and
+step count for every run. Change only the serving stage config:
+
+```yaml
+runtime:
+  devices: "0,1,2,3"
+  num_replicas: 4
+```
+
+Then run a fixed benchmark and write one JSON file per replica count:
+
+```bash
+python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
+  --base-url http://localhost:8099 \
+  --model Qwen/Qwen-Image \
+  --task t2i \
+  --dataset random \
+  --num-prompts 32 \
+  --max-concurrency 8 \
+  --request-rate inf \
+  --num-inference-steps 10 \
+  --width 512 --height 512 \
+  --output-file diffusion_replica_4.json
+```
+
+Repeat for `num_replicas=1`, `2`, `3`, and `4`, using matching `runtime.devices`.
+
 ### Basic flags
 
 - `--base-url`: Server address (the script calls `.../v1/chat/completions`).
@@ -130,9 +161,21 @@ Warmup flags:
 
 - `--warmup-requests`: Number of warmup requests.
 - `--warmup-num-inference-steps`: Steps used during warmup.
+- `--warmup-concurrency`: Maximum concurrent warmup requests. Use this to warm
+  the same batch shape as the measured run instead of warming only batch=`1`.
 - For `--task t2v`: warmup requests are forced to use `num_frames=1` to make warmup faster and less noisy.
 
 Traffic / concurrency flags:
 
 - `--request-rate`: Target request rate (requests/second). If set to `inf`, the script sends all requests immediately.
 - `--max-concurrency`: Max number of in-flight requests (default: `1`). This can hard-cap the achieved QPS: if it is too small, requests will queue behind the semaphore, and both achieved throughput and observed SLO attainment can be skewed.
+
+### Batched warmup note
+
+For batched serving runs, warm the same in-flight shape you plan to measure.
+For example, a run with `--max-concurrency 8` should usually also use
+`--warmup-requests 8 --warmup-concurrency 8`; otherwise the first measured
+batch may still pay compile or CUDA-graph capture cost.
+
+For a Qwen-Image continuous-batching replay example, see
+[`performance_dashboard/qwen_image_serving_performance.md`](./performance_dashboard/qwen_image_serving_performance.md).

benchmarks/diffusion/diffusion_benchmark_serving.py

@@ -751,6 +751,52 @@ async def iter_requests(
         yield req
 
 
+def _make_warmup_request(
+    requests_list: list[RequestFuncInput],
+    index: int,
+    args,
+) -> RequestFuncInput:
+    warm_req = requests_list[index % len(requests_list)]
+    if args.warmup_num_inference_steps is not None:
+        warm_req = replace(
+            warm_req,
+            num_inference_steps=args.warmup_num_inference_steps,
+        )
+    if args.task == "t2v":
+        warm_req = replace(warm_req, num_frames=1)
+    return warm_req
+
+
+async def _run_warmups(
+    requests_list: list[RequestFuncInput],
+    args,
+    session: aiohttp.ClientSession,
+    request_func,
+) -> list[tuple[RequestFuncInput, RequestFuncOutput]]:
+    if not args.warmup_requests or not requests_list:
+        return []
+
+    warmup_requests = [_make_warmup_request(requests_list, i, args) for i in range(args.warmup_requests)]
+    warmup_concurrency = min(int(args.warmup_concurrency), len(warmup_requests))
+    warmup_semaphore = asyncio.Semaphore(warmup_concurrency)
+
+    print(
+        f"Running {len(warmup_requests)} warmup request(s) "
+        f"with num_inference_steps={args.warmup_num_inference_steps} "
+        f"and warmup_concurrency={warmup_concurrency}..."
+    )
+
+    async def limited_warmup_request_func(
+        req: RequestFuncInput,
+    ) -> RequestFuncOutput:
+        async with warmup_semaphore:
+            return await request_func(req, session, None)
+
+    warmup_tasks = [asyncio.create_task(limited_warmup_request_func(req)) for req in warmup_requests]
+    warmup_outputs = await asyncio.gather(*warmup_tasks)
+    return list(zip(warmup_requests, warmup_outputs))
+
+
 def calculate_metrics(
     outputs: list[RequestFuncOutput],
     total_duration: float,
@@ -900,23 +946,12 @@ async def benchmark(args):
     pbar = tqdm(total=len(requests_list), disable=args.disable_tqdm)
 
     async with aiohttp.ClientSession() as session:
-        warmup_pairs: list[tuple[RequestFuncInput, RequestFuncOutput]] = []
-        if args.warmup_requests and requests_list:
-            print(
-                f"Running {args.warmup_requests} warmup request(s) \
-                with num_inference_steps={args.warmup_num_inference_steps}..."
-            )
-            for i in range(args.warmup_requests):
-                warm_req = requests_list[i % len(requests_list)]
-                if args.warmup_num_inference_steps is not None:
-                    warm_req = replace(
-                        warm_req,
-                        num_inference_steps=args.warmup_num_inference_steps,
-                    )
-                if args.task == "t2v":
-                    warm_req = replace(warm_req, num_frames=1)
-                warm_out = await limited_request_func(warm_req, session, None)
-                warmup_pairs.append((warm_req, warm_out))
+        warmup_pairs = await _run_warmups(
+            requests_list=requests_list,
+            args=args,
+            session=session,
+            request_func=request_func,
+        )
 
         if args.slo:
             # Prefer trace-provided per-request slo_ms. Only populate when missing.
@@ -1072,6 +1107,13 @@ if __name__ == "__main__":
         default=1,
         help="num_inference_steps used for warmup requests.",
     )
+    parser.add_argument(
+        "--warmup-concurrency",
+        type=int,
+        default=1,
+        help="Maximum number of warmup requests to run concurrently. "
+        "Set this to match the real batch shape when warming up torch.compile or CUDA graphs.",
+    )
     parser.add_argument("--width", type=int, default=None, help="Image/Video width.")
     parser.add_argument("--height", type=int, default=None, help="Image/Video height.")
     parser.add_argument("--num-frames", type=int, default=None, help="Number of frames (for video).")
@@ -1135,5 +1177,4 @@ if __name__ == "__main__":
     )
 
     args = parser.parse_args()
-
     asyncio.run(benchmark(args))

benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md

@@ -33,6 +33,12 @@ vllm serve Qwen/Qwen-Image --omni \
     --port 8091
 ```
 
+To replay step-wise continuous batching, compare
+`vllm serve Qwen/Qwen-Image --omni --port 8089 --step-execution --max-num-seqs 1`
+against the same command with `--max-num-seqs 8`. `--step-execution` is the
+feature gate, and increasing `--max-num-seqs` above `1` lets the scheduler
+keep more compatible requests active at once.
+
 ## 3.2 Key Parameters
 
 | Parameter             | Description              |
@@ -70,6 +76,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
 | `--task`               | Task type (e.g., `t2i`)           |
 | `--num-prompts`        | Total number of requests          |
 | `--max-concurrency`    | Client-side concurrency           |
+| `--warmup-concurrency` | Warmup concurrency for batch-shape prewarming |
 | `--random-request-config`| JSON string defining random request |
 
 ---
@@ -133,6 +140,12 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
     ]'
 ```
 
+For a continuous-batching A/B replay, run the same benchmark command twice
+against the two serving configs above and keep traffic plus warmup settings
+identical across both runs. If the measured run uses `--max-concurrency 8`,
+warm with `--warmup-requests 8 --warmup-concurrency 8` so the first measured
+batch does not include compile or CUDA-graph capture overhead.
+
 ---
 
 # 6. Performance Metrics
@@ -163,6 +176,7 @@ To ensure consistent and comparable benchmark results:
 * Record parallel configuration
 * Record benchmark parameters (resolution, concurrency, number of prompts)
 * Ensure no background workload on GPUs during testing
+* When comparing continuous batching, keep warmup settings the same across runs
 
 ---
 

benchmarks/fish-speech/bench_speaker_cache.py

@@ -3,11 +3,11 @@
 Measures TTFP improvement from DAC-code caching when using uploaded voices.
 
 Setup:
-  1. Start vllm-omni with Fish Speech S2 Pro (use our feat branch)
+  1. Start vllm-omni with Fish Speech S2 Pro
   2. Provide a reference audio file for voice cloning
 
 Usage:
-    python bench_voice_cache.py \
+    python bench_speaker_cache.py \
         --ref-audio /path/to/reference.wav \
         --ref-text "Transcript of the reference audio." \
         --num-prompts 20 \

docker/Dockerfile.ci

@@ -1,5 +1,5 @@
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai
-ARG VLLM_BASE_TAG=v0.20.0-cu130
+ARG VLLM_BASE_TAG=v0.20.0
 FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
 ARG APP_DIR=/workspace/vllm-omni
 WORKDIR ${APP_DIR}

docker/Dockerfile.cuda

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai:v0.19.0
+ARG BASE_IMAGE=vllm/vllm-openai:v0.20.0
 FROM ${BASE_IMAGE}
 
 ARG COMMON_WORKDIR=/app

docker/Dockerfile.rocm

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.19.0
+ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.20.0
 FROM ${BASE_IMAGE} AS base
 
 # Declare a variable to know if we want to use the nightly build or the stable build.

docker/Dockerfile.xpu

@@ -14,8 +14,9 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO
 RUN apt clean && apt-get update -y && \
     apt-get install -y --no-install-recommends --fix-missing \
     curl \
-    espeak-ng \
+    ffmpeg \
     git \
+    libsndfile1 \
     libsm6 \
     libxext6 \
     libgl1 \
@@ -28,8 +29,20 @@ RUN apt clean && apt-get update -y && \
     python3.12-dev \
     python3-pip
 
-RUN apt update && apt upgrade -y && \
-    apt install -y intel-oneapi-compiler-dpcpp-cpp-2025.3
+# Add oneAPI repo, pin oneAPI to 2025.3, then install pinned packages in one layer.
+RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \
+    echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \
+    printf '%s\n' \
+    'Package: intel-oneapi-* intel-deep-learning-essentials* intel-pti*' \
+    'Pin: version 2025.3*' \
+    'Pin-Priority: 1001' \
+    > /etc/apt/preferences.d/oneapi-2025.3.pref && \
+    apt-get update -y && \
+    apt-get install -y --no-install-recommends \
+    intel-oneapi-compiler-dpcpp-cpp-2025.3 \
+    intel-oneapi-mkl-devel-2025.3 \
+    intel-oneapi-dnnl-devel-2025.3 && \
+    rm -rf /var/lib/apt/lists/*
 
 # Install UMD with fixed version
 RUN mkdir neo && \
@@ -52,9 +65,9 @@ RUN curl -LsSf https://astral.sh/uv/install.sh | sh
 RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV}
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-# This oneccl contains the BMG support which is not the case for default version of oneapi 2025.2.
-ARG ONECCL_INSTALLER="intel-oneccl-2021.15.7.8_offline.sh"
-RUN wget "https://github.com/uxlfoundation/oneCCL/releases/download/2021.15.7/${ONECCL_INSTALLER}" && \
+# This oneccl contains the BMG support which is not the case for default version of oneapi 2025.3.
+ARG ONECCL_INSTALLER="intel-oneccl-2021.15.9.14_offline.sh"
+RUN wget "https://github.com/uxlfoundation/oneCCL/releases/download/2021.15.9/${ONECCL_INSTALLER}" && \
     bash "${ONECCL_INSTALLER}" -a --silent --eula accept && \
     rm "${ONECCL_INSTALLER}" && \
     echo "source /opt/intel/oneapi/setvars.sh --force" >> /root/.bashrc && \
@@ -66,6 +79,7 @@ SHELL ["bash", "-c"]
 CMD ["bash", "-c", "source /root/.bashrc && exec bash"]
 
 WORKDIR /workspace/
+
 ENV UV_HTTP_TIMEOUT=500
 
 # Configure package index for XPU
@@ -74,24 +88,18 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
 ENV UV_INDEX_STRATEGY="unsafe-best-match"
 ENV UV_LINK_MODE="copy"
 
-ARG VLLM_VERSION=v0.19.0
+ARG VLLM_VERSION=v0.20.0
 RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
 WORKDIR /workspace/vllm
 
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install --upgrade pip && \
-    uv pip install -r requirements/xpu.txt
-
- # used for suffix method speculative decoding
- # build deps for proto + nanobind-based extensions to set up the build environment
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip install grpcio-tools protobuf nanobind
- # arctic-inference is built from source which needs torch-xpu properly installed first
-RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install -r requirements/xpu.txt && \
+    uv pip install setuptools_scm &&\
+    uv pip install grpcio-tools protobuf nanobind && \
     source /opt/intel/oneapi/setvars.sh --force && \
     source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force && \
-    export CMAKE_PREFIX_PATH="$(python -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH}" && \
-    uv pip install --no-build-isolation arctic-inference==0.1.1
+    export CMAKE_PREFIX_PATH="$(python3 -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH}"
 
 ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/"
 
@@ -105,6 +113,9 @@ CMD ["/bin/bash"]
 
 FROM vllm-base AS vllm-openai
 
+# install nixl from source code
+RUN uv pip install nixl
+
 # install additional dependencies for openai api server
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install accelerate hf_transfer pytest pytest_asyncio lm_eval[api] modelscope
@@ -112,10 +123,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 # install development dependencies (for testing)
 RUN uv pip install -e tests/vllm_test_utils
 
-# install nixl from source code
-ENV NIXL_VERSION=0.7.0
-RUN python /workspace/vllm/tools/install_nixl_from_source_ubuntu.py
-
 # ensure vllm is properly installed
 RUN python -c "import vllm, inspect; print(vllm.__file__)"
 RUN uv pip show vllm
@@ -135,7 +142,7 @@ RUN uv pip install --no-cache-dir ".[dev]" --no-build-isolation
 # FIX triton
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip uninstall triton triton-xpu && \
-    uv pip install triton-xpu==3.6.0
+    uv pip install triton-xpu==3.7.0
 
 # remove torch bundled oneccl to avoid conflicts
 RUN --mount=type=cache,target=/root/.cache/uv \

docs/.nav.yml

@@ -16,8 +16,6 @@ nav:
     - examples/README.md
     - Offline Inference:
       - BAGEL-7B-MoT: user_guide/examples/offline_inference/bagel.md
-      - CosyVoice3: user_guide/examples/offline_inference/cosyvoice3.md
-      - Fish Speech S2 Pro: user_guide/examples/offline_inference/fish_speech.md
       - GLM-Image Multistage End-to-End Inference: user_guide/examples/offline_inference/glm_image.md
       - Helios Video Generation: user_guide/examples/offline_inference/helios.md
       - HunyuanImage-3.0 Image-to-Text Inference: user_guide/examples/offline_inference/hunyuan_image3.md
@@ -28,24 +26,22 @@ nav:
       - MiMo-Audio Offline Inference: user_guide/examples/offline_inference/mimo_audio.md
       - Qwen2.5-Omni: user_guide/examples/offline_inference/qwen2_5_omni.md
       - Qwen3-Omni: user_guide/examples/offline_inference/qwen3_omni.md
-      - Qwen3-TTS: user_guide/examples/offline_inference/qwen3_tts.md
       - Text-To-Audio: user_guide/examples/offline_inference/text_to_audio.md
       - Text-To-Image: user_guide/examples/offline_inference/text_to_image.md
+      - Text-To-Speech: user_guide/examples/offline_inference/text_to_speech.md
       - Text-To-Video: user_guide/examples/offline_inference/text_to_video.md
-      - Voxtral TTS Offline Inference: user_guide/examples/offline_inference/voxtral_tts.md
       - X-To-Video-Audio: user_guide/examples/offline_inference/x_to_video_audio.md
     - Online Serving:
       - BAGEL-7B-MoT: user_guide/examples/online_serving/bagel.md
       - vLLM-Omni Helm Chart: user_guide/examples/online_serving/chart-helm.md
-      - Diffusers Backend Adapter Example: user_guide/examples/online_serving/diffusers_pipeline_adapter.md
-      - Fish Speech S2 Pro: user_guide/examples/online_serving/fish_speech.md
+      - Diffusers Backend Adapter: user_guide/examples/online_serving/diffusers_pipeline_adapter.md
       - GLM-Image Online Serving: user_guide/examples/online_serving/glm_image.md
       - Image-To-Image: user_guide/examples/online_serving/image_to_image.md
       - Image-To-Video: user_guide/examples/online_serving/image_to_video.md
       - Online serving Example of vLLM-Omni for MiMo-Audio: user_guide/examples/online_serving/mimo_audio.md
       - Qwen2.5-Omni: user_guide/examples/online_serving/qwen2_5_omni.md
       - Qwen3-Omni: user_guide/examples/online_serving/qwen3_omni.md
-      - Qwen3-TTS: user_guide/examples/online_serving/qwen3_tts.md
+      - Text-To-Speech: user_guide/examples/online_serving/text_to_speech.md
       - Text-To-Image: user_guide/examples/online_serving/text_to_image.md
       - Text-To-Video: user_guide/examples/online_serving/text_to_video.md
   - General:
@@ -116,6 +112,7 @@ nav:
       - design/feature/async_chunk.md
       - design/feature/vae_parallel.md
       - design/feature/diffusion_step_execution.md
+      - design/feature/diffusion_continuous_batching.md
     - Module Design:
       - design/module/ar_module.md
       - design/module/dit_module.md

docs/contributing/model/adding_diffusion_model.md

@@ -737,6 +737,16 @@ step-scoped phases. The reference implementation is
 Do not enable `step_execution=True` until those four methods are implemented
 and validated against the request-level path.
 
+If you want the pipeline to work with the experimental batched step-wise path
+(`max_num_seqs > 1`), also see:
+[Continuous Batching for Step-Wise Diffusion](../../design/feature/diffusion_continuous_batching.md).
+
+If you expose this in example scripts or recipes, keep it opt-in. Surface
+runtime features like `step_execution` as optional flags instead of silently
+turning them on. For Qwen-Image-style serving examples, document
+`--step-execution` as the feature gate and `--max-num-seqs N` as the
+companion batching knob.
+
 ### Cache Acceleration
 
 #### TeaCache

docs/contributing/model/adding_tts_model.md

@@ -186,6 +186,17 @@ vllm_omni/model_executor/stage_configs/
   your_model_name_async_chunk.yaml   # Streaming mode config
 ```
 
+### Example placement
+
+TTS examples live in the consolidated text-to-speech hub, **not** in their
+own top-level directory. Place per-model scripts under
+`examples/offline_inference/text_to_speech/<your_model>/` and
+`examples/online_serving/text_to_speech/<your_model>/`, and add a section
+to the hub `README.md` files (table row + per-model section) instead of a
+new per-model `README.md`. The mkdocs `generate_examples` hook treats the
+`text_to_speech/` parent as a single example, so per-model READMEs inside
+it would not be picked up — the hub README is the documented surface.
+
 **Qwen3-TTS reference files:**
 
 | File | Purpose |

docs/contributing/profiling.md

@@ -37,6 +37,8 @@ Supported fields:
 | `warmup_iterations` | Torch-profiler warmup iterations. |
 | `active_iterations` | Torch-profiler active iterations. |
 
+For detailed explanations of the fields, please refer to upstream vLLM implementation [vllm/config/profiler.py](https://github.com/vllm-project/vllm/blob/v0.20.1/vllm/config/profiler.py)
+
 ### Minimal configurations by output
 
 Only collect trace output:
@@ -124,7 +126,7 @@ profiler_config = {
     "max_iterations": 0,
     "wait_iterations": 0,
     "warmup_iterations": 0,
-    "active_iterations": 0,
+    "active_iterations": 1,
 }
 ```
 
@@ -220,7 +222,7 @@ vllm serve Wan-AI/Wan2.2-I2V-A14B-Diffusers \
     "max_iterations": 0,
     "wait_iterations": 0,
     "warmup_iterations": 0,
-    "active_iterations": 0
+    "active_iterations": 1
   }'
 ```
 

docs/design/feature/diffusion_continuous_batching.md

@@ -0,0 +1,156 @@
+# Continuous Batching for Step-Wise Diffusion
+
+!!! warning "Experimental Feature"
+    This feature is experimental. It currently applies only to native
+    diffusion pipelines running with `step_execution=True`.
+
+This document describes the batching extension built on top of
+[Diffusion Step Execution](diffusion_step_execution.md). The base
+step-execution contract is unchanged. The batching work is mainly in the
+scheduler and runner layers.
+
+## Why It Helps
+
+Step-wise execution breaks a long denoise loop into scheduler-visible units.
+That gives the runtime a place to admit other compatible requests between
+steps instead of waiting for an entire request to finish.
+
+This matters most in low-MFU or bursty serving scenarios:
+
+- one request's denoise step may not fully saturate the GPU
+- several compatible requests can share the same denoise forward pass
+- throughput and device utilization can improve without changing request-local
+  scheduler state
+
+This is **not** a guaranteed single-request latency win. The main benefit is
+usually higher utilization and better throughput when the workload contains
+multiple in-flight compatible requests.
+
+## Overview
+
+With continuous batching enabled:
+
+- the scheduler may keep multiple compatible requests active at the same time
+- the runner packs request-local step state into one `InputBatch`
+- `denoise_step()` runs on that batch
+- `step_scheduler()` and `post_decode()` still run per request
+
+The current implementation is conservative:
+
+- only compatible requests are batched together
+- request-mode diffusion still runs with `max_num_seqs=1`
+- per-request progress and completion remain independent
+
+## Enablement
+
+Use `--step-execution` as the feature gate, then increase `--max-num-seqs`
+above `1` if you want batching:
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --step-execution \
+  --max-num-seqs 8
+```
+
+`--max-num-seqs 1` keeps the step-wise path without enabling batching.
+
+For a reproducible replay flow using the bundled serving benchmark, see the
+Qwen-Image replay commands in
+[`benchmarks/diffusion/README.md`](gh-file:benchmarks/diffusion/README.md)
+and
+[`benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md`](gh-file:benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md).
+
+## Scheduler
+
+The scheduler derives its batch capacity from `max_num_seqs` through
+`max_num_running_reqs`.
+
+Batch admission is gated by
+[`SamplingParamsKey`](gh-file:vllm_omni/diffusion/sched/interface.py),
+which is built from shape-sensitive and CFG-sensitive sampling fields. This is
+the core correctness rule for batching: requests are only co-batched when they
+share the same denoise tensor contract.
+
+There are two important details:
+
+- `num_inference_steps` is not part of the key, so requests with different
+  total step counts can still share a batch
+- requests also do not need to be at the same current denoise progress; active
+  requests can continue batching even when their current step indices diverge
+- admission is still FIFO, so an incompatible request at the head of the
+  waiting queue blocks later compatible requests
+
+Today that compatibility rule is still shape-sensitive. `height`, `width`,
+`num_frames`, and CFG-related fields remain part of the key, so different
+resolutions or incompatible guidance settings do **not** co-batch yet.
+
+The current batching unit is one `OmniDiffusionRequest`. Requests with
+multiple prompts do not participate in batching today.
+
+## Runner
+
+The runner keeps persistent per-request execution state in
+[`DiffusionRequestState`](gh-file:vllm_omni/diffusion/worker/utils.py),
+while the scheduler owns a separate lightweight request state for queueing and
+lifecycle tracking.
+
+For each step, the runner builds an
+[`InputBatch`](gh-file:vllm_omni/diffusion/worker/input_batch.py) from the
+active request states:
+
+- prompt embeddings and masks are normalized and padded
+- dynamic tensors such as `latents` and `timesteps` are gathered each step
+- buffers are reused when batch composition stays the same
+
+The step-wise batched path is:
+
+1. Run `prepare_encode()` for newly admitted requests.
+2. Build or refresh `InputBatch`.
+3. Run one batched `denoise_step(input_batch)`.
+4. Slice the batched `noise_pred` back per request.
+5. Run per-request `step_scheduler()`.
+6. Run `post_decode()` only for requests that finished denoising.
+7. Scatter updated latents back into persistent request state with
+   [`scatter_latents()`](gh-file:vllm_omni/diffusion/worker/input_batch.py).
+
+This keeps the shared work limited to the denoise forward pass while preserving
+request-local scheduler state and outputs.
+
+## Engine
+
+[`DiffusionEngine`](gh-file:vllm_omni/diffusion/diffusion_engine.py) provides
+the background loop and async add-request path needed for multiple requests to
+accumulate in the scheduler.
+
+This is supporting infrastructure, not the main design point. The batching
+behavior is defined by scheduler-side compatibility gating and runner-side
+batch packing.
+
+## Current Limitations
+
+- Experimental feature; use `max_num_seqs=1` for the older conservative path.
+- Only native pipelines that already support `step_execution=True`.
+- Request-mode diffusion still clamps `max_num_seqs` back to `1`.
+- Only homogeneous batches keyed by `SamplingParamsKey` are supported.
+- Multi-prompt requests are not batched.
+- `cache_backend`, KV transfer, and other request-mode extras are not wired
+  into the batched step-wise path yet.
+- Future work can relax the current same-shape restriction with richer
+  heterogeneous batching policies such as bucketing or padded execution for
+  different resolutions.
+
+## Related Files
+
+- Scheduler base:
+  [`vllm_omni/diffusion/sched/base_scheduler.py`](gh-file:vllm_omni/diffusion/sched/base_scheduler.py)
+- Scheduler interface:
+  [`vllm_omni/diffusion/sched/interface.py`](gh-file:vllm_omni/diffusion/sched/interface.py)
+- Step scheduler:
+  [`vllm_omni/diffusion/sched/step_scheduler.py`](gh-file:vllm_omni/diffusion/sched/step_scheduler.py)
+- Runner:
+  [`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py)
+- Input batch:
+  [`vllm_omni/diffusion/worker/input_batch.py`](gh-file:vllm_omni/diffusion/worker/input_batch.py)
+- Tests:
+  [`tests/diffusion/test_diffusion_scheduler.py`](gh-file:tests/diffusion/test_diffusion_scheduler.py)

docs/design/feature/diffusion_step_execution.md

@@ -1,4 +1,4 @@
-# Diffusion Step Execution
+# Adding Step Execution Support for Diffusion Pipelines
 
 This guide documents vLLM-Omni's stepwise diffusion contract for model authors
 and contributors implementing `step_execution=True` support for a diffusion
@@ -7,12 +7,20 @@ pipeline.
 For end-user enablement, supported models, and current limitations, see
 [Step Execution](../../user_guide/diffusion/step_execution.md).
 
+This document describes the base step-execution contract only. For the
+experimental batching policy layered on top of the step-wise path, see
+[Continuous Batching for Step-Wise Diffusion](diffusion_continuous_batching.md).
+
 ## Current Support Scope
 
 `step_execution` is **not** a generic diffusion toggle. It only works for
 pipelines that implement the segmented stateful contract in
 [`vllm_omni/diffusion/models/interface.py`](gh-file:vllm_omni/diffusion/models/interface.py).
 
+This page is intentionally author-facing. Treat runtime enablement
+(`step_execution=True` in Python or `--step-execution` in serving) as an
+opt-in user knob layered on top of the implementation contract below.
+
 Current in-tree support:
 
 | Pipeline | Example models | Step execution |
@@ -22,7 +30,9 @@ Current in-tree support:
 
 Current engine/runtime limitations:
 
-- `StepScheduler` only schedules `batch_size=1`.
+- Continuous batching with `max_num_seqs > 1` is experimental and documented in
+  [Continuous Batching for Step-Wise Diffusion](diffusion_continuous_batching.md).
+  Keep `max_num_seqs=1` if you want the older conservative behavior.
 - `cache_backend` is not supported in step mode.
 - Request-mode extras such as KV transfer are not wired into step mode yet.
 - Unsupported pipelines now fail early during model loading instead of failing on the first request.

docs/design/index.md

@@ -10,6 +10,8 @@ This section contains design documents and architecture specifications for vLLM-
 
 - [Disaggregated Inference](feature/disaggregated_inference.md)
 - [Ray-based Execution](feature/ray_based_execution.md)
+- [Adding Step Execution Support for Diffusion Pipelines](feature/diffusion_step_execution.md)
+- [Continuous Batching for Step-Wise Diffusion](feature/diffusion_continuous_batching.md)
 
 ## Module Design Documents
 

docs/design/module/dit_module.md

@@ -201,7 +201,7 @@ class _BaseScheduler(SchedulerInterface):
 
 - **Shared cleanup logic**: Request-id registration, finish handling, and state removal are centralized instead of duplicated in each policy.
 
-- **Current constraint**: `max_num_running_reqs` remains `1` because the current engine path is still synchronous request-mode execution.
+- **Current constraint boundary**: `_BaseScheduler` derives `max_num_running_reqs` from `max_num_seqs`, but request-mode diffusion is still clamped back to `1` by the engine. The step-wise path can keep this above `1` for compatible-request batching.
 
 #### 2.4 Current `RequestScheduler` Policy
 
@@ -217,7 +217,7 @@ class RequestScheduler(_BaseScheduler):
 
 - **FIFO request scheduling**: Waiting requests are promoted in queue order.
 
-- **Single-request admission**: The current policy only admits one active request at a time.
+- **Single-request admission**: `RequestScheduler` still admits one active request at a time because request-mode execution completes a whole request per dispatch.
 
 - **Executor result feedback**: `update_from_output()` converts executor output into `FINISHED_COMPLETED` or `FINISHED_ERROR` and returns finished scheduler ids.
 
@@ -237,7 +237,7 @@ while True:
 
 - **No scheduler-owned IPC**: Scheduler no longer talks to workers directly.
 
-- **Conservative concurrency**: The current request-mode implementation still allows only one active request at a time.
+- **Split concurrency model**: Request-mode diffusion remains single-active-request, while the step-wise path can keep multiple compatible requests running and advance them independently between denoise steps.
 
 ---
 

docs/getting_started/installation/gpu/rocm.inc.md

@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700
+uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
 ```
 
 #### Installation of vLLM-Omni
@@ -37,10 +37,10 @@ uv pip install onnxruntime-rocm
 If you do not need to modify source code of vLLM, you can directly install the stable 0.20.0 release version of the library
 
 ```bash
-uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700
+uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
 ```
 
-The pre-built 0.20.0 vLLM wheel targets ROCm 7.0. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The pre-built 0.20.0 vLLM wheel targets ROCm 7.2.1. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
 
 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source

docs/getting_started/quickstart.md

@@ -22,7 +22,7 @@ source .venv/bin/activate
 uv pip install vllm==0.20.0 --torch-backend=auto
 
 # On ROCm
-uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700
+uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
 
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
@@ -31,6 +31,12 @@ uv pip install -e .
 
 For additional installation methods — please see the [installation guide](installation/README.md).
 
+
+!!! note
+    It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.
+
+    If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.20.0` and vLLM Omni `0.20.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
+
 ## Offline Inference
 
 Text-to-image generation quickstart with vLLM-Omni:

docs/models/supported_models.md

@@ -32,6 +32,7 @@ th {
 | `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
+| `WanSpeechToVideoPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `LTX2Pipeline` | LTX-2-T2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
 | `LTX2ImageToVideoPipeline` | LTX-2-I2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
@@ -53,6 +54,7 @@ th {
 | `FluxPipeline` | FLUX.1-schnell | `black-forest-labs/FLUX.1-schnell` | ✅︎ | ✅︎ | | ✅︎ |
 | `OmniGen2Pipeline` | OmniGen2 | `OmniGen2/OmniGen2` | ✅︎ | ✅︎ | | ✅︎ |
 | `StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` | ✅︎ | ✅︎ | | ✅︎ |
+| `AudioXPipeline` | AudioX | `zhangj1an/AudioX` | ✅︎ | ✅︎ | | |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
@@ -66,5 +68,5 @@ th {
 | `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | |
 | `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | |
 |`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | |
-
+| `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 ✅︎ indicates the model is supported on that backend. Empty cells mean not listed as supported on that backend.

docs/serving/speech_api.md

@@ -358,6 +358,31 @@ curl -X POST http://localhost:8091/v1/audio/speech \
     }' --output cloned.wav
 ```
 
+### Voice Storage & Caching
+
+Uploaded voices are persisted to disk as a single `.safetensors` file per voice
+(audio samples + metadata — name, consent, ref_text, sample_rate, created_at —
+in the file header). On server restart the directory is scanned and all
+previously uploaded voices are restored automatically, so uploads survive
+process restarts.
+
+Uploading an existing name overwrites the previous entry (a warning is logged).
+
+Feature extraction artifacts (ref_code, speaker_embedding, DAC codes, etc.)
+are cached in-process with a shared LRU so repeated requests with the same
+`voice=...` skip the extraction pipeline. The cache is a true singleton across
+all TTS model types; deleting a voice invalidates every model-type slot at
+once.
+
+**Configuration (environment variables):**
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `SPEAKER_SAMPLES_DIR` | `~/.cache/vllm-omni/speakers` | Directory for persisted uploaded speakers (`.safetensors` files). |
+| `SPEAKER_MAX_UPLOADED` | `1000` | Maximum number of uploaded speakers kept on disk. Upload requests past the cap return 400. |
+
+The in-memory LRU has a fixed 512 MiB byte budget.
+
 ## Batch Speech Generation
 
 The batch endpoint synthesizes multiple texts in a single request, returning all results as JSON with base64-encoded audio.
@@ -535,7 +560,7 @@ The bundled `qwen3_tts.yaml` uses `max_num_seqs: 1` (single request) on both sta
 |-------|-------------|
 | `fishaudio/s2-pro` | 4B dual-AR TTS with DAC codec (44.1 kHz). Supports text-to-speech and voice cloning. |
 
-Fish Speech uses `ref_audio` and `ref_text` for voice cloning (no `task_type` needed). The `voice` field should be set to `"default"`. See the [Fish Speech online serving example](../user_guide/examples/online_serving/fish_speech.md) for details.
+Fish Speech uses `ref_audio` and `ref_text` for voice cloning (no `task_type` needed). The `voice` field should be set to `"default"`. See the [Fish Speech section of the online TTS hub](../user_guide/examples/online_serving/text_to_speech.md#fish-speech-s2-pro) for details.
 
 ### Voxtral TTS
 
@@ -543,6 +568,30 @@ Fish Speech uses `ref_audio` and `ref_text` for voice cloning (no `task_type` ne
 |-------|-------------|
 | `mistralai/Voxtral-4B-TTS-2603` | 3B AR + FlowMatching TTS. Supports text-to-speech with preset voices. |
 
+### CosyVoice3
+
+| Model | Description |
+|-------|-------------|
+| `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | Voice cloning from `ref_audio` + `ref_text`. No built-in voice presets — upload a voice or pass `ref_audio`/`ref_text` per request. |
+
+### OmniVoice
+
+| Model | Description |
+|-------|-------------|
+| `k2-fsa/OmniVoice` | Pure-diffusion TTS. Supports voice cloning via `ref_audio` (with optional `ref_text`); no built-in voice presets. |
+
+### VoxCPM2
+
+| Model | Description |
+|-------|-------------|
+| `openbmb/VoxCPM2` | TTS + voice cloning with built-in speaker presets and uploaded-voice support. Accepts `voice` (preset or uploaded) or `ref_audio` + optional `ref_text`. |
+
+### MOSS-TTS-Nano
+
+| Model | Description |
+|-------|-------------|
+| `OpenMOSS-Team/MOSS-TTS-Nano` | Voice cloning only. Requires `ref_audio` (or an uploaded `voice`); no built-in voice presets. `ref_text` is accepted but ignored — upstream's `voice_clone` mode does not consume a transcript. |
+
 ## Error Responses
 
 ### 400 Bad Request

docs/user_guide/diffusion/cpu_offload_diffusion.md

@@ -17,7 +17,7 @@ Both strategies use pinned memory for faster CPU-GPU transfers. The strategies a
 Model-level offloading implements mutual exclusion between DiT transformer and encoder modules using pre forward hooks:
 
 - **When encoders run**: DiT transformer is offloaded to CPU
-- **When DiT runs**: Encoders are offloaded to CPU
+- **When DiT runs**: Encoders are offloaded to CPU, if more than one dit models, only one loaded on GPU, others get offloaded to CPU.
 - **VAE**: Stays resident on GPU
 
 Before each module's forward pass, the hook automatically moves it to GPU while offloading the other module group to CPU. Transfers use pinned memory for speed.

docs/user_guide/diffusion/multi_replica_serving.md

@@ -0,0 +1,80 @@
+# Diffusion Multi-Replica Serving
+
+Diffusion stages can be replicated in online serving by setting `runtime.num_replicas` on a `stage_type: diffusion` stage. Each replica owns an independent diffusion engine process and receives requests through round-robin routing with request affinity.
+
+## When to Use
+
+Use multi-replica serving when one diffusion process does not saturate the available GPUs, or when you want higher request throughput for many independent image/video generation requests. It improves concurrency throughput; it does not make a single request faster.
+
+## Configuration
+
+A replica consumes the devices required by that diffusion stage. For single-device diffusion pipelines, set `runtime.devices` to one device per replica:
+
+```yaml
+stages:
+  - stage_id: 0
+    stage_type: diffusion
+    runtime:
+      devices: "0,1,2,3"
+      num_replicas: 4
+    engine_args:
+      model: Qwen/Qwen-Image
+      parallel_config:
+        world_size: 1
+```
+
+For multi-device diffusion pipelines, `parallel_config.world_size` is treated as the number of devices per replica. For example, four 2-GPU replicas need eight devices:
+
+```yaml
+stages:
+  - stage_id: 0
+    stage_type: diffusion
+    runtime:
+      devices: "0,1,2,3,4,5,6,7"
+      num_replicas: 4
+    engine_args:
+      model: Wan-AI/Wan2.2-T2V-A14B-Diffusers
+      parallel_config:
+        world_size: 2
+```
+
+The runtime splits the device list into contiguous chunks per replica. With the second example, replica 0 uses `0,1`, replica 1 uses `2,3`, replica 2 uses `4,5`, and replica 3 uses `6,7`.
+
+## Serving
+
+Start the server with the stage config:
+
+```bash
+vllm serve Qwen/Qwen-Image \
+  --omni \
+  --port 8099 \
+  --stage-configs-path /path/to/diffusion_multi_replica.yaml
+```
+
+Then send normal OpenAI-compatible diffusion chat requests. The orchestrator routes new requests across replicas and keeps updates/abort calls on the same replica for each request.
+
+## Benchmarking
+
+Benchmark each replica count by changing `runtime.num_replicas` and `runtime.devices`, restarting the server, then running:
+
+```bash
+python benchmarks/diffusion/diffusion_benchmark_serving.py \
+  --base-url http://localhost:8099 \
+  --model Qwen/Qwen-Image \
+  --task t2i \
+  --dataset random \
+  --num-prompts 8 \
+  --request-rate inf \
+  --num-inference-steps 2 \
+  --width 256 --height 256 \
+  --output-file diffusion_replica_4.json
+```
+
+For a quick scaling sweep, test `num_replicas` values `1`, `2`, `3`, and `4` with the same prompt count and generation parameters. Compare throughput and end-to-end latency from the JSON outputs.
+
+## Notes and Limitations
+
+- Multi-replica serving is currently supported for diffusion stages. Single-stage mode still rejects non-diffusion replicas.
+- `runtime.devices` must provide exactly `num_replicas * devices_per_replica` devices.
+- Request-level latency may stay similar or slightly increase because each request still runs on one replica.
+- Throughput improves only when enough concurrent requests are issued and each replica has enough GPU memory.

docs/user_guide/diffusion/step_execution.md

@@ -8,6 +8,8 @@ implement the stepwise contract support it today.
 
 ## Quick Start
 
+### Python API
+
 ```python
 from vllm_omni import Omni
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
@@ -25,6 +27,18 @@ outputs = omni.generate(
 )
 ```
 
+### Serving
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --step-execution \
+  --max-num-seqs 8
+```
+
+For serving, `--step-execution` enables the step-wise runtime. Continuous
+batching only becomes relevant when `--max-num-seqs > 1`.
+
 ## Supported Pipelines
 
 | Pipeline | Example models | Step execution |
@@ -32,9 +46,17 @@ outputs = omni.generate(
 | `QwenImagePipeline` | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes |
 | All other diffusion pipelines | `QwenImageEditPipeline`, `QwenImageEditPlusPipeline`, `QwenImageLayeredPipeline`, GLM-Image, Wan, Flux, etc. | No |
 
+!!! warning "Experimental continuous batching"
+    When `--step-execution` is enabled and `max_num_seqs > 1` is configured,
+    the step-wise path can batch
+    compatible requests together. This is experimental. Requests with
+    incompatible sampling parameters are intentionally kept in separate batches,
+    and `max_num_seqs=1` remains the conservative default.
+
 ## Current Limitations
 
-- `step_execution` currently supports `batch_size=1` only.
+- Continuous batching under `step_execution` is experimental and only batches
+  compatible requests.
 - `cache_backend` is not supported together with step execution.
 - Unsupported pipelines fail early during model loading.
 - Request-mode extras such as KV transfer are not wired into step mode yet.
@@ -45,6 +67,11 @@ Use step execution only when you specifically need the pipeline to run through
 its stepwise request state machine. For normal diffusion inference, leave it
 disabled unless your workflow depends on this mode.
 
+For Qwen-Image online serving, the usual progression is:
+
+- start with `--step-execution --max-num-seqs 1` if you only need the step-wise path
+- increase `--max-num-seqs` after that if you want the experimental compatible-request batching behavior
+
 If you are looking for general diffusion speedups, see
 [Diffusion Features Overview](../diffusion_features.md).
 
@@ -59,3 +86,7 @@ pipeline does not support step execution.
 If you want to add step execution support to a new diffusion pipeline, see the
 implementation guide:
 [Diffusion Step Execution Design](../../design/feature/diffusion_step_execution.md).
+
+If you also want that pipeline to participate in the experimental batched
+step-wise path, see:
+[Continuous Batching for Step-Wise Diffusion](../../design/feature/diffusion_continuous_batching.md).

docs/user_guide/diffusion_features.md

@@ -15,7 +15,7 @@ vLLM-Omni supports various advanced features for diffusion models:
 - Acceleration: **cache methods**, **parallelism methods**, **startup optimizations**
 - Memory optimization: **cpu offloading**, **quantization**
 - Extensions: **LoRA inference**, **frame interpolation**
-- Execution modes: **step execution**
+- Execution modes: **step execution**, **multi-replica serving**
 
 ## Supported Features
 
@@ -43,6 +43,7 @@ Parallelism methods distribute computation across GPUs without quality loss (mat
 | **[Tensor Parallelism](diffusion/parallelism/tensor_parallel.md)** | Shards model weights across devices | Large models that don't fit in single GPU, with 2+ GPUs |
 | **[HSDP](diffusion/parallelism/hsdp.md)** | Weight sharding via FSDP2, redistributed on-demand at runtime | Very large models (14B+) on limited VRAM, combinable with SP |
 | **[Expert Parallelism](diffusion/parallelism/expert_parallel.md)** | Shards MoE expert MLP blocks across devices | MoE diffusion models (e.g., HunyuanImage3.0) |
+| **[Multi-Replica Serving](diffusion/multi_replica_serving.md)** | Runs multiple independent diffusion stage replicas and routes requests across them | Online serving throughput with many concurrent requests |
 
 #### Startup Optimization
 
@@ -129,6 +130,7 @@ The following tables show which models support each feature:
 | **Qwen-Image-Layered** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
 | **Stable-Diffusion3.5** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ (decode) | ❌ | ❌ |
 | **Z-Image** | ✅ | ✅ | ✅ | ❓ | ✅ (TP=2 only) | ✅ | ❌ | ✅ (decode) | ✅ | ❌ |
+| **ERNIE-Image** | ❌ | ✅ | ✅ | ❓ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
 
 > Notes:
 > 1. Nextstep_1(T2I) does not support cache acceleration methods such as TeaCache or Cache-DiT.
@@ -144,7 +146,7 @@ The following tables show which models support each feature:
 | **LTX-2.3** | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
 | **Helios** | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
 | **HunyuanVideo-1.5 T2V I2V** | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ✅ | ❌ |
-| **DreamID-Omni** | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
+| **DreamID-Omni** | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 
 **Frame Interpolation Support**
 
@@ -155,7 +157,7 @@ The following tables show which models support each feature:
 
 | Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
 |-------|:----------:|:-----------:|:---------------------:|:--------------:|:-----------------:|:------:|:------------------------:|:--------------------:|:--------------:|:----------------:|
-| **Stable-Audio-Open** | ✅ | ❌ | ❓ | ❓ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ |
+| **Stable-Audio-Open** | ✅ | ❌ | ❓ | ❓ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ |
 
 
 ## Feature Compatibility

docs/user_guide/examples/offline_inference/cosyvoice3.md

@@ -1,81 +0,0 @@
-# CosyVoice3
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/cosyvoice3>.
-
-
-## Setup
-
-Install dependencies:
-```
-uv pip install -e .
-```
-
-> **Note:** This includes required libraries such as `soundfile`,
-> `onnxruntime`, `x-transformers`, and `einops` via
-> `requirements/common.txt` and platform-specific requirements files.
-
-Download the model snapshot:
-```
-from huggingface_hub import snapshot_download
-snapshot_download('FunAudioLLM/Fun-CosyVoice3-0.5B-2512', local_dir='pretrained_models/Fun-CosyVoice3-0.5B')
-```
-
-Add `config.json` in `pretrained_models/Fun-CosyVoice3-0.5B/`:
-```json
-{
-    "model_type": "cosyvoice3",
-    "architectures": [
-        "CosyVoice3Model"
-    ]
-}
-```
-
-> **Why `config.json` is required:**
-> `AutoConfig.register("cosyvoice3", CosyVoice3Config)` only registers a class mapping.
-> The loader still needs `model_type: "cosyvoice3"` from `config.json` to select that class.
-> If no `config.json` is present, model type cannot be inferred automatically.
-> If your downloaded checkpoint already includes a valid `config.json` with
-> `model_type: "cosyvoice3"`, this manual step can be skipped.
-
-Run the offline verification script:
-```
-python examples/offline_inference/cosyvoice3/verify_e2e_cosyvoice.py \
-  --model pretrained_models/Fun-CosyVoice3-0.5B \
-  --tokenizer pretrained_models/Fun-CosyVoice3-0.5B/CosyVoice-BlankEN
-```
-
-## Implementation Overview
-
-CosyVoice3 runs as a **2-stage Omni pipeline**.
-
-- **Stage 0** (`talker`) converts text + prompt audio to speech tokens.
-- **Stage 1** (`code2wav`) converts speech tokens + prompt features to waveform via flow matching and HiFiGAN.
-
-Key components live in `vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py`.
-
-- `CosyVoice3MultiModalProcessor` builds the multimodal inputs:
-    - Tokenizes `prompt` and `prompt_text`.
-    - Extracts speech tokens and mel features from the prompt audio.
-    - Extracts a speaker embedding.
-- `CosyVoice3Model` implements both stages:
-    - Stage 0 uses `CosyVoice3LM` and outputs speech tokens + conditioning features.
-    - Stage 1 runs the flow model (DiT-based CFM) and HiFiGAN to synthesize waveform.
-
-Pipeline topology lives in `vllm_omni/model_executor/models/cosyvoice3/pipeline.py`;
-runtime tunables (batch size, memory limits, sampling) live in
-`vllm_omni/deploy/cosyvoice3.yaml`. The deploy config auto-loads by
-HF `model_type` and defaults to `async_chunk: true` (shared-memory
-streaming). Pass `--no-async-chunk` on `vllm serve` to switch to the
-legacy sync path where stage 1 runs `text2flow` over the full
-speech-token sequence.
-
-- Stage 0 emits latent speech tokens.
-- Stage 1 consumes them via `sync_process_input_func` (sync mode) or the
-  shared-memory connector (async-chunk mode) and outputs audio.
-
-## Example materials
-
-??? abstract "verify_e2e_cosyvoice.py"
-    ``````py
-    --8<-- "examples/offline_inference/cosyvoice3/verify_e2e_cosyvoice.py"
-    ``````

docs/user_guide/examples/offline_inference/fish_speech.md

@@ -1,85 +0,0 @@
-# Fish Speech S2 Pro
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/fish_speech>.
-
-
-This directory contains an offline demo for running Fish Speech S2 Pro (`fishaudio/s2-pro`) with vLLM Omni. It supports text-only synthesis and voice cloning with reference audio.
-
-## Model Overview
-
-[Fish Speech S2 Pro](https://huggingface.co/fishaudio/s2-pro) is a 4B dual-AR text-to-speech model by FishAudio. It uses:
-
-- **Slow AR**: Generates semantic tokens autoregressively (Qwen3-based backbone)
-- **Fast AR**: Predicts residual codebook tokens from semantic tokens
-- **DAC Decoder**: Converts codec codes to 44.1 kHz audio waveform
-
-The model produces high-quality speech with voice cloning capabilities.
-
-## Prerequisites
-
-Install the `fish-speech` package for the DAC codec:
-
-```bash
-pip install fish-speech
-```
-
-## Setup
-
-Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
-
-## Quick Start
-
-### Text-Only Synthesis
-
-```bash
-python end2end.py --text "Hello, this is a test of the Fish Speech text to speech system."
-```
-
-Generated audio files are saved to `output_audio/` by default.
-
-### Voice Cloning
-
-Provide a reference audio file and its transcript to clone a voice:
-
-```bash
-python end2end.py \
-    --text "Hello, this is a cloned voice." \
-    --ref-audio /path/to/reference.wav \
-    --ref-text "Transcript of the reference audio."
-```
-
-## Streaming Mode
-
-Add `--streaming` to stream audio chunks progressively via `AsyncOmni` (requires `async_chunk: true` in the stage config):
-
-```bash
-python end2end.py --text "Hello, this is a test." --streaming
-```
-
-Each DAC decoder chunk is logged as it arrives. The final WAV file is written once generation completes.
-
-## Configuration
-
-| Argument | Default | Description |
-|----------|---------|-------------|
-| `--text` | `"Hello, this is a test..."` | Text to synthesize |
-| `--ref-audio` | None | Path to reference audio for voice cloning |
-| `--ref-text` | None | Transcript of the reference audio |
-| `--model` | `fishaudio/s2-pro` | HuggingFace model path |
-| `--stage-configs-path` | `fish_speech_s2_pro.yaml` | Path to stage configs YAML |
-| `--output-dir` | `output_audio` | Output directory for WAV files |
-| `--streaming` | False | Enable streaming via AsyncOmni |
-| `--stage-init-timeout` | 600 | Stage initialization timeout (seconds) |
-
-## Notes
-
-- Output audio is 44.1 kHz mono WAV.
-- The DAC codec weights (`codec.pth`) are loaded lazily from the model directory.
-- Voice cloning encodes the reference audio using the DAC codec to extract semantic codes, then prepends them as a system message.
-
-## Example materials
-
-??? abstract "end2end.py"
-    ``````py
-    --8<-- "examples/offline_inference/fish_speech/end2end.py"
-    ``````

docs/user_guide/examples/offline_inference/qwen3_tts.md

@@ -1,172 +0,0 @@
-# Qwen3-TTS
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen3_tts>.
-
-
-This directory contains an offline demo for running Qwen3 TTS models with vLLM Omni. It builds task-specific inputs and generates WAV files locally.
-
-## Model Overview
-
-Qwen3 TTS provides multiple task variants for speech generation:
-
-- **CustomVoice**: Generate speech with a known speaker identity (speaker ID) and optional instruction.
-- **VoiceDesign**: Generate speech from text plus a descriptive instruction that designs a new voice.
-- **Base**: Voice cloning using a reference audio + reference transcript, with optional mode selection. The `ref_audio` field accepts a local file path, HTTP URL, or base64 data URL.
-
-## Setup
-Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
-
-### ROCm Dependencies
-
-You will need to install the dependency `onnxruntime-rocm`.
-
-```
-pip uninstall onnxruntime # should be removed before we can install onnxruntime-rocm
-pip install onnxruntime-rocm
-```
-
-## Quick Start
-
-Run a single sample for a task:
-
-```
-python end2end.py --query-type CustomVoice
-```
-
-Generated audio files are saved to `output_audio/` by default.
-
-## Task Usage
-
-### CustomVoice
-
-Single sample:
-
-```
-python end2end.py --query-type CustomVoice
-```
-
-Batch sample (multiple prompts in one run):
-
-```
-python end2end.py --query-type CustomVoice --use-batch-sample
-```
-
-### VoiceDesign
-
-Single sample:
-
-```
-python end2end.py --query-type VoiceDesign
-```
-
-Batch sample:
-
-```
-python end2end.py --query-type VoiceDesign --use-batch-sample
-```
-
-### Base (Voice Clone)
-
-Single sample:
-
-```
-python end2end.py --query-type Base
-```
-
-Batch sample:
-
-```
-python end2end.py --query-type Base --use-batch-sample
-```
-
-Mode selection for Base:
-
-- `--mode-tag icl` (default): standard mode
-- `--mode-tag xvec_only`: enable `x_vector_only_mode` in the request
-
-Examples:
-
-```
-python end2end.py --query-type Base --mode-tag icl
-```
-
-## Voice and Language Control
-
-### Supported Voices (CustomVoice)
-
-Predefined speaker voices are set via the `speaker` (or `voice_type`) field in `additional_information`. Available speakers depend on the loaded checkpoint; check `talker_config.spk_id` in the model config for the full list. Common voices include `vivian`, `ryan`, `aiden`, `ethan`, `serena` (case-insensitive).
-
-Pass the speaker name in your request:
-
-```python
-additional_information = {
-    "text": ["你好，我是通义千问"],
-    "task_type": ["CustomVoice"],
-    "speaker": ["Vivian"],  
-    "language": ["Chinese"],
-}
-```
-
-### Supported Languages
-
-The `language` field controls the codec-level language tag. Use `"Auto"` (default) for automatic detection.
-
-Supported values: `Auto`, `Chinese`, `English`, `Japanese`, `Korean`, `German`, `French`, `Russian`, `Portuguese`, `Spanish`, `Italian`.
-
-```python
-additional_information = {
-    "text": ["Hello, nice to meet you."],
-    "task_type": ["CustomVoice"],
-    "speaker": ["Aiden"],
-    "language": ["English"],
-}
-```
-
-### VoiceDesign and Base
-
-- **VoiceDesign**: Use `instruct` for natural-language voice description; no `speaker` needed.
-- **Base**: Use `ref_audio` and `ref_text` for voice cloning; `language` is optional.
-
-## Streaming Mode
-
-Add `--streaming` to stream audio chunks progressively via `AsyncOmni` (requires `async_chunk: true` in the stage config):
-
-```bash
-python end2end.py --query-type CustomVoice --streaming --output-dir /tmp/out_stream
-```
-
-Each Code2Wav chunk is logged as it arrives (default 25 frames; configurable via `codec_chunk_frames`
-in the stage config). The initial chunk size is dynamically selected based on server load for reduced
-TTFA, and can be overridden per-request via the `initial_codec_chunk_frames` API field. The final WAV file is written once generation
-completes. This demonstrates that audio data is available progressively rather than only at the end.
-
-> **Note:** Streaming uses `AsyncOmni` internally. The non-streaming path (`Omni`) is unchanged.
-
-## Batched Decoding
-
-The Code2Wav stage (stage 1) supports batched decoding, where multiple requests are decoded in a single forward pass through the SpeechTokenizer. To use it, set `max_num_seqs > 1` on both stages via `--stage-overrides` and pass multiple prompts via `--txt-prompts` with a matching `--batch-size`.
-
-```
-python end2end.py --query-type CustomVoice \
-    --txt-prompts benchmark_prompts.txt \
-    --batch-size 4 \
-    --stage-overrides '{"0":{"max_num_seqs":4,"gpu_memory_utilization":0.2},"1":{"max_num_seqs":4,"gpu_memory_utilization":0.2}}'
-```
-
-**Important:** `--batch-size` must match a CUDA graph capture size (1, 2, 4, 8, 16...) because the Talker's code predictor KV cache is sized to `max_num_seqs`, and CUDA graphs pad the batch to the next capture size. Both stages need `max_num_seqs >= batch_size` in the stage config for batching to take effect. If only stage 1 has a higher `max_num_seqs`, it won't help — stage 1 can only batch chunks from requests that are in-flight simultaneously, which requires stage 0 to also process multiple requests concurrently.
-
-## Notes
-
-- The script uses the model paths embedded in `end2end.py`. Update them if your local cache path differs.
-- Use `--output-dir` to change the output folder.
-
-## Example materials
-
-??? abstract "benchmark_prompts.txt"
-    ``````txt
-    --8<-- "examples/offline_inference/qwen3_tts/benchmark_prompts.txt"
-    ``````
-??? abstract "end2end.py"
-    ``````py
-    --8<-- "examples/offline_inference/qwen3_tts/end2end.py"
-    ``````

docs/user_guide/examples/offline_inference/text_to_audio.md

@@ -57,6 +57,8 @@ Key arguments:
 - `--hsdp-shard-size`: number of GPUs used for HSDP sharding.
 - `--hsdp-replicate-size`: number of HSDP replica groups.
 - `--output`: path to save the generated WAV file.
+- `--enable-cpu-offload`: enabling model-wise offloading to save gpu memory
+- `--enable-layerwise-offload`: enabling layerwise offloading to save gpu memory
 
 ## Example materials
 

docs/user_guide/examples/offline_inference/text_to_speech.md

@@ -0,0 +1,395 @@
+# Text-To-Speech
+
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech>.
+
+
+vLLM-Omni supports several autoregressive TTS models. They share a common
+CLI shape (`--text`, `--ref-audio`, `--ref-text`, `--output-dir`) and live
+together in this hub. Each model has its own subdirectory containing a
+single `end2end.py` script; this README is the single doc entry point.
+
+For online serving, see `examples/online_serving/<model>/`. For the full
+list of supported architectures across all modalities, see
+[Supported Models](https://github.com/vllm-project/vllm-omni/tree/main/docs/models/supported_models.md).
+
+## Supported Models
+
+| Model | HuggingFace repo | Stages | Voice cloning | Streaming | Special modes | Sample rate |
+|---|---|---|---|---|---|---|
+| VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | — | continuation (`--ref-audio` + `--ref-text`) | 48 kHz |
+| VoxCPM | local model dir | split | ✓ | ✓ (`voxcpm_async_chunk.yaml`) | — | 24 kHz |
+| CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ (`async_chunk: true` default) | — | 22.05 kHz |
+| Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ (`--streaming`) | — | 44.1 kHz |
+| OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design (`--instruct`), language (`--lang`) | 24 kHz |
+| Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | 2 (talker + code2wav) | ✓ (Base) | ✓ | 3 task variants (`--query-type`) | 24 kHz |
+| Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | varies | ✓ | ✓ | voice presets (`--voice`) | 24 kHz |
+
+## Common Quick Start
+
+Most models share this invocation shape:
+
+```bash
+python examples/offline_inference/text_to_speech/<model>/end2end.py \
+    --text "Hello, this is a test." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text  "Transcript of the reference audio."
+```
+
+`--ref-audio` and `--ref-text` are optional (text-only synthesis works without them) and must be provided together for voice cloning. The exotic scripts — Qwen3-TTS, Voxtral TTS, CosyVoice3 — accept additional model-specific flags documented in their per-model section below.
+
+---
+
+## VoxCPM2
+
+Single-stage native AR TTS at 48 kHz. Pipeline: `feat_encoder → MiniCPM4 → FSQ → residual_lm → LocDiT → AudioVAE`.
+
+### Prerequisites
+```bash
+pip install voxcpm
+# or, for a local source checkout:
+export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/voxcpm
+```
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
+    --model openbmb/VoxCPM2 \
+    --text "Hello, this is a VoxCPM2 demo."
+```
+
+### Voice cloning
+Pass a reference audio for isolated cloning, or both `--ref-audio` + `--ref-text` for prompt continuation:
+```bash
+python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
+    --text "Hello, this is a voice clone demo." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text  "Transcript of the reference audio."
+```
+
+### Notes
+- Output: 48 kHz mono WAV.
+- Stage config: `vllm_omni/model_executor/stage_configs/voxcpm2.yaml` (default).
+
+---
+
+## VoxCPM
+
+Split-stage TTS. The hub example covers single TTS, single voice cloning, and streaming. Use `benchmarks/voxcpm/` for warmup, batch JSONL prompts, profiler injection, and offline TTFP / RTF measurement.
+
+### Prerequisites
+```bash
+pip install voxcpm soundfile
+# or use a local source tree:
+export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
+```
+
+If the native VoxCPM `config.json` does not contain HF metadata such as `model_type`, prepare a persistent HF-compatible config directory and point the stage configs to it via `VLLM_OMNI_VOXCPM_HF_CONFIG_PATH`:
+
+```bash
+export VOXCPM_MODEL=/path/to/voxcpm-model
+export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config
+mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"
+cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json"
+cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true
+python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)'
+```
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This is a split-stage VoxCPM synthesis example running on vLLM Omni."
+```
+
+### Voice cloning
+```bash
+python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This sentence is synthesized with a cloned voice." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text  "The exact transcript spoken in reference.wav."
+```
+
+### Streaming
+Pass the async-chunk stage config:
+```bash
+python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
+    --model "$VOXCPM_MODEL" \
+    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
+    --text "This is a split-stage VoxCPM streaming example running on vLLM Omni."
+```
+
+### Notes
+- `voxcpm.yaml` is the default non-streaming stage config; `voxcpm_async_chunk.yaml` enables streaming.
+- Streaming is currently single-request oriented.
+- `--ref-text` must be the real transcript of `--ref-audio`; mismatched text degrades quality.
+- For online serving, see [`examples/online_serving/voxcpm`](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/voxcpm/README.md). For benchmark reporting, see [`benchmarks/voxcpm`](https://github.com/vllm-project/vllm-omni/tree/main/benchmarks/voxcpm/README.md).
+
+---
+
+## CosyVoice3
+
+2-stage TTS pipeline (`talker` + `code2wav`) at 22.05 kHz.
+
+### Prerequisites
+```bash
+uv pip install -e .
+# Includes soundfile, onnxruntime, x-transformers, einops via requirements.
+```
+
+Download the model snapshot:
+```python
+from huggingface_hub import snapshot_download
+snapshot_download('FunAudioLLM/Fun-CosyVoice3-0.5B-2512',
+                  local_dir='pretrained_models/Fun-CosyVoice3-0.5B')
+```
+
+If your downloaded checkpoint lacks `config.json`, add it:
+```json
+{
+    "model_type": "cosyvoice3",
+    "architectures": ["CosyVoice3Model"]
+}
+```
+This is required because `AutoConfig.register("cosyvoice3", CosyVoice3Config)` only registers the class mapping; the loader still reads `model_type` from `config.json` to select the class.
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
+    --model pretrained_models/Fun-CosyVoice3-0.5B \
+    --tokenizer pretrained_models/Fun-CosyVoice3-0.5B/CosyVoice-BlankEN
+```
+
+### Voice cloning
+Pass a reference audio. Note that CosyVoice3's `--prompt-text` is a system-style prompt for the GPT stage, not a reference transcript:
+```bash
+python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
+    --model pretrained_models/Fun-CosyVoice3-0.5B \
+    --tokenizer pretrained_models/Fun-CosyVoice3-0.5B/CosyVoice-BlankEN \
+    --ref-audio prompt.wav \
+    --prompt-text "You are a helpful assistant.<|endofprompt|>Testing my voices. Why should I not?"
+```
+
+### Notes
+- Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform.
+- Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config <path>` to override.
+- `async_chunk: true` is the default; pass `--no-async-chunk` to switch to the legacy synchronous path.
+
+---
+
+## Fish Speech S2 Pro
+
+4B dual-AR text-to-speech model from FishAudio with the DAC codec at 44.1 kHz.
+
+### Prerequisites
+```bash
+pip install fish-speech
+```
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
+    --text "Hello, this is a test of the Fish Speech text to speech system."
+```
+
+### Voice cloning
+```bash
+python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
+    --text "Hello, this is a cloned voice." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text  "Transcript of the reference audio."
+```
+
+### Streaming
+```bash
+python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
+    --text "Hello, this is a streaming test." \
+    --streaming
+```
+Streaming requires `async_chunk: true` in the stage config.
+
+### Notes
+- Output: 44.1 kHz mono WAV.
+- DAC codec weights (`codec.pth`) are loaded lazily from the model directory.
+
+---
+
+## OmniVoice
+
+Zero-shot multilingual TTS supporting 600+ languages, with three modes (auto / clone / design).
+
+### Prerequisites
+```bash
+huggingface-cli download k2-fsa/OmniVoice
+```
+Voice cloning requires `transformers>=5.3.0`. Auto and design modes work with `transformers>=4.57.0`.
+
+### Quick start (auto voice)
+```bash
+python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
+    --model k2-fsa/OmniVoice \
+    --text "Hello, this is a test."
+```
+
+### Voice cloning
+```bash
+python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
+    --model k2-fsa/OmniVoice \
+    --text "Hello, this is a test." \
+    --ref-audio ref.wav \
+    --ref-text  "This is the reference transcription."
+```
+
+### Voice design
+```bash
+python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
+    --model k2-fsa/OmniVoice \
+    --text "Hello, this is a test." \
+    --instruct "female, low pitch, british accent"
+```
+
+### Language hint
+```bash
+python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
+    --model k2-fsa/OmniVoice \
+    --text "你好，这是一个测试。" \
+    --lang zh
+```
+
+### Notes
+- Stage 0 (Generator): Qwen3-0.6B with 32-step iterative unmasking.
+- Stage 1 (Decoder): HiggsAudioV2 RVQ + DAC at 24 kHz.
+
+---
+
+## Qwen3-TTS
+
+3-task-variant TTS with 24 kHz output. Has its own argparse surface (this script does not follow the common `--text` / `--ref-audio` shape).
+
+### Prerequisites
+For ROCm builds, replace `onnxruntime` with `onnxruntime-rocm`:
+```bash
+pip uninstall onnxruntime
+pip install onnxruntime-rocm
+```
+
+### Task variants
+- `CustomVoice`: predefined speaker (speaker ID) with optional style instruction.
+- `VoiceDesign`: text + descriptive instruction designs a new voice.
+- `Base`: voice cloning from reference audio + transcript.
+
+```bash
+# Single sample
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type CustomVoice
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type VoiceDesign
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type Base
+
+# Base variant has an additional mode flag:
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type Base --mode-tag icl       # default
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type Base --mode-tag xvec_only # x_vector_only_mode
+
+# Batch (multiple prompts in one run)
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type CustomVoice --use-batch-sample
+```
+
+### Streaming
+```bash
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
+    --query-type CustomVoice \
+    --streaming \
+    --output-dir /tmp/out_stream
+```
+Streaming requires `async_chunk: true` in the stage config.
+
+### Batched decoding
+The Code2Wav stage supports batched decoding through the SpeechTokenizer. Configure both stages with `max_num_seqs > 1` via `--stage-overrides` and pass multiple prompts via `--txt-prompts`:
+```bash
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
+    --query-type CustomVoice \
+    --txt-prompts examples/offline_inference/text_to_speech/qwen3_tts/benchmark_prompts.txt \
+    --batch-size 4 \
+    --stage-overrides '{"0":{"max_num_seqs":4,"gpu_memory_utilization":0.2},"1":{"max_num_seqs":4,"gpu_memory_utilization":0.2}}'
+```
+`--batch-size` must match a CUDA-graph capture size (1, 2, 4, 8, 16…).
+
+### Notes
+- Run `--help` for the full argument surface.
+- See `qwen3_tts/end2end.py` for the prompt-length-estimation logic the Talker uses.
+
+---
+
+## Voxtral TTS
+
+Voxtral-4B-TTS (Mistral). Has its own argparse surface; uses voice presets and the `mistral_common` `SpeechRequest` protocol.
+
+### Prerequisites
+Latest `mistral_common` with `SpeechRequest` support:
+```bash
+pip install -e /path/to/mistral-common  # or upgrade from PyPI when available
+```
+
+### Quick start (voice preset)
+```bash
+python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
+    --write-audio --voice cheerful_female \
+    --model mistralai/Voxtral-4B-TTS-2603 \
+    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
+```
+
+### Voice cloning (capability gated upstream)
+```bash
+python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
+    --write-audio \
+    --model mistralai/Voxtral-4B-TTS-2603 \
+    --text "This is a test message." \
+    --ref-audio path/to/reference_audio.wav
+```
+
+### Streaming + concurrency
+```bash
+python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
+    --num-prompts 32 --concurrency 8 --streaming --write-audio --voice neutral_female \
+    --model mistralai/Voxtral-4B-TTS-2603 \
+    --text "..."
+```
+Available voice presets are listed on the HF model card (`mistralai/Voxtral-4B-TTS-2603`).
+
+### Notes
+- `--num-prompts N` replicates the prompt for performance measurement.
+- `--concurrency M` requires `--streaming` and must evenly divide `--num-prompts`.
+- Run `--help` for the full argument surface.
+
+## Example materials
+
+??? abstract "cosyvoice3/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/cosyvoice3/end2end.py"
+    ``````
+??? abstract "fish_speech/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/fish_speech/end2end.py"
+    ``````
+??? abstract "omnivoice/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/omnivoice/end2end.py"
+    ``````
+??? abstract "qwen3_tts/benchmark_prompts.txt"
+    ``````txt
+    --8<-- "examples/offline_inference/text_to_speech/qwen3_tts/benchmark_prompts.txt"
+    ``````
+??? abstract "qwen3_tts/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/qwen3_tts/end2end.py"
+    ``````
+??? abstract "voxcpm/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/voxcpm/end2end.py"
+    ``````
+??? abstract "voxcpm2/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/voxcpm2/end2end.py"
+    ``````
+??? abstract "voxtral_tts/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/voxtral_tts/end2end.py"
+    ``````

docs/user_guide/examples/offline_inference/voxtral_tts.md

@@ -1,69 +0,0 @@
-# Voxtral TTS Offline Inference
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/voxtral_tts>.
-
-
-`end2end.py` runs Voxtral TTS end-to-end offline inference using vLLM. It supports both blocking (`Omni`) and streaming (`AsyncOmni`) generation, batched prompts with configurable concurrency, and voice selection via preset name or reference audio file.
-
-When `mistral_common` has `SpeechRequest` support, prompt token IDs are built via `encode_speech_request`. Otherwise, the script falls back to manual token construction.
-
-## Usage Examples
-
-
-```bash
-# Basic single-prompt with cheerful_female voice preset
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --write-audio --voice cheerful_female \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
-
-# 32 replicate prompts with cheerful_female voice preset
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --num-prompts 32 --write-audio --voice cheerful_female \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
-
-# Streaming with neutral_female voice preset
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --streaming --write-audio --voice neutral_female \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
-
-# 32 prompts, 8 concurrent requests per wave, streaming with neutral_female voice
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --num-prompts 32 --concurrency 8 --streaming --write-audio --voice neutral_female \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
-
-# Short debug prompt with reference audio
-# Note: Reference audio capability is not yet released.
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --write-audio \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "This is a test message." \
-    --audio-path path/to/reference_audio.wav
-```
-
-## Arguments
-
-| Argument | Description |
-|---|---|
-| `--model PATH` | HuggingFace repo ID or local directory path (default: `mistralai/Voxtral-4B-TTS-2603`) |
-| `--text TEXT` | Text to synthesize (default: `"This is a test message."`) |
-| `--audio-path PATH` | Path to reference audio file for voice cloning |
-| `--output-dir DIR` | Directory to write output WAV files (default: `output_audio`) |
-| `--deploy-config PATH` | Override the deploy config path. If unset, auto-loads `vllm_omni/deploy/voxtral_tts.yaml` from the HF `model_type`. |
-| `--num-prompts N` | Number of replicate prompts to run for measuring performance (default: 1) |
-| `--streaming` | Use streaming generation via `AsyncOmni` (default: blocking `Omni`) |
-| `--concurrency N` | Max concurrent requests per wave (must be used with `--streaming`, must evenly divide `--num-prompts`) |
-| `--voice NAME` | Voice preset to use instead of reference audio file (e.g., casual_female, casual_male, cheerful_female, neutral_female, neutral_male) |
-| `--write-audio` | Write generated audio to WAV files |
-| `--profiling-mode` | Enable profiling mode (reduces max tokens to 50) |
-| `--log-stats` | Enable detailed statistics logging |
-
-## Example materials
-
-??? abstract "end2end.py"
-    ``````py
-    --8<-- "examples/offline_inference/voxtral_tts/end2end.py"
-    ``````

docs/user_guide/examples/offline_inference/x_to_video_audio.md

@@ -47,6 +47,24 @@ python x_to_video_audio.py \
 ```
 In the current test scenario (2 images + 2 audio inputs), the VRAM requirement is 72GB, regardless of whether cfg-parallel is enabled or disabled.
 The VRAM usage can be reduced by enabling CPU offload via --enable-cpu-offload.
+For multi-GPU memory reduction on the fused DreamID-Omni transformer, you can also enable HSDP:
+
+```python
+python x_to_video_audio.py \
+  --model /path/to/dreamid_omni \
+  --prompt "Two people walking together and singing happily" \
+  --image-path ./example0.png ./example1.png \
+  --audio-path ./example0.wav ./example1.wav \
+  --video-negative-prompt "jitter, bad hands, blur, distortion" \
+  --audio-negative-prompt "robotic, muffled, echo, distorted" \
+  --cfg-parallel-size 2 \
+  --num-inference-steps 45 \
+  --height 704 \
+  --width 1280 \
+  --use-hsdp \
+  --hsdp-shard-size 2 \
+  --output out_dreamid_omni_twoip.mp4
+```
 
 
 You could take reference images/audios from the test cases in the official repo: https://github.com/Guoxu1233/DreamID-Omni
@@ -77,6 +95,9 @@ Key arguments:
 - `--image-path`: path to the input image list.
 - `--audio-path`: path to the input audio list, indicate the timbre of the output video.
 - `--cfg-parallel-size`: number of parallel cfg parallel (defaults 1).
+- `--use-hsdp`: enable HSDP weight sharding for DreamID-Omni fused blocks.
+- `--hsdp-shard-size`: number of GPUs used for HSDP sharding.
+- `--hsdp-replicate-size`: number of HSDP replica groups.
 - `--num-inference-steps`: number of denoising steps (defaults 45).
 - `--video-negative-prompt`: negative prompt for video generation.
 - `--audio-negative-prompt`: negative prompt for audio generation.

docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md

@@ -1,18 +1,14 @@
-# Diffusers Backend Adapter Example
+# Diffusers Backend Adapter
 
 Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/diffusers_pipeline_adapter>.
 
 
-This example demonstrates how to serve any 🤗 Diffusers pipeline through vLLM-Omni
-using the `diffusers` load format.
-
-## Supported Models
-
-Any model loadable via `DiffusionPipeline.from_pretrained()` should be supported, including text-to-image, image-to-image, text-to-video, image-to-video, and text-to-audio.
+vLLM-Omni supports running diffusion models with the diffusers backend, directly serving any 🤗 Diffusers pipeline online without implementing them natively.
 
 ## Limitations
 
-The diffusers backend is a black-box adapter. The following features are NOT yet supported.
+The diffusers backend is a black-box adapter. Its primary focus is to serve diffusion models online.
+Currently, the following features are NOT yet supported.
 It is not guaranteed whether they will be supported in the future.
 
 - CFG parallel execution
@@ -22,72 +18,87 @@ It is not guaranteed whether they will be supported in the future.
 
 For these features, it is recommended to use natively supported pipelines instead.
 
-## Usage
+## Model Support
 
-### Option 1: CLI arguments
+Any model loadable via `DiffusionPipeline.from_pretrained()` should run, including text-to-image, image-to-image, text-to-video, image-to-video, and text-to-audio.
+
+However, as we strive to ensure output similarity between vLLM-Omni's diffuser backend and plain diffusers library, the following models are particularly verified:
+
+- Qwen/Qwen-Image
+- Tongyi-MAI/Z-Image-Turbo
+- Wan2.2-I2V-A14B-Diffusers
+
+If you find that a model not listed above also produces different outputs from running diffusers model directly.
+Please consider file an issue.
+
+## Usage
 
 ```bash
 vllm serve "stable-diffusion-v1-5/stable-diffusion-v1-5" \
     --omni \
-    --diffusion-load-format diffusers \
-    --diffusers-load-kwargs '{"use_safetensors": true}' \
-    --diffusers-call-kwargs '{"num_inference_steps": 30, "guidance_scale": 7.5}'
+    --diffusion-load-format diffusers
 ```
 
-`--diffusers-load-kwargs` and `--diffusers-call-kwargs` are only valid together with `--diffusion-load-format diffusers`.
+Users turn on the diffusers backend primarily through `--diffusion-load-format diffusers` argument.
+There are two more optional arguments, `--diffusers-load-kwargs` and `--diffusers-call-kwargs`,
+which are only valid together with `--diffusion-load-format diffusers`.
 
-### Option 2: Stage config YAML
-
-```bash
-vllm serve stable-diffusion-v1-5/stable-diffusion-v1-5 --stage-configs-path examples/online_serving/diffusers_pipeline_adapter/stage_config.yaml --omni
-```
-
-The particular fields of interest are `model`, `diffusion_load_format`, `diffusers_load_kwargs`, and `diffusers_call_kwargs` under `engine_args`. They are the same as the CLI arguments.
-
-## Send a Request
-
-```bash
-curl http://localhost:8000/v1/images/generations \
-  -H "Content-Type: application/json" \
-  -d '{
-    "model": "stable-diffusion-v1-5/stable-diffusion-v1-5",
-    "prompt": "a photo of an astronaut riding a horse on mars",
-    "n": 1,
-    "size": "512x512"
-  }'
-```
-
-Or refer to other documentation pages on how to request a particular input/output modality, such as `examples/online_serving/text_to_image/openai_chat_client.py`.
+After launching the model, users send a request as usual. Refer to other documentation pages on how to request a particular input/output modality, such as `examples/online_serving/text_to_image/openai_chat_client.py`.
 
 ## Configuration Reference
 
-For the diffusers adapter, set options under **`engine_args`**:
+### `--diffusers-load-kwargs`
 
-### `diffusion_load_format: "diffusers"`
+Passed as-is to `DiffusionPipeline.from_pretrained()`.
 
-This field selects the Hugging Face diffusers adapter path (see `DiffusersPipelineLoader`).
-
-### `diffusers_load_kwargs`
-
-Passed to `DiffusionPipeline.from_pretrained()`.
-
-This is suitable for model-specific configurations not available through the vLLM-Omni interface (such as `Omni.__init__()`, `vllm serve` CLI arguments, and stage config YAML fields outside `diffusers_load_kwargs`).
+This is suitable for model-specific configurations not available through the vLLM-Omni interface.
+For example: `--diffusers-load-kwargs '{"use_safetensors": true}'`.
 
 When a parameter is available in the vLLM-Omni interface, it will be adapted here.
 But if that parameter is simultaneously set in both the vLLM-Omni interface and `diffusers_load_kwargs`, the **latter** will take precedence.
 
-### `diffusers_call_kwargs`
+### `--diffusers-call-kwargs`
 
 Passed to `pipeline.__call__()`.
 
-This is suitable for sampling parameters not available through the vLLM-Omni interface (such as `Omni.generate()` and online serving payloads).
+This is suitable for sampling parameters not available through the vLLM-Omni interface (such as online serving payloads).
 
 When a parameter is available in the vLLM-Omni interface, it will be adapted here.
 But if that parameter is simultaneously set in both the vLLM-Omni interface and `diffusers_call_kwargs`, the **former** will take precedence (because it is set at request time).
 
-## Example materials
+### Attention Backends
 
-??? abstract "stage_config.yaml"
-    ``````yaml
-    --8<-- "examples/online_serving/diffusers_pipeline_adapter/stage_config.yaml"
-    ``````
+The diffusers backend converts
+[vLLM-Omni standard of attention backend setting](../../../docs/user_guide/diffusion/attention_backends.md)
+to [diffusers standard](https://huggingface.co/docs/diffusers/optimization/attention_backends#available-backends).
+
+Specifically for `FLASH_ATTN`, it will first attempt to use FlashAttention-3 and then FlashAttention-2.
+
+For each attempted version of `FLASH_ATTN` and `SAGE_ATTN`, it will first try to load the attention backend from HuggingFace `kernels` library, then without.
+
+For unsuccessful attention selection or `TORCH_SDPA`, it will use the PyTorch's default attention backend.
+
+The loaded attention backend and the failed attempts (if any) are logged to console.
+
+### Model Specific Settings
+
+The model loading and inference strictly follows the diffusers library, and they may be different from vLLM-Omni's native interface for some specific models.
+Users are encouraged to double-check the model pipeline's interface in [diffusers' official documentation](https://huggingface.co/docs/diffusers/api/pipelines/overview).
+Some particular examples are below.
+
+#### Wan Series
+
+The Wan series video generation models takes `boundary_ratio` and `flow_shift` during model initialization ([ref](https://huggingface.co/docs/diffusers/api/pipelines/wan)), not during inference.
+
+Since our `OmniDiffusionConfig` contains these two values ([source](https://github.com/vllm-project/vllm-omni/blob/main/vllm_omni/diffusion/data.py)), we can directly pass `--boundary-ratio` and `--flow-shift` arguments to `vllm serve` command.
+
+```bash
+vllm serve "Wan2.2-T2V-A14B-Diffusers" \
+    --omni \
+    --boundary-ratio 0.875 \
+    --flow-shift 3 \
+    --diffusion-load-format diffusers
+```
+
+These extra CLI args will be attempted to pass as-is to the `OmniDiffusionConfig` dataclass and being accessible during model loading time.
+Special routines inside the pipeline adapter ensures that they are set properly.

docs/user_guide/examples/online_serving/fish_speech.md

@@ -1,199 +0,0 @@
-# Fish Speech S2 Pro
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/fish_speech>.
-
-
-## Model
-
-| Model | Description |
-|-------|-------------|
-| `fishaudio/s2-pro` | Fish Speech S2 Pro -- 4B dual-AR TTS model with DAC codec (44.1 kHz) |
-
-## Gradio Demo
-
-!!! note "Gradio is an optional dependency"
-    The Gradio demo requires the `[demo]` extras. Install them first:
-
-    ```bash
-    pip install 'vllm-omni[demo]'
-    ```
-
-    Or, if installing from source: `pip install -e '.[demo]'`
-
-An interactive Gradio demo is available with text-to-speech synthesis, voice cloning, and streaming support.
-
-```bash
-# Option 1: Launch server + Gradio together
-./run_gradio_demo.sh
-
-# Option 2: If server is already running
-python gradio_demo.py --api-base http://localhost:8091
-```
-
-Then open http://localhost:7860 in your browser.
-
-Features:
-
-- Text-to-speech synthesis
-- Voice cloning from uploaded audio or URL
-- Streaming mode (progressive PCM playback)
-
-## Launch the Server
-
-```bash
-vllm serve fishaudio/s2-pro --omni --port 8091
-```
-
-The deploy config is auto-loaded from `vllm_omni/deploy/fish_qwen3_omni.yaml`.
-
-Or use the convenience script:
-
-```bash
-./run_server.sh
-```
-
-## Send TTS Request
-
-### Using curl
-
-```bash
-# Basic TTS
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, how are you?",
-        "voice": "default",
-        "response_format": "wav"
-    }' --output output.wav
-```
-
-### Voice Cloning
-
-Provide a reference audio (URL or base64 data URL) and its transcript:
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, this is a cloned voice.",
-        "voice": "default",
-        "ref_audio": "https://example.com/reference.wav",
-        "ref_text": "Transcript of the reference audio."
-    }' --output cloned.wav
-```
-
-### Using Python
-
-```python
-import httpx
-
-# Basic TTS
-response = httpx.post(
-    "http://localhost:8091/v1/audio/speech",
-    json={
-        "input": "Hello, how are you?",
-        "voice": "default",
-        "response_format": "wav",
-    },
-    timeout=300.0,
-)
-
-with open("output.wav", "wb") as f:
-    f.write(response.content)
-```
-
-### Using the CLI Client
-
-```bash
-cd examples/online_serving/fish_speech
-
-# Basic TTS
-python speech_client.py --text "Hello, how are you?"
-
-# Voice cloning
-python speech_client.py \
-    --text "Hello, this is a cloned voice." \
-    --ref-audio /path/to/reference.wav \
-    --ref-text "Transcript of the reference audio."
-
-# Streaming PCM output
-python speech_client.py --text "Hello world" --stream --output output.pcm
-```
-
-The CLI client supports:
-
-- `--api-base`: API base URL (default: `http://localhost:8091`)
-- `--model` (or `-m`): Model name (default: `fishaudio/s2-pro`)
-- `--text`: Text to synthesize (required)
-- `--ref-audio`: Reference audio for voice cloning (local path or URL)
-- `--ref-text`: Transcript of the reference audio
-- `--stream`: Enable streaming (PCM output)
-- `--response-format`: Audio format: wav, mp3, flac, pcm, aac, opus (default: wav)
-- `--output` (or `-o`): Output file path
-
-## Streaming
-
-Set `stream=true` with `response_format="pcm"` to receive raw PCM audio chunks as they are decoded:
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, how are you?",
-        "voice": "default",
-        "stream": true,
-        "response_format": "pcm"
-    }' --no-buffer | play -t raw -r 44100 -e signed -b 16 -c 1 -
-```
-
-**Note:** Fish Speech outputs at 44.1 kHz (unlike Qwen3-TTS which outputs at 24 kHz).
-
-## API Parameters
-
-Fish Speech uses the same `/v1/audio/speech` endpoint as Qwen3-TTS. See the [Speech API reference](https://docs.vllm.ai/projects/vllm-omni/en/latest/serving/speech_api/) for full parameter documentation.
-
-Key parameters for Fish Speech:
-
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `input` | string | **required** | Text to synthesize |
-| `voice` | string | "default" | Voice name (use "default" for Fish Speech) |
-| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus |
-| `ref_audio` | string | null | Reference audio URL or base64 data URL for voice cloning |
-| `ref_text` | string | null | Transcript of reference audio (required for voice cloning) |
-| `max_new_tokens` | int | 4096 | Maximum tokens to generate |
-| `stream` | bool | false | Stream raw PCM chunks |
-
-## Prerequisites
-
-Install the `fish-speech` package for the DAC codec:
-
-```bash
-pip install fish-speech
-```
-
-## Troubleshooting
-
-1. **No audio output**: Make sure the `fish-speech` package is installed for the DAC decoder
-2. **Connection refused**: Ensure the server is running on the correct port
-3. **Flashinfer version mismatch**: Set `FLASHINFER_DISABLE_VERSION_CHECK=1` if you see version warnings
-4. **Out of memory**: Reduce `--gpu-memory-utilization` or use a GPU with more VRAM
-
-## Example materials
-
-??? abstract "gradio_demo.py"
-    ``````py
-    --8<-- "examples/online_serving/fish_speech/gradio_demo.py"
-    ``````
-??? abstract "run_gradio_demo.sh"
-    ``````sh
-    --8<-- "examples/online_serving/fish_speech/run_gradio_demo.sh"
-    ``````
-??? abstract "run_server.sh"
-    ``````sh
-    --8<-- "examples/online_serving/fish_speech/run_server.sh"
-    ``````
-??? abstract "speech_client.py"
-    ``````py
-    --8<-- "examples/online_serving/fish_speech/speech_client.py"
-    ``````

docs/user_guide/examples/online_serving/qwen3_tts.md

@@ -1,451 +0,0 @@
-# Qwen3-TTS
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/qwen3_tts>.
-
-
-## 🛠️ Installation
-
-Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md)
-
-## Supported Models
-
-| Model                                  | Task Type   | Description                                           |
-| -------------------------------------- | ----------- | ----------------------------------------------------- |
-| `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | CustomVoice | Predefined speaker voices with optional style control |
-| `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | VoiceDesign | Natural language voice style description              |
-| `Qwen/Qwen3-TTS-12Hz-1.7B-Base`        | Base        | Voice cloning from reference audio                    |
-| `Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice` | CustomVoice | Smaller/faster variant                                |
-| `Qwen/Qwen3-TTS-12Hz-0.6B-Base`        | Base        | Smaller/faster variant for voice cloning              |
-
-## Gradio Demo
-
-!!! note "Gradio is an optional dependency"
-    The Gradio demo requires the `[demo]` extras. Install them first:
-
-    ```bash
-    pip install 'vllm-omni[demo]'
-    ```
-
-    Or, if installing from source: `pip install -e '.[demo]'`
-
-Two interactive Gradio demos are available, both supporting all 3 task types:
-
-| Demo     | File                     | Transport    | Streaming Quality                                  |
-| -------- | ------------------------ | ------------ | -------------------------------------------------- |
-| Standard | `gradio_demo.py`         | HTTP chunked | May have small gaps between chunks                 |
-| FastRTC  | `gradio_fastrtc_demo.py` | WebRTC       | Gapless streaming (requires `pip install fastrtc`) |
-
-```bash
-# Option 1: Launch server + Standard Gradio together
-./run_gradio_demo.sh                                # CustomVoice (default)
-./run_gradio_demo.sh --task-type VoiceDesign        # VoiceDesign
-./run_gradio_demo.sh --task-type Base               # Voice cloning
-
-# Option 2: If server is already running
-python gradio_demo.py --api-base http://localhost:8000
-
-# Option 3: FastRTC demo (gapless streaming)
-pip install fastrtc
-python gradio_fastrtc_demo.py --api-base http://localhost:8000
-```
-
-Then open http://localhost:7860 in your browser.
-
-## Run examples (Qwen3-TTS)
-
-### Launch the Server
-
-```bash
-# CustomVoice model (predefined speakers)
-vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
-    --deploy-config vllm_omni/deploy/qwen3_tts.yaml \
-    --omni \
-    --port 8091 \
-    --trust-remote-code \
-    --enforce-eager
-
-# VoiceDesign model
-vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign \
-    --deploy-config vllm_omni/deploy/qwen3_tts.yaml \
-    --omni \
-    --port 8091 \
-    --trust-remote-code \
-    --enforce-eager
-
-# Base model (voice cloning)
-vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base \
-    --deploy-config vllm_omni/deploy/qwen3_tts.yaml \
-    --omni \
-    --port 8091 \
-    --trust-remote-code \
-    --enforce-eager
-```
-
-If you have custom stage configs file, launch the server with command below
-```bash
-vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
-    --stage-configs-path /path/to/stage_configs_file \
-    --omni \
-    --port 8091 \
-    --trust-remote-code \
-    --enforce-eager
-```
-
-Alternatively, use the convenience script:
-```bash
-./run_server.sh                  # Default: CustomVoice model
-./run_server.sh CustomVoice      # CustomVoice model
-./run_server.sh VoiceDesign      # VoiceDesign model
-./run_server.sh Base             # Base (voice clone) model
-```
-
-### Send TTS Request
-
-Get into the example folder
-```bash
-cd examples/online_serving/qwen3_tts
-```
-
-####  Send request via python
-
-```bash
-# CustomVoice: Use predefined speaker
-python openai_speech_client.py \
-    --text "你好，我是通义千问" \
-    --speaker vivian \
-    --language Chinese
-
-# CustomVoice with style instruction
-python openai_speech_client.py \
-    --text "今天天气真好" \
-    --speaker ryan \
-    --instructions "用开心的语气说"
-
-# VoiceDesign: Describe the voice style
-python openai_speech_client.py \
-    --model Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign \
-    --task-type VoiceDesign \
-    --text "哥哥，你回来啦" \
-    --instructions "体现撒娇稚嫩的萝莉女声，音调偏高"
-
-# Base: Voice cloning
-python openai_speech_client.py \
-    --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
-    --task-type Base \
-    --text "Hello, this is a cloned voice" \
-    --ref-audio /path/to/reference.wav \
-    --ref-text "Original transcript of the reference audio"
-```
-
-The Python client supports the following command-line arguments:
-
-- `--api-base`: API base URL (default: `http://localhost:8091`)
-- `--model` (or `-m`): Model name/path (default: `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice`)
-- `--task-type` (or `-t`): TTS task type. Options: `CustomVoice`, `VoiceDesign`, `Base`
-- `--text`: Text to synthesize (required)
-- `--speaker`: Speaker name (default: `vivian`). Options: `vivian`, `ryan`, `aiden`, etc.
-- `--language`: Language. Options: `Auto`, `Chinese`, `English`, `Japanese`, `Korean`, `German`, `French`, `Russian`, `Portuguese`, `Spanish`, `Italian`
-- `--instructions`: Voice style/emotion instructions
-- `--ref-audio`: Reference audio file path or URL for voice cloning (Base task). Local paths are automatically base64-encoded by the client before sending to the server.
-- `--ref-text`: Reference audio transcript for voice cloning (Base task).
-- `--response-format`: Audio output format (default: `wav`). Options: `wav`, `mp3`, `flac`, `pcm`, `aac`, `opus`
-- `--output` (or `-o`): Output audio file path (default: `tts_output.wav`)
-
-####  Send request via curl
-
-```bash
-# Simple TTS request
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, how are you?",
-        "voice": "vivian",
-        "language": "English"
-    }' --output output.wav
-
-# With style instruction
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "I am so excited!",
-        "voice": "vivian",
-        "instructions": "Speak with great enthusiasm"
-    }' --output excited.wav
-
-# List available voices in CustomVoice models
-curl http://localhost:8091/v1/audio/voices
-```
-
-### Using OpenAI SDK
-
-```python
-from openai import OpenAI
-
-client = OpenAI(base_url="http://localhost:8091/v1", api_key="none")
-
-response = client.audio.speech.create(
-    model="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
-    voice="vivian",
-    input="Hello, how are you?",
-)
-
-response.stream_to_file("output.wav")
-```
-
-### Using Python httpx
-
-```python
-import httpx
-
-response = httpx.post(
-    "http://localhost:8091/v1/audio/speech",
-    json={
-        "input": "Hello, how are you?",
-        "voice": "vivian",
-        "language": "English",
-    },
-    timeout=300.0,
-)
-
-with open("output.wav", "wb") as f:
-    f.write(response.content)
-```
-
-## API Reference
-
-### Voices Endpoint
-
-#### GET /v1/audio/voices
-
-List all available voices/speakers from the loaded model, including both built-in model voices and uploaded custom voices.
-
-**Response Example:**
-```json
-{
-  "voices": ["vivian", "ryan", "custom_voice_1"],
-  "uploaded_voices": [
-    {
-      "name": "custom_voice_1",
-      "consent": "user_consent_id",
-      "created_at": 1738660000,
-      "file_size": 1024000,
-      "mime_type": "audio/wav",
-      "ref_text": "The exact transcript of the audio sample.",
-      "speaker_description": "warm narrator"
-    }
-  ]
-}
-```
-
-Fields `ref_text` and `speaker_description` are omitted per-entry when not provided at upload time.
-
-#### POST /v1/audio/voices
-
-Upload a new voice sample for voice cloning in Base task TTS requests.
-
-**Form Parameters:**
-- `audio_sample` (required): Audio file (max 10MB, supported formats: wav, mp3, flac, ogg, aac, webm, mp4)
-- `consent` (required): Consent recording ID
-- `name` (required): Name for the new voice
-- `ref_text` (optional): Transcript of the audio. Enables in-context voice cloning (higher quality).
-- `speaker_description` (optional): Free-form description of the voice (e.g. "warm narrator", "energetic presenter"). Stored as metadata.
-
-**Response Example:**
-```json
-{
-  "success": true,
-  "voice": {
-    "name": "custom_voice_1",
-    "consent": "user_consent_id",
-    "created_at": 1738660000,
-    "mime_type": "audio/wav",
-    "file_size": 1024000,
-    "ref_text": "The exact transcript of the audio sample.",
-    "speaker_description": "warm narrator"
-  }
-}
-```
-
-Fields `ref_text` and `speaker_description` are omitted when not provided at upload time.
-
-**Usage Example:**
-```bash
-curl -X POST http://localhost:8091/v1/audio/voices \
-  -F "audio_sample=@/path/to/voice_sample.wav" \
-  -F "consent=user_consent_id" \
-  -F "name=custom_voice_1" \
-  -F "ref_text=The exact transcript of the audio sample." \
-  -F "speaker_description=warm narrator"
-```
-
-### Endpoint
-
-```
-POST /v1/audio/speech
-Content-Type: application/json
-```
-
-This endpoint follows the [OpenAI Audio Speech API](https://platform.openai.com/docs/api-reference/audio/createSpeech) format with additional Qwen3-TTS parameters.
-
-### Request Body
-
-```json
-{
-    "input": "Text to synthesize",
-    "voice": "vivian",
-    "response_format": "wav",
-    "task_type": "CustomVoice",
-    "language": "Auto",
-    "instructions": "Optional style instructions",
-    "ref_audio":  "HTTP URL, base64 data URL, or file:// URI for voice cloning",
-    "ref_text": "Reference audio transcript",
-    "x_vector_only_mode": false,
-    "max_new_tokens": 2048
-}
-```
-
-> **Note:** The `model` field is optional when serving a single model, as the server already knows which model is loaded.
-
-### Response
-
-Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/wav`).
-
-### Voice and language (summary)
-
-- **Speaker**: Use the `voice` request field to select the speaker (e.g., `vivian`, `ryan`, `aiden`). List available speakers with `GET /v1/audio/voices`.
-- **Language**: Use the `language` field for the codec language tag (`Auto`, `Chinese`, `English`, etc.). Default is `Auto` for automatic detection.
-- **CustomVoice**: Requires a valid `voice` from the model’s speaker set. **VoiceDesign**: Use `instructions` to describe the voice. **Base**: Use `ref_audio` and `ref_text` for voice cloning.
-
-## Parameters
-
-### OpenAI Standard Parameters
-
-| Parameter         | Type   | Default        | Description                                                 |
-| ----------------- | ------ | -------------- | ----------------------------------------------------------- |
-| `input`           | string | **required**   | Text to synthesize                                          |
-| `model`           | string | server's model | Model to use (optional, should match server if specified)   |
-| `voice`           | string | "vivian"       | Speaker name (e.g., vivian, ryan, aiden)                    |
-| `response_format` | string | "wav"          | Audio format: wav, mp3, flac, pcm, aac, opus                |
-| `speed`           | float  | 1.0            | Playback speed (0.25-4.0, not supported with `stream=true`) |
-
-### vLLM-Omni Extension Parameters
-
-| Parameter                    | Type   | Default       | Description                                                                                                          |
-| ---------------------------- | ------ | ------------- | -------------------------------------------------------------------------------------------------------------------- |
-| `task_type`                  | string | "CustomVoice" | Task: CustomVoice, VoiceDesign, or Base                                                                              |
-| `language`                   | string | "Auto"        | Language (see supported languages below)                                                                             |
-| `instructions`               | string | ""            | Voice style/emotion instructions                                                                                     |
-| `max_new_tokens`             | int    | 2048          | Maximum tokens to generate                                                                                           |
-| `initial_codec_chunk_frames` | int    | null          | Per-request initial chunk size override for TTFA tuning. When null, IC is computed dynamically based on server load. |
-| `stream`                     | bool   | false         | Stream raw PCM chunks as they are decoded (requires `response_format="pcm"`)                                         |
-
-**Supported languages:** Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian
-
-### Voice Clone Parameters (Base task)
-
-| Parameter            | Type   | Required | Description                                                                                     |
-| -------------------- | ------ | -------- | ----------------------------------------------------------------------------------------------- |
-| `ref_audio`          | string | **Yes**  | Reference audio (HTTP URL, base64 data URL, or `file://` URI with `--allowed-local-media-path`) |
-| `ref_text`           | string | No       | Transcript of reference audio (for ICL mode)                                                    |
-| `x_vector_only_mode` | bool   | No       | Use speaker embedding only (no ICL)                                                             |
-
-## Streaming
-
-Set `stream=true` with `response_format="pcm"` to receive raw PCM audio chunks as they are decoded
-(one chunk per Code2Wav window, default 25 frames; configurable in the stage config):
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, how are you?",
-        "voice": "vivian",
-        "language": "English",
-        "stream": true,
-        "response_format": "pcm"
-    }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 -
-```
-
-**Constraints:**
-- `stream=true` requires `response_format="pcm"` (raw 16-bit signed PCM, 24 kHz mono).
-- `speed` adjustment is not supported when streaming.
-- Requires the server stage config to have `async_chunk: true` (default in `qwen3_tts.yaml`).
-
-## Streaming Text Input (WebSocket)
-
-The `/v1/audio/speech/stream` WebSocket endpoint accepts text incrementally, buffers and splits it at sentence boundaries, and generates audio per sentence.
-
-When `stream_audio=true`, each sentence is emitted as `audio.start`, one or more binary PCM frames, and `audio.done`.
-
-### Quick Start
-
-```bash
-python streaming_speech_client.py \
-    --text "Hello world. How are you? I am fine."
-
-python streaming_speech_client.py \
-    --text "Hello world. How are you? I am fine." \
-    --simulate-stt --stt-delay 0.1
-```
-
-### WebSocket Protocol
-
-Client -> Server:
-
-```jsonc
-{"type": "session.config", "voice": "Vivian", "task_type": "CustomVoice", "language": "Auto", "split_granularity": "sentence", "stream_audio": true, "response_format": "pcm"}
-{"type": "input.text", "text": "Hello, how are you? "}
-{"type": "input.done"}
-```
-
-Server -> Client:
-
-```jsonc
-{"type": "audio.start", "sentence_index": 0, "sentence_text": "Hello, how are you?", "format": "pcm", "sample_rate": 24000}
-// binary PCM frame(s)
-{"type": "audio.done", "sentence_index": 0, "total_bytes": 96000, "error": false}
-{"type": "session.done", "total_sentences": 1}
-```
-
-## Limitations
-
-- **Single request**: Batch processing is not yet optimized for online serving.
-
-## Troubleshooting
-
-1. **TTS model did not produce audio output**: Ensure you're using the correct model variant for your task type (CustomVoice task → CustomVoice model, etc.)
-2. **Connection refused**: Make sure the server is running on the correct port
-3. **Out of memory**: Use smaller model variant (`Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice`) or reduce `--gpu-memory-utilization`
-4. **Unsupported speaker**: Use `/v1/audio/voices` to list available voices for the loaded model
-5. **Voice clone fails**: Ensure you're using the Base model variant for voice cloning
-
-## Example materials
-
-??? abstract "gradio_demo.py"
-    ``````py
-    --8<-- "examples/online_serving/qwen3_tts/gradio_demo.py"
-    ``````
-??? abstract "gradio_fastrtc_demo.py"
-    ``````py
-    --8<-- "examples/online_serving/qwen3_tts/gradio_fastrtc_demo.py"
-    ``````
-??? abstract "openai_speech_client.py"
-    ``````py
-    --8<-- "examples/online_serving/qwen3_tts/openai_speech_client.py"
-    ``````
-??? abstract "run_gradio_demo.sh"
-    ``````sh
-    --8<-- "examples/online_serving/qwen3_tts/run_gradio_demo.sh"
-    ``````
-??? abstract "run_server.sh"
-    ``````sh
-    --8<-- "examples/online_serving/qwen3_tts/run_server.sh"
-    ``````
-??? abstract "streaming_speech_client.py"
-    ``````py
-    --8<-- "examples/online_serving/qwen3_tts/streaming_speech_client.py"
-    ``````
-??? abstract "tts_common.py"
-    ``````py
-    --8<-- "examples/online_serving/qwen3_tts/tts_common.py"
-    ``````

docs/user_guide/examples/online_serving/text_to_speech.md

@@ -0,0 +1,489 @@
+# Text-To-Speech (Online Serving)
+
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/text_to_speech>.
+
+
+vLLM-Omni exposes TTS models through the OpenAI-compatible
+[`POST /v1/audio/speech`](https://github.com/vllm-project/vllm-omni/tree/main/docs/serving/speech_api.md) endpoint,
+launched with `vllm serve <model> --omni`. Each TTS model has its own
+subdirectory containing client snippets, gradio demos, and helper
+scripts; this README is the single doc entry point for all of them.
+
+For offline inference, see [`examples/offline_inference/text_to_speech`](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md).
+For the full list of supported architectures across all modalities, see
+[Supported Models](https://github.com/vllm-project/vllm-omni/tree/main/docs/models/supported_models.md).
+
+## Supported Models
+
+| Model | HuggingFace repo | Voice cloning | Streaming | Voice presets / upload | Gradio demo |
+|---|---|---|---|---|---|
+| Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
+| OmniVoice | `k2-fsa/OmniVoice` | (offline only) | — | — | — |
+| Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | ✓ (Base) | ✓ (PCM + WebSocket) | ✓ (presets + `/v1/audio/voices` upload) | ✓ (standard + FastRTC) |
+| VoxCPM | local model dir | ✓ | ✓ (PCM stream) | — | — |
+| VoxCPM2 | `openbmb/VoxCPM2` | ✓ | ✓ (AudioWorklet via gradio) | — | ✓ |
+| Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✓ (gated upstream) | ✓ | ✓ (presets) | ✓ |
+
+CosyVoice3 is intentionally absent: no online example exists for it yet. See its [offline section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#cosyvoice3) instead.
+
+## Common Quick Start
+
+Launch the server (defaults shown — adjust `--port`, `--gpu-memory-utilization`, etc. as needed):
+
+```bash
+vllm serve <hf-repo-or-local-path> --omni --port 8091
+```
+
+Send a TTS request via curl:
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Hello, how are you?",
+        "voice": "default",
+        "response_format": "wav"
+    }' --output output.wav
+```
+
+Or via Python httpx:
+
+```python
+import httpx
+
+response = httpx.post(
+    "http://localhost:8091/v1/audio/speech",
+    json={
+        "input": "Hello, how are you?",
+        "voice": "default",
+        "response_format": "wav",
+    },
+    timeout=300.0,
+)
+open("output.wav", "wb").write(response.content)
+```
+
+Or via the OpenAI SDK:
+
+```python
+from openai import OpenAI
+
+client = OpenAI(base_url="http://localhost:8091/v1", api_key="none")
+response = client.audio.speech.create(
+    model="<hf-repo>",
+    voice="default",
+    input="Hello, how are you?",
+)
+response.stream_to_file("output.wav")
+```
+
+Streaming PCM output (where supported) — set `stream=true` with `response_format="pcm"`:
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Hello, how are you?",
+        "voice": "default",
+        "stream": true,
+        "response_format": "pcm"
+    }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 -
+```
+
+Adjust the player's sample rate to match the model (44.1 kHz for Fish Speech, 48 kHz for VoxCPM2, 24 kHz for the others).
+
+For full request-shape documentation (all parameters, response formats, error codes), see the [Speech API reference](https://github.com/vllm-project/vllm-omni/tree/main/docs/serving/speech_api.md).
+
+---
+
+## Fish Speech S2 Pro
+
+4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec.
+
+### Prerequisites
+```bash
+pip install fish-speech
+```
+
+### Launch
+```bash
+vllm serve fishaudio/s2-pro --omni --port 8091
+# or:
+./fish_speech/run_server.sh
+```
+The deploy config auto-loads from `vllm_omni/deploy/fish_qwen3_omni.yaml` (the HF `model_type` on the fishaudio checkpoint is `fish_qwen3_omni`).
+
+### Voice cloning
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Hello, this is a cloned voice.",
+        "voice": "default",
+        "ref_audio": "https://example.com/reference.wav",
+        "ref_text": "Transcript of the reference audio."
+    }' --output cloned.wav
+```
+
+### CLI client
+```bash
+cd examples/online_serving/text_to_speech/fish_speech
+python speech_client.py --text "Hello, how are you?"
+python speech_client.py --text "Hello world" --stream --output output.pcm
+```
+
+### Gradio demo
+```bash
+./fish_speech/run_gradio_demo.sh             # launches server + Gradio
+python fish_speech/gradio_demo.py --api-base http://localhost:8091  # if server already running
+```
+
+### Notes
+- Output: 44.1 kHz mono.
+- Streaming PCM player command must use `-r 44100`.
+
+---
+
+## OmniVoice
+
+Zero-shot multilingual TTS (600+ languages). Online serving currently exposes **auto voice** only; voice cloning and voice design are available offline.
+
+### Prerequisites
+```bash
+huggingface-cli download k2-fsa/OmniVoice
+```
+Voice cloning (offline) needs `transformers>=5.3.0`; auto voice works with `transformers>=4.57.0`.
+
+### Launch
+```bash
+vllm serve k2-fsa/OmniVoice --omni --port 8091 --trust-remote-code
+# or:
+./omnivoice/run_server.sh
+```
+
+### CLI client
+```bash
+cd examples/online_serving/text_to_speech/omnivoice
+python speech_client.py --text "Hello, how are you?"
+python speech_client.py --text "Bonjour, comment allez-vous?" --language French
+```
+
+The client supports `--api-base`, `--model`, `--text`, `--response-format`, `--language`, `--output`.
+
+### Notes
+- Voice cloning and voice design require offline inference; see the [offline OmniVoice section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#omnivoice).
+
+---
+
+## Qwen3-TTS
+
+Three model variants exposed via separate checkpoints:
+
+| Variant | HF repo | Use |
+|---|---|---|
+| CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | Predefined speakers (`vivian`, `ryan`, …) with optional style instructions |
+| VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | Natural-language voice style description |
+| Base | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | Voice cloning from a reference audio |
+
+Each variant ships smaller `0.6B` companions where available.
+
+### Launch
+```bash
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni --port 8091
+# or:
+./qwen3_tts/run_server.sh                # default: CustomVoice
+./qwen3_tts/run_server.sh VoiceDesign
+./qwen3_tts/run_server.sh Base
+```
+
+### Choosing an executor backend (uniproc vs mp)
+Stage configs ship with the chunked-streaming default. To use the uniproc executor (lower IPC overhead for the Base cloning task), pass `--stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts_uniproc.yaml`. See [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) and [#2604](https://github.com/vllm-project/vllm-omni/pull/2604) for the full investigation.
+
+To opt out of chunked streaming, pass `--no-async-chunk` instead — the pipeline auto-dispatches to the end-to-end codec processor.
+
+### Sending requests
+```bash
+# CustomVoice with a predefined speaker
+python qwen3_tts/openai_speech_client.py \
+    --model Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
+    --text "今天天气真好" \
+    --voice ryan \
+    --instructions "用开心的语气说"
+
+# VoiceDesign with a style description
+python qwen3_tts/openai_speech_client.py \
+    --model Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign \
+    --task-type VoiceDesign \
+    --text "哥哥，你回来啦" \
+    --instructions "体现撒娇稚嫩的萝莉女声，音调偏高"
+
+# Base voice cloning
+python qwen3_tts/openai_speech_client.py \
+    --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
+    --task-type Base \
+    --text "Hello, this is a cloned voice" \
+    --ref-audio /path/to/reference.wav \
+    --ref-text "Original transcript of the reference audio"
+```
+
+### Voices endpoint
+List available voices, or upload a custom one for Base cloning:
+```bash
+# List
+curl http://localhost:8091/v1/audio/voices
+
+# Upload
+curl -X POST http://localhost:8091/v1/audio/voices \
+    -F "audio_sample=@/path/to/voice_sample.wav" \
+    -F "consent=user_consent_id" \
+    -F "name=custom_voice_1" \
+    -F "ref_text=The exact transcript of the audio sample." \
+    -F "speaker_description=warm narrator"
+```
+Uploaded voices are then usable as `voice="custom_voice_1"` on subsequent requests.
+
+### Streaming PCM
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Hello, how are you?",
+        "voice": "vivian",
+        "language": "English",
+        "stream": true,
+        "response_format": "pcm"
+    }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 -
+```
+Streaming requires `response_format="pcm"` and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming.
+
+### Streaming WebSocket
+The `/v1/audio/speech/stream` endpoint accepts text incrementally, splits it at sentence boundaries, and emits one PCM stream per sentence:
+```bash
+python qwen3_tts/streaming_speech_client.py --text "Hello world. How are you? I am fine."
+python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-delay 0.1
+```
+
+### Gradio demos
+```bash
+./qwen3_tts/run_gradio_demo.sh                              # CustomVoice (default)
+./qwen3_tts/run_gradio_demo.sh --task-type VoiceDesign
+./qwen3_tts/run_gradio_demo.sh --task-type Base
+
+# FastRTC variant (gapless WebRTC streaming):
+pip install fastrtc
+python qwen3_tts/gradio_fastrtc_demo.py --api-base http://localhost:8000
+```
+
+### Speaker embedding interpolation
+`qwen3_tts/speaker_embedding_interpolation.py` blends two predefined speakers' embeddings to produce intermediate voices. See the script for usage.
+
+### Batch client
+`qwen3_tts/batch_speech_client.py` issues many concurrent requests for throughput measurement.
+
+### Notes
+- Base voice cloning has uniproc-vs-mp tradeoffs depending on per-request reference audio cost; see the executor-backend section above.
+- `vllm_omni/deploy/qwen3_tts.yaml` is the default deploy config (loaded by HF `model_type`); per-stage runtime overrides are available via `--stage-N-<field> <value>`.
+
+---
+
+## VoxCPM
+
+Split-stage TTS at 24 kHz.
+
+### Prerequisites
+```bash
+pip install voxcpm
+# or use a local source tree:
+export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
+```
+
+If the native VoxCPM `config.json` lacks HF `model_type`, set up an HF-compatible config dir:
+```bash
+export VOXCPM_MODEL=/path/to/voxcpm-model
+export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config
+mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"
+cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json"
+cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true
+python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)'
+```
+
+### Launch
+```bash
+export VOXCPM_MODEL=/path/to/voxcpm-model
+./voxcpm/run_server.sh                # async-chunk streaming (default)
+./voxcpm/run_server.sh sync           # non-streaming
+```
+Or directly:
+```bash
+vllm serve "$VOXCPM_MODEL" \
+    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
+    --trust-remote-code --enforce-eager --omni --port 8091
+```
+
+### Sending requests
+```bash
+# Basic TTS
+python voxcpm/openai_speech_client.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This is a VoxCPM online text-to-speech example."
+
+# Voice cloning
+python voxcpm/openai_speech_client.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This sentence is synthesized with a cloned voice." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text "The exact transcript spoken in reference.wav."
+
+# Streaming PCM
+python voxcpm/openai_speech_client.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This is a streaming VoxCPM request." \
+    --stream --output voxcpm_stream.pcm
+```
+
+### Notes
+- `voxcpm.yaml` for one-shot decode; `voxcpm_async_chunk.yaml` for single-request streaming. Do not use the async-chunk config for concurrent requests or `/v1/audio/speech/batch`.
+- Generic TTS fields not supported by VoxCPM: `voice`, `instructions`, `language`, `speaker_embedding`, `x_vector_only_mode`.
+- For benchmark measurement, see [`benchmarks/voxcpm`](https://github.com/vllm-project/vllm-omni/tree/main/benchmarks/voxcpm/README.md).
+
+---
+
+## VoxCPM2
+
+Single-stage native AR TTS at 48 kHz.
+
+### Launch
+```bash
+vllm serve openbmb/VoxCPM2 --omni --host 0.0.0.0 --port 8000
+```
+Deploy config auto-loads from `vllm_omni/deploy/voxcpm2.yaml`. Pass `--deploy-config <path>` to override or `--stage-N-<field> <value>` for per-stage runtime tweaks.
+
+### Sending requests
+```bash
+# Zero-shot synthesis
+python voxcpm2/openai_speech_client.py --text "Hello, this is VoxCPM2."
+
+# Voice cloning
+python voxcpm2/openai_speech_client.py \
+    --text "This should sound like the reference speaker." \
+    --ref-audio /path/to/reference.wav
+```
+The `ref_audio` field accepts local file paths (auto-base64), HTTP URLs, or `data:audio/wav;base64,...` data URIs.
+
+### Gradio demo (gapless streaming via AudioWorklet)
+```bash
+python voxcpm2/gradio_demo.py
+```
+Uses an AudioWorklet-based player adapted from the Qwen3-TTS demo for gap-free playback. Audio is streamed from the OpenAI Speech endpoint with `stream=true`.
+
+---
+
+## Voxtral TTS
+
+Voxtral-4B-TTS (Mistral). Uses the `mistral_common` `SpeechRequest` protocol; voice presets are model-specific.
+
+### Prerequisites
+Latest `mistral_common` with `SpeechRequest` support:
+```bash
+pip install -e /path/to/mistral-common  # or upgrade from PyPI when available
+```
+
+### Launch
+```bash
+vllm serve mistralai/Voxtral-4B-TTS-2603 --omni --port 8091
+```
+Deploy config auto-loads from `vllm_omni/deploy/voxtral_tts.yaml`.
+
+### Gradio demo
+```bash
+python voxtral_tts/gradio_demo.py
+```
+The demo handles voice-preset selection and reference-audio upload. `voxtral_tts/text_preprocess.py` provides the text-normalization helpers used by the demo (also available for other clients).
+
+### Notes
+- Voice presets are listed on the HF model card (`mistralai/Voxtral-4B-TTS-2603`).
+- Voice cloning is gated upstream and may require a recent `mistral_common`.
+- A standalone CLI client is not yet shipped; the gradio demo is the canonical reference for now.
+
+## Example materials
+
+??? abstract "fish_speech/gradio_demo.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/fish_speech/gradio_demo.py"
+    ``````
+??? abstract "fish_speech/run_gradio_demo.sh"
+    ``````sh
+    --8<-- "examples/online_serving/text_to_speech/fish_speech/run_gradio_demo.sh"
+    ``````
+??? abstract "fish_speech/run_server.sh"
+    ``````sh
+    --8<-- "examples/online_serving/text_to_speech/fish_speech/run_server.sh"
+    ``````
+??? abstract "fish_speech/speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/fish_speech/speech_client.py"
+    ``````
+??? abstract "omnivoice/run_server.sh"
+    ``````sh
+    --8<-- "examples/online_serving/text_to_speech/omnivoice/run_server.sh"
+    ``````
+??? abstract "omnivoice/speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/omnivoice/speech_client.py"
+    ``````
+??? abstract "qwen3_tts/batch_speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/batch_speech_client.py"
+    ``````
+??? abstract "qwen3_tts/gradio_demo.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/gradio_demo.py"
+    ``````
+??? abstract "qwen3_tts/openai_speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/openai_speech_client.py"
+    ``````
+??? abstract "qwen3_tts/run_gradio_demo.sh"
+    ``````sh
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/run_gradio_demo.sh"
+    ``````
+??? abstract "qwen3_tts/run_server.sh"
+    ``````sh
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/run_server.sh"
+    ``````
+??? abstract "qwen3_tts/speaker_embedding_interpolation.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/speaker_embedding_interpolation.py"
+    ``````
+??? abstract "qwen3_tts/streaming_speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/streaming_speech_client.py"
+    ``````
+??? abstract "qwen3_tts/tts_common.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/tts_common.py"
+    ``````
+??? abstract "voxcpm/openai_speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/voxcpm/openai_speech_client.py"
+    ``````
+??? abstract "voxcpm/run_server.sh"
+    ``````sh
+    --8<-- "examples/online_serving/text_to_speech/voxcpm/run_server.sh"
+    ``````
+??? abstract "voxcpm2/gradio_demo.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/voxcpm2/gradio_demo.py"
+    ``````
+??? abstract "voxcpm2/openai_speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/voxcpm2/openai_speech_client.py"
+    ``````
+??? abstract "voxtral_tts/gradio_demo.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/voxtral_tts/gradio_demo.py"
+    ``````
+??? abstract "voxtral_tts/text_preprocess.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/voxtral_tts/text_preprocess.py"
+    ``````

docs/user_guide/quantization/fp8.md

@@ -32,15 +32,15 @@ guide. FP8 on Ampere may use a weight-only path where available.
 
 ### Diffusion Model (Qwen-Image, Wan2.2)
 
-| Model | HF models | Online | Pre-calibrated | Recommendation | `ignored_layers` |
-|-------|-----------|:-------:|:------:|----------------|------------------|
-| Qwen-Image | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes | Yes | Skip sensitive image-stream MLPs when quality regresses | `img_mlp` |
-| Wan2.2 | Wan2.2 diffusion pipelines | Not validated | Not validated | Validate against BF16 before documenting as supported | TBD |
-| Z-Image | `Tongyi-MAI/Z-Image-Turbo` | Yes | Yes | All layers | None |
-| FLUX.1 | `black-forest-labs/FLUX.1-dev`, `black-forest-labs/FLUX.1-schnell` | Yes | Yes | All layers | None |
-| FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B` | Yes | Yes | All layers | None |
-| HunyuanImage-3.0 | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | Yes | Yes | All layers; use the Hunyuan stage config for multi-stage runs | None |
-| HunyuanVideo-1.5 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `720p_t2v`, `480p_i2v` | Yes | Yes | All layers | None |
+| Model | HF models | Online | Pre-calibrated | Recommendation | `ignored_layers` | Text-Encoder quantization |
+|-------|-----------|:-------:|:------:|----------------|------------------|------------------|
+| Qwen-Image | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes | Yes | Skip sensitive image-stream MLPs when quality regresses | `img_mlp` | |
+| Wan2.2 | Wan2.2 diffusion pipelines | Not validated | Not validated | Validate against BF16 before documenting as supported | TBD | |
+| Z-Image | `Tongyi-MAI/Z-Image-Turbo` | Yes | Yes | All layers | None | ✅︎ |
+| FLUX.1 | `black-forest-labs/FLUX.1-dev`, `black-forest-labs/FLUX.1-schnell` | Yes | Yes | All layers | None | |
+| FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B` | Yes | Yes | All layers | None | |
+| HunyuanImage-3.0 | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | Yes | Yes | All layers; use the Hunyuan stage config for multi-stage runs | None | |
+| HunyuanVideo-1.5 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `720p_t2v`, `480p_i2v` | Yes | Yes | All layers | None | |
 
 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
 

examples/offline_inference/audiox/README.md

@@ -0,0 +1,40 @@
+# AudioX offline inference
+
+Generate audio with the [AudioX](https://zeyuet.github.io/AudioX/) MMDiT diffusion
+pipeline (`AudioXPipeline`). Six tasks: `t2a`, `t2m`, `v2a`, `v2m`, `tv2a`, `tv2m`.
+
+## Prerequisites
+
+Download a vLLM-Omni weight bundle (component-sharded safetensors):
+
+```bash
+huggingface-cli download zhangj1an/AudioX --local-dir ./audiox_weights
+```
+
+The Hugging Face id `zhangj1an/AudioX` also works directly without prefetching.
+
+## Usage
+
+```bash
+# Text-to-audio only (default uses zhangj1an/AudioX from the Hub):
+python end2end.py --tasks t2a
+
+# All six tasks against a local bundle and a sample video for v2*/tv2*:
+python end2end.py \
+  --model ./audiox_weights \
+  --video https://zeyuet.github.io/AudioX/static/samples/V2M/1XeBotOFqHA.mp4
+
+# Subset of tasks, custom seed and steps:
+python end2end.py --tasks t2a tv2a --num-inference-steps 100 --seed 0
+```
+
+## Arguments
+
+- `--model`: HF id or local bundle path (default: `zhangj1an/AudioX`).
+- `--tasks`: any subset of `t2a t2m v2a v2m tv2a tv2m` (default: all).
+- `--video`: video file/URL — required for `v2*` and `tv2*`.
+- `--reference-audio`: optional audio prompt (audio-conditioned generation).
+- `--num-inference-steps`, `--guidance-scale`, `--seed`, `--seconds-total`,
+  `--sample-rate`, `--output-dir`: generation knobs.
+
+Outputs land in `<output-dir>/<task>.wav` as 16-bit stereo WAV.

examples/offline_inference/audiox/end2end.py

@@ -0,0 +1,110 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""End-to-end AudioX offline example covering the 6 t2*/v2*/tv2* tasks.
+
+Provide a directory with the **vLLM-Omni AudioX safetensors bundle** (e.g. from
+``zhangj1an/AudioX`` on Hugging Face)::
+
+    huggingface-cli download zhangj1an/AudioX --local-dir ./audiox_weights
+    python end2end.py --model ./audiox_weights
+    python end2end.py --model ./audiox_weights --tasks t2a tv2a
+"""
+
+from __future__ import annotations
+
+import argparse
+import time
+from pathlib import Path
+
+import soundfile
+import torch
+import torchaudio.functional as TF
+
+from vllm_omni.entrypoints.omni import Omni
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+from vllm_omni.platforms import current_omni_platform
+
+ROOT = Path(__file__).resolve().parent
+
+SAMPLE_PROMPTS: dict[str, str] = {
+    "t2a": "Fireworks burst twice, followed by a period of silence before a clock begins ticking.",
+    "t2m": "Uplifting ukulele tune for a travel vlog",
+    "v2a": "",
+    "v2m": "",
+    "tv2a": "drum beating sound and human talking",
+    "tv2m": "uplifting music matching the scene",
+}
+
+ALL_TASKS = ("t2a", "t2m", "v2a", "v2m", "tv2a", "tv2m")
+VIDEO_TASKS = frozenset({"v2a", "v2m", "tv2a", "tv2m"})
+TEXT_TASKS = frozenset({"t2a", "t2m", "tv2a", "tv2m"})
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description="AudioX offline end-to-end (6 t2*/v2*/tv2* tasks).")
+    p.add_argument("--model", default="zhangj1an/AudioX", help="HF id or local AudioX bundle path.")
+    p.add_argument("--tasks", nargs="+", default=list(ALL_TASKS), choices=ALL_TASKS)
+    p.add_argument("--video", default="", help="Video path / URL (required for v2*/tv2*).")
+    p.add_argument("--reference-audio", default="", help="Optional audio prompt for audio-conditioned generation.")
+    p.add_argument("--output-dir", default=str(ROOT / "audiox_task_outputs"))
+    p.add_argument("--num-inference-steps", type=int, default=250)
+    p.add_argument("--seconds-total", type=float, default=10.0)
+    p.add_argument("--guidance-scale", type=float, default=6.0)
+    p.add_argument("--seed", type=int, default=42)
+    p.add_argument("--sample-rate", type=int, default=48000, help="Output WAV rate (resampled if != model rate).")
+    return p.parse_args()
+
+
+def save_wav(audio: torch.Tensor, path: Path, sample_rate: int) -> None:
+    """Write 16-bit PCM WAV. ``audio`` is ``[channels, samples]`` float in [-1, 1]."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    soundfile.write(str(path), audio.clamp(-1.0, 1.0).cpu().T.numpy(), sample_rate, subtype="PCM_16")
+
+
+def main() -> None:
+    args = parse_args()
+
+    omni = Omni(model=args.model, model_class_name="AudioXPipeline")
+
+    for task in args.tasks:
+        if task in VIDEO_TASKS and not args.video:
+            raise SystemExit(f"task={task!r} requires --video")
+        prompt = SAMPLE_PROMPTS[task] if task in TEXT_TASKS else ""
+        extra: dict = {"audiox_task": task, "seconds_start": 0.0, "seconds_total": float(args.seconds_total)}
+        if task in VIDEO_TASKS:
+            extra["video_path"] = args.video
+        if args.reference_audio:
+            extra["audio_path"] = args.reference_audio
+
+        generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
+        t0 = time.perf_counter()
+        outputs = omni.generate(
+            prompt,
+            OmniDiffusionSamplingParams(
+                generator=generator,
+                guidance_scale=args.guidance_scale,
+                num_inference_steps=args.num_inference_steps,
+                seed=args.seed,
+                extra_args=extra,
+            ),
+        )
+        audio = outputs[0].request_output.multimodal_output.get("audio")
+        if audio is None:
+            raise RuntimeError(f"No audio produced for task {task!r}")
+        audio = torch.as_tensor(audio).detach().cpu().float()
+        if audio.ndim == 3:
+            audio = audio[0]
+
+        model_sr = int(outputs[0].request_output.multimodal_output.get("audio_sample_rate") or 44100)
+        if model_sr != args.sample_rate:
+            audio = TF.resample(audio, model_sr, args.sample_rate)
+
+        out_path = Path(args.output_dir) / f"{task}.wav"
+        save_wav(audio, out_path, args.sample_rate)
+        print(f"[{task}] saved {out_path} ({time.perf_counter() - t0:.2f}s)")
+
+    omni.close()
+
+
+if __name__ == "__main__":
+    main()

examples/offline_inference/bagel/end2end.py

@@ -153,7 +153,7 @@ def main():
 
     from vllm_omni.entrypoints.omni import Omni
 
-    omni_kwargs = {}
+    omni_kwargs = vars(args).copy()
     deploy_config = args.deploy_config
     if args.think and deploy_config is None:
         deploy_config = "vllm_omni/deploy/bagel_think.yaml"
@@ -161,22 +161,12 @@ def main():
     if deploy_config:
         omni_kwargs["deploy_config"] = deploy_config
 
-    omni_kwargs.update(
-        {
-            "log_stats": args.log_stats,
-            "init_sleep_seconds": args.init_sleep_seconds,
-            "batch_timeout": args.batch_timeout,
-            "init_timeout": args.init_timeout,
-            "shm_threshold_bytes": args.shm_threshold_bytes,
-            "worker_backend": args.worker_backend,
-            "ray_address": args.ray_address,
-            "enable_diffusion_pipeline_profiler": args.enable_diffusion_pipeline_profiler,
-        }
-    )
     if args.quantization:
         omni_kwargs["quantization_config"] = args.quantization
 
-    omni = Omni.from_cli_args(args, model=model_name, **omni_kwargs)
+    # Override CLI --model with the derived model_name.
+    omni_kwargs["model"] = model_name
+    omni = Omni(**omni_kwargs)
 
     formatted_prompts = []
     for p in prompts:

examples/offline_inference/cosyvoice3/README.md

@@ -1,71 +0,0 @@
-# CosyVoice3
-
-## Setup
-
-Install dependencies:
-```
-uv pip install -e .
-```
-
-> **Note:** This includes required libraries such as `soundfile`,
-> `onnxruntime`, `x-transformers`, and `einops` via
-> `requirements/common.txt` and platform-specific requirements files.
-
-Download the model snapshot:
-```
-from huggingface_hub import snapshot_download
-snapshot_download('FunAudioLLM/Fun-CosyVoice3-0.5B-2512', local_dir='pretrained_models/Fun-CosyVoice3-0.5B')
-```
-
-Add `config.json` in `pretrained_models/Fun-CosyVoice3-0.5B/`:
-```json
-{
-    "model_type": "cosyvoice3",
-    "architectures": [
-        "CosyVoice3Model"
-    ]
-}
-```
-
-> **Why `config.json` is required:**
-> `AutoConfig.register("cosyvoice3", CosyVoice3Config)` only registers a class mapping.
-> The loader still needs `model_type: "cosyvoice3"` from `config.json` to select that class.
-> If no `config.json` is present, model type cannot be inferred automatically.
-> If your downloaded checkpoint already includes a valid `config.json` with
-> `model_type: "cosyvoice3"`, this manual step can be skipped.
-
-Run the offline verification script:
-```
-python examples/offline_inference/cosyvoice3/verify_e2e_cosyvoice.py \
-  --model pretrained_models/Fun-CosyVoice3-0.5B \
-  --tokenizer pretrained_models/Fun-CosyVoice3-0.5B/CosyVoice-BlankEN
-```
-
-## Implementation Overview
-
-CosyVoice3 runs as a **2-stage Omni pipeline**.
-
-- **Stage 0** (`talker`) converts text + prompt audio to speech tokens.
-- **Stage 1** (`code2wav`) converts speech tokens + prompt features to waveform via flow matching and HiFiGAN.
-
-Key components live in `vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py`.
-
-- `CosyVoice3MultiModalProcessor` builds the multimodal inputs:
-    - Tokenizes `prompt` and `prompt_text`.
-    - Extracts speech tokens and mel features from the prompt audio.
-    - Extracts a speaker embedding.
-- `CosyVoice3Model` implements both stages:
-    - Stage 0 uses `CosyVoice3LM` and outputs speech tokens + conditioning features.
-    - Stage 1 runs the flow model (DiT-based CFM) and HiFiGAN to synthesize waveform.
-
-Pipeline topology lives in `vllm_omni/model_executor/models/cosyvoice3/pipeline.py`;
-runtime tunables (batch size, memory limits, sampling) live in
-`vllm_omni/deploy/cosyvoice3.yaml`. The deploy config auto-loads by
-HF `model_type` and defaults to `async_chunk: true` (shared-memory
-streaming). Pass `--no-async-chunk` on `vllm serve` to switch to the
-legacy sync path where stage 1 runs `text2flow` over the full
-speech-token sequence.
-
-- Stage 0 emits latent speech tokens.
-- Stage 1 consumes them via `sync_process_input_func` (sync mode) or the
-  shared-memory connector (async-chunk mode) and outputs audio.

examples/offline_inference/custom_pipeline/image_to_image/image_edit.py

@@ -118,9 +118,6 @@ def parse_args() -> argparse.Namespace:
         help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
     )
 
-    from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
-
-    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/dynin_omni/end2end.py

@@ -18,6 +18,8 @@ import numpy as np
 import torch
 from PIL import Image
 
+from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
+
 TASK_CHOICES = ("t2t", "t2i", "t2s", "i2i", "i2t", "s2t", "v2t")
 
 TASK_DEFAULT_RUNTIME = {
@@ -970,7 +972,6 @@ def parse_args(repo_root: Path) -> argparse.Namespace:
     parser.add_argument("--vq-model-audio-local-files-only", action=argparse.BooleanOptionalAction, default=None)
 
     parser.add_argument("--disable-hf-xet", action=argparse.BooleanOptionalAction, default=True)
-    from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 
     nullify_stage_engine_defaults(parser)
     return parser.parse_args()

examples/offline_inference/fish_speech/README.md

@@ -1,75 +0,0 @@
-# Fish Speech S2 Pro
-
-This directory contains an offline demo for running Fish Speech S2 Pro (`fishaudio/s2-pro`) with vLLM Omni. It supports text-only synthesis and voice cloning with reference audio.
-
-## Model Overview
-
-[Fish Speech S2 Pro](https://huggingface.co/fishaudio/s2-pro) is a 4B dual-AR text-to-speech model by FishAudio. It uses:
-
-- **Slow AR**: Generates semantic tokens autoregressively (Qwen3-based backbone)
-- **Fast AR**: Predicts residual codebook tokens from semantic tokens
-- **DAC Decoder**: Converts codec codes to 44.1 kHz audio waveform
-
-The model produces high-quality speech with voice cloning capabilities.
-
-## Prerequisites
-
-Install the `fish-speech` package for the DAC codec:
-
-```bash
-pip install fish-speech
-```
-
-## Setup
-
-Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
-
-## Quick Start
-
-### Text-Only Synthesis
-
-```bash
-python end2end.py --text "Hello, this is a test of the Fish Speech text to speech system."
-```
-
-Generated audio files are saved to `output_audio/` by default.
-
-### Voice Cloning
-
-Provide a reference audio file and its transcript to clone a voice:
-
-```bash
-python end2end.py \
-    --text "Hello, this is a cloned voice." \
-    --ref-audio /path/to/reference.wav \
-    --ref-text "Transcript of the reference audio."
-```
-
-## Streaming Mode
-
-Add `--streaming` to stream audio chunks progressively via `AsyncOmni` (requires `async_chunk: true` in the stage config):
-
-```bash
-python end2end.py --text "Hello, this is a test." --streaming
-```
-
-Each DAC decoder chunk is logged as it arrives. The final WAV file is written once generation completes.
-
-## Configuration
-
-| Argument | Default | Description |
-|----------|---------|-------------|
-| `--text` | `"Hello, this is a test..."` | Text to synthesize |
-| `--ref-audio` | None | Path to reference audio for voice cloning |
-| `--ref-text` | None | Transcript of the reference audio |
-| `--model` | `fishaudio/s2-pro` | HuggingFace model path |
-| `--stage-configs-path` | `fish_speech_s2_pro.yaml` | Path to stage configs YAML |
-| `--output-dir` | `output_audio` | Output directory for WAV files |
-| `--streaming` | False | Enable streaming via AsyncOmni |
-| `--stage-init-timeout` | 600 | Stage initialization timeout (seconds) |
-
-## Notes
-
-- Output audio is 44.1 kHz mono WAV.
-- The DAC codec weights (`codec.pth`) are loaded lazily from the model directory.
-- Voice cloning encodes the reference audio using the DAC codec to extract semantic codes, then prepends them as a system message.

examples/offline_inference/helios/end2end.py

@@ -196,9 +196,6 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
     parser.add_argument("--tensor-parallel-size", type=int, default=1, help="Tensor parallelism size.")
 
-    from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
-
-    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/hunyuan_image3/README.md

@@ -31,6 +31,14 @@ python end2end.py --model tencent/HunyuanImage-3.0-Instruct \
                   --prompts "A cute cat sitting on a windowsill watching the sunset"
 ```
 
+**With VAE tiling (required on A100 GPUs):**
+```bash
+python end2end.py --model tencent/HunyuanImage-3.0-Instruct \
+                  --modality text2img \
+                  --prompts "A cute cat sitting on a windowsill watching the sunset" \
+                  --vae-use-tiling
+```
+
 #### Image to Image (img2img)
 
 - **Pipeline**: Image + Text → AR (CoT + recaption + latent) → DiT → Edited Image
@@ -103,6 +111,7 @@ python end2end.py --modality text2img \
 | `--stage-configs-path` | string | auto                                 | Custom stage config YAML path                                |
 | `--enforce-eager`      | flag   | `False`                              | Disable torch.compile                                        |
 | `--init-timeout`       | int    | `300`                                | Initialization timeout (seconds)                             |
+| `--vae-use-tiling`     | flag   | `False`                              | Enable VAE tiling for memory optimization (required to avoid OOM on A100) |
 
 ------
 
@@ -135,23 +144,25 @@ python end2end.py --model tencent/HunyuanImage-3.0-Instruct \
 
 ## Prompt Format
 
-HunyuanImage-3.0 uses a pretrain template format:
+HunyuanImage-3.0-Instruct uses an instruct chat template:
 
 ```
-<|startoftext|>{system_prompt}{<img>}{trigger_tag}{user_prompt}
+<|startoftext|>{system_prompt}\n\nUser: {<img>?}{user_prompt}\n\nAssistant: {trigger_tag?}
 ```
 
-- `<img>`: Placeholder for each input image (auto-inserted by `prompt_utils.py`)
-- Trigger tags: `<think>` (CoT), `<recaption>` (recaptioning)
+- `<img>`: Placeholder for each input image (single token; expanded by the multimodal pipeline)
+- Trigger tags: `<think>` (CoT), `<recaption>` (recaptioning) — placed AFTER `Assistant: `
 - System prompt: Auto-selected based on task
+- `t2i_vanilla` is the only task that uses the bare pretrain template (no chat structure)
 
-The `prompt_utils.build_prompt()` handles this formatting automatically.
+The shared `vllm_omni.diffusion.models.hunyuan_image3.prompt_utils.build_prompt_tokens()`
+helper handles segment-by-segment tokenization (matches HF `apply_chat_template` byte-for-byte).
 
 ------
 
 ## FAQ
 
-- **OOM errors**: Decrease `gpu_memory_utilization` in the YAML stage config, or use a smaller `max_num_batched_tokens`.
+- **OOM errors**: Decrease `gpu_memory_utilization` in the YAML stage config, use a smaller `max_num_batched_tokens`, or enable VAE tiling with `--vae-use-tiling` (required on A100 GPUs).
 - **Custom image sizes**: Use `--height` and `--width` flags (multiples of 16 recommended).
 
 | Stage             | VRAM (approx)        |

examples/offline_inference/hunyuan_image3/end2end.py

@@ -16,13 +16,13 @@ Usage:
 import argparse
 import os
 
-from vllm_omni.diffusion.models.hunyuan_image3.system_prompt import (
-    get_system_prompt,
+from vllm_omni.diffusion.models.hunyuan_image3.prompt_utils import (
+    build_prompt_tokens,
 )
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.inputs.data import OmniPromptType
 
-# task → (sys_type, bot_task, trigger_tag)
+# task -> (sys_type, bot_task, trigger_tag)
 _TASK_PRESETS: dict[str, tuple[str, str | None, str | None]] = {
     "t2t": ("en_unified", None, None),
     "i2t": ("en_unified", None, None),
@@ -42,36 +42,6 @@ _MODALITY_TASK_MAP = {
 }
 
 
-def build_prompt(
-    user_prompt: str,
-    task: str = "it2i_think",
-    sys_type: str | None = None,
-    custom_system_prompt: str | None = None,
-) -> str:
-    """Build a HunyuanImage-3.0 prompt using pretrain template format."""
-    if task not in _TASK_PRESETS:
-        raise ValueError(f"Unknown task {task!r}. Choose from: {sorted(_TASK_PRESETS)}")
-
-    preset_sys_type, preset_bot_task, trigger_tag = _TASK_PRESETS[task]
-    effective_sys_type = sys_type or preset_sys_type
-
-    system_prompt = get_system_prompt(effective_sys_type, preset_bot_task, custom_system_prompt)
-    sys_text = system_prompt.strip() if system_prompt else ""
-
-    has_image_input = task.startswith("i2t") or task.startswith("it2i")
-
-    parts = ["<|startoftext|>"]
-    if sys_text:
-        parts.append(sys_text)
-    if has_image_input:
-        parts.append("<img>")
-    if trigger_tag:
-        parts.append(trigger_tag)
-    parts.append(user_prompt)
-
-    return "".join(parts)
-
-
 # Modality → default stage config
 _MODALITY_DEFAULT_CONFIG = {
     "text2img": "hunyuan_image3_t2i.yaml",
@@ -114,6 +84,11 @@ def parse_args():
     parser.add_argument("--seed", type=int, default=42, help="Random seed.")
     parser.add_argument("--height", type=int, default=1024, help="Output image height.")
     parser.add_argument("--width", type=int, default=1024, help="Output image width.")
+    parser.add_argument(
+        "--vae-use-tiling",
+        action="store_true",
+        help="Enable VAE tiling for memory optimization.",
+    )
 
     # Prompt configuration
     parser.add_argument(
@@ -154,6 +129,7 @@ def main():
     # Build Omni
     omni_kwargs = {
         "model": args.model,
+        "vae_use_tiling": args.vae_use_tiling,
         "stage_configs_path": stage_configs_path,
         "log_stats": args.log_stats,
         "init_timeout": args.init_timeout,
@@ -179,12 +155,28 @@ def main():
 
         input_image = Image.open(args.image_path).convert("RGB")
 
+    # Load tokenizer for segment-wise prompt tokenization (matches HF
+    # apply_chat_template byte-for-byte; see build_prompt_tokens docstring).
+    from transformers import AutoTokenizer
+
+    tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
+
     # Format prompts
     formatted_prompts: list[OmniPromptType] = []
     for p in prompts:
-        formatted_text = build_prompt(p, task=task, sys_type=args.sys_type)
+        token_ids = build_prompt_tokens(p, tokenizer, task=task, sys_type=args.sys_type)
+        preset_sys_type, _, _ = _TASK_PRESETS[task]
+        effective_sys_type = args.sys_type or preset_sys_type
 
-        prompt_dict: dict = {"prompt": formatted_text}
+        # `prompt_token_ids` drives the AR stage (matches HF byte-for-byte).
+        # `prompt` and `use_system_prompt` are forwarded by ar2diffusion to
+        # the DiT stage so the diffusion pipeline can rebuild the same
+        # system prefix when constructing its model inputs.
+        prompt_dict: dict = {
+            "prompt_token_ids": token_ids,
+            "prompt": p,
+            "use_system_prompt": effective_sys_type,
+        }
 
         if args.modality == "text2img":
             prompt_dict["modalities"] = ["image"]

examples/offline_inference/image_to_image/image_edit.py

@@ -378,9 +378,6 @@ def parse_args() -> argparse.Namespace:
         default=None,
         help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
     )
-    from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
-
-    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/image_to_video/image_to_video.py

@@ -211,9 +211,6 @@ def parse_args() -> argparse.Namespace:
         default=None,
         help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
     )
-    from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
-
-    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/magi_human/end2end.py

@@ -24,9 +24,6 @@ def parse_args():
     parser.add_argument("--width", type=int, default=448, help="Video width.")
     parser.add_argument("--num-inference-steps", type=int, default=8, help="Number of denoising steps.")
     parser.add_argument("--seed", type=int, default=52, help="Random seed for generation.")
-    from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
-
-    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/mimo_audio/end2end.py

@@ -23,6 +23,7 @@ from message_convert import (
 from vllm import SamplingParams
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 
+from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.inputs.data import OmniTokensPrompt
 
@@ -438,6 +439,7 @@ def parse_args():
         "vllm_omni/deploy/mimo_audio.yaml based on the HF model_type.",
     )
 
+    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/ming_flash_omni/README.md

@@ -4,41 +4,41 @@
 
 vLLM-Omni supports two deployment modes:
 
-| Mode | Stage config | Output |
-|------|-------------|--------|
-| Thinker only (multimodal understanding) | `ming_flash_omni_thinker.yaml` (default `--omni`) | Text |
-| Thinker + Talker (omni-speech) | `ming_flash_omni.yaml` | Text + Audio |
+| Mode | Deploy config | Output |
+|------|--------------|--------|
+| Thinker + Talker (omni-speech, default) | `vllm_omni/deploy/ming_flash_omni.yaml` | Text + Audio |
+| Thinker only (multimodal understanding) | `vllm_omni/deploy/ming_flash_omni_thinker_only.yaml` | Text |
 
-For standalone TTS (talker only), see [`examples/offline_inference/ming_flash_omni_tts/`](../ming_flash_omni_tts/).
+For standalone TTS (talker only), see the [Ming-flash-omni-TTS section in the Text-To-Speech hub](../text_to_speech/README.md#ming-flash-omni-tts).
 
 ## Setup
 
 Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
 
-The default `--omni` flag runs thinker only.  For omni-speech, pass the two-stage config explicitly:
+When no `--deploy-config` is passed, the model registry auto-loads the full thinker+talker `vllm_omni/deploy/ming_flash_omni.yaml` (See [Omni-Speech](#omni-speech-thinker--talker)).
+
+For text-only output without spinning up the talker, pass:
 
 ```bash
---stage-configs-path vllm_omni/model_executor/stage_configs/ming_flash_omni.yaml
+--deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml
 ```
 
 ## Run examples
 
-The end-to-end script defaults to built-in assets; pass `--image-path`,
-`--audio-path`, or `--video-path` to override.
+The end-to-end script defaults to built-in assets; pass `--image-path`, `--audio-path`, or `--video-path` to override.
+
+### Multi-Modality Understanding (Standalone Thinker)
+
+Here we pass thinker-only deploy yaml:
 
 ```bash
-# Text-only
-python examples/offline_inference/ming_flash_omni/end2end.py --query-type text
-
-# Image / audio / video / mixed understanding
-python examples/offline_inference/ming_flash_omni/end2end.py --query-type use_image
-python examples/offline_inference/ming_flash_omni/end2end.py --query-type use_audio
-python examples/offline_inference/ming_flash_omni/end2end.py --query-type use_video --num-frames 16
-python examples/offline_inference/ming_flash_omni/end2end.py --query-type use_mixed_modalities \
-    --image-path /path/to/image.jpg --audio-path /path/to/audio.wav
+python examples/offline_inference/ming_flash_omni/end2end.py --deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml --query-type text
+python examples/offline_inference/ming_flash_omni/end2end.py --deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml --query-type use_image
+python examples/offline_inference/ming_flash_omni/end2end.py --deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml --query-type use_audio
+python examples/offline_inference/ming_flash_omni/end2end.py --deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml --query-type use_video --num-frames 16
 ```
 
-#### Reasoning (Thinking Mode)
+### Reasoning (Thinking Mode)
 
 Reasoning ("detailed thinking on") is applied by the script when
 `--query-type reasoning` is set. The default prompt matches Ming's cookbook
@@ -46,19 +46,21 @@ and expects the reference figure from the upstream repo — see
 `get_reasoning_query` in `end2end.py`.
 
 ```bash
-python examples/offline_inference/ming_flash_omni/end2end.py -q reasoning --image-path ./3_0.png
+python examples/offline_inference/ming_flash_omni/end2end.py \
+    --deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml \
+    --query-type reasoning \
+    --image-path ./3_0.png
 ```
 
-### Omni-speech (thinker + talker)
+### Omni-Speech (Thinker + Talker)
 
-To enable spoken output, use the two-stage config and request `audio` (or `text,audio`) modalities.
+The default deploy YAML already runs thinker+talker, so spoken output only requires requesting `audio` (or `text,audio`) modalities.
 The thinker processes your multimodal input, generates text, then the talker synthesises the response as speech.
 
 **Audio-only output** (speech response, no text):
 ```bash
 python examples/offline_inference/ming_flash_omni/end2end.py \
     --query-type text \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/ming_flash_omni.yaml \
     --modalities audio \
     --output-dir output_ming_omni_speech
 ```
@@ -67,14 +69,13 @@ python examples/offline_inference/ming_flash_omni/end2end.py \
 ```bash
 python examples/offline_inference/ming_flash_omni/end2end.py \
     --query-type use_audio \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/ming_flash_omni.yaml \
     --modalities text,audio \
     --output-dir output_ming_omni_speech
 ```
 
 Generated `.wav` files are saved to `--output-dir` (default `output_ming`), one per request.
 
-The stage config allocates thinker on GPUs 0–3 and talker on GPU 3 by default. Adjust `devices` in the YAML to match your hardware.
+The default deploy YAML allocates thinker on GPUs 0–3 and talker on GPU 3 for a common device topology (4 rather than 5 devices that talker on its own device). Adjust `devices` in a copied YAML and pass it via `--deploy-config` to match your hardware or requirements.
 
 ### Modality control
 
@@ -84,8 +85,7 @@ The stage config allocates thinker on GPUs 0–3 and talker on GPU 3 by default.
 | `audio` | Text (internal) | Runs | `<id>.wav` |
 | `text,audio` | Text | Runs | `<id>.txt` + `<id>.wav` |
 
-Pass `--stage-configs-path /path/to/your_config.yaml` to any of the commands
-above to override the stage config.
+Pass `--deploy-config /path/to/your_deploy.yaml` to any of the commands above to override the bundled deploy config.
 
 ## Online serving
 

examples/offline_inference/ming_flash_omni/end2end.py

@@ -20,6 +20,7 @@ from vllm.multimodal.media.audio import load_audio
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 
 import vllm_omni
+from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 from vllm_omni.entrypoints.omni import Omni
 
 # Imports the processor also registers itself
@@ -302,14 +303,10 @@ def main(args):
     else:
         query_result = query_func(processor)
 
-    # Initialize Omni (with thinker-only stage config)
-    omni = Omni(
-        model=MODEL_NAME,
-        stage_configs_path=args.stage_configs_path,
-        log_stats=args.log_stats,
-        init_timeout=args.init_timeout,
-        stage_init_timeout=args.stage_init_timeout,
-    )
+    omni_kwargs = vars(args).copy()
+    # override CLI --model with derived model_name
+    omni_kwargs["model"] = MODEL_NAME
+    omni = Omni(**omni_kwargs)
 
     # Thinker sampling params
     thinker_sampling_params = SamplingParams(
@@ -410,10 +407,10 @@ def parse_args():
         help="Query type.",
     )
     parser.add_argument(
-        "--stage-configs-path",
+        "--deploy-config",
         type=str,
         default=None,
-        help="Path to a stage configs YAML file.",
+        help="Path to a deploy YAML; leave unset to auto-load full thinker+talker. Pass custom for text-only",
     )
     parser.add_argument(
         "--log-stats",
@@ -499,6 +496,7 @@ def parse_args():
         help="Output directory for results.",
     )
 
+    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/ming_flash_omni_tts/README.md

@@ -1,47 +0,0 @@
-# Ming-flash-omni Standalone TTS (Offline)
-
-This example runs **Ming-flash-omni-2.0 talker-only** offline inference with:
-
-- `model`: `Jonathan1909/Ming-flash-omni-2.0`
-- `stage config`: `vllm_omni/model_executor/stage_configs/ming_flash_omni_tts.yaml`
-
-It follows the Ming cookbook parameter style:
-
-- `prompt`: `"Please generate speech based on the following description.\n"`
-- `max_decode_steps`: `200`
-- `cfg`: `2.0`
-- `sigma`: `0.25`
-- `temperature`: `0.0`
-
-## Quick Start
-
-```bash
-python examples/offline_inference/ming_flash_omni_tts/end2end.py --case style
-```
-
-## Cases
-
-```bash
-# Style
-python examples/offline_inference/ming_flash_omni_tts/end2end.py --case style
-
-# IP
-python examples/offline_inference/ming_flash_omni_tts/end2end.py --case ip
-
-# Basic (speed/pitch/volume control)
-python examples/offline_inference/ming_flash_omni_tts/end2end.py --case basic
-```
-
-## Useful Arguments
-
-- `--text`: override default text in the selected case
-- `--output`: custom output wav path
-- `--model`: local model path or HF repo id
-- `--stage-configs-path`: custom talker stage config path
-- `--log-stats`: enable runtime stats logs
-
-## Notes
-
-- This directory is for **standalone talker deployment (TTS)**.
-- For Ming thinker multimodal understanding examples, see:
-  `examples/offline_inference/ming_flash_omni/`.

examples/offline_inference/moss_tts_nano/README.md

@@ -1,101 +0,0 @@
-# MOSS-TTS-Nano Offline Inference
-
-## Overview
-
-Single-stage offline TTS pipeline using the 0.1B MOSS-TTS-Nano AR LM and MOSS-Audio-Tokenizer-Nano codec. Outputs 48 kHz stereo WAV.
-
-## Quick Start
-
-```bash
-python end2end.py --text "Hello, this is MOSS-TTS-Nano."
-```
-
-The first run downloads `OpenMOSS-Team/MOSS-TTS-Nano` and `OpenMOSS-Team/MOSS-Audio-Tokenizer-Nano` from Hugging Face.
-
-## Usage
-
-```
-python end2end.py [OPTIONS]
-
-Options:
-  --text TEXT               Text to synthesize (default: "Hello, this is MOSS-TTS-Nano speaking.")
-  --voice VOICE             Built-in voice preset (default: Junhao). See voice table below.
-  --mode MODE               voice_clone (default) or continuation
-  --prompt-audio PATH       Reference WAV/MP3 for custom voice cloning
-  --prompt-text TEXT        Reference transcript (continuation mode)
-  --max-new-frames N        Max AR frames, default 375 (~14 s audio)
-  --seed INT                Random seed for reproducibility
-  --audio-temperature F     Audio sampling temperature (default: 0.8)
-  --audio-top-k N           Audio top-k sampling (default: 25)
-  --audio-top-p F           Audio top-p sampling (default: 0.95)
-  --text-temperature F      Text layer temperature (default: 1.0)
-  --batch                   Run a built-in batch of diverse samples (ZH/EN/FR)
-  --output-dir DIR          Directory for WAV outputs (default: /tmp/moss_tts_nano_output)
-  --deploy-config PATH      Override deploy YAML (defaults to vllm_omni/deploy/moss_tts_nano.yaml)
-  --stage-init-timeout INT  Timeout in seconds for stage init (default: 120)
-```
-
-## Examples
-
-```bash
-# Built-in Chinese voice
-python end2end.py --text "你好，这是MOSS-TTS-Nano的语音合成演示。" --voice Junhao
-
-# Built-in English voice
-python end2end.py --text "Hello from MOSS-TTS-Nano." --voice Ava
-
-# Custom voice clone
-python end2end.py \
-    --text "Hello, this is a cloned voice." \
-    --prompt-audio /path/to/reference.wav \
-    --prompt-text "Exact transcript of the reference audio."
-
-# Batch synthesis (ZH/EN/FR)
-python end2end.py --batch --output-dir /tmp/batch_output
-
-# Reproducible output
-python end2end.py --text "Deterministic test." --seed 42
-```
-
-## Built-in Voice Presets
-
-| Voice | Language |
-|-------|----------|
-| `Junhao` | ZH |
-| `Zhiming` | ZH |
-| `Weiguo` | ZH |
-| `Xiaoyu` | ZH |
-| `Yuewen` | ZH |
-| `Lingyu` | ZH |
-| `Ava` | EN |
-| `Bella` | EN |
-| `Adam` | EN |
-| `Nathan` | EN |
-| `Sakura` | JA |
-| `Yui` | JA |
-| `Aoi` | JA |
-| `Hina` | JA |
-| `Mei` | JA |
-
-## Deploy Config
-
-Runtime knobs live in `vllm_omni/deploy/moss_tts_nano.yaml` (auto-loaded;
-override with `--deploy-config PATH`). Key stage-level settings:
-
-```yaml
-stages:
-  - stage_id: 0
-    gpu_memory_utilization: 0.3   # ~2 GB VRAM; increase for faster init
-    max_num_seqs: 4               # concurrent requests
-    max_model_len: 4096
-```
-
-## Output Format
-
-WAV files, 48 kHz, stereo (2-channel). The codec interleaves stereo as `[L, R, L, R, ...]` in the flat tensor returned by the model.
-
-## Troubleshooting
-
-- **`libnvrtc.so.13: cannot open shared object file`**: torchaudio 2.10+ torchcodec backend requires NVRTC. The model patches `torchaudio.load/save` automatically at load time to fall back to soundfile.
-- **`flash_attn not installed`**: The model falls back to `sdpa` attention automatically.
-- **Empty audio**: Check that `--text` is non-empty and the model loaded successfully (look for "MOSS-TTS-Nano LM loaded" in logs).

examples/offline_inference/omnivoice/README.md

@@ -1,73 +0,0 @@
-# OmniVoice
-
-This directory contains an offline demo for running OmniVoice TTS models with vLLM Omni. It generates speech from text and saves WAV files locally.
-
-## Model Overview
-
-[OmniVoice](https://huggingface.co/k2-fsa/OmniVoice) is a zero-shot multilingual TTS model supporting 600+ languages. It uses a diffusion language model (Qwen3-0.6B backbone) with iterative masked unmasking to generate speech.
-
-Three inference modes are supported:
-
-- **Auto Voice**: Generate speech without any reference — the model picks a voice automatically.
-- **Voice Clone**: Clone a voice from a reference audio + transcription.
-- **Voice Design**: Control voice style via natural language instruction (e.g., "female, low pitch, british accent").
-
-## Setup
-
-Ensure the model is downloaded:
-
-```bash
-huggingface-cli download k2-fsa/OmniVoice
-```
-
-> **Note:** Voice cloning requires `transformers>=5.3.0` for `HiggsAudioV2TokenizerModel`. Auto voice and voice design modes work with `transformers>=4.57.0`.
-
-## Quick Start
-
-Auto voice (text only):
-
-```bash
-python end2end.py --model k2-fsa/OmniVoice --text "Hello, this is a test."
-```
-
-Voice design (with style instruction):
-
-```bash
-python end2end.py --model k2-fsa/OmniVoice \
-    --text "Hello, this is a test." \
-    --instruct "female, low pitch, british accent"
-```
-
-Voice clone (with reference audio):
-
-```bash
-python end2end.py --model k2-fsa/OmniVoice \
-    --text "Hello, this is a test." \
-    --ref-audio ref.wav \
-    --ref-text "This is the reference transcription."
-```
-
-## Language Support
-
-Specify a language for improved quality:
-
-```bash
-python end2end.py --model k2-fsa/OmniVoice \
-    --text "你好，这是一个测试。" \
-    --lang zh
-```
-
-## Architecture
-
-OmniVoice uses a two-stage pipeline:
-
-- **Stage 0 (Generator)**: Qwen3-0.6B transformer with 32-step iterative unmasking and classifier-free guidance. Generates 8-codebook audio tokens from text.
-- **Stage 1 (Decoder)**: HiggsAudioV2 RVQ quantizer + DAC acoustic decoder. Converts tokens to 24kHz waveform.
-
-Both stages use `GPUGenerationWorker` with `OmniGenerationScheduler`.
-
-## Notes
-
-- Output audio is saved to `output.wav` by default. Use `--output` to change the path.
-- The model estimates duration from text automatically via `RuleDurationEstimator`.
-- Use `--stage-init-timeout` to increase the stage initialization timeout for first-time model downloads.

examples/offline_inference/qwen2_5_omni/end2end.py

@@ -21,6 +21,7 @@ from vllm.multimodal.media.audio import load_audio
 from vllm.sampling_params import SamplingParams
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 
+from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 from vllm_omni.entrypoints.omni import Omni
 
 SEED = 42
@@ -325,7 +326,10 @@ def main(args):
     else:
         query_result = query_func()
     args.quantization_config = quantization_config
-    omni = Omni.from_cli_args(args, model=model_name)
+    omni_kwargs = vars(args).copy()
+    # Override CLI --model with the derived model_name.
+    omni_kwargs["model"] = model_name
+    omni = Omni(**omni_kwargs)
     thinker_sampling_params = SamplingParams(
         temperature=0.0,  # Deterministic - no randomness
         top_p=1.0,  # Disable nucleus sampling
@@ -550,6 +554,7 @@ def parse_args():
         default=False,
         help="Use py_generator mode. The returned type of Omni.generate() is a Python Generator object.",
     )
+    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/qwen3_omni/end2end.py

@@ -295,7 +295,10 @@ def main(args):
     else:
         query_result = query_func()
 
-    omni = Omni.from_cli_args(args, model=model_name)
+    omni_kwargs = vars(args).copy()
+    # Override CLI --model with the derived model_name.
+    omni_kwargs["model"] = model_name
+    omni = Omni(**omni_kwargs)
 
     thinker_sampling_params = SamplingParams(
         temperature=0.9,
@@ -389,6 +392,13 @@ def main(args):
             output_wav = os.path.join(output_dir, f"output_{request_id}.wav")
 
             # Convert to numpy array and ensure correct format
+            # In async_chunk mode, audio may arrive as a list of chunks
+            if isinstance(audio_tensor, list):
+                import torch
+
+                audio_tensor = torch.cat(
+                    [(t if isinstance(t, torch.Tensor) else torch.tensor(t)).flatten() for t in audio_tensor]
+                )
             audio_numpy = audio_tensor.float().detach().cpu().numpy()
 
             # Ensure audio is 1D (flatten if needed)

examples/offline_inference/qwen3_omni/end2end_async_chunk.py

@@ -41,6 +41,7 @@ from vllm.multimodal.image import convert_image_mode
 from vllm.multimodal.media.audio import load_audio
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 
+from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 from vllm_omni.entrypoints.async_omni import AsyncOmni
 
 logger = logging.getLogger(__name__)
@@ -382,13 +383,7 @@ async def run_all(args):
     print(f"[Info] Creating AsyncOmni with deploy_config={args.deploy_config}")
     async_omni = None
     try:
-        # ``from_cli_args`` expands vars(args) into kwargs and auto-captures
-        # ``_cli_explicit_keys`` from ``sys.argv[1:]`` so argparse defaults
-        # do not silently override deploy YAML values. Mirrors the
-        # ``EngineArgs.from_cli_args`` pattern used throughout vllm /
-        # vllm-omni. ``deploy_config=None`` (the default) falls through to
-        # the bundled ``vllm_omni/deploy/qwen3_omni_moe.yaml``.
-        async_omni = AsyncOmni.from_cli_args(args)
+        async_omni = AsyncOmni(**vars(args))
 
         # Use default sampling params from stage config (they are pre-configured
         # in the YAML for each stage).
@@ -594,6 +589,7 @@ def parse_args():
         default=16000,
         help="Sampling rate for audio loading.",
     )
+    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/qwen3_tts/README.md

@@ -1,121 +0,0 @@
-# Qwen3-TTS
-
-This directory contains an offline demo for running Qwen3 TTS models with vLLM Omni. It builds task-specific inputs and generates WAV files locally.
-
-## Model Overview
-
-Qwen3 TTS provides multiple task variants for speech generation:
-
-- **CustomVoice**: Generate speech with a known speaker identity (speaker ID) and optional instruction.
-- **VoiceDesign**: Generate speech from text plus a descriptive instruction that designs a new voice.
-- **Base**: Voice cloning using a reference audio + reference transcript, with optional mode selection. The `ref_audio` field accepts a local file path, HTTP URL, or base64 data URL.
-
-## Setup
-Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
-
-### ROCm Dependencies
-
-You will need to install the dependency `onnxruntime-rocm`.
-
-```
-pip uninstall onnxruntime # should be removed before we can install onnxruntime-rocm
-pip install onnxruntime-rocm
-```
-
-## Quick Start
-
-Run a single sample for a task:
-
-```
-python end2end.py --query-type CustomVoice
-```
-
-Generated audio files are saved to `output_audio/` by default.
-
-## Task Usage
-
-### CustomVoice
-
-Single sample:
-
-```
-python end2end.py --query-type CustomVoice
-```
-
-Batch sample (multiple prompts in one run):
-
-```
-python end2end.py --query-type CustomVoice --use-batch-sample
-```
-
-### VoiceDesign
-
-Single sample:
-
-```
-python end2end.py --query-type VoiceDesign
-```
-
-Batch sample:
-
-```
-python end2end.py --query-type VoiceDesign --use-batch-sample
-```
-
-### Base (Voice Clone)
-
-Single sample:
-
-```
-python end2end.py --query-type Base
-```
-
-Batch sample:
-
-```
-python end2end.py --query-type Base --use-batch-sample
-```
-
-Mode selection for Base:
-
-- `--mode-tag icl` (default): standard mode
-- `--mode-tag xvec_only`: enable `x_vector_only_mode` in the request
-
-Examples:
-
-```
-python end2end.py --query-type Base --mode-tag icl
-```
-
-## Streaming Mode
-
-Add `--streaming` to stream audio chunks progressively via `AsyncOmni` (requires `async_chunk: true` in the stage config):
-
-```bash
-python end2end.py --query-type CustomVoice --streaming --output-dir /tmp/out_stream
-```
-
-Each Code2Wav chunk is logged as it arrives (default 25 frames; configurable via `codec_chunk_frames`
-in the stage config). The initial chunk size is dynamically selected based on server load for reduced
-TTFA, and can be overridden per-request via the `initial_codec_chunk_frames` API field. The final WAV file is written once generation
-completes. This demonstrates that audio data is available progressively rather than only at the end.
-
-> **Note:** Streaming uses `AsyncOmni` internally. The non-streaming path (`Omni`) is unchanged.
-
-## Batched Decoding
-
-The Code2Wav stage (stage 1) supports batched decoding, where multiple requests are decoded in a single forward pass through the SpeechTokenizer. To use it, set `max_num_seqs > 1` on both stages via `--stage-overrides` and pass multiple prompts via `--txt-prompts` with a matching `--batch-size`.
-
-```
-python end2end.py --query-type CustomVoice \
-    --txt-prompts benchmark_prompts.txt \
-    --batch-size 4 \
-    --stage-overrides '{"0":{"max_num_seqs":4,"gpu_memory_utilization":0.2},"1":{"max_num_seqs":4,"gpu_memory_utilization":0.2}}'
-```
-
-**Important:** `--batch-size` must match a CUDA graph capture size (1, 2, 4, 8, 16...) because the Talker's code predictor KV cache is sized to `max_num_seqs`, and CUDA graphs pad the batch to the next capture size. Both stages need `max_num_seqs >= batch_size` in the stage config for batching to take effect. If only stage 1 has a higher `max_num_seqs`, it won't help — stage 1 can only batch chunks from requests that are in-flight simultaneously, which requires stage 0 to also process multiple requests concurrently.
-
-## Notes
-
-- The script uses the model paths embedded in `end2end.py`. Update them if your local cache path differs.
-- Use `--output-dir` to change the output folder.

examples/offline_inference/speech_to_video/README.md

@@ -0,0 +1,135 @@
+# Speech-To-Video
+
+This example demonstrates how to generate talking-head videos from a reference image and an audio clip using the Wan2.2 Speech-to-Video (S2V) pipeline with vLLM-Omni's offline inference API.
+
+The S2V pipeline generates multiple video clips autoregressively — the last frames of each clip become the motion context for the next — producing a seamless video that spans the full audio duration.
+
+## Prerequisites
+
+The S2V pipeline requires `librosa` for audio loading:
+
+```bash
+pip install librosa decord
+```
+
+## Local CLI Usage
+
+### Singing Example (480p)
+
+```bash
+wget -O "Five Hundred Miles.png" "https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/examples/Five%20Hundred%20Miles.png"
+wget -O "Five Hundred Miles.MP3" "https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/examples/Five%20Hundred%20Miles.MP3"
+python speech_to_video.py \
+  --model Wan-AI/Wan2.2-S2V-14B \
+  --image "Five Hundred Miles.png" \
+  --audio "Five Hundred Miles.MP3" \
+  --prompt "A person singing" \
+  --height 448 --width 832 \
+  --num-inference-steps 5 \
+  --output s2v_singing_480p.mp4
+```
+
+## Key Arguments
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `--model` | *(required)* | Path to Wan2.2 S2V model (local path or HuggingFace ID) |
+| `--image` | *(required)* | Path to reference image (face/portrait) |
+| `--audio` | *(required)* | Path to audio file (wav/mp3) |
+| `--prompt` | `"A person speaking naturally"` | Text prompt describing the scene |
+| `--negative-prompt` | *(S2V default)* | Negative prompt (built-in Chinese quality filter if not set) |
+| `--height` / `--width` | *auto* | Output resolution (auto-calculated from image aspect ratio if not set, must be divisible by 64) |
+| `--num-frames` | `80` | Number of frames per clip (should be divisible by 4) |
+| `--num-inference-steps` | `40` | Number of denoising steps |
+| `--guidance-scale` | `4.5` | Classifier-free guidance scale |
+| `--flow-shift` | `3.0` | Scheduler flow shift |
+| `--seed` | `42` | Random seed for reproducibility |
+| `--fps` | `16` | Frames per second for the saved MP4 |
+| `--init-first-frame` | `False` | Use reference image as the first frame |
+| `--output` | `s2v_output.mp4` | Output video file path |
+| `--vae-use-slicing` | `False` | Enable VAE slicing for memory optimization |
+| `--vae-use-tiling` | `False` | Enable VAE tiling for memory optimization |
+| `--tensor-parallel-size` | `1` | Number of GPUs for tensor parallelism (TP) inside the DiT |
+| `--cfg-parallel-size` | `1` | Number of GPUs for CFG parallelism (1 or 2) |
+| `--vae-patch-parallel-size` | `1` | Number of GPUs for VAE patch parallelism |
+| `--enable-cpu-offload` | `False` | Enable CPU offloading |
+| `--enable-layerwise-offload` | `False` | Enable layerwise offloading on DiT |
+
+> If you encounter OOM errors, try `--vae-use-slicing` and `--vae-use-tiling` to reduce memory usage, or `--enable-cpu-offload` for more aggressive memory saving.
+
+## Performance Tips
+
+The dominant cost in S2V is self-attention across the full token sequence (~80K at 720p), which scales quadratically with resolution. Strategies to speed up generation:
+
+| Strategy | Speedup | How |
+|---|---|---|
+| **Lower resolution** (480p) | ~3.5x | `--height 448 --width 832` — reduces tokens from 80K to 33K |
+| **Tensor parallelism** (2 GPUs) | ~1.4x | `--tensor-parallel-size 2` — splits attention/GEMM across GPUs |
+| **Fewer steps** | Linear | `--num-inference-steps 5` — trades quality for speed |
+| **Combine all above** | ~5x+ | 480p + TP=2 + 5 steps |
+
+## Profiling
+
+To profile the S2V pipeline and identify performance bottlenecks:
+
+### Enable Built-in Profiler
+
+Use `--enable-diffusion-pipeline-profiler` to display stage durations:
+
+```bash
+python speech_to_video.py \
+  --model Wan-AI/Wan2.2-S2V-14B \
+  --image "Five Hundred Miles.png" \
+  --audio "Five Hundred Miles.MP3" \
+  --prompt "A person singing" \
+  --height 448 --width 832 \
+  --num-inference-steps 5 \
+  --enable-diffusion-pipeline-profiler \
+  --output s2v_output.mp4
+```
+
+### Torch Profiler (Device-Level Profiling)
+
+For detailed GPU/XPU kernel-level profiling with Chrome trace visualization:
+
+```bash
+python speech_to_video.py \
+  --model Wan-AI/Wan2.2-S2V-14B \
+  --image "Five Hundred Miles.png" \
+  --audio "Five Hundred Miles.MP3" \
+  --prompt "A person singing" \
+  --height 448 --width 832 \
+  --num-inference-steps 1 \
+  --profile-dir /tmp/s2v_profile \
+  --profile-record-shapes \
+  --profile-with-memory \
+  ----profile-with-stack \
+  --output s2v_profiled.mp4
+```
+
+**Profiler Arguments:**
+
+| Parameter | Description |
+|-----------|-------------|
+| `--profile-dir` | Enable torch profiler and save traces to this directory |
+| `--profile-record-shapes` | Record tensor shapes (increases trace size) |
+| `--profile-with-stack` | Record stack traces (increases overhead) |
+| `--profile-with-memory` | Profile memory usage |
+| `--profile-with-flops` | Estimate FLOPs for operations |
+
+**Output Files:**
+- `trace_rank*.json.gz` - Chrome trace (open in `chrome://tracing`)
+- `ops_rank*.xlsx` - Excel workbook with operation statistics
+
+> **Note:** Use `--num-inference-steps 1` for faster profiling. The trace captures one full denoising step with all transformer blocks and attention operations.
+
+## How It Works
+
+The S2V pipeline processes in the following stages:
+
+1. **Pre-process**: The reference image is resized/cropped to fit the target resolution. The audio path is validated.
+2. **Text encoding**: The prompt is encoded using the UMT5 text encoder.
+3. **Audio encoding**: The audio file is loaded via `librosa`, processed through Wav2Vec2, and bucketed to align with video frame-rate. The audio length determines the number of clips (`num_repeat`).
+4. **Reference image encoding**: The reference image is VAE-encoded into latent tokens.
+5. **Multi-clip denoising**: For each clip, random noise is generated and denoised through the `WanModel_S2V` transformer with audio injection. The last frames of each decoded clip become the motion context for the next.
+6. **Post-process**: All clips are concatenated and converted to pixel-space video frames.

examples/offline_inference/speech_to_video/speech_to_video.py

@@ -0,0 +1,402 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""
+Speech-to-Video generation example using Wan2.2 S2V.
+
+Generates talking-head videos from a reference image and an audio clip
+using the Wan2.2 S2V pipeline with multi-clip autoregressive generation.
+
+Usage:
+    python speech_to_video.py \
+        --model /path/to/Wan2.2-S2V-14B \
+        --image reference.jpg \
+        --audio speech.wav \
+        --prompt "A person speaking naturally"
+"""
+
+import argparse
+import os
+import time
+from pathlib import Path
+
+import torch
+
+from vllm_omni.diffusion.data import DiffusionParallelConfig
+from vllm_omni.entrypoints.omni import Omni
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+from vllm_omni.outputs import OmniRequestOutput
+from vllm_omni.platforms import current_omni_platform
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Generate a talking-head video from a reference image and audio (Wan2.2 S2V)."
+    )
+    parser.add_argument(
+        "--model",
+        required=True,
+        help="Path to Wan2.2 S2V model (local path or HuggingFace ID).",
+    )
+    parser.add_argument("--image", required=True, help="Path to reference image.")
+    parser.add_argument("--audio", required=True, help="Path to audio file (wav/mp3).")
+    parser.add_argument(
+        "--prompt",
+        default="A person speaking naturally",
+        help="Text prompt describing the scene.",
+    )
+    parser.add_argument(
+        "--negative-prompt",
+        default=None,
+        help="Negative prompt (uses S2V default if not set).",
+    )
+    parser.add_argument("--seed", type=int, default=42, help="Random seed.")
+    parser.add_argument(
+        "--guidance-scale",
+        type=float,
+        default=4.5,
+        help="CFG scale (default: 4.5).",
+    )
+    parser.add_argument(
+        "--height",
+        type=int,
+        default=None,
+        help="Video height (auto-calculated from reference image if not set).",
+    )
+    parser.add_argument(
+        "--width",
+        type=int,
+        default=None,
+        help="Video width (auto-calculated from reference image if not set).",
+    )
+    parser.add_argument(
+        "--num-frames",
+        type=int,
+        default=80,
+        help="Frames per clip (should be divisible by 4, default: 80).",
+    )
+    parser.add_argument(
+        "--num-inference-steps",
+        type=int,
+        default=40,
+        help="Number of denoising steps (default: 40).",
+    )
+    parser.add_argument(
+        "--flow-shift",
+        type=float,
+        default=3.0,
+        help="Scheduler flow shift (default: 3.0).",
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default="s2v_output.mp4",
+        help="Path to save the output video.",
+    )
+    parser.add_argument(
+        "--fps",
+        type=int,
+        default=16,
+        help="Frames per second for the output video (default: 16).",
+    )
+    parser.add_argument(
+        "--init-first-frame",
+        action="store_true",
+        help="Use the reference image as the first frame of the video.",
+    )
+    parser.add_argument(
+        "--vae-use-slicing",
+        action="store_true",
+        help="Enable VAE slicing for memory optimization.",
+    )
+    parser.add_argument(
+        "--vae-use-tiling",
+        action="store_true",
+        help="Enable VAE tiling for memory optimization.",
+    )
+    parser.add_argument(
+        "--enable-cpu-offload",
+        action="store_true",
+        help="Enable CPU offloading for diffusion models.",
+    )
+    parser.add_argument(
+        "--enable-layerwise-offload",
+        action="store_true",
+        help="Enable layerwise (blockwise) offloading on DiT modules.",
+    )
+    parser.add_argument(
+        "--tensor-parallel-size",
+        type=int,
+        default=1,
+        help="Number of GPUs used for tensor parallelism (TP) inside the DiT.",
+    )
+    parser.add_argument(
+        "--cfg-parallel-size",
+        type=int,
+        default=1,
+        choices=[1, 2],
+        help="Number of GPUs used for classifier free guidance parallel size.",
+    )
+    parser.add_argument(
+        "--vae-patch-parallel-size",
+        type=int,
+        default=1,
+        help="Number of GPUs used for VAE patch/tile parallelism (decode).",
+    )
+    parser.add_argument(
+        "--ring-degree",
+        type=int,
+        default=1,
+        help="Number of GPUs used for ring sequence parallelism.",
+    )
+    parser.add_argument(
+        "--ulysses-degree",
+        type=int,
+        default=1,
+        help="Number of GPUs used for ulysses sequence parallelism.",
+    )
+    parser.add_argument(
+        "--enforce-eager",
+        action="store_true",
+        help="Disable torch.compile and force eager execution.",
+    )
+    parser.add_argument(
+        "--enable-diffusion-pipeline-profiler",
+        action="store_true",
+        help="Enable diffusion pipeline profiler to display stage durations.",
+    )
+    parser.add_argument(
+        "--cache-backend",
+        type=str,
+        default=None,
+        choices=["cache_dit"],
+        help="Cache backend for acceleration. Default: None.",
+    )
+    parser.add_argument(
+        "--enable-cache-dit-summary",
+        action="store_true",
+        help="Enable cache-dit summary logging after diffusion forward passes.",
+    )
+    parser.add_argument(
+        "--profile-dir",
+        type=str,
+        default=None,
+        help="Enable torch profiler and save traces to this directory.",
+    )
+    parser.add_argument(
+        "--profile-record-shapes",
+        action="store_true",
+        help="Record tensor shapes in profiler (increases trace size).",
+    )
+    parser.add_argument(
+        "--profile-with-stack",
+        action="store_true",
+        help="Record stack traces in profiler (increases overhead).",
+    )
+    parser.add_argument(
+        "--profile-with-memory",
+        action="store_true",
+        help="Profile memory usage.",
+    )
+    parser.add_argument(
+        "--profile-with-flops",
+        action="store_true",
+        help="Estimate FLOPs for operations.",
+    )
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
+
+    # Load reference image
+    import PIL.Image
+
+    image = PIL.Image.open(args.image).convert("RGB")
+
+    # Cache-dit config
+    cache_config = None
+    if args.cache_backend == "cache_dit":
+        cache_config = {
+            "Fn_compute_blocks": 1,
+            "Bn_compute_blocks": 0,
+            "max_warmup_steps": 4,
+            "max_cached_steps": 20,
+            "residual_diff_threshold": 0.24,
+            "max_continuous_cached_steps": 3,
+            "enable_taylorseer": False,
+            "taylorseer_order": 1,
+            "scm_steps_mask_policy": None,
+            "scm_steps_policy": "dynamic",
+        }
+
+    parallel_config = DiffusionParallelConfig(
+        tensor_parallel_size=args.tensor_parallel_size,
+        cfg_parallel_size=args.cfg_parallel_size,
+        vae_patch_parallel_size=args.vae_patch_parallel_size,
+        ring_degree=args.ring_degree,
+        ulysses_degree=args.ulysses_degree,
+    )
+
+    # Check if profiling is requested via CLI args or environment variable
+    profile_dir = args.profile_dir or os.getenv("VLLM_TORCH_PROFILER_DIR")
+    profiler_enabled = bool(profile_dir)
+    profiler_config = None
+    if profiler_enabled:
+        from vllm.config import ProfilerConfig
+
+        profiler_config = ProfilerConfig(
+            profiler="torch",
+            torch_profiler_dir=profile_dir,
+            torch_profiler_record_shapes=args.profile_record_shapes,
+            torch_profiler_with_stack=args.profile_with_stack,
+            torch_profiler_with_memory=args.profile_with_memory,
+            torch_profiler_with_flops=args.profile_with_flops,
+        )
+
+    omni = Omni(
+        model=args.model,
+        model_class_name="WanS2VPipeline",
+        flow_shift=args.flow_shift,
+        vae_use_slicing=args.vae_use_slicing,
+        vae_use_tiling=args.vae_use_tiling,
+        enable_cpu_offload=args.enable_cpu_offload,
+        enable_layerwise_offload=args.enable_layerwise_offload,
+        parallel_config=parallel_config,
+        enforce_eager=args.enforce_eager,
+        enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
+        cache_backend=args.cache_backend,
+        cache_config=cache_config,
+        enable_cache_dit_summary=args.enable_cache_dit_summary,
+        profiler_config=profiler_config,
+    )
+
+    # Print generation configuration
+    print(f"\n{'=' * 60}")
+    print("Generation Configuration:")
+    print(f"  Model: {args.model}")
+    print(f"  Reference image: {args.image}")
+    print(f"  Audio: {args.audio}")
+    print(f"  Inference steps: {args.num_inference_steps}")
+    print(f"  Frames per clip: {args.num_frames}")
+    print(f"  Guidance scale: {args.guidance_scale}")
+    print(f"  Flow shift: {args.flow_shift}")
+    print(f"  Init first frame: {args.init_first_frame}")
+    if args.height and args.width:
+        print(f"  Video size: {args.width}x{args.height}")
+    else:
+        print("  Video size: auto (from reference image)")
+    print(f"{'=' * 60}\n")
+
+    # Start profiling if enabled
+    if profiler_enabled:
+        print("[Profiler] Starting profiling...")
+        omni.start_profile()
+
+    generation_start = time.perf_counter()
+
+    result = omni.generate(
+        {
+            "prompt": args.prompt,
+            "negative_prompt": args.negative_prompt,
+            "multi_modal_data": {
+                "image": image,
+                "audio": args.audio,
+                "init_first_frame": args.init_first_frame,
+            },
+        },
+        OmniDiffusionSamplingParams(
+            height=args.height,
+            width=args.width,
+            num_frames=args.num_frames,
+            num_inference_steps=args.num_inference_steps,
+            guidance_scale=args.guidance_scale,
+            seed=args.seed,
+            generator=generator,
+        ),
+    )
+
+    generation_end = time.perf_counter()
+    generation_time = generation_end - generation_start
+    print(f"Total generation time: {generation_time:.4f} seconds ({generation_time * 1000:.2f} ms)")
+
+    # Stop profiling if enabled
+    if profiler_enabled:
+        print("\n[Profiler] Stopping profiler and collecting results...")
+        profile_results = omni.stop_profile()
+        if profile_results and isinstance(profile_results, list):
+            print("\n" + "=" * 60)
+            print("PROFILING RESULTS:")
+            for i, result in enumerate(profile_results):
+                print(f"\nStage {i}:")
+                if result:
+                    print(f"  {result}")
+            print("=" * 60)
+        else:
+            print("[Profiler] No valid profiling data returned.")
+
+    # Extract output from result
+    output = OmniRequestOutput.unwrap_result(result)
+
+    if not output.images:
+        raise ValueError("No video frames found in OmniRequestOutput.")
+
+    # Extract audio from multimodal_output (set by pipeline post-processor)
+    mm = output.multimodal_output or {}
+    audio_waveform = mm.get("audio")
+    output_fps = float(mm.get("fps", args.fps))
+    output_sr = int(mm.get("audio_sample_rate", 16000))
+
+    if audio_waveform is not None:
+        print(f"Audio waveform: shape={audio_waveform.shape}, sr={output_sr}")
+    else:
+        print("Warning: no audio waveform in pipeline output")
+
+    # Normalize frames to (T, H, W, C) uint8 numpy array
+    import numpy as np
+
+    def _flatten_to_array(data):
+        """Unwrap to a single (T, H, W, C) numpy array."""
+        if isinstance(data, np.ndarray):
+            if data.ndim == 5:
+                return data[0]  # (B, T, H, W, C) → (T, H, W, C)
+            if data.ndim == 4:
+                return data  # already (T, H, W, C)
+        if isinstance(data, list) and data:
+            first_elem = data[0]
+            if isinstance(first_elem, np.ndarray) and first_elem.ndim >= 4:
+                return _flatten_to_array(first_elem)
+            if isinstance(first_elem, np.ndarray) and first_elem.ndim == 3:
+                return np.stack(data)  # list of (H, W, C) → (T, H, W, C)
+        return data
+
+    video_frames = _flatten_to_array(output.images)
+    # postprocess_video returns float32 in [0,1]; mux_video_audio_bytes needs uint8
+    if isinstance(video_frames, np.ndarray) and video_frames.dtype != np.uint8:
+        video_frames = (np.clip(video_frames, 0, 1) * 255).astype(np.uint8)
+    num_frames = video_frames.shape[0] if isinstance(video_frames, np.ndarray) else len(video_frames)
+
+    # Save output video with audio
+    output_path = Path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes
+
+    video_bytes = mux_video_audio_bytes(
+        video_frames,
+        audio_waveform,
+        fps=output_fps,
+        audio_sample_rate=output_sr,
+    )
+    with open(output_path, "wb") as f:
+        f.write(video_bytes)
+
+    print(f"Saved generated video to {output_path}")
+    print(f"Video has {num_frames} frames at {output_fps} fps ({num_frames / output_fps:.1f}s)")
+    if audio_waveform is not None:
+        print(f"Audio: {len(audio_waveform) / output_sr:.1f}s at {output_sr} Hz")
+
+
+if __name__ == "__main__":
+    main()

examples/offline_inference/text_to_audio/text_to_audio.py

@@ -114,6 +114,12 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="Enable diffusion pipeline profiler to display stage durations.",
     )
+    parser.add_argument(
+        "--enable-cpu-offload", action="store_true", help="Enable ModelWise cpu offloading to save gpu memory"
+    )
+    parser.add_argument(
+        "--enable-layerwise-offload", action="store_true", help="Enable Layerwise cpu offloading to save gpu memory"
+    )
     parser.add_argument(
         "--use-hsdp",
         action="store_true",
@@ -214,6 +220,8 @@ def main():
     print(f"  Inference steps: {args.num_inference_steps}")
     print(f"  Guidance scale: {args.guidance_scale}")
     print(f"  Cache backend: {args.cache_backend if args.cache_backend else 'None (no acceleration)'}")
+    print(f"  ModelWise Offload: {'Enabled' if args.enable_cpu_offload else 'None'}")
+    print(f"  LayerWise Offload: {'Enabled' if args.enable_layerwise_offload else 'None'}")
     if args.use_hsdp:
         print(f"  HSDP: enabled (shard_size={args.hsdp_shard_size}, replicate_size={args.hsdp_replicate_size})")
     else:
@@ -234,6 +242,8 @@ def main():
         cache_backend=args.cache_backend,
         cache_config=cache_config,
         enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
+        enable_cpu_offload=args.enable_cpu_offload,
+        enable_layerwise_offload=args.enable_layerwise_offload,
     )
 
     # Calculate audio end time

examples/offline_inference/text_to_image/text_to_image.py

@@ -9,7 +9,7 @@ from typing import Any
 
 import torch
 
-from vllm_omni.diffusion.data import DiffusionParallelConfig, logger
+from vllm_omni.diffusion.data import logger
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 from vllm_omni.lora.request import LoRARequest
@@ -342,17 +342,6 @@ def main():
             #       (e.g., QwenImagePipeline or FluxPipeline)
         }
 
-    # assert args.ring_degree == 1, "Ring attention is not supported yet"
-    parallel_config = DiffusionParallelConfig(
-        ulysses_degree=args.ulysses_degree,
-        ring_degree=args.ring_degree,
-        ulysses_mode=args.ulysses_mode,
-        cfg_parallel_size=args.cfg_parallel_size,
-        tensor_parallel_size=args.tensor_parallel_size,
-        vae_patch_parallel_size=args.vae_patch_parallel_size,
-        enable_expert_parallel=args.enable_expert_parallel,
-    )
-
     profiler_enabled = args.profiler_config is not None
 
     # Prepare LoRA kwargs for Omni initialization
@@ -388,7 +377,13 @@ def main():
         "cache_backend": args.cache_backend,
         "cache_config": cache_config,
         "enable_cache_dit_summary": args.enable_cache_dit_summary,
-        "parallel_config": parallel_config,
+        "ulysses_degree": args.ulysses_degree,
+        "ring_degree": args.ring_degree,
+        "ulysses_mode": args.ulysses_mode,
+        "cfg_parallel_size": args.cfg_parallel_size,
+        "tensor_parallel_size": args.tensor_parallel_size,
+        "vae_patch_parallel_size": args.vae_patch_parallel_size,
+        "enable_expert_parallel": args.enable_expert_parallel,
         "enforce_eager": args.enforce_eager,
         "enable_cpu_offload": args.enable_cpu_offload,
         "mode": "text-to-image",

examples/offline_inference/text_to_speech/README.md

@@ -0,0 +1,439 @@
+# Text-To-Speech
+
+vLLM-Omni supports several autoregressive TTS models. They share a mostly
+common CLI shape (`--text`, `--ref-audio`, `--ref-text`, plus an
+output-path flag — `--output-dir` for most, `--output` for OmniVoice) and
+live together in this hub. Each model has its own subdirectory containing
+a single `end2end.py` script; this README is the single doc entry point.
+
+For online serving, see [`examples/online_serving/text_to_speech/`](../../online_serving/text_to_speech/README.md). For the full
+list of supported architectures across all modalities, see
+[Supported Models](../../../docs/models/supported_models.md).
+
+## Supported Models
+
+| Model | HuggingFace repo | Stages | Voice cloning | Streaming | Special modes | Sample rate |
+|---|---|---|---|---|---|---|
+| CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ | — | 22.05 kHz |
+| Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ | — | 44.1 kHz |
+| Ming-flash-omni-TTS | `Jonathan1909/Ming-flash-omni-2.0` | single (talker only) | — (caption-controlled) | — | style / IP / basic captions | 44.1 kHz |
+| MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | single (AR + codec) | ✓ (required) | ✓ | voice_clone, continuation | 48 kHz |
+| OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design, language hint | 24 kHz |
+| Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | 2 (talker + code2wav) | ✓ (Base) | ✓ | 3 task variants | 24 kHz |
+| VoxCPM | `openbmb/VoxCPM-0.5B` | split | ✓ | ✓ | — | 24 kHz |
+| VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | ✓ (online) | continuation | 48 kHz |
+| Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | varies | ✓ | ✓ | voice presets | 24 kHz |
+
+## Common Quick Start
+
+Most models share this invocation shape:
+
+```bash
+python examples/offline_inference/text_to_speech/<model>/end2end.py \
+    --text "Hello, this is a test." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text  "Transcript of the reference audio."
+```
+
+`--ref-audio` and `--ref-text` are optional (text-only synthesis works without them) and must be provided together for voice cloning. The exotic scripts — Qwen3-TTS, Voxtral TTS, CosyVoice3 — accept additional model-specific flags documented in their per-model section below. Qwen3-TTS in particular uses its own argparse surface (`--query-type`, `--audio-path`, etc.) and does not follow the common shape; see its section.
+
+---
+
+## CosyVoice3
+
+2-stage TTS pipeline (`talker` + `code2wav`) at 22.05 kHz.
+
+### Prerequisites
+```bash
+uv pip install -e .
+# Includes soundfile, onnxruntime, x-transformers, einops via requirements.
+```
+
+Download the model snapshot:
+```python
+from huggingface_hub import snapshot_download
+snapshot_download('FunAudioLLM/Fun-CosyVoice3-0.5B-2512',
+                  local_dir='pretrained_models/Fun-CosyVoice3-0.5B')
+```
+
+If your downloaded checkpoint lacks `config.json`, add it:
+```json
+{
+    "model_type": "cosyvoice3",
+    "architectures": ["CosyVoice3Model"]
+}
+```
+This is required because `AutoConfig.register("cosyvoice3", CosyVoice3Config)` only registers the class mapping; the loader still reads `model_type` from `config.json` to select the class.
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
+    --model pretrained_models/Fun-CosyVoice3-0.5B \
+    --tokenizer pretrained_models/Fun-CosyVoice3-0.5B/CosyVoice-BlankEN
+```
+
+### Voice cloning
+Pass a reference audio. Note that CosyVoice3's `--prompt-text` is a system-style prompt for the GPT stage, not a reference transcript:
+```bash
+python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
+    --model pretrained_models/Fun-CosyVoice3-0.5B \
+    --tokenizer pretrained_models/Fun-CosyVoice3-0.5B/CosyVoice-BlankEN \
+    --ref-audio prompt.wav \
+    --prompt-text "You are a helpful assistant.<|endofprompt|>Testing my voices. Why should I not?"
+```
+
+### Streaming
+Streaming is enabled by default via `async_chunk: true` in `vllm_omni/deploy/cosyvoice3.yaml`. Pass `--no-async-chunk` on `vllm serve` to switch to the legacy synchronous path.
+
+### Notes
+- Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform.
+- Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config <path>` to override.
+
+---
+
+## Fish Speech S2 Pro
+
+4B dual-AR text-to-speech model from FishAudio with the DAC codec at 44.1 kHz.
+
+### Prerequisites
+```bash
+pip install fish-speech
+```
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
+    --text "Hello, this is a test of the Fish Speech text to speech system."
+```
+
+### Voice cloning
+```bash
+python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
+    --text "Hello, this is a cloned voice." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text  "Transcript of the reference audio."
+```
+
+### Streaming
+```bash
+python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
+    --text "Hello, this is a streaming test." \
+    --streaming
+```
+Streaming requires `async_chunk: true` in the stage config.
+
+### Notes
+- Output: 44.1 kHz mono WAV.
+- DAC codec weights (`codec.pth`) are loaded lazily from the model directory.
+
+---
+
+## Ming-flash-omni-TTS
+
+Standalone talker-only deployment of Ming-flash-omni-2.0 at 44.1 kHz. Voice is controlled through caption fields (`风格` / `IP` / `语速`/`基频`/`音量`) rather than reference audio.
+
+### Prerequisites
+The example calls into `vllm_omni.model_executor.models.ming_flash_omni.prompt_utils` for the default prompt and instruction builder; no extra pip install on top of the base vLLM-Omni install.
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/ming_flash_omni_tts/end2end.py --case style
+```
+
+### Cases
+```bash
+# ASMR-style whisper (caption-driven)
+python examples/offline_inference/text_to_speech/ming_flash_omni_tts/end2end.py --case style
+
+# IP voice (preset character voice via caption)
+python examples/offline_inference/text_to_speech/ming_flash_omni_tts/end2end.py --case ip
+
+# Basic speed/pitch/volume control
+python examples/offline_inference/text_to_speech/ming_flash_omni_tts/end2end.py --case basic
+```
+
+Override the default text per case with `--text`, write to a custom path with `--output`.
+
+### Notes
+- Talker-only deployment — for the multimodal Ming-flash-omni example, see [`examples/offline_inference/ming_flash_omni/`](../../ming_flash_omni/).
+- Deploy config: `vllm_omni/deploy/ming_flash_omni_tts.yaml` (single GPU, `enforce_eager`, `max_num_seqs: 1`).
+- Decode defaults from the Ming cookbook: `max_decode_steps=200`, `cfg=2.0`, `sigma=0.25`, `temperature=0.0`, `use_zero_spk_emb=True`.
+
+---
+
+## MOSS-TTS-Nano
+
+Single-stage 0.1B AR LM + MOSS-Audio-Tokenizer-Nano codec at 48 kHz mono (mixed down from upstream stereo). ZH / EN / JA. Every request requires a reference clip via `--ref-audio`.
+
+> **No built-in speaker presets.** `--ref-audio` is required on every call. Default `--mode voice_clone` matches upstream's recommended workflow; `--mode continuation` is exposed for completeness but upstream's continuation-with-prompt path emits very short / near-silent output, so it is rarely useful in practice. Sample reference clips ship in the upstream repo under [`assets/audio/`](https://github.com/OpenMOSS/MOSS-TTS-Nano/tree/main/assets/audio) (e.g. `zh_1.wav`, `en_2.wav`, `jp_2.wav`).
+
+### Quick start
+```bash
+# Fetch a sample reference clip (one-off, user-scoped cache).
+REF_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/moss-tts-nano"
+mkdir -p "$REF_DIR"
+[ -s "$REF_DIR/zh_1.wav" ] || \
+    curl -L -o "$REF_DIR/zh_1.wav" https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS-Nano/main/assets/audio/zh_1.wav
+
+python examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py \
+    --text "你好，这是MOSS-TTS-Nano的语音合成演示。" \
+    --ref-audio "$REF_DIR/zh_1.wav"
+```
+The first run downloads `OpenMOSS-Team/MOSS-TTS-Nano` and `OpenMOSS-Team/MOSS-Audio-Tokenizer-Nano` from Hugging Face.
+
+### Reproducible runs
+```bash
+python examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py \
+    --text "Deterministic test." \
+    --ref-audio "$REF_DIR/en_2.wav" \
+    --seed 42
+```
+
+### Notes
+- Output: 48 kHz mono WAV (the tokenizer is internally stereo at 48 kHz; the wrapper averages to mono before reaching the engine).
+- Deploy config: `vllm_omni/deploy/moss_tts_nano.yaml` (auto-loaded; override with `--deploy-config`).
+- Default `--max-new-frames 375` ≈ 14 s of audio; raise for longer outputs.
+- `--ref-text` is rejected in `voice_clone` mode and required only with `--mode continuation`.
+- Run `--help` for the full sampling-knob surface (`--audio-temperature`, `--audio-top-k`, `--audio-top-p`, `--text-temperature`).
+
+---
+
+## OmniVoice
+
+Zero-shot multilingual TTS supporting 600+ languages, with three modes (auto / clone / design).
+
+### Prerequisites
+```bash
+huggingface-cli download k2-fsa/OmniVoice
+```
+Voice cloning requires `transformers>=5.3.0`. Auto and design modes work with `transformers>=4.57.0`.
+
+### Quick start (auto voice)
+```bash
+python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
+    --model k2-fsa/OmniVoice \
+    --text "Hello, this is a test."
+```
+
+### Voice cloning
+```bash
+python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
+    --model k2-fsa/OmniVoice \
+    --text "Hello, this is a test." \
+    --ref-audio ref.wav \
+    --ref-text  "This is the reference transcription."
+```
+
+### Voice design
+```bash
+python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
+    --model k2-fsa/OmniVoice \
+    --text "Hello, this is a test." \
+    --instruct "female, low pitch, british accent"
+```
+
+### Language hint
+```bash
+python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
+    --model k2-fsa/OmniVoice \
+    --text "你好，这是一个测试。" \
+    --lang zh
+```
+
+### Notes
+- Stage 0 (Generator): Qwen3-0.6B with 32-step iterative unmasking.
+- Stage 1 (Decoder): HiggsAudioV2 RVQ + DAC at 24 kHz.
+
+---
+
+## Qwen3-TTS
+
+3-task-variant TTS with 24 kHz output. Has its own argparse surface (this script does not follow the common `--text` / `--ref-audio` shape).
+
+### Prerequisites
+For ROCm builds, replace `onnxruntime` with `onnxruntime-rocm`:
+```bash
+pip uninstall onnxruntime
+pip install onnxruntime-rocm
+```
+
+### Task variants
+- `CustomVoice`: predefined speaker (speaker ID) with optional style instruction.
+- `VoiceDesign`: text + descriptive instruction designs a new voice.
+- `Base`: voice cloning from reference audio + transcript.
+
+```bash
+# Single sample
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type CustomVoice
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type VoiceDesign
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type Base
+
+# Base with a custom reference audio (Qwen3-TTS uses --audio-path, not --ref-audio):
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
+    --query-type Base --audio-path /path/to/reference.wav
+
+# Base variant has an additional mode flag:
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type Base --mode-tag icl       # default
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type Base --mode-tag xvec_only # x_vector_only_mode
+
+# Batch (multiple prompts in one run)
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type CustomVoice --use-batch-sample
+```
+
+### Streaming
+```bash
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
+    --query-type CustomVoice \
+    --streaming \
+    --output-dir /tmp/out_stream
+```
+Streaming requires `async_chunk: true` in the stage config.
+
+### Batched decoding
+The Code2Wav stage supports batched decoding through the SpeechTokenizer. Pass multiple prompts via `--txt-prompts` and set `--batch-size` accordingly. To raise `max_num_seqs` on either stage, point `--stage-configs-path` at a stage configs YAML with the desired values (see `vllm_omni/model_executor/stage_configs/` for templates):
+```bash
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
+    --query-type CustomVoice \
+    --txt-prompts examples/offline_inference/text_to_speech/qwen3_tts/benchmark_prompts.txt \
+    --batch-size 4 \
+    --stage-configs-path /path/to/qwen3_tts_batched.yaml
+```
+`--batch-size` must match a CUDA-graph capture size (1, 2, 4, 8, 16…).
+
+### Notes
+- Run `--help` for the full argument surface.
+- See `qwen3_tts/end2end.py` for the prompt-length-estimation logic the Talker uses.
+
+---
+
+## VoxCPM
+
+Split-stage TTS. The hub example covers single TTS, single voice cloning, and streaming. Use `benchmarks/voxcpm/` for warmup, batch JSONL prompts, profiler injection, and offline TTFP / RTF measurement.
+
+### Prerequisites
+```bash
+pip install voxcpm soundfile
+# or use a local source tree:
+export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
+```
+
+If the native VoxCPM `config.json` does not contain HF metadata such as `model_type`, prepare a persistent HF-compatible config directory and point the stage configs to it via `VLLM_OMNI_VOXCPM_HF_CONFIG_PATH`:
+
+```bash
+export VOXCPM_MODEL=/path/to/voxcpm-model
+export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config
+mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"
+cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json"
+cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true
+python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)'
+```
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This is a split-stage VoxCPM synthesis example running on vLLM Omni."
+```
+
+### Voice cloning
+```bash
+python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This sentence is synthesized with a cloned voice." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text  "The exact transcript spoken in reference.wav."
+```
+
+### Streaming
+Pass the async-chunk stage config:
+```bash
+python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
+    --model "$VOXCPM_MODEL" \
+    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
+    --text "This is a split-stage VoxCPM streaming example running on vLLM Omni."
+```
+
+### Notes
+- `voxcpm.yaml` is the default non-streaming stage config; `voxcpm_async_chunk.yaml` enables streaming.
+- Streaming is currently single-request oriented.
+- `--ref-text` must be the real transcript of `--ref-audio`; mismatched text degrades quality.
+- For online serving, see the [VoxCPM section in the online hub](../../online_serving/text_to_speech/README.md#voxcpm). For benchmark reporting, see [`benchmarks/voxcpm`](../../../benchmarks/voxcpm/README.md).
+
+---
+
+## VoxCPM2
+
+Single-stage native AR TTS at 48 kHz. Pipeline: `feat_encoder → MiniCPM4 → FSQ → residual_lm → LocDiT → AudioVAE`.
+
+### Prerequisites
+```bash
+pip install voxcpm
+# or, for a local source checkout:
+export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/voxcpm
+```
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
+    --model openbmb/VoxCPM2 \
+    --text "Hello, this is a VoxCPM2 demo."
+```
+
+### Voice cloning
+Pass a reference audio for isolated cloning, or both `--ref-audio` + `--ref-text` for prompt continuation:
+```bash
+python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
+    --text "Hello, this is a voice clone demo." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text  "Transcript of the reference audio."
+```
+
+### Streaming
+Streaming is exposed through the online OpenAI Speech API (`stream=true`). See [`examples/online_serving/text_to_speech/voxcpm2/gradio_demo.py`](../../online_serving/text_to_speech/voxcpm2/gradio_demo.py) for an AudioWorklet-based gapless streaming player; the offline `end2end.py` script does not expose a streaming path.
+
+### Notes
+- Output: 48 kHz mono WAV.
+- Deploy config: `vllm_omni/deploy/voxcpm2.yaml` (auto-loaded by HF `model_type`).
+
+---
+
+## Voxtral TTS
+
+Voxtral-4B-TTS (Mistral). Has its own argparse surface; uses voice presets and the `mistral_common` `SpeechRequest` protocol.
+
+### Prerequisites
+Latest `mistral_common` with `SpeechRequest` support:
+```bash
+pip install -e /path/to/mistral-common  # or upgrade from PyPI when available
+```
+
+### Quick start (voice preset)
+```bash
+python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
+    --write-audio --voice cheerful_female \
+    --model mistralai/Voxtral-4B-TTS-2603 \
+    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
+```
+
+### Voice cloning (capability gated upstream)
+```bash
+python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
+    --write-audio \
+    --model mistralai/Voxtral-4B-TTS-2603 \
+    --text "This is a test message." \
+    --ref-audio path/to/reference_audio.wav
+```
+
+### Streaming + concurrency
+```bash
+python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
+    --num-prompts 32 --concurrency 8 --streaming --write-audio --voice neutral_female \
+    --model mistralai/Voxtral-4B-TTS-2603 \
+    --text "..."
+```
+Available voice presets are listed on the HF model card (`mistralai/Voxtral-4B-TTS-2603`).
+
+### Notes
+- `--num-prompts N` replicates the prompt for performance measurement.
+- `--concurrency M` requires `--streaming` and must evenly divide `--num-prompts`.
+- Run `--help` for the full argument surface.

examples/offline_inference/text_to_speech/cosyvoice3/end2end.py

@@ -9,6 +9,7 @@ from vllm import SamplingParams
 from vllm.assets.audio import AudioAsset
 from vllm.multimodal.media.audio import load_audio
 
+from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.model_executor.models.cosyvoice3.config import CosyVoice3Config
 from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
@@ -31,19 +32,20 @@ def run_e2e():
         help="Override the deploy config path. If unset, auto-loads "
         "vllm_omni/deploy/cosyvoice3.yaml based on the HF model_type.",
     )
-    parser.add_argument("--prompt", type=str, default="Hello, this is a test of the CosyVoice system capability.")
+    parser.add_argument("--text", type=str, default="Hello, this is a test of the CosyVoice system capability.")
     parser.add_argument(
         "--prompt-text",
         type=str,
         default="You are a helpful assistant.<|endofprompt|>Testing my voices. Why should I not?",
     )
-    parser.add_argument("--audio-path", type=str, default="prompt.wav")
+    parser.add_argument("--ref-audio", type=str, default="prompt.wav")
     parser.add_argument(
         "--tokenizer",
         type=str,
         required=True,
         help="Path to tokenizer directory (e.g., <model_path>/CosyVoice-BlankEN).",
     )
+    nullify_stage_engine_defaults(parser)
     args = parser.parse_args()
     # Ensure tokenizer directory exists
     if not os.path.exists(args.tokenizer):
@@ -64,11 +66,11 @@ def run_e2e():
     sampling_cfg = {"top_p": 0.8, "top_k": 25, "eos_token_id": 6561 + 1}
 
     print("Model initialized. Preparing inputs...")
-    if args.audio_path:
-        if not os.path.exists(args.audio_path):
-            raise FileNotFoundError(f"Audio file not found: {args.audio_path}")
+    if args.ref_audio:
+        if not os.path.exists(args.ref_audio):
+            raise FileNotFoundError(f"Audio file not found: {args.ref_audio}")
         # Load at native sample rate
-        audio_signal, sr = load_audio(args.audio_path, sr=None)
+        audio_signal, sr = load_audio(args.ref_audio, sr=None)
 
         # Validate sample rate before processing (similar to original CosyVoice)
         min_sr = 16000
@@ -84,7 +86,7 @@ def run_e2e():
         audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
 
     prompts = {
-        "prompt": args.prompt,
+        "prompt": args.text,
         "multi_modal_data": {
             "audio": audio_data,
         },
@@ -94,7 +96,7 @@ def run_e2e():
         },
     }
 
-    print(f"Generating for prompt: {args.prompt}")
+    print(f"Generating for prompt: {args.text}")
 
     config = CosyVoice3Config()
     tokenizer = get_qwen_tokenizer(
@@ -102,7 +104,7 @@ def run_e2e():
         skip_special_tokens=config.skip_special_tokens,
         version=config.version,
     )
-    _, text_token_len = extract_text_token(args.prompt, tokenizer, config.allowed_special)
+    _, text_token_len = extract_text_token(args.text, tokenizer, config.allowed_special)
     base_len = int(text_token_len)
     min_len = int(base_len * config.min_token_text_ratio)
     max_len = int(base_len * config.max_token_text_ratio)

examples/offline_inference/text_to_speech/fish_speech/end2end.py

@@ -29,6 +29,7 @@ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 
 from vllm_omni import AsyncOmni, Omni
+from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 from vllm_omni.model_executor.models.fish_speech.dac_utils import DAC_HOP_LENGTH, DAC_SAMPLE_RATE
 from vllm_omni.model_executor.models.fish_speech.prompt_utils import (
     build_fish_text_only_prompt_ids,
@@ -44,6 +45,7 @@ DEFAULT_STAGE_CONFIG = os.path.join(
     "..",
     "..",
     "..",
+    "..",
     "vllm_omni",
     "model_executor",
     "stage_configs",
@@ -265,6 +267,7 @@ def parse_args():
         default=False,
         help="Stream audio chunks as they arrive via AsyncOmni.",
     )
+    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/text_to_speech/ming_flash_omni_tts/end2end.py

@@ -9,6 +9,7 @@ from vllm.utils.argparse_utils import FlexibleArgumentParser
 
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
 
+from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.inputs.data import OmniTokensPrompt
 from vllm_omni.model_executor.models.ming_flash_omni.prompt_utils import (
@@ -17,7 +18,6 @@ from vllm_omni.model_executor.models.ming_flash_omni.prompt_utils import (
 )
 
 MODEL_NAME = "Jonathan1909/Ming-flash-omni-2.0"
-DEFAULT_STAGE_CONFIG = "vllm_omni/model_executor/stage_configs/ming_flash_omni_tts.yaml"
 
 
 def get_messages(case: str, text_override: str | None) -> dict[str, Any]:
@@ -71,10 +71,10 @@ def parse_args():
     parser = FlexibleArgumentParser(description="Ming-flash-omni standalone talker offline e2e example")
     parser.add_argument("--model", type=str, default=MODEL_NAME, help="Model name or local path.")
     parser.add_argument(
-        "--stage-configs-path",
+        "--deploy-config",
         type=str,
-        default=DEFAULT_STAGE_CONFIG,
-        help="Path to stage configs yaml for standalone talker deployment.",
+        default="vllm_omni/deploy/ming_flash_omni_tts.yaml",
+        help="Path to a custom deploy YAML for the TTS deployment. ",
     )
     parser.add_argument(
         "--case",
@@ -88,19 +88,15 @@ def parse_args():
     parser.add_argument("--log-stats", action="store_true", default=False, help="Enable stats logging.")
     parser.add_argument("--init-timeout", type=int, default=600, help="Engine init timeout in seconds.")
     parser.add_argument("--stage-init-timeout", type=int, default=300, help="Single stage init timeout in seconds.")
+
+    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 
 def main():
     args = parse_args()
 
-    omni = Omni(
-        model=args.model,
-        stage_configs_path=args.stage_configs_path,
-        log_stats=args.log_stats,
-        init_timeout=args.init_timeout,
-        stage_init_timeout=args.stage_init_timeout,
-    )
+    omni = Omni(**vars(args))
 
     messages = get_messages(args.case, args.text)
     decode_args = {

examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py

@@ -3,23 +3,31 @@
 """Offline inference example for MOSS-TTS-Nano via vLLM-Omni.
 
 Single-stage pipeline: the 0.1B AR LM and MOSS-Audio-Tokenizer-Nano codec
-both run inside one generation stage.  Output is 48 kHz stereo WAV.
+both run inside one generation stage. Output is 48 kHz mono WAV (the
+upstream tokenizer is stereo at 48 kHz; the wrapper mixes down to mono so
+the existing single-channel audio writer in vLLM-Omni stays correct).
 
-Supports:
-  - Built-in voice presets (--voice)
-  - Custom voice cloning via reference audio (--prompt-audio)
-  - Batch synthesis of multiple texts
-  - Reproducible output via --seed
+MOSS-TTS-Nano upstream supports two modes (matching ``infer.py``):
 
-Usage examples:
-  # Built-in voice
-  python end2end.py --text "Hello from MOSS-TTS-Nano."
+* ``voice_clone`` (recommended): only ``--ref-audio`` is required.
+* ``continuation``: ``--ref-audio`` + ``--ref-text`` together.
 
-  # Voice clone with reference audio
-  python end2end.py --text "Hello!" --prompt-audio /path/to/ref.wav
+Usage:
+  # Voice clone (recommended): ref audio only, no transcript needed.
+  python end2end.py \\
+    --text "Hello!" \\
+    --ref-audio /path/to/ref.wav
 
-  # Batch with different voices
-  python end2end.py --batch --output-dir /tmp/moss_output
+  # Continuation: ref audio + its transcript.
+  python end2end.py \\
+    --text "Hello!" \\
+    --ref-audio /path/to/ref.wav \\
+    --ref-text "Transcript of the reference clip." \\
+    --mode continuation
+
+  # Sample reference clips ship in the upstream repo:
+  #   https://github.com/OpenMOSS/MOSS-TTS-Nano/tree/main/assets/audio
+  # e.g. zh_1.wav (Chinese), en_2.wav (English), jp_2.wav (Japanese).
 """
 
 from __future__ import annotations
@@ -39,19 +47,12 @@ from vllm_omni import Omni  # noqa: E402
 
 MODEL = "OpenMOSS-Team/MOSS-TTS-Nano"
 
-BATCH_SAMPLES = [
-    {"text": "Hello, this is a test of MOSS-TTS-Nano.", "voice": "Ava", "label": "en_hello"},
-    {"text": "你好，这是 MOSS-TTS-Nano 的语音合成测试。", "voice": "Junhao", "label": "zh_hello"},
-    {"text": "Bonjour, ceci est un test de synthèse vocale.", "voice": "Bella", "label": "fr_hello"},
-]
-
 
 def build_request(
     text: str,
-    voice: str = "Junhao",
-    mode: str = "voice_clone",
-    prompt_audio_path: str | None = None,
+    prompt_audio_path: str,
     prompt_text: str | None = None,
+    mode: str = "voice_clone",
     max_new_frames: int = 375,
     seed: int | None = None,
     audio_temperature: float = 0.8,
@@ -59,20 +60,24 @@ def build_request(
     audio_top_p: float = 0.95,
     text_temperature: float = 1.0,
 ) -> dict:
-    """Build an Omni request payload for MOSS-TTS-Nano."""
+    """Build an Omni request payload for MOSS-TTS-Nano.
+
+    Upstream's ``_resolve_inference_mode`` forbids ``prompt_text`` in
+    ``voice_clone`` mode and requires it in ``continuation`` mode (with
+    ``prompt_audio_path``), so we only forward ``prompt_text`` when it is
+    actually supplied.
+    """
     additional: dict = {
         "text": [text],
-        "voice": [voice],
         "mode": [mode],
+        "prompt_audio_path": [str(prompt_audio_path)],
         "max_new_frames": [max_new_frames],
         "audio_temperature": [audio_temperature],
         "audio_top_k": [audio_top_k],
         "audio_top_p": [audio_top_p],
         "text_temperature": [text_temperature],
     }
-    if prompt_audio_path:
-        additional["prompt_audio_path"] = [str(prompt_audio_path)]
-    if prompt_text:
+    if prompt_text is not None and prompt_text.strip():
         additional["prompt_text"] = [prompt_text]
     if seed is not None:
         additional["seed"] = [seed]
@@ -84,10 +89,13 @@ def build_request(
 
 
 def save_audio(waveform: torch.Tensor, path: str, sample_rate: int = 48000) -> None:
+    """Write the model's mono waveform to ``path`` at ``sample_rate``.
+
+    The model wrapper mixes the upstream tokenizer's stereo output down to
+    mono before reaching the engine, so ``waveform`` is always 1-D here —
+    no extra interleave/reshape is needed.
+    """
     audio_np = waveform.float().numpy()
-    # Reshape stereo: inference_stream yields interleaved [samples*2]; reshape to [samples, 2]
-    if audio_np.ndim == 1 and audio_np.shape[0] % 2 == 0:
-        audio_np = audio_np.reshape(-1, 2)
     sf.write(path, audio_np, sample_rate)
     print(f"  Saved {path} ({audio_np.shape}, {sample_rate} Hz)")
 
@@ -111,26 +119,21 @@ def main(args) -> None:
     output_dir = Path(args.output_dir)
     output_dir.mkdir(parents=True, exist_ok=True)
 
-    if args.batch:
-        print(f"Running batch synthesis ({len(BATCH_SAMPLES)} samples)...")
-        inputs = [build_request(s["text"], voice=s["voice"], seed=args.seed) for s in BATCH_SAMPLES]
-        params_list = [sampling_params] * len(inputs)
-    else:
-        print(f"Synthesizing: {args.text!r}")
-        inputs = build_request(
-            text=args.text,
-            voice=args.voice,
-            mode=args.mode,
-            prompt_audio_path=args.prompt_audio,
-            prompt_text=args.prompt_text,
-            max_new_frames=args.max_new_frames,
-            seed=args.seed,
-            audio_temperature=args.audio_temperature,
-            audio_top_k=args.audio_top_k,
-            audio_top_p=args.audio_top_p,
-            text_temperature=args.text_temperature,
-        )
-        params_list = sampling_params
+    print(f"Synthesizing: {args.text!r}")
+    print(f"  ref_audio: {args.ref_audio}")
+    inputs = build_request(
+        text=args.text,
+        prompt_audio_path=args.ref_audio,
+        prompt_text=args.ref_text,
+        mode=args.mode,
+        max_new_frames=args.max_new_frames,
+        seed=args.seed,
+        audio_temperature=args.audio_temperature,
+        audio_top_k=args.audio_top_k,
+        audio_top_p=args.audio_top_p,
+        text_temperature=args.text_temperature,
+    )
+    params_list = sampling_params
 
     for stage_outputs in omni.generate(inputs, params_list):
         for i, req_output in enumerate(stage_outputs.request_output):
@@ -145,8 +148,7 @@ def main(args) -> None:
                     print(f"  [req {i}] No waveform in multimodal_output.")
                     continue
                 sr = int(sr_tensor.item()) if sr_tensor is not None else 48000
-                label = BATCH_SAMPLES[i]["label"] if args.batch else f"output_{i}_{j}"
-                out_path = str(output_dir / f"{label}.wav")
+                out_path = str(output_dir / f"output_{i}_{j}.wav")
                 save_audio(audio.cpu(), out_path, sr)
 
     print("Done.")
@@ -155,18 +157,34 @@ def main(args) -> None:
 def parse_args():
     parser = FlexibleArgumentParser(description="MOSS-TTS-Nano offline inference")
     parser.add_argument("--text", default="Hello, this is MOSS-TTS-Nano speaking.", help="Text to synthesize.")
-    parser.add_argument("--voice", default="Junhao", help="Built-in voice preset name.")
+    parser.add_argument(
+        "--ref-audio",
+        required=True,
+        help="Path to reference audio for voice cloning / continuation (required).",
+    )
+    parser.add_argument(
+        "--ref-text",
+        default=None,
+        help=(
+            "Optional transcript of --ref-audio. Required (and only meaningful) "
+            "in --mode continuation; rejected by upstream in --mode voice_clone."
+        ),
+    )
     parser.add_argument("--mode", default="voice_clone", choices=["voice_clone", "continuation"])
-    parser.add_argument("--prompt-audio", default=None, help="Path to reference audio for voice cloning.")
-    parser.add_argument("--prompt-text", default=None, help="Reference transcript (continuation mode).")
     parser.add_argument("--max-new-frames", type=int, default=375, help="Max AR frames (~14s at default).")
     parser.add_argument("--seed", type=int, default=None, help="Random seed.")
     parser.add_argument("--audio-temperature", type=float, default=0.8)
     parser.add_argument("--audio-top-k", type=int, default=25)
     parser.add_argument("--audio-top-p", type=float, default=0.95)
     parser.add_argument("--text-temperature", type=float, default=1.0)
-    parser.add_argument("--batch", action="store_true", help="Run built-in batch of diverse samples.")
-    parser.add_argument("--output-dir", default="/tmp/moss_tts_nano_output", help="Directory for WAV outputs.")
+    parser.add_argument(
+        "--output-dir",
+        default=os.path.join(
+            os.environ.get("XDG_CACHE_HOME", os.path.join(os.path.expanduser("~"), ".cache")),
+            "moss_tts_nano_output",
+        ),
+        help="Directory for WAV outputs (default: ~/.cache/moss_tts_nano_output).",
+    )
     parser.add_argument(
         "--deploy-config",
         default=None,

examples/offline_inference/text_to_speech/omnivoice/end2end.py

@@ -21,6 +21,7 @@ import os
 import numpy as np
 import soundfile as sf
 
+from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 
@@ -79,6 +80,7 @@ def run_e2e():
         default=600,
         help="Stage initialization timeout in seconds",
     )
+    nullify_stage_engine_defaults(parser)
     args = parser.parse_args()
 
     if not os.path.exists(args.stage_config):

examples/offline_inference/text_to_speech/qwen3_tts/end2end.py

@@ -18,6 +18,7 @@ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 
 from vllm_omni import AsyncOmni, Omni
+from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 
 logger = logging.getLogger(__name__)
 
@@ -383,7 +384,10 @@ def main(args):
     output_dir = args.output_dir
     os.makedirs(output_dir, exist_ok=True)
 
-    omni = Omni.from_cli_args(args, model=model_name)
+    omni_kwargs = vars(args).copy()
+    # Override CLI --model with the derived model_name.
+    omni_kwargs["model"] = model_name
+    omni = Omni(**omni_kwargs)
 
     batch_size = args.batch_size
     for batch_start in range(0, len(inputs), batch_size):
@@ -399,7 +403,10 @@ async def main_streaming(args):
     output_dir = args.output_dir
     os.makedirs(output_dir, exist_ok=True)
 
-    omni = AsyncOmni.from_cli_args(args, model=model_name)
+    omni_kwargs = vars(args).copy()
+    # Override CLI --model with the derived model_name.
+    omni_kwargs["model"] = model_name
+    omni = AsyncOmni(**omni_kwargs)
 
     for i, prompt in enumerate(inputs):
         request_id = str(i)
@@ -541,6 +548,7 @@ def parse_args():
         help="Number of prompts per batch (default: 1, sequential).",
     )
 
+    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/text_to_speech/voxcpm/end2end.py

@@ -12,8 +12,9 @@ import torch
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 
 from vllm_omni import AsyncOmni, Omni
+from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 
-REPO_ROOT = Path(__file__).resolve().parents[3]
+REPO_ROOT = Path(__file__).resolve().parents[4]
 DEFAULT_SYNC_STAGE_CONFIG = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm.yaml"
 
 
@@ -185,6 +186,7 @@ def parse_args():
     )
     parser.add_argument("--stage-init-timeout", type=int, default=600, help="Stage initialization timeout in seconds.")
     parser.add_argument("--log-stats", action="store_true", help="Enable vLLM Omni stats logging.")
+    nullify_stage_engine_defaults(parser)
     args = parser.parse_args()
     if (args.ref_audio is None) != (args.ref_text is None):
         raise ValueError("Voice cloning requires --ref-audio and --ref-text together.")

examples/offline_inference/text_to_speech/voxcpm2/end2end.py

@@ -15,8 +15,9 @@ import torch
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 
 from vllm_omni import Omni
+from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 
-REPO_ROOT = Path(__file__).resolve().parents[3]
+REPO_ROOT = Path(__file__).resolve().parents[4]
 DEFAULT_STAGE_CONFIGS_PATH = str(REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm2.yaml")
 SAMPLE_RATE = 48_000
 
@@ -48,29 +49,18 @@ def parse_args():
         help="Path to the stage config YAML file.",
     )
     parser.add_argument(
-        "--reference-audio",
+        "--ref-audio",
         type=str,
         default=None,
-        help="Path to reference audio for voice cloning (isolated ref mode).",
-    )
-    parser.add_argument(
-        "--prompt-audio",
-        type=str,
-        default=None,
-        help="Path to prompt audio for continuation mode (requires --prompt-text).",
-    )
-    parser.add_argument(
-        "--prompt-text",
-        type=str,
-        default=None,
-        help="Text matching --prompt-audio for continuation mode.",
+        help="Path to reference audio for voice cloning.",
     )
     parser.add_argument(
         "--ref-text",
         type=str,
         default=None,
-        help="Optional transcript of --reference-audio (enables ref_continuation mode).",
+        help="Optional transcript of --ref-audio (enables continuation mode).",
     )
+    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 
@@ -120,8 +110,8 @@ def main():
     split_map = build_cjk_split_map(tokenizer)
     hf_config = engine.engine.stage_vllm_configs[0].model_config.hf_config
 
-    ref_audio_arg = args.reference_audio or args.prompt_audio
-    ref_text_arg = args.ref_text or args.prompt_text
+    ref_audio_arg = args.ref_audio
+    ref_text_arg = args.ref_text
     ref_wav, ref_sr = (None, None)
     if ref_audio_arg:
         ref_wav_arr, ref_sr = sf.read(ref_audio_arg)

examples/offline_inference/text_to_speech/voxtral_tts/end2end.py

@@ -30,6 +30,7 @@ from vllm import SamplingParams
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 
 from vllm_omni import AsyncOmni
+from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 from vllm_omni.entrypoints.omni import Omni
 
 logger = logging.getLogger(__name__)
@@ -241,7 +242,7 @@ def parse_args() -> Namespace:
         help="Text to synthesize.",
     )
     parser.add_argument(
-        "--audio-path",
+        "--ref-audio",
         type=str,
         default=None,
         help="Path to reference audio file for voice cloning.",
@@ -304,6 +305,7 @@ def parse_args() -> Namespace:
         default=None,
         help="CFG alpha for flow-matching guidance (default: use value from stage config, typically 1.2).",
     )
+    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 
@@ -344,10 +346,10 @@ def main(args: Any) -> None:
     model_name = args.model
     output_dir = args.output_dir
 
-    if args.voice is None and args.audio_path is None:
-        raise ValueError("Either --voice or --audio-path must be provided.")
+    if args.voice is None and args.ref_audio is None:
+        raise ValueError("Either --voice or --ref-audio must be provided.")
 
-    audio_prompt_file = args.audio_path
+    audio_prompt_file = args.ref_audio
     text_chunk = TextChunk(text=args.text)
 
     if args.write_audio:

examples/offline_inference/text_to_video/text_to_video.py

@@ -226,9 +226,6 @@ def parse_args() -> argparse.Namespace:
         default=1,
         help="Number of HSDP replica groups.",
     )
-    from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
-
-    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/vace/vace_video_generation.py

@@ -71,9 +71,6 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--ulysses-degree", type=int, default=1, help="Ulysses SP degree.")
     parser.add_argument("--ring-degree", type=int, default=1, help="Ring attention degree.")
     parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
-    from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
-
-    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/voxcpm/README.md

@@ -1,123 +0,0 @@
-# VoxCPM Offline Example
-
-This directory contains the minimal offline VoxCPM example for vLLM Omni.
-
-`end2end.py` is intentionally small and only covers:
-
-- single text-to-speech
-- single voice cloning with `ref_audio` + `ref_text`
-- non-streaming with `vllm_omni/model_executor/stage_configs/voxcpm.yaml`
-- streaming with `vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml`
-
-Advanced workflows were moved out of the getting-started example:
-
-- `benchmarks/voxcpm/vllm_omni/bench_tts_offline.py`: warmup, batch prompts, profiler, offline TTFP / RTF
-- `benchmarks/voxcpm/vllm_omni/run_offline_matrix.py`: fixed offline smoke matrix
-- `benchmarks/voxcpm/`: benchmark scripts and benchmark docs
-
-## Prerequisites
-
-Install VoxCPM in one of these ways:
-
-```bash
-pip install voxcpm
-```
-
-or point vLLM Omni to the local VoxCPM source tree:
-
-```bash
-export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
-```
-
-The example writes WAV files with `soundfile`:
-
-```bash
-pip install soundfile
-```
-
-## Model Path
-
-Pass the native VoxCPM model directory directly:
-
-```bash
-export VOXCPM_MODEL=/path/to/voxcpm-model
-```
-
-If the native VoxCPM `config.json` does not contain HuggingFace metadata such as
-`model_type`, prepare a persistent HF-compatible config directory and point the
-stage configs to it with `VLLM_OMNI_VOXCPM_HF_CONFIG_PATH`:
-
-```bash
-export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config
-mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"
-cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json"
-cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true
-python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)'
-```
-
-If the model directory itself already has `model_type`, this extra directory is
-not required.
-
-## Quick Start
-
-Single text-to-speech, non-streaming:
-
-```bash
-python examples/offline_inference/voxcpm/end2end.py \
-  --model "$VOXCPM_MODEL" \
-  --text "This is a split-stage VoxCPM synthesis example running on vLLM Omni."
-```
-
-Single voice cloning, non-streaming:
-
-```bash
-python examples/offline_inference/voxcpm/end2end.py \
-  --model "$VOXCPM_MODEL" \
-  --text "This sentence is synthesized with a cloned voice." \
-  --ref-audio /path/to/reference.wav \
-  --ref-text "The exact transcript spoken in reference.wav."
-```
-
-Streaming:
-
-```bash
-python examples/offline_inference/voxcpm/end2end.py \
-  --model "$VOXCPM_MODEL" \
-  --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
-  --text "This is a split-stage VoxCPM streaming example running on vLLM Omni."
-```
-
-By default, `end2end.py` writes to `output_audio/` for non-streaming and
-`output_audio_streaming/` for streaming.
-
-## Advanced Workflows
-
-Use `benchmarks/voxcpm/vllm_omni/bench_tts_offline.py` when you need:
-
-- warmup runs
-- prompt files
-- batch JSONL inputs
-- profiler injection
-- offline TTFP / RTF emission
-
-Use `benchmarks/voxcpm/vllm_omni/run_offline_matrix.py` when you need the fixed offline smoke matrix that previously lived in `test.py`.
-
-Full matrix benchmark example:
-
-```bash
-python benchmarks/voxcpm/vllm_omni/run_offline_matrix.py \
-  --model "$VOXCPM_MODEL" \
-  --ref-audio /path/to/reference.wav \
-  --ref-text "The exact transcript spoken in reference.wav."
-```
-
-For online serving examples, see [examples/online_serving/voxcpm](../../online_serving/voxcpm/README.md).
-
-For benchmark reporting, see [benchmarks/voxcpm](../../../benchmarks/voxcpm/README.md).
-
-## Notes
-
-- `voxcpm.yaml` is the default non-streaming stage config.
-- `voxcpm_async_chunk.yaml` is the streaming stage config.
-- Streaming is currently single-request oriented; the fixed smoke matrix now lives in `benchmarks/voxcpm/vllm_omni/run_offline_matrix.py`.
-- `ref_text` must be the real transcript of the reference audio. Mismatched text usually causes obvious quality degradation.

examples/offline_inference/voxcpm2/README.md

@@ -1,83 +0,0 @@
-# VoxCPM2 Offline Inference (Native AR)
-
-VoxCPM2 is a 2B-parameter tokenizer-free diffusion AR TTS model. It produces 48kHz audio and supports 30+ languages with a single-stage native AR pipeline backed by MiniCPM4.
-
-## Prerequisites
-
-Install the `voxcpm` package, or set the environment variable pointing to the source tree:
-
-```bash
-# Option A: install package
-pip install voxcpm
-
-# Option B: use source checkout
-export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/voxcpm
-```
-
-## Quick Start
-
-Zero-shot synthesis:
-
-```bash
-python examples/offline_inference/voxcpm2/end2end.py \
-    --model openbmb/VoxCPM2 \
-    --text "Hello, this is a VoxCPM2 demo." \
-    --output-dir output_audio
-```
-
-Voice cloning with a reference audio:
-
-```bash
-python examples/offline_inference/voxcpm2/end2end.py \
-    --text "Hello, this is a voice clone demo." \
-    --reference-audio /path/to/reference.wav \
-    --output-dir output_clone
-```
-
-Prompt continuation (matched audio + text prefix):
-
-```bash
-python examples/offline_inference/voxcpm2/end2end.py \
-    --text "Continuation target sentence." \
-    --prompt-audio /path/to/prompt.wav \
-    --prompt-text "Transcript of the prompt audio." \
-    --output-dir output_cont
-```
-
-The script accepts the following arguments:
-
-| Argument | Default | Description |
-|---|---|---|
-| `--model` | `openbmb/VoxCPM2` | HuggingFace repo ID or local path |
-| `--text` | (example sentence) | Text to synthesize |
-| `--output-dir` | `output_audio` | Directory for output WAV files |
-| `--stage-configs-path` | `voxcpm2.yaml` | Stage config YAML path |
-| `--reference-audio` | `None` | Reference audio for voice cloning (isolated) |
-| `--prompt-audio` | `None` | Prompt audio for continuation mode |
-| `--prompt-text` | `None` | Transcript matching `--prompt-audio` |
-
-## Performance
-
-Measured on a single H20 GPU (80 GB):
-
-| Input length | RTF | Sample rate |
-|---|---|---|
-| Short (~10 tokens) | ~0.28 | 48 kHz |
-| Long (~100 tokens) | ~0.34 | 48 kHz |
-
-RTF < 1.0 means faster than real time.
-
-## Architecture
-
-VoxCPM2 uses a single-stage native AR pipeline:
-
-```
-feat_encoder
-└─► MiniCPM4 (base LM)
-     └─► FSQ (finite scalar quantization)
-          └─► residual_lm (residual AR)
-               └─► LocDiT (local diffusion transformer)
-                    └─► AudioVAE → 48 kHz waveform
-```
-
-All stages are fused into one vllm-native execution graph via `voxcpm2.yaml`, eliminating inter-stage coordination overhead and enabling true end-to-end batching.

examples/offline_inference/voxtral_tts/README.md

@@ -1,59 +0,0 @@
-# Voxtral TTS Offline Inference
-
-`end2end.py` runs Voxtral TTS end-to-end offline inference using vLLM. It supports both blocking (`Omni`) and streaming (`AsyncOmni`) generation, batched prompts with configurable concurrency, and voice selection via preset name or reference audio file.
-
-When `mistral_common` has `SpeechRequest` support, prompt token IDs are built via `encode_speech_request`. Otherwise, the script falls back to manual token construction.
-
-## Usage Examples
-
-
-```bash
-# Basic single-prompt with cheerful_female voice preset
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --write-audio --voice cheerful_female \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
-
-# 32 replicate prompts with cheerful_female voice preset
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --num-prompts 32 --write-audio --voice cheerful_female \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
-
-# Streaming with neutral_female voice preset
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --streaming --write-audio --voice neutral_female \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
-
-# 32 prompts, 8 concurrent requests per wave, streaming with neutral_female voice
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --num-prompts 32 --concurrency 8 --streaming --write-audio --voice neutral_female \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
-
-# Short debug prompt with reference audio
-# Note: Reference audio capability is not yet released.
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --write-audio \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "This is a test message." \
-    --audio-path path/to/reference_audio.wav
-```
-
-## Arguments
-
-| Argument | Description |
-|---|---|
-| `--model PATH` | HuggingFace repo ID or local directory path (default: `mistralai/Voxtral-4B-TTS-2603`) |
-| `--text TEXT` | Text to synthesize (default: `"This is a test message."`) |
-| `--audio-path PATH` | Path to reference audio file for voice cloning |
-| `--output-dir DIR` | Directory to write output WAV files (default: `output_audio`) |
-| `--deploy-config PATH` | Override the deploy config path. If unset, auto-loads `vllm_omni/deploy/voxtral_tts.yaml` from the HF `model_type`. |
-| `--num-prompts N` | Number of replicate prompts to run for measuring performance (default: 1) |
-| `--streaming` | Use streaming generation via `AsyncOmni` (default: blocking `Omni`) |
-| `--concurrency N` | Max concurrent requests per wave (must be used with `--streaming`, must evenly divide `--num-prompts`) |
-| `--voice NAME` | Voice preset to use instead of reference audio file. Check Huggingface `mistralai/Voxtral-4B-TTS-2603` to get the list of available voices |
-| `--write-audio` | Write generated audio to WAV files |
-| `--profiling-mode` | Enable profiling mode (reduces max tokens to 50) |
-| `--log-stats` | Enable detailed statistics logging |

examples/offline_inference/x_to_video_audio/x_to_video_audio.md

@@ -46,6 +46,24 @@ python x_to_video_audio.py \
 ```
 In the current test scenario (2 images + 2 audio inputs), the VRAM requirement is 72GB, regardless of whether cfg-parallel is enabled or disabled.
 The VRAM usage can be reduced by enabling CPU offload via --enable-cpu-offload.
+For multi-GPU memory reduction on the fused DreamID-Omni transformer, you can also enable HSDP:
+
+```python
+python x_to_video_audio.py \
+  --model /path/to/dreamid_omni \
+  --prompt "Two people walking together and singing happily" \
+  --image-path ./example0.png ./example1.png \
+  --audio-path ./example0.wav ./example1.wav \
+  --video-negative-prompt "jitter, bad hands, blur, distortion" \
+  --audio-negative-prompt "robotic, muffled, echo, distorted" \
+  --cfg-parallel-size 2 \
+  --num-inference-steps 45 \
+  --height 704 \
+  --width 1280 \
+  --use-hsdp \
+  --hsdp-shard-size 2 \
+  --output out_dreamid_omni_twoip.mp4
+```
 
 
 You could take reference images/audios from the test cases in the official repo: https://github.com/Guoxu1233/DreamID-Omni
@@ -76,6 +94,9 @@ Key arguments:
 - `--image-path`: path to the input image list.
 - `--audio-path`: path to the input audio list, indicate the timbre of the output video.
 - `--cfg-parallel-size`: number of parallel cfg parallel (defaults 1).
+- `--use-hsdp`: enable HSDP weight sharding for DreamID-Omni fused blocks.
+- `--hsdp-shard-size`: number of GPUs used for HSDP sharding.
+- `--hsdp-replicate-size`: number of HSDP replica groups.
 - `--num-inference-steps`: number of denoising steps (defaults 45).
 - `--video-negative-prompt`: negative prompt for video generation.
 - `--audio-negative-prompt`: negative prompt for audio generation.

examples/offline_inference/x_to_video_audio/x_to_video_audio.py

@@ -63,6 +63,23 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="Enable layerwise (blockwise) offloading on DiT modules.",
     )
+    parser.add_argument(
+        "--use-hsdp",
+        action="store_true",
+        help="Enable HSDP for DreamID-Omni fused transformer blocks.",
+    )
+    parser.add_argument(
+        "--hsdp-shard-size",
+        type=int,
+        default=1,
+        help="Number of GPUs used for HSDP sharding when HSDP is enabled.",
+    )
+    parser.add_argument(
+        "--hsdp-replicate-size",
+        type=int,
+        default=1,
+        help="Number of HSDP replica groups. Default 1 means pure sharding.",
+    )
     return parser.parse_args()
 
 
@@ -124,6 +141,9 @@ def main() -> None:
 
     parallel_config = DiffusionParallelConfig(
         cfg_parallel_size=args.cfg_parallel_size,
+        use_hsdp=args.use_hsdp,
+        hsdp_shard_size=args.hsdp_shard_size,
+        hsdp_replicate_size=args.hsdp_replicate_size,
     )
 
     omni = Omni(

examples/online_serving/audiox/README.md

@@ -0,0 +1,65 @@
+# AudioX online serving
+
+Launches the `AudioXPipeline` behind vLLM-Omni's OpenAI-compatible chat endpoint and provides a
+minimal Python client that covers all six tasks (`t2a`, `t2m`, `v2a`, `v2m`, `tv2a`, `tv2m`).
+
+## Start the server
+
+```bash
+cd examples/online_serving/audiox
+bash run_server.sh                 # defaults: MODEL=zhangj1an/AudioX, PORT=8099
+```
+
+Environment overrides: `MODEL`, `PORT`, `DIFFUSION_ATTENTION_BACKEND`.
+
+## Call from Python
+
+```bash
+# text-to-audio
+python openai_chat_client.py --task t2a \
+    --prompt "Fireworks burst twice, followed by a period of silence before a clock begins ticking." \
+    --output t2a.wav
+
+# text-to-music
+python openai_chat_client.py --task t2m \
+    --prompt "Uplifting ukulele tune for a travel vlog" \
+    --output t2m.wav
+
+# video-to-audio (no text)
+python openai_chat_client.py --task v2a --video path/to/clip.mp4 --output v2a.wav
+
+# text+video-to-audio
+python openai_chat_client.py --task tv2a \
+    --prompt "drum beating sound and human talking" \
+    --video path/to/clip.mp4 \
+    --output tv2a.wav
+```
+
+The client sends:
+
+- `num_inference_steps`, `guidance_scale`, `seed` as first-class OpenAI chat-completion fields
+- `audiox_task`, `seconds_start`, `seconds_total`, `sigma_min`, `sigma_max` nested under
+  `extra_args` (a reserved dict on the request body that the server forwards verbatim into
+  the pipeline's `sampling_params.extra_args` — the same escape hatch `serving_video.py` exposes
+  as `extra_params` on /v1/videos)
+- For `v2*` / `tv2*` tasks, the video as a `video_url` content item (data URI for local files)
+
+## curl
+
+```bash
+curl -sS -X POST http://localhost:8099/v1/chat/completions \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "model": "zhangj1an/AudioX",
+    "messages": [{"role": "user", "content": [{"type": "text", "text": "Uplifting ukulele"}]}],
+    "num_inference_steps": 250,
+    "guidance_scale": 7.0,
+    "seed": 42,
+    "extra_args": {
+      "audiox_task": "t2m",
+      "seconds_total": 10.0,
+      "sigma_min": 0.3,
+      "sigma_max": 500.0
+    }
+  }' > t2m.json
+```

examples/online_serving/audiox/openai_chat_client.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""AudioX OpenAI-compatible chat client.
+
+AudioX supports 6 tasks (t2a, t2m, v2a, v2m, tv2a, tv2m). Text-only tasks send the prompt as the
+chat message; video-conditioned tasks additionally attach the video via a ``video_url`` content
+item (data URI for local files). Task + generation knobs (steps, cfg, sigma range, seconds, seed)
+are sent via the OpenAI SDK's ``extra_body`` as ``extra_args`` — the same pipeline-agnostic escape
+hatch used by the /v1/videos endpoint's ``extra_params`` field.
+
+Usage:
+  python openai_chat_client.py --task t2a --prompt "Fireworks burst twice..." --output t2a.wav
+  python openai_chat_client.py --task tv2a --prompt "drum beating" --video clip.mp4 -o tv2a.wav
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import io
+import mimetypes
+import sys
+from pathlib import Path
+
+import requests
+import soundfile
+import torch
+
+VIDEO_TASKS = frozenset({"v2a", "v2m", "tv2a", "tv2m"})
+TEXT_TASKS = frozenset({"t2a", "t2m", "tv2a", "tv2m"})
+
+
+def _to_data_url(path: str) -> str:
+    mime, _ = mimetypes.guess_type(path)
+    mime = mime or "video/mp4"
+    with open(path, "rb") as f:
+        data = base64.b64encode(f.read()).decode("ascii")
+    return f"data:{mime};base64,{data}"
+
+
+def _save_wav(audio: torch.Tensor, path: Path, sample_rate: int) -> None:
+    audio = audio.to(torch.float32)
+    audio = audio / audio.abs().max().clamp(min=1e-8)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    # soundfile expects channels-last (T, C); project convention is (C, T).
+    soundfile.write(str(path), audio.clamp(-1.0, 1.0).cpu().T.numpy(), sample_rate, subtype="PCM_16")
+
+
+def _decode_audio_from_response(body: dict) -> tuple[torch.Tensor, int]:
+    for choice in body.get("choices", []):
+        audio_obj = choice.get("message", {}).get("audio")
+        if not (isinstance(audio_obj, dict) and audio_obj.get("data")):
+            continue
+        data, sr = soundfile.read(io.BytesIO(base64.b64decode(audio_obj["data"])), dtype="float32", always_2d=True)
+        return torch.from_numpy(data).transpose(0, 1), sr
+    brief = {k: v for k, v in body.items() if k != "choices"}
+    raise RuntimeError(f"no audio in response message.audio: {brief}")
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description="AudioX OpenAI chat client")
+    p.add_argument("--task", required=True, choices=["t2a", "t2m", "v2a", "v2m", "tv2a", "tv2m"])
+    p.add_argument("--prompt", "-p", default="", help="Text prompt (required for t2*/tv2*).")
+    p.add_argument("--video", help="Video path or URL (required for v2*/tv2*).")
+    p.add_argument("--output", "-o", default="audiox_out.wav")
+    p.add_argument("--server", "-s", default="http://localhost:8099")
+    p.add_argument("--model", default="zhangj1an/AudioX")
+    p.add_argument("--steps", type=int, default=250)
+    p.add_argument("--guidance-scale", type=float, default=7.0)
+    p.add_argument("--seed", type=int, default=42)
+    p.add_argument("--seconds-total", type=float, default=10.0)
+    p.add_argument("--seconds-start", type=float, default=0.0)
+    p.add_argument("--sigma-min", type=float, default=0.03)
+    p.add_argument("--sigma-max", type=float, default=1000.0)
+    args = p.parse_args()
+
+    if args.task in VIDEO_TASKS and not args.video:
+        print(f"ERROR: task {args.task!r} requires --video", file=sys.stderr)
+        return 2
+    if args.task in TEXT_TASKS and not args.prompt.strip() and args.task not in {"v2a", "v2m"}:
+        print(f"ERROR: task {args.task!r} requires --prompt", file=sys.stderr)
+        return 2
+
+    content: list[dict] = [{"type": "text", "text": args.prompt}]
+    if args.task in VIDEO_TASKS:
+        vurl = args.video if args.video.startswith(("http://", "https://")) else _to_data_url(args.video)
+        content.append({"type": "video_url", "video_url": {"url": vurl}})
+
+    payload = {
+        "model": args.model,
+        "messages": [{"role": "user", "content": content}],
+        "num_inference_steps": args.steps,
+        "guidance_scale": args.guidance_scale,
+        "seed": args.seed,
+        "extra_args": {
+            "audiox_task": args.task,
+            "seconds_start": args.seconds_start,
+            "seconds_total": args.seconds_total,
+            "sigma_min": args.sigma_min,
+            "sigma_max": args.sigma_max,
+        },
+    }
+
+    print(f"POST {args.server}/v1/chat/completions  task={args.task} steps={args.steps}")
+    r = requests.post(
+        f"{args.server}/v1/chat/completions",
+        headers={"Content-Type": "application/json"},
+        json=payload,
+        timeout=600,
+    )
+    r.raise_for_status()
+    audio, sr = _decode_audio_from_response(r.json())
+    _save_wav(audio, Path(args.output), sr)
+    dur = audio.shape[-1] / sr
+    print(f"saved {args.output}  sr={sr}Hz  duration={dur:.2f}s  channels={audio.shape[0]}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

examples/online_serving/audiox/run_server.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# AudioX online serving startup script.
+
+MODEL="${MODEL:-zhangj1an/AudioX}"
+PORT="${PORT:-8099}"
+DIFFUSION_ATTENTION_BACKEND="${DIFFUSION_ATTENTION_BACKEND:-FLASH_ATTN}"
+
+echo "Starting AudioX server..."
+echo "Model: $MODEL"
+echo "Port: $PORT"
+echo "Diffusion attention backend: $DIFFUSION_ATTENTION_BACKEND"
+
+DIFFUSION_ATTENTION_BACKEND="$DIFFUSION_ATTENTION_BACKEND" \
+    vllm serve "$MODEL" --omni \
+        --model-class-name AudioXPipeline \
+        --port "$PORT"

examples/online_serving/diffusers_pipeline_adapter/README.md

@@ -1,15 +1,11 @@
-# Diffusers Backend Adapter Example
+# Diffusers Backend Adapter
 
-This example demonstrates how to serve any 🤗 Diffusers pipeline through vLLM-Omni
-using the `diffusers` load format.
-
-## Supported Models
-
-Any model loadable via `DiffusionPipeline.from_pretrained()` should be supported, including text-to-image, image-to-image, text-to-video, image-to-video, and text-to-audio.
+vLLM-Omni supports running diffusion models with the diffusers backend, directly serving any 🤗 Diffusers pipeline online without implementing them natively.
 
 ## Limitations
 
-The diffusers backend is a black-box adapter. The following features are NOT yet supported.
+The diffusers backend is a black-box adapter. Its primary focus is to serve diffusion models online.
+Currently, the following features are NOT yet supported.
 It is not guaranteed whether they will be supported in the future.
 
 - CFG parallel execution
@@ -19,65 +15,92 @@ It is not guaranteed whether they will be supported in the future.
 
 For these features, it is recommended to use natively supported pipelines instead.
 
-## Usage
+## Model Support
 
-### Option 1: CLI arguments
+Any model loadable via `DiffusionPipeline.from_pretrained()` should run, including text-to-image, image-to-image, text-to-video, image-to-video, and text-to-audio.
+
+However, as we strive to ensure output similarity between vLLM-Omni's diffuser backend and plain diffusers library, the following models are particularly verified:
+
+- Qwen/Qwen-Image
+- Tongyi-MAI/Z-Image-Turbo
+- Wan2.2-I2V-A14B-Diffusers
+
+If you find that a model not listed above also produces different outputs from running diffusers model directly.
+Please consider file an issue or submit a PR to fix.
+
+## Usage
 
 ```bash
 vllm serve "stable-diffusion-v1-5/stable-diffusion-v1-5" \
     --omni \
-    --diffusion-load-format diffusers \
-    --diffusers-load-kwargs '{"use_safetensors": true}' \
-    --diffusers-call-kwargs '{"num_inference_steps": 30, "guidance_scale": 7.5}'
+    --diffusion-load-format diffusers
 ```
 
-`--diffusers-load-kwargs` and `--diffusers-call-kwargs` are only valid together with `--diffusion-load-format diffusers`.
+Users turn on the diffusers backend primarily through `--diffusion-load-format diffusers` argument.
+There are two more optional arguments, `--diffusers-load-kwargs` and `--diffusers-call-kwargs`,
+which are only valid together with `--diffusion-load-format diffusers`.
 
-### Option 2: Stage config YAML
-
-```bash
-vllm serve stable-diffusion-v1-5/stable-diffusion-v1-5 --stage-configs-path examples/online_serving/diffusers_pipeline_adapter/stage_config.yaml --omni
-```
-
-The particular fields of interest are `model`, `diffusion_load_format`, `diffusers_load_kwargs`, and `diffusers_call_kwargs` under `engine_args`. They are the same as the CLI arguments.
-
-## Send a Request
-
-```bash
-curl http://localhost:8000/v1/images/generations \
-  -H "Content-Type: application/json" \
-  -d '{
-    "model": "stable-diffusion-v1-5/stable-diffusion-v1-5",
-    "prompt": "a photo of an astronaut riding a horse on mars",
-    "n": 1,
-    "size": "512x512"
-  }'
-```
-
-Or refer to other documentation pages on how to request a particular input/output modality, such as `examples/online_serving/text_to_image/openai_chat_client.py`.
+After launching the model, users send a request as usual. Refer to other documentation pages on how to request a particular input/output modality, such as `examples/online_serving/text_to_image/openai_chat_client.py`.
 
 ## Configuration Reference
 
-For the diffusers adapter, set options under **`engine_args`**:
+### `--diffusers-load-kwargs`
 
-### `diffusion_load_format: "diffusers"`
+Passed as-is to `DiffusionPipeline.from_pretrained()`.
 
-This field selects the Hugging Face diffusers adapter path (see `DiffusersPipelineLoader`).
-
-### `diffusers_load_kwargs`
-
-Passed to `DiffusionPipeline.from_pretrained()`.
-
-This is suitable for model-specific configurations not available through the vLLM-Omni interface (such as `Omni.__init__()`, `vllm serve` CLI arguments, and stage config YAML fields outside `diffusers_load_kwargs`).
+This is suitable for model-specific configurations not available through the vLLM-Omni interface.
+For example: `--diffusers-load-kwargs '{"use_safetensors": true}'`.
 
 When a parameter is available in the vLLM-Omni interface, it will be adapted here.
 But if that parameter is simultaneously set in both the vLLM-Omni interface and `diffusers_load_kwargs`, the **latter** will take precedence.
 
-### `diffusers_call_kwargs`
+### `--diffusers-call-kwargs`
 
 Passed to `pipeline.__call__()`.
 
-This is suitable for sampling parameters not available through the vLLM-Omni interface (such as `Omni.generate()` and online serving payloads).
+This is suitable for sampling parameters not available through the vLLM-Omni interface (such as online serving payloads).
 
 When a parameter is available in the vLLM-Omni interface, it will be adapted here.
 But if that parameter is simultaneously set in both the vLLM-Omni interface and `diffusers_call_kwargs`, the **former** will take precedence (because it is set at request time).
+
+!!! note
+    In vLLM-Omni, the default values for some sampling parameters may be different from diffusers.
+    Consider referring to [`OmniDiffusionSamplingParams`](../../../vllm_omni/inputs/data.py) for the default sampling parameters in the vLLM-Omni interface,
+    and the corresponding [diffusers pipeline](https://huggingface.co/docs/diffusers/main/en/api/pipelines/overview)'s `__call__` function documentation.
+
+### Attention Backends
+
+The diffusers backend converts
+[vLLM-Omni standard of attention backend setting](../../../docs/user_guide/diffusion/attention_backends.md)
+to [diffusers standard](https://huggingface.co/docs/diffusers/optimization/attention_backends#available-backends).
+
+Specifically for `FLASH_ATTN`, it will first attempt to use FlashAttention-3 and then FlashAttention-2.
+
+For each attempted version of `FLASH_ATTN` and `SAGE_ATTN`, it will first try to load the attention backend from HuggingFace `kernels` library, then without.
+
+For unsuccessful attention selection or `TORCH_SDPA`, it will use the PyTorch's default attention backend.
+
+The loaded attention backend and the failed attempts (if any) are logged to console.
+
+### Model Specific Settings
+
+The model loading and inference strictly follows the diffusers library, and they may be different from vLLM-Omni's native interface for some specific models.
+Users are encouraged to double-check the model pipeline's interface in [diffusers' official documentation](https://huggingface.co/docs/diffusers/api/pipelines/overview).
+Some particular examples are below.
+
+#### Wan Series
+
+The Wan series video generation models takes `boundary_ratio` and `flow_shift` during model initialization ([ref](https://huggingface.co/docs/diffusers/api/pipelines/wan)), not during inference.
+
+Since our `OmniDiffusionConfig` contains these two values ([source](https://github.com/vllm-project/vllm-omni/blob/main/vllm_omni/diffusion/data.py)), we can directly pass `--boundary-ratio` and `--flow-shift` arguments to `vllm serve` command.
+
+```bash
+vllm serve "Wan2.2-T2V-A14B-Diffusers" \
+    --omni \
+    --boundary-ratio 0.875 \
+    --flow-shift 3 \
+    --diffusion-load-format diffusers
+```
+
+These extra CLI args will be attempted to pass as-is to the `OmniDiffusionConfig` dataclass and being accessible during model loading time.
+Special routines inside the pipeline adapter ensures that they are set properly.

examples/online_serving/diffusers_pipeline_adapter/stage_config.yaml

@@ -1,31 +0,0 @@
-# Example stage config for diffusers backend
-# This config demonstrates serving Stable Diffusion 1.5 via the diffusers adapter.
-# Users should copy and modify this for their own models.
-
-model_type: diffusion
-
-stage_args:
-  - stage_id: 0
-    stage_type: diffusion
-    engine_args:
-      model_stage: diffusion
-      model: "stable-diffusion-v1-5/stable-diffusion-v1-5"
-      distributed_executor_backend: "mp"
-      # gpu_memory_utilization: 0.9
-      engine_output_type: image
-      # Select the HF diffusers adapter
-      diffusion_load_format: "diffusers"
-      # model_class_name: "DiffusersAdapterPipeline" # default when diffusion_load_format is diffusers
-      diffusers_load_kwargs:
-        # Passed to DiffusionPipeline.from_pretrained().
-        # Good for model-specific loading parameters not covered by OmniDiffusionConfig.
-        # During model load time, parameters here override their counterparts in the vLLM-Omni interface.
-        use_safetensors: true
-      diffusers_call_kwargs:
-        # Passed to pipeline.__call__().
-        # Good for model-specific sampling parameters not covered by OmniDiffusionSamplingParams.
-        # During request time, parameters here are overridden by the counterparts in OmniDiffusionSamplingParams.
-        num_inference_steps: 30
-        guidance_scale: 7.5
-    final_output: true
-    final_output_type: image

examples/online_serving/fish_speech/README.md

@@ -1,169 +0,0 @@
-# Fish Speech S2 Pro
-
-## Model
-
-| Model | Description |
-|-------|-------------|
-| `fishaudio/s2-pro` | Fish Speech S2 Pro -- 4B dual-AR TTS model with DAC codec (44.1 kHz) |
-
-## Gradio Demo
-
-An interactive Gradio demo is available with text-to-speech synthesis, voice cloning, and streaming support.
-
-```bash
-# Option 1: Launch server + Gradio together
-./run_gradio_demo.sh
-
-# Option 2: If server is already running
-python gradio_demo.py --api-base http://localhost:8091
-```
-
-Then open http://localhost:7860 in your browser.
-
-Features:
-
-- Text-to-speech synthesis
-- Voice cloning from uploaded audio or URL
-- Streaming mode (progressive PCM playback)
-
-## Launch the Server
-
-```bash
-vllm serve fishaudio/s2-pro --omni --port 8091
-```
-
-The deploy config is auto-loaded from `vllm_omni/deploy/fish_qwen3_omni.yaml`
-(the HF `model_type` on the fishaudio checkpoints is `fish_qwen3_omni`).
-
-Or use the convenience script:
-
-```bash
-./run_server.sh
-```
-
-## Send TTS Request
-
-### Using curl
-
-```bash
-# Basic TTS
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, how are you?",
-        "voice": "default",
-        "response_format": "wav"
-    }' --output output.wav
-```
-
-### Voice Cloning
-
-Provide a reference audio (URL or base64 data URL) and its transcript:
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, this is a cloned voice.",
-        "voice": "default",
-        "ref_audio": "https://example.com/reference.wav",
-        "ref_text": "Transcript of the reference audio."
-    }' --output cloned.wav
-```
-
-### Using Python
-
-```python
-import httpx
-
-# Basic TTS
-response = httpx.post(
-    "http://localhost:8091/v1/audio/speech",
-    json={
-        "input": "Hello, how are you?",
-        "voice": "default",
-        "response_format": "wav",
-    },
-    timeout=300.0,
-)
-
-with open("output.wav", "wb") as f:
-    f.write(response.content)
-```
-
-### Using the CLI Client
-
-```bash
-cd examples/online_serving/fish_speech
-
-# Basic TTS
-python speech_client.py --text "Hello, how are you?"
-
-# Voice cloning
-python speech_client.py \
-    --text "Hello, this is a cloned voice." \
-    --ref-audio /path/to/reference.wav \
-    --ref-text "Transcript of the reference audio."
-
-# Streaming PCM output
-python speech_client.py --text "Hello world" --stream --output output.pcm
-```
-
-The CLI client supports:
-
-- `--api-base`: API base URL (default: `http://localhost:8091`)
-- `--model` (or `-m`): Model name (default: `fishaudio/s2-pro`)
-- `--text`: Text to synthesize (required)
-- `--ref-audio`: Reference audio for voice cloning (local path or URL)
-- `--ref-text`: Transcript of the reference audio
-- `--stream`: Enable streaming (PCM output)
-- `--response-format`: Audio format: wav, mp3, flac, pcm, aac, opus (default: wav)
-- `--output` (or `-o`): Output file path
-
-## Streaming
-
-Set `stream=true` with `response_format="pcm"` to receive raw PCM audio chunks as they are decoded:
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, how are you?",
-        "voice": "default",
-        "stream": true,
-        "response_format": "pcm"
-    }' --no-buffer | play -t raw -r 44100 -e signed -b 16 -c 1 -
-```
-
-**Note:** Fish Speech outputs at 44.1 kHz (unlike Qwen3-TTS which outputs at 24 kHz).
-
-## API Parameters
-
-Fish Speech uses the same `/v1/audio/speech` endpoint as Qwen3-TTS. See the [Speech API reference](https://docs.vllm.ai/projects/vllm-omni/en/latest/serving/speech_api/) for full parameter documentation.
-
-Key parameters for Fish Speech:
-
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `input` | string | **required** | Text to synthesize |
-| `voice` | string | "default" | Voice name (use "default" for Fish Speech) |
-| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus |
-| `ref_audio` | string | null | Reference audio URL or base64 data URL for voice cloning |
-| `ref_text` | string | null | Transcript of reference audio (required for voice cloning) |
-| `max_new_tokens` | int | 4096 | Maximum tokens to generate |
-| `stream` | bool | false | Stream raw PCM chunks |
-
-## Prerequisites
-
-Install the `fish-speech` package for the DAC codec:
-
-```bash
-pip install fish-speech
-```
-
-## Troubleshooting
-
-1. **No audio output**: Make sure the `fish-speech` package is installed for the DAC decoder
-2. **Connection refused**: Ensure the server is running on the correct port
-3. **Flashinfer version mismatch**: Set `FLASHINFER_DISABLE_VERSION_CHECK=1` if you see version warnings
-4. **Out of memory**: Reduce `--gpu-memory-utilization` or use a GPU with more VRAM

examples/online_serving/ming_flash_omni/README.md

@@ -8,27 +8,29 @@ Please refer to [README.md](../../../README.md)
 
 | Mode | Launch command | Output |
 |------|---------------|--------|
-| Thinker only (multimodal understanding) | `vllm serve ... --omni` | Text |
-| Thinker + Talker (omni-speech) | `vllm serve ... --omni --stage-configs-path ming_flash_omni.yaml` | Text + Audio |
+| Thinker + Talker (omni-speech, default) | `vllm serve ... --omni` | Text + Audio |
+| Thinker only (multimodal understanding) | `vllm serve ... --omni --deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml` | Text |
 
-For standalone TTS (talker only), see [`examples/online_serving/ming_flash_omni_tts/`](../ming_flash_omni_tts/).
+For standalone TTS (talker only), see the [Ming-flash-omni-TTS section in the Text-To-Speech hub](../text_to_speech/README.md#ming-flash-omni-tts).
 
 ## Run examples (Ming-flash-omni 2.0)
 
 ### Launch the Server
 
-**Thinker only (text output):**
+**Thinker + Talker (omni-speech, text + audio output):**
 ```bash
 vllm serve Jonathan1909/Ming-flash-omni-2.0 --omni --port 8091
 ```
 
-**Thinker + Talker (omni-speech, text + audio output):**
+The model registry auto-loads corresponding deploy yaml.
+
+**Thinker-only (text output):**
 ```bash
 vllm serve Jonathan1909/Ming-flash-omni-2.0 --omni --port 8091 \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/ming_flash_omni.yaml
+    --deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml
 ```
 
-Pass `--stage-configs-path /path/to/your_config.yaml` to use a custom stage
+Pass `--deploy-config /path/to/your_deploy.yaml` to use a custom deploy
 config.
 
 ### Send Multi-modal Request

examples/online_serving/ming_flash_omni/run_curl_multimodal_generation.sh

@@ -18,17 +18,6 @@ if [[ ! "$QUERY_TYPE" =~ ^(text|use_audio|use_image|use_video|use_mixed_modaliti
     exit 1
 fi
 
-thinker_sampling_params='{
-  "temperature": 0.4,
-  "top_p": 0.9,
-  "top_k": -1,
-  "max_tokens": 16384,
-  "seed": 42,
-  "detokenize": true,
-  "repetition_penalty": 1.05
-}'
-# Above is optional, it has a default setting in stage_configs of the corresponding model.
-
 # Define URLs for assets
 MARY_HAD_LAMB_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/mary_had_lamb.ogg"
 CHERRY_BLOSSOM_IMAGE_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
@@ -114,9 +103,6 @@ echo ""
 request_body=$(cat <<EOF
 {
   "model": "Jonathan1909/Ming-flash-omni-2.0",
-  "sampling_params_list": [
-    $thinker_sampling_params
-  ],
   "modalities": ["text"],
   "messages": [
     {

examples/online_serving/ming_flash_omni_tts/README.md

@@ -1,54 +0,0 @@
-# Ming-flash-omni Standalone TTS (Online Serving)
-
-This directory contains online e2e examples for **Ming-flash-omni-2.0 standalone talker deployment**.
-
-Server uses:
-
-- `model`: `Jonathan1909/Ming-flash-omni-2.0`
-- `stage config`: `vllm_omni/model_executor/stage_configs/ming_flash_omni_tts.yaml`
-
-## Launch the Server
-
-```bash
-# from repo root
-bash examples/online_serving/ming_flash_omni_tts/run_server.sh
-```
-
-Equivalent manual command:
-
-```bash
-vllm serve Jonathan1909/Ming-flash-omni-2.0 \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/ming_flash_omni_tts.yaml \
-    --host 0.0.0.0 \
-    --port 8091 \
-    --trust-remote-code \
-    --omni
-```
-
-## Send TTS Request
-
-Python client:
-
-```bash
-python examples/online_serving/ming_flash_omni_tts/speech_client.py \
-    --text "我们当迎着阳光辛勤耕作，去摘取，去制作，去品尝，去馈赠。" \
-    --output ming_online.wav
-```
-
-Long-form `instructions` (e.g. ASMR whisper style) via the client:
-
-```bash
-python examples/online_serving/ming_flash_omni_tts/speech_client.py \
-    --text "我会一直在这里陪着你，直到你慢慢、慢慢地沉入那个最温柔的梦里……好吗？" \
-    --instructions "这是一种ASMR耳语，属于一种旨在引发特殊感官体验的创意风格。这个女性使用轻柔的普通话进行耳语，声音气音成分重。音量极低，紧贴麦克风，语速极慢，旨在制造触发听者颅内快感的声学刺激。" \
-    --output ming_online_asmr.wav
-```
-
-## Notes
-
-- This is the **online serving** counterpart of `examples/offline_inference/ming_flash_omni_tts/`.
-- The server uses `use_zero_spk_emb=True` and the default decode args
-  (`max_decode_steps=200`, `cfg=2.0`, `sigma=0.25`, `temperature=0.0`).
-  For other caption fields (`语速`, `基频`, `IP`, BGM, etc.) or overriding
-  decode args, use the offline e2e example where `additional_information`
-  is set explicitly.

examples/online_serving/moss_tts_nano/README.md

@@ -1,158 +0,0 @@
-# MOSS-TTS-Nano
-
-## Model checkpoint
-
-| Model | Description |
-|-------|-------------|
-| `OpenMOSS-Team/MOSS-TTS-Nano` | 0.1B AR LM + MOSS-Audio-Tokenizer-Nano codec, 48 kHz stereo, ZH/EN/JA |
-
-## Gradio Demo
-
-An interactive Gradio demo is available with multilingual voice presets, custom voice cloning, and streaming support.
-
-```bash
-# Option 1: Launch server + Gradio together
-./run_gradio_demo.sh
-
-# Option 2: If server is already running
-python gradio_demo.py --api-base http://localhost:8091
-```
-
-Then open http://localhost:7860 in your browser.
-
-Features:
-
-- 15 built-in voice presets (6 ZH · 4 EN · 5 JA)
-- Custom voice cloning from uploaded reference audio
-- Streaming mode (progressive PCM playback)
-
-## Launch the Server
-
-```bash
-vllm serve OpenMOSS-Team/MOSS-TTS-Nano --omni --port 8091
-```
-
-The deploy config at `vllm_omni/deploy/moss_tts_nano.yaml` auto-loads; no
-`--stage-configs-path`, `--trust-remote-code`, or `--enforce-eager` flags
-are needed.
-
-Or use the convenience script:
-
-```bash
-./run_server.sh
-```
-
-## Send TTS Request
-
-### Using curl
-
-```bash
-# Built-in voice preset
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "你好，这是语音合成测试。",
-        "voice": "Junhao",
-        "response_format": "wav"
-    }' --output output.wav
-
-# English preset
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, this is MOSS-TTS-Nano.",
-        "voice": "Ava",
-        "response_format": "wav"
-    }' --output output_en.wav
-```
-
-### Custom Voice Cloning
-
-Provide a reference audio (base64 data URL) and its transcript:
-
-```bash
-REF_AUDIO=$(base64 -w 0 /path/to/reference.wav)
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d "{
-        \"input\": \"Hello, this is a cloned voice.\",
-        \"voice\": \"Junhao\",
-        \"ref_audio\": \"data:audio/wav;base64,${REF_AUDIO}\",
-        \"ref_text\": \"Exact transcript of the reference audio.\"
-    }" --output cloned.wav
-```
-
-### Using Python
-
-```python
-import httpx
-
-response = httpx.post(
-    "http://localhost:8091/v1/audio/speech",
-    json={
-        "input": "你好，这是语音合成测试。",
-        "voice": "Junhao",
-        "response_format": "wav",
-    },
-    timeout=300.0,
-)
-
-with open("output.wav", "wb") as f:
-    f.write(response.content)
-```
-
-### Streaming
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, streaming output from MOSS-TTS-Nano.",
-        "voice": "Ava",
-        "stream": true,
-        "response_format": "pcm"
-    }' --no-buffer | play -t raw -r 48000 -e signed -b 16 -c 2 -
-```
-
-**Note:** MOSS-TTS-Nano outputs at 48 kHz stereo (2-channel).
-
-## API Parameters
-
-MOSS-TTS-Nano uses the standard `/v1/audio/speech` endpoint.
-
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `input` | string | **required** | Text to synthesize (ZH / EN / JA) |
-| `voice` | string | `"Junhao"` | Built-in voice name (see table below) |
-| `response_format` | string | `"wav"` | Audio format: wav, mp3, flac, pcm |
-| `ref_audio` | string | null | Base64 data URL for custom voice cloning |
-| `ref_text` | string | null | Transcript of reference audio (required when `ref_audio` is set) |
-| `stream` | bool | false | Stream raw PCM chunks |
-| `max_new_tokens` | int | 4096 | Maximum tokens to generate |
-
-## Built-in Voice Presets
-
-| Voice | Language | Description |
-|-------|----------|-------------|
-| `Junhao` | ZH | Male, standard Mandarin |
-| `Zhiming` | ZH | Male |
-| `Weiguo` | ZH | Male |
-| `Xiaoyu` | ZH | Female |
-| `Yuewen` | ZH | Female |
-| `Lingyu` | ZH | Female |
-| `Ava` | EN | Female, American English |
-| `Bella` | EN | Female |
-| `Adam` | EN | Male |
-| `Nathan` | EN | Male |
-| `Sakura` | JA | Female |
-| `Yui` | JA | Female |
-| `Aoi` | JA | Female |
-| `Hina` | JA | Female |
-| `Mei` | JA | Female |
-
-## Troubleshooting
-
-1. **`libnvrtc.so.13: cannot open shared object file`**: torchaudio 2.10+ defaults to torchcodec which requires NVRTC. vLLM-Omni patches this automatically at model load time to use soundfile instead.
-2. **Connection refused**: Ensure the server is running on the correct port.
-3. **Flashinfer version mismatch**: Set `FLASHINFER_DISABLE_VERSION_CHECK=1` if you see version warnings.
-4. **Out of memory**: The default `gpu_memory_utilization=0.3` is conservative. Increase it in the stage config if you have more VRAM available.