docs: restore quantization overview page

Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>

.buildkite/amd/test-amd-merge.yml

@@ -89,7 +89,7 @@ steps:
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
-        - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py
+        - pytest -s -v tests/diffusion/distributed/test_tensor_parallel.py
 
     - label: "Diffusion Model Test"
       agent_pool: mi325_1

.buildkite/amd/test-amd-ready.yml

@@ -35,7 +35,7 @@ steps:
   mirror_hardwares: [amdproduction]
   grade: Blocking
   commands:
-    - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model"
+    - timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"
 
 
 - label: "Voxtral TTS CUDA Unit Test"

.buildkite/cuda/test-merge.yml

@@ -8,14 +8,24 @@ steps:
   - group: ":card_index_dividers: Simple Test"
     depends_on: upload-merge-pipeline
     steps:
-      - label: "Simple · Diffusion & Model Executor Test"
+      - label: "Simple · Model Executor Test"
         commands:
-          - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+          - "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        mirror_hardwares: l4_1
+
+      - label: "Simple · Diffusion Test"
+        commands:
+          - "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        mirror_hardwares: l4_1
+
+      - label: "Simple · Engine&Entrypoints Test"
+        commands:
+          - "timeout 20m pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
         mirror_hardwares: l4_1
 
       - label: "Simple · Other Test"
         commands:
-          - "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+          - "timeout 20m pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
         mirror_hardwares: l4_1
 
   - group: ":card_index_dividers: Diffusion Test"
@@ -27,12 +37,6 @@ steps:
           - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model"
         mirror_hardwares: l4_1
 
-      - label: "Diffusion · Offloader Test"
-        timeout_in_minutes: 40
-        commands:
-          - timeout 40m pytest -s -v tests/diffusion/offloader -m "advanced_model and cuda"
-        mirror_hardwares: l4_1
-
   - label: "Distributed Test"
     depends_on: upload-merge-pipeline
     timeout_in_minutes: 60
@@ -135,39 +139,32 @@ steps:
         timeout_in_minutes: 50
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
+        mirror_hardwares: h100_2
+
+      - label: "Omni · MiniCPM-o 4.5 Duplex Test"
+        source_file_dependencies:
+          - tests/assets/minicpmo_4_5/response_required_16k.wav
+          - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav
+          - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
+          - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+          - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
+          - tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py
+          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
+          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
+          - tests/helpers/minicpmo_4_5_duplex.py
+          - vllm_omni/deploy/minicpmo_4_5_duplex.yaml
+          - vllm_omni/experimental/fullduplex/
+          - vllm_omni/model_executor/models/minicpmo_4_5/
+          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+          - vllm_omni/core/sched/
+          - vllm_omni/outputs/multimodal_accumulation.py
+          - vllm_omni/engine/stage_init_utils.py
+          - vllm_omni/engine/async_omni_engine.py
+          - vllm_omni/entrypoints/cli/serve.py
+        timeout_in_minutes: 50
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+        mirror_hardwares: h100_2
 
       - label: "TTS · VoxCPM2 Test"
         source_file_dependencies:
@@ -185,22 +182,6 @@ steps:
             "
         mirror_hardwares: l4_1
 
-      - label: "TTS · SoulX-Singer Test"
-        source_file_dependencies:
-          - tests/e2e/offline_inference/test_soulxsinger.py
-          - vllm_omni/diffusion/models/soulx_singer/
-          - vllm_omni/deploy/soulxsinger_svc.yaml
-          - vllm_omni/deploy/soulxsinger_svs.yaml
-        timeout_in_minutes: 30
-        commands:
-          - |
-            timeout 30m bash -c "
-              export VLLM_LOGGING_LEVEL=DEBUG
-              export VLLM_WORKER_MULTIPROC_METHOD=spawn
-              pytest -s -v tests/e2e/offline_inference/test_soulxsinger.py -k 'svc or precomputed' -m 'advanced_model and cuda' --run-level 'advanced_model'
-            "
-        mirror_hardwares: l4_1
-
       - label: "Diffusion · Qwen Image Edit Test"
         source_file_dependencies:
           - tests/e2e/online_serving/test_qwen_image_edit.py

.buildkite/cuda/test-nightly.yml

@@ -15,7 +15,13 @@ steps:
       - label: ":full_moon: Omni · Function Test with H100"
         timeout_in_minutes: 120
         commands:
-          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
+          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
+        timeout_in_minutes: 50
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni" --run-level "full_model"
         mirror_hardwares: h100_2
 
       - label: ":full_moon: Omni · Doc Test with L4"
@@ -421,6 +427,12 @@ steps:
           - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
         mirror_hardwares: h100_2
 
+      - label: ":full_moon: Diffusion X2V · MiniMax H3 T2VA Accuracy Test"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_t2va_similarity.py -m full_model --run-level full_model
+        mirror_hardwares: h100_4
+
       - label: ":full_moon: Diffusion X2V · Other Accuracy Test"
         timeout_in_minutes: 180
         commands:
@@ -449,36 +461,47 @@ steps:
         mirror_hardwares: h100_2
 
   - group: ":card_index_dividers: Diffusion Test"
+    key: nightly-diffusion-test-group
     depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test"
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     steps:
+      - label: ":full_moon: Diffusion · Distributed Test with L4"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4' --run-level "full_model"
+        mirror_hardwares: l4_4
+
+      - label: ":full_moon: Diffusion · Distributed Test with H100"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100' --run-level "full_model"
+        mirror_hardwares: h100_4
 
       - label: ":full_moon: Diffusion · Offloader Test"
-        timeout_in_minutes: 40
+        timeout_in_minutes: 60
         commands:
-          - timeout 40m pytest -s -v tests/diffusion/offloader -m "full_model and cuda"
+          - pytest -sv tests/diffusion/offloader -m "full_model and cuda"
         mirror_hardwares: l4_1
 
-      - label: ":full_moon: Diffusion · Quantization Test with H100"
+      - label: ":full_moon: Diffusion · Quantization Test with H100 · Single-GPU"
+        timeout_in_minutes: 60
         commands:
-          - timeout 60m pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model"
+          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model"
         mirror_hardwares: h100_1
 
+      - label: ":full_moon: Diffusion · MiniMax H3 FP8 Quality Test with H100 · 2-GPU"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py -m 'full_model and cuda and H100' --run-level "full_model"
+        mirror_hardwares: h100_2
+
       - label: ":full_moon: Diffusion · Quantization Test with L4"
         timeout_in_minutes: 60
         commands:
+          - pip install "vllm-gguf-plugin>=0.0.3"
           - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model"
         mirror_hardwares: l4_1
 
-      - label: ":full_moon: Diffusion · Wan VAE Spatial Shard Correctness Test"
-        timeout_in_minutes: 60
-        commands:
-          - >-
-            pytest -sv tests/diffusion/distributed/test_wan_spatial_shard.py
-            -m "full_model and diffusion and H100 and distributed_cuda"
-            --run-level "full_model"
-        mirror_hardwares: h100_2
-
   - label: ":bar_chart: Testcase Statistics"
     key: nightly-testcase-statistics
     timeout_in_minutes: 120

.buildkite/cuda/test-ready.yml

@@ -8,20 +8,24 @@ steps:
   - group: ":card_index_dividers: Simple Test"
     depends_on: upload-ready-pipeline
     steps:
-      - label: "Simple · Diffusion & Model Executor Test"
+      - label: "Simple · Model Executor Test"
         commands:
-          - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+          - "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        mirror_hardwares: l4_1
+
+      - label: "Simple · Diffusion Test"
+        commands:
+          - "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        mirror_hardwares: l4_1
+
+      - label: "Simple · Engine&Entrypoints Test"
+        commands:
+          - "timeout 20m pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
         mirror_hardwares: l4_1
 
       - label: "Simple · Other Test"
         commands:
-          - "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        mirror_hardwares: l4_1
-
-  - label: "Custom Pipeline Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model"
+          - "timeout 20m pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
         mirror_hardwares: l4_1
 
   - group: ":card_index_dividers: Diffusion Test"
@@ -104,11 +108,34 @@ steps:
   - group: ":card_index_dividers: E2E Test"
     depends_on: upload-ready-pipeline
     steps:
+      - label: "Custom Pipeline Test"
+        source_file_dependencies:
+          - tests/e2e/features/custom_pipeline/
+          - tests/e2e/features/helpers/custom_pipeline.py
+          - vllm_omni/diffusion/
+          - vllm_omni/engine/
+          - vllm_omni/entrypoints/
+          - vllm_omni/inputs/
+          - vllm_omni/outputs/
+        commands:
+          - timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"
+        agents:
+          queue: "gpu_1_queue"
+        plugins:
+          - docker#v5.2.0:
+              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              always-pull: true
+              propagate-environment: true
+              environment:
+                - "HF_HOME=/fsx/hf_cache"
+              volumes:
+                - "/fsx/hf_cache:/fsx/hf_cache"
 
       - label: "RLHF · VeRL-Omni E2E Test"
         source_file_dependencies:
-          - tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py
-          - tests/e2e/offline_inference/custom_pipeline/
+          - tests/e2e/features/rlhf_test/test_verl_omni_e2e.py
+          - tests/e2e/features/helpers/verl_omni_server.py
+          - tests/e2e/features/helpers/custom_pipeline.py
           - vllm_omni/diffusion/
           - vllm_omni/engine/
           - vllm_omni/entrypoints/
@@ -117,7 +144,7 @@ steps:
         commands:
           - |
             timeout 20m bash -c "
-              pytest -s -v tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py
+              pytest -s -v tests/e2e/features/rlhf_test/test_verl_omni_e2e.py
             "
         mirror_hardwares: l4_1
 
@@ -143,34 +170,7 @@ steps:
             timeout 20m bash -c "
               pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
             "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
+        mirror_hardwares: h100_2
 
       - label: "Omni · MiniCPM-o 4.5 Online Test"
         source_file_dependencies:
@@ -181,34 +181,34 @@ steps:
             timeout 20m bash -c "
               pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
             "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
+        mirror_hardwares: h100_2
+
+      - label: "Omni · MiniCPM-o 4.5 Duplex Test"
+        source_file_dependencies:
+          - tests/assets/minicpmo_4_5/response_required_16k.wav
+          - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav
+          - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
+          - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+          - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
+          - tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py
+          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
+          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
+          - tests/helpers/minicpmo_4_5_duplex.py
+          - vllm_omni/deploy/minicpmo_4_5_duplex.yaml
+          - vllm_omni/experimental/fullduplex/
+          - vllm_omni/model_executor/models/minicpmo_4_5/
+          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+          - vllm_omni/core/sched/
+          - vllm_omni/outputs/multimodal_accumulation.py
+          - vllm_omni/engine/stage_init_utils.py
+          - vllm_omni/engine/async_omni_engine.py
+          - vllm_omni/entrypoints/cli/serve.py
+        commands:
+          - |
+            timeout 20m bash -c "
+              pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and cuda' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_2
 
       - label: "TTS · VoxCPM2 Test"
         source_file_dependencies:
@@ -344,13 +344,3 @@ steps:
               pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model'
             "
         mirror_hardwares: h100_1
-
-      - label: "Diffusion · GGUF Plugin Test"
-        soft_fail: true
-        source_file_dependencies:
-          - tests/plugin/test_gguf_diffusion.py
-          - vllm_omni/diffusion/model_loader
-        commands:
-          - pip install "vllm-gguf-plugin>=0.0.3"
-          - timeout 30m pytest -sv tests/plugin/test_gguf_diffusion.py
-        mirror_hardwares: l4_1

.buildkite/intel/pipeline-intel.yml

@@ -10,7 +10,7 @@ steps:
           DOCKER_BUILDKIT: "1"
           # Buildkite will automatically replace this with the actual commit hash
           VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.25.0"
+          VLLM_VERSION: "v0.26.0"
         priority: 100
         timeout_in_minutes: 60
         soft_fail: false

.buildkite/npu/test-npu-nightly.yml

@@ -95,6 +95,80 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
+  - group: ":card_index_dividers: Diffusion I2I Model Test"
+    key: nightly-HunyuanImage3-x2i-test-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test" ||
+      build.pull_request.labels includes "diffusion-x2iat-test"
+    steps:
+      - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HUNYUAN_IMAGE3_MODEL: "tencent/HunyuanImage-3.0-Instruct"
+          HUNYUAN_IMAGE3_DEVICES: "0,1,2,3"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m "full_model and A3" --run-level full_model
+  - group: ":card_index_dividers: Diffusion Benchmark Perf Test"
+    key: nightly-diffusion-benchmark-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test" ||
+      build.pull_request.labels includes "diffusion-x2iat-test"
+    steps:
+      - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 CFGP=2"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HF_TOKEN: "${HF_TOKEN}"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+      - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 SP=2"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HF_TOKEN: "${HF_TOKEN}"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+      - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=4"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HF_TOKEN: "${HF_TOKEN}"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp4.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
   - group: ":card_index_dividers: Diffusion X2V Model Test"
     key: nightly-diffusion-x2v-group
     depends_on: upload-nightly-pipeline
@@ -103,6 +177,14 @@ steps:
       build.pull_request.labels includes "nightly-test" ||
       build.pull_request.labels includes "diffusion-x2v-test"
     steps:
+      - label: ":full_moon: Diffusion X2V · Function Test"
+        mirror_hardwares: a2b3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"
       - label: ":full_moon: Diffusion X2V · Perf Test"
         key: nightly-diffusion-x2v-performance
         timeout_in_minutes: 180

.claude/skills/add-diffusion-model/SKILL.md

@@ -309,7 +309,7 @@ For Omni or custom models, create:
 
 Required updates:
 1. `docs/user_guide/diffusion/parallelism/overview.md` — parallelism support overview/table
-2. `docs/user_guide/diffusion/cpu_offload_diffusion.md` — if CPU offload supported (add to supported models table)
+2. `docs/user_guide/diffusion/cpu_offload.md` — if CPU offload supported (add to supported models table)
 3. `docs/user_guide/diffusion/cache_acceleration/teacache.md` — if TeaCache supported
 4. `docs/user_guide/diffusion/cache_acceleration/cache_dit.md` — if Cache-DiT supported
 5. Offline example docs under `examples/offline_inference/<name>/` (`README.md` or category-specific `.md`)

.github/CODEOWNERS

@@ -16,15 +16,15 @@
 /.claude/ @hsliuustc0106
 
 # Tests
-/tests/ @yenuo26
+/tests/ @yenuo26 @NickCao
 
 # CI
-/.buildkite/ @yenuo26 @congw729
+/.buildkite/ @yenuo26 @congw729 @NickCao
 
 # Engine and model_executor
 /vllm_omni/core/ @tzhouam @yinpeiqi @fake0fan @Sy0307
 /vllm_omni/engine/ @tzhouam @yinpeiqi @fake0fan @Sy0307
-/vllm_omni/entrypoints/ @tzhouam @yinpeiqi @fake0fan @alex-jw-brooks
+/vllm_omni/entrypoints/ @tzhouam @yinpeiqi @fake0fan @alex-jw-brooks @NickCao
 /vllm_omni/model_executor/ @tzhouam @gcanlin
 /vllm_omni/worker/ @tzhouam @gcanlin
 
@@ -37,7 +37,7 @@
 
 # Omni and TTS support
 /vllm_omni/transformers_utils/ @tzhouam @yuanheng-zhao
-/vllm_omni/model_executor/models/ @linyueqian @ZeldaHuang @princepride @yuanheng-zhao @amy-why-3459 @Bounty-hunter @Sy0307 @gcanlin
+/vllm_omni/model_executor/models/ @linyueqian @ZeldaHuang @princepride @yuanheng-zhao @amy-why-3459 @Bounty-hunter @Sy0307 @gcanlin @NickCao
 
 # Benchmarks
 /benchmarks/accuracy/ @david6666666

README.md

@@ -16,14 +16,10 @@ Easy, fast, and cheap omni-modality model serving for everyone
 ---
 
 *Latest News* 🔥
+- [2026/08] We released [0.26.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.26.0) - aligned with the vLLM 0.26 release line, featuring [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) joint video/audio generation, an experimental full-duplex realtime runtime for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md), distributed layerwise diffusion offload, and broader model, hardware, streaming, TTS, and quantization support.
 - [2026/07] We released [0.24.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.24.0) - aligned with the vLLM 0.24 release line, expanding production-ready coverage across TTS, speech, diffusion, image/video generation, and robot-policy serving, with major Omni stage runtime refactoring, diffusion request-level batching, async output materialization, quantization/cache/memory improvements, and broad CUDA/ROCm/XPU/NPU support.
-- [2026/06] We released [0.22.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.22.0) - an **omnimodal world-model** release aligned with vLLM 0.22, featuring [Nvidia Cosmos3](recipes/cosmos3/Cosmos3-Nano.md)/DreamZero world model support, expanded quantization coverage across Blackwell/NPU/XPU, TTS production improvements, new models including MiniCPM-o 4.5, MOSS-TTS, and Lance, plus RL integration with [VeRL-Omni](https://github.com/verl-project/verl-omni).
-- [2026/05] We released [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0) - refreshes the serving/runtime stack for large-scale omni workloads, and improves diffusion model performance, quantization, and hardware readiness across CUDA, ROCm, MUSA, NPU, and XPU backends.
-- [2026/03] We released [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0) - strengthens the core runtime through a large entrypoint refactor and scheduler/runtime cleanups, expands unified quantization and diffusion execution, broadens multimodal model coverage, and improves production readiness across audio, omni, image, video, RL, and multi-platform deployments.
+- [2026/06] Starting with [0.14.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.14.0), vLLM-Omni publishes a stable release aligned with every even-numbered upstream vLLM minor version. [0.16.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.16.0), [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0), [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0), and [0.22.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.22.0) continued this cadence, expanding omni and world-model support with [NVIDIA Cosmos3](recipes/cosmos3/Cosmos3-Nano.md) and DreamZero, adding models such as MiniCPM-o 4.5, MOSS-TTS, and Lance, and advancing TTS, diffusion, distributed execution, quantization, RL integration through [VeRL-Omni](https://github.com/verl-project/verl-omni), and CUDA/ROCm/MUSA/NPU/XPU coverage.
 - [2026/03] Check out our first public [project deepdive](https://youtu.be/sgwNfsNnR9I) at the vLLM Hong Kong Meetup!
-- [2026/03] **[vllm-omni-skills](https://github.com/hsliuustc0106/vllm-omni-skills)** is a community-driven collection of AI assistant skills that help developers work with vLLM-Omni more effectively. These skills can be used with popular agentic AI coding assistants like **Cursor IDE**, **Claude**, **Codex**, and more.
-- [2026/02] We released [0.16.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.16.0) - A major alignment + capability release that rebases onto **upstream vLLM v0.16.0** and significantly expands performance, distributed execution, and production readiness across **Qwen3-Omni / Qwen3-TTS**, **Bagel**, **MiMo-Audio**, **GLM-Image** and the **Diffusion (DiT) image/video stack**—while also improving platform coverage (CUDA / ROCm / NPU / XPU), CI quality, and documentation.
-- [2026/02] We released [0.14.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.14.0) - This is the first **stable release** of vLLM-Omni that expands Omni’s diffusion / image-video generation and audio / TTS stack, improves distributed execution and memory efficiency, and broadens platform/backend coverage (GPU/ROCm/NPU/XPU). It also brings meaningful upgrades to serving APIs, profiling & benchmarking, and overall stability. Please check our latest [paper](https://arxiv.org/abs/2602.02204) for architecture design and performance results.
 - [2025/11] vLLM community officially released [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni) in order to support omni-modality models serving.
 
 ---
@@ -55,12 +51,13 @@ vLLM-Omni is flexible and easy to use with:
 - Tensor, pipeline, data and expert parallelism support for distributed inference
 - Streaming outputs
 - OpenAI-compatible API server
+- Full-duplex realtime serving with streaming audio input and output (experimental)
 
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
-- **Omni-modality models** (e.g. Qwen3-Omni, Cosmos3, HunyuanImage, BAGEL)
+- **Omni-modality models** (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
 - **TTS models** (e.g. Qwen3-TTS, VoxCPM2, Ming-Omni-TTS, CosyVoice3)
-- **Diffusion models** — image, video, and audio generation (e.g. Qwen-Image, Wan2.2, FLUX)
+- **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, Qwen-Image, Wan2.2, FLUX)
 - **Robot-policy and action models** (e.g. GR00T-N1.7, DreamZero-DROID, InternVLA-A1, Cosmos3 action policy)
 
 ## Getting Started

apps/ComfyUI-vLLM-Omni/README.md

@@ -47,7 +47,7 @@ If no, check your shell running the ComfyUI process. There may be some error mes
 This extension offers the following nodes based on the output modalities (at **ComfyUI sidebar -> Node Library**):
 
 - **Generate Image** for text-to-image and image-to-image tasks
-- **Generate Video** for text-to-video and image-to-video tasks
+- **Generate Video** for text-to-video, first-frame/image-to-video, and reference-conditioned video
 - **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks
 - **TTS** and **TTS Voice Clone** for TTS tasks
 
@@ -105,7 +105,7 @@ You can configure per-stage sampling parameters for multi-stage models.
 > [!TIP]
 > The node automatically choose text-to-image or image-to-image API endpoints depending on whether you connect an image input or not.
 
-### Text-to-video and image-to-video generation (e.g., Wan)
+### Text-to-video and image-to-video generation (e.g., Wan, MiniMax-H3)
 
 (Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni Video Generation**)
 
@@ -117,7 +117,11 @@ You can configure per-stage sampling parameters for multi-stage models.
 </p>
 
 > [!TIP]
-> The node automatically choose text-to-video or image-to-video API endpoints depending on whether you connect an image input or not.
+> Connect a **frame** image for first-frame / image-to-video (e.g., Wan I2V, MiniMax-H3 FL2VA).
+>
+> For reference-conditioned generation (MiniMax-H3 Ref2VA), connect a **Video References** node instead.
+>
+> Do not use `frame` and `references` together. Task routing is automatic from which inputs you connect.
 
 ### TTS (e.g., Qwen TTS series)
 

apps/ComfyUI-vLLM-Omni/__init__.py

@@ -15,11 +15,13 @@ from .comfyui_vllm_omni.nodes import (
     VLLMOmniDiffusionSampling,
     VLLMOmniGenerateImage,
     VLLMOmniGenerateVideo,
+    VLLMOmniMiniMaxH3Params,
     VLLMOmniQwenTTSParams,
     VLLMOmniRemoteLoRA,
     VLLMOmniSamplingParamsList,
     VLLMOmniTTS,
     VLLMOmniUnderstanding,
+    VLLMOmniVideoReferences,
     VLLMOmniVoiceClone,
     VLLMOmniWanParams,
 )
@@ -32,6 +34,7 @@ NODE_CLASS_MAPPINGS = {
     "VLLMOmniUnderstanding": VLLMOmniUnderstanding,
     "VLLMOmniTTS": VLLMOmniTTS,
     "VLLMOmniVoiceClone": VLLMOmniVoiceClone,
+    "VLLMOmniVideoReferences": VLLMOmniVideoReferences,
     # === Params ===
     "VLLMOmniARSampling": VLLMOmniARSampling,
     "VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling,
@@ -39,6 +42,7 @@ NODE_CLASS_MAPPINGS = {
     "VLLMOmniRemoteLoRA": VLLMOmniRemoteLoRA,
     "VLLMOmniQwenTTSParams": VLLMOmniQwenTTSParams,
     "VLLMOmniWanParams": VLLMOmniWanParams,
+    "VLLMOmniMiniMaxH3Params": VLLMOmniMiniMaxH3Params,
 }
 
 # A dictionary that contains the friendly/humanly readable titles for the nodes
@@ -49,6 +53,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "VLLMOmniUnderstanding": "Multimodality Understanding",
     "VLLMOmniTTS": "TTS (Text to Speech)",
     "VLLMOmniVoiceClone": "TTS Voice Cloning",
+    "VLLMOmniVideoReferences": "Video References",
     # === Params ===
     "VLLMOmniARSampling": "AR Sampling Params",
     "VLLMOmniDiffusionSampling": "Diffusion Sampling Params",
@@ -56,6 +61,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "VLLMOmniRemoteLoRA": "LoRA",
     "VLLMOmniQwenTTSParams": "Qwen TTS Params",
     "VLLMOmniWanParams": "Wan Video Params",
+    "VLLMOmniMiniMaxH3Params": "MiniMax-H3 Video Params",
 }
 
 WEB_DIRECTORY = "./web"

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py

@@ -10,7 +10,9 @@ from .utils.types import (
     AudioFormat,
     AutoregressionSamplingParams,
     DiffusionSamplingParams,
+    MiniMaxH3ModelSpecificParams,
     QwenTTSModelSpecificParams,
+    VideoReferences,
     WanModelSpecificParams,
 )
 from .utils.validators import (
@@ -163,7 +165,8 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
                 "num_frames": ("INT", {"default": 41, "min": 1}),
             },
             "optional": {
-                "image": ("IMAGE",),
+                "frame": ("IMAGE",),
+                "references": ("VIDEO_REFERENCES",),
                 "sampling_params": ("SAMPLING_PARAMS",),
                 "lora": ("REMOTE_LORA",),
                 "model_params": ("VIDEO_PARAMS",),
@@ -174,6 +177,15 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
     RETURN_NAMES = ("video",)
     FUNCTION = "generate"
 
+    @classmethod
+    def VALIDATE_INPUTS(cls, url, model, frame=None, references=None, **_kwargs) -> str | Literal[True]:
+        base = super().VALIDATE_INPUTS(url, model)
+        if base is not True:
+            return base
+        if frame is not None and references is not None:
+            return "Provide only one of frame or references, not both."
+        return True
+
     async def generate(
         self,
         url: str,
@@ -184,7 +196,8 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
         fps: int,
         num_frames: int,
         negative_prompt: str | None = None,
-        image: torch.Tensor | None = None,
+        frame: torch.Tensor | None = None,
+        references: dict | None = None,
         sampling_params: dict | list[dict] | None = None,
         model_params: dict | None = None,
         lora: dict | None = None,
@@ -207,14 +220,16 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
 
         if sampling_params is not None:
             sampling_params.pop("type", None)  # internal fields
-        if model_params is not None:
-            model_params.pop("type", None)  # internal fields
+        # model_params["type"] is also an internal field, but is used in api_client
+        # if model_params is not None:
+        #     model_params.pop("type", None)  # internal fields
 
         client = VLLMOmniClient(url)
         output = await client.generate_video(
             model=model,
             prompt=prompt,
-            image=image,  # image present => i2v, absent => t2v
+            frame=frame,  # frame present => fl2va / Wan I2V
+            references=references,
             width=width,
             height=height,
             num_frames=num_frames,
@@ -734,3 +749,78 @@ class VLLMOmniWanParams:
 
     def get_params(self, **kwargs):
         return (WanModelSpecificParams(kwargs),)
+
+
+class VLLMOmniMiniMaxH3Params:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "audio_flow_shift": (
+                    "FLOAT",
+                    {"default": 3.0, "min": 0.0, "max": 100.0, "step": 0.1},
+                ),
+                "flow_shift": (
+                    "FLOAT",
+                    {"default": 12.0, "min": 0.0, "max": 100.0, "step": 0.1},
+                ),
+            }
+        }
+
+    RETURN_TYPES = ("VIDEO_PARAMS",)
+    RETURN_NAMES = ("MiniMax-H3 video params",)
+    FUNCTION = "get_params"
+    CATEGORY = "vLLM-Omni/Video Params"
+
+    def get_params(self, **kwargs):
+        params = MiniMaxH3ModelSpecificParams(kwargs)
+        params["type"] = "minimax_h3"
+        return (params,)
+
+
+class VLLMOmniVideoReferences:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {},
+            "optional": {
+                "image_1": ("IMAGE",),
+                "image_2": ("IMAGE",),
+                "audio_1": ("AUDIO",),
+                "audio_2": ("AUDIO",),
+                "video_1": ("VIDEO",),
+                "video_2": ("VIDEO",),
+            },
+        }
+
+    RETURN_TYPES = ("VIDEO_REFERENCES",)
+    RETURN_NAMES = ("references",)
+    FUNCTION = "get_references"
+    CATEGORY = "vLLM-Omni"
+
+    def get_references(
+        self,
+        image_1: torch.Tensor | None = None,
+        image_2: torch.Tensor | None = None,
+        audio_1: AudioInput | None = None,
+        audio_2: AudioInput | None = None,
+        video_1: VideoInput | None = None,
+        video_2: VideoInput | None = None,
+        **kwargs,
+    ):
+        if kwargs:
+            logger.info("Uncaught kwargs: %s", kwargs)
+        refs = VideoReferences()
+        if image_1 is not None:
+            refs["image_1"] = image_1
+        if image_2 is not None:
+            refs["image_2"] = image_2
+        if audio_1 is not None:
+            refs["audio_1"] = audio_1
+        if audio_2 is not None:
+            refs["audio_2"] = audio_2
+        if video_1 is not None:
+            refs["video_1"] = video_1
+        if video_2 is not None:
+            refs["video_2"] = video_2
+        return (refs,)

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py

@@ -24,6 +24,7 @@ from .format import (
     image_tensor_to_base64,
     image_tensor_to_png_bytes,
     video_to_base64,
+    video_to_bytes,
 )
 from .logger import get_logger, pretty_printer
 from .models import lookup_model_spec
@@ -63,7 +64,11 @@ async def url_bytes(session: aiohttp.ClientSession, url: str, verb: str = "get",
 
 class VLLMOmniClient:
     def __init__(
-        self, base_url: str, timeout: float | None = None, poll_interval: float = 5.0, max_poll_duration: float = 60 * 5
+        self,
+        base_url: str,
+        timeout: float | None = None,
+        poll_interval: float = 5.0,
+        max_poll_duration: float = 60 * 30,
     ):
         self.base_url = base_url
         self.timeout = aiohttp.ClientTimeout(total=timeout)
@@ -261,12 +266,17 @@ class VLLMOmniClient:
         num_frames: int,
         fps: int,
         negative_prompt: str | None = None,
-        image: torch.Tensor | None = None,
+        frame: torch.Tensor | None = None,
+        references: dict | None = None,
         sampling_params: dict | None = None,
         model_params: dict | None = None,
         lora: dict | None = None,
-        **extra_body,
+        **extra_params,
     ) -> VideoInput:
+        if frame is not None and references is not None:
+            raise ValueError("Provide only one of frame or references, not both.")
+
+        # === regular payload fields ===
         form = aiohttp.FormData()
         form.add_field("model", model)
         form.add_field("prompt", prompt)
@@ -279,23 +289,91 @@ class VLLMOmniClient:
         if sampling_params is not None:
             for k, v in sampling_params.items():
                 form.add_field(k, str(v))
-        if model_params is not None:
-            for k, v in model_params.items():
-                form.add_field(k, str(v))
         if lora is not None:
             form.add_field("lora", json.dumps(lora, ensure_ascii=False))
-        if extra_body:
-            form.add_field("extra_body", json.dumps(extra_body, ensure_ascii=False))
 
-        if image is not None:
+        # === multimodal inputs (first-last-frames, references, etc.) ===
+        input_reference_image: torch.Tensor | None = None
+        audio_reference: AudioInput | None = None
+        reference_videos: list[VideoInput] = []
+        video_task: str | None = None
+
+        if frame is not None:
+            input_reference_image = frame
+            video_task = "fl2va"
+        elif references is not None:
+            images = [references[k] for k in ("image_1", "image_2") if k in references and references[k] is not None]
+            audios = [references[k] for k in ("audio_1", "audio_2") if k in references and references[k] is not None]
+            videos = [references[k] for k in ("video_1", "video_2") if k in references and references[k] is not None]
+
+            if not images and not audios and not videos:
+                raise ValueError("references is empty; connect at least one image/audio/video.")
+
+            if videos:
+                if images or audios:
+                    raise ValueError(
+                        "Video references cannot be combined with image or audio references. "
+                        "Connect only video_1/video_2 for multi-video Ref2VA."
+                    )
+                reference_videos = videos
+                video_task = "ref2va"
+            elif len(images) == 1 and len(audios) == 1:
+                input_reference_image = images[0]
+                audio_reference = audios[0]
+                video_task = "ref2va"
+            else:
+                raise ValueError(
+                    "Invalid references combination. Supported modes: "
+                    "(1) one or more videos only, or (2) exactly one image and one audio."
+                )
+        else:
+            video_task = "t2va"
+
+        if input_reference_image is not None:
             image_filename = "image.png"  # Required for multipart form
             form.add_field(
                 "input_reference",
-                image_tensor_to_png_bytes(image, image_filename),
+                image_tensor_to_png_bytes(input_reference_image, image_filename),
                 filename=image_filename,
                 content_type="image/png",
             )
 
+        if audio_reference is not None:
+            form.add_field(
+                "audio_reference",
+                json.dumps({"audio_url": audio_to_base64(audio_reference)}, ensure_ascii=False),
+            )
+
+        for idx, video in enumerate(reference_videos, start=1):
+            video_filename = f"reference_{idx}.mp4"
+            form.add_field(
+                "input_references",
+                video_to_bytes(video, video_filename),
+                filename=video_filename,
+                content_type="video/mp4",
+            )
+
+        # === model specific params. Either use a specialized builder, or add flattened fields as-is ===
+        if model_params is not None:
+            model_params = dict(model_params)
+            model_params.pop("type", None)
+
+        spec, _ = lookup_model_spec(model)
+        params_builder = spec.get("params_builder") if spec else None
+        if params_builder is not None:
+            form_fields = params_builder(
+                model_params or {},
+                extra_params={**extra_params, "task": video_task},
+            )
+            for k, v in form_fields.items():
+                form.add_field(k, v if isinstance(v, str) else str(v))
+        else:
+            if model_params is not None:
+                for k, v in model_params.items():
+                    form.add_field(k, str(v))
+            if extra_params:
+                form.add_field("extra_params", json.dumps(extra_params, ensure_ascii=False))
+
         async with aiohttp.ClientSession(timeout=self.timeout) as session:
             # Start the video generation job
             url = f"{self.base_url}/videos"

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py

@@ -1,7 +1,12 @@
+import json
 import re
+from typing import Any
 
+from .logger import get_logger
 from .types import Modality, ModelMode, Spec
 
+logger = get_logger(__name__)
+
 
 def _bagel_payload_preprocessor(payload: dict) -> dict:
     try:
@@ -29,6 +34,30 @@ def _qwen25_payload_preprocessor(payload: dict) -> dict:
     return payload
 
 
+def _minimaxh3_params_builder(
+    model_params: dict[str, Any],
+    *,
+    extra_params: dict[str, Any],
+) -> dict[str, Any]:
+    """Build multipart form fields for MiniMax-H3 from model_params + routed task."""
+    params = dict(model_params)
+    params.pop("type", None)
+    form_fields: dict[str, Any] = {}
+    merged_extra_params: dict[str, Any] = dict(extra_params or {})
+
+    if "flow_shift" in params:
+        form_fields["flow_shift"] = params.pop("flow_shift")
+    for key in ("audio_flow_shift",):
+        if key in params:
+            merged_extra_params[key] = params.pop(key)
+    if params:
+        logger.warning("Unused MiniMax-H3 model params ignored: %s", sorted(params))
+
+    if merged_extra_params:
+        form_fields["extra_params"] = json.dumps(merged_extra_params, ensure_ascii=False)
+    return form_fields
+
+
 _MODEL_PIPELINE_SPECS: dict[str, Spec] = {
     r"BAGEL-7B-MoT": {
         "stages": [
@@ -71,6 +100,21 @@ _MODEL_PIPELINE_SPECS: dict[str, Spec] = {
             }
         ],
     },
+    r"MiniMax-H3(/FL2VA|/Ref2VA)?": {
+        "stages": ["diffusion"],
+        "modes": [
+            {
+                "mode": ModelMode.VIDEO_GENERATION,
+                "input_modalities": [
+                    Modality.TEXT,
+                    Modality.IMAGE,
+                    Modality.VIDEO,
+                    Modality.AUDIO,
+                ],
+            }
+        ],
+        "params_builder": _minimaxh3_params_builder,
+    },
 }
 # Convert dict keys to regex patterns
 MODEL_PIPELINE_SPECS: dict[re.Pattern, Spec] = {}
@@ -80,12 +124,10 @@ del _MODEL_PIPELINE_SPECS
 
 
 def lookup_model_spec(model: str) -> tuple[Spec | None, str | None]:
-    try:
-        last_component = model.rstrip("/").rsplit("/", 1)[-1]
-    except IndexError:
-        last_component = model
+    normalized = model.rstrip("/").lstrip("/")
+    lookup_key = normalized.split("/", 1)[-1]
     for pattern, spec in MODEL_PIPELINE_SPECS.items():
-        if pattern.search(last_component):
+        if pattern.search(lookup_key):
             return spec, pattern.pattern
     return None, None
 
@@ -101,6 +143,9 @@ if __name__ == "__main__":
         "Qwen/Qwen3-Omni-30B-A3B-Instruct",
         "Custom/Path/UnknownModel-Instruct",
         "Not/Matching/Anything",
+        "/path/to/MiniMax-H3/FL2VA",
+        "/path/to/MiniMax-H3/Ref2VA",
+        "MiniMaxAI/MiniMax-H3",
     ]
 
     test_payload = {"messages": [{"role": "user", "content": "prompt"}]}
@@ -112,7 +157,13 @@ if __name__ == "__main__":
             if preprocessor := spec.get("payload_preprocessor"):
                 result = preprocessor(test_payload)
                 print(f"✓ {path:<40} → {result}")
+            elif params_builder := spec.get("params_builder"):
+                result = params_builder(
+                    {"audio_flow_shift": 3.0, "flow_shift": 12.0, "type": "minimax_h3"},
+                    extra_params={"task": "t2va"},
+                )
+                print(f"✓ {path:<40} → {result}")
             else:
-                print(f"✓ {path:<40} → No preprocessor")
+                print(f"✓ {path:<40} → No preprocessor/params_builder")
         else:
             print(f"✗ {path:<40} → No match")

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py

@@ -1,12 +1,8 @@
 from collections.abc import Callable
 from enum import Enum, auto
-from typing import (
-    Any,
-    Literal,
-    NotRequired,
-    TypeAlias,
-    TypedDict,
-)
+from typing import Any, Literal, TypeAlias
+
+from typing_extensions import NotRequired, TypedDict
 
 AudioFormat: TypeAlias = Literal["mp3", "opus", "aac", "flac", "wav", "pcm"]
 
@@ -27,6 +23,14 @@ class WanModelSpecificParams(dict):
     pass
 
 
+class MiniMaxH3ModelSpecificParams(dict):
+    pass
+
+
+class VideoReferences(dict):
+    pass
+
+
 class ModelMode(Enum):
     IMAGE_GENERATION = auto()
     VIDEO_GENERATION = auto()
@@ -47,9 +51,11 @@ class ModelModeSpec(TypedDict):
 
 
 PayloadPreprocessor: TypeAlias = Callable[[dict[str, Any]], dict[str, Any]]
+ParamsBuilder: TypeAlias = Callable[..., dict[str, Any]]
 
 
 class Spec(TypedDict):
     stages: list[Literal["diffusion", "autoregression"]]
     modes: list[ModelModeSpec]
     payload_preprocessor: NotRequired[PayloadPreprocessor]
+    params_builder: NotRequired[ParamsBuilder]

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json

@@ -1,513 +1 @@
-{
-    "id": "3aea1d08-7de5-4997-ba71-b2f66ad72a21",
-    "revision": 0,
-    "last_node_id": 14,
-    "last_link_id": 28,
-    "nodes": [
-        {
-            "id": 10,
-            "type": "MarkdownNote",
-            "pos": [
-                227.99306819575278,
-                -231.24143306069843
-            ],
-            "size": [
-                240.3326478922474,
-                136.3505820791881
-            ],
-            "flags": {},
-            "order": 0,
-            "mode": 0,
-            "inputs": [],
-            "outputs": [],
-            "title": "Note: Task and Input",
-            "properties": {},
-            "widgets_values": [
-                "vLLM-Omni nodes are categorized based on the output modality. The \"Generate Video\" node supports both text-to-video generation and image-to-video generation. The node will create corresponding payload depending on whether an input image is present or not."
-            ],
-            "color": "#432",
-            "bgcolor": "#000"
-        },
-        {
-            "id": 4,
-            "type": "LoadImage",
-            "pos": [
-                496.31859627609606,
-                -229.71277089860084
-            ],
-            "size": [
-                270,
-                314
-            ],
-            "flags": {},
-            "order": 1,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "image",
-                    "name": "image",
-                    "type": "COMBO",
-                    "widget": {
-                        "name": "image"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "choose file to upload",
-                    "name": "upload",
-                    "type": "IMAGEUPLOAD",
-                    "widget": {
-                        "name": "upload"
-                    },
-                    "link": null
-                }
-            ],
-            "outputs": [
-                {
-                    "localized_name": "IMAGE",
-                    "name": "IMAGE",
-                    "type": "IMAGE",
-                    "links": [
-                        25
-                    ]
-                },
-                {
-                    "localized_name": "MASK",
-                    "name": "MASK",
-                    "type": "MASK",
-                    "links": null
-                }
-            ],
-            "properties": {
-                "Node name for S&R": "LoadImage"
-            },
-            "widgets_values": [
-                "cute-cat.jpg",
-                "image"
-            ]
-        },
-        {
-            "id": 8,
-            "type": "VLLMOmniDiffusionSampling",
-            "pos": [
-                465.5140218220927,
-                148.80072646828987
-            ],
-            "size": [
-                284.205078125,
-                226
-            ],
-            "flags": {},
-            "order": 2,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "n",
-                    "name": "n",
-                    "type": "INT",
-                    "widget": {
-                        "name": "n"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "num_inference_steps",
-                    "name": "num_inference_steps",
-                    "type": "INT",
-                    "widget": {
-                        "name": "num_inference_steps"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "guidance_scale",
-                    "name": "guidance_scale",
-                    "type": "FLOAT",
-                    "widget": {
-                        "name": "guidance_scale"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "true_cfg_scale",
-                    "name": "true_cfg_scale",
-                    "type": "FLOAT",
-                    "widget": {
-                        "name": "true_cfg_scale"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "vae_use_slicing",
-                    "name": "vae_use_slicing",
-                    "type": "BOOLEAN",
-                    "widget": {
-                        "name": "vae_use_slicing"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "vae_use_tiling",
-                    "name": "vae_use_tiling",
-                    "type": "BOOLEAN",
-                    "widget": {
-                        "name": "vae_use_tiling"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "seed",
-                    "name": "seed",
-                    "type": "INT",
-                    "widget": {
-                        "name": "seed"
-                    },
-                    "link": null
-                }
-            ],
-            "outputs": [
-                {
-                    "localized_name": "diffusion sampling params",
-                    "name": "diffusion sampling params",
-                    "type": "SAMPLING_PARAMS",
-                    "links": [
-                        27
-                    ]
-                }
-            ],
-            "properties": {
-                "Node name for S&R": "VLLMOmniDiffusionSampling"
-            },
-            "widgets_values": [
-                1,
-                50,
-                1,
-                1,
-                false,
-                false,
-                1992,
-                "randomize"
-            ]
-        },
-        {
-            "id": 14,
-            "type": "VLLMOmniWanParams",
-            "pos": [
-                518.6005521845747,
-                428.4455663719664
-            ],
-            "size": [
-                270,
-                106
-            ],
-            "flags": {},
-            "order": 3,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "guidance_scale_2",
-                    "name": "guidance_scale_2",
-                    "type": "FLOAT",
-                    "widget": {
-                        "name": "guidance_scale_2"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "boundary_ratio",
-                    "name": "boundary_ratio",
-                    "type": "FLOAT",
-                    "widget": {
-                        "name": "boundary_ratio"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "flow_shift",
-                    "name": "flow_shift",
-                    "type": "FLOAT",
-                    "widget": {
-                        "name": "flow_shift"
-                    },
-                    "link": null
-                }
-            ],
-            "outputs": [
-                {
-                    "localized_name": "Wan video params",
-                    "name": "Wan video params",
-                    "type": "VIDEO_PARAMS",
-                    "links": [
-                        28
-                    ]
-                }
-            ],
-            "properties": {
-                "Node name for S&R": "VLLMOmniWanParams"
-            },
-            "widgets_values": [
-                4,
-                0.875,
-                5
-            ]
-        },
-        {
-            "id": 12,
-            "type": "VLLMOmniGenerateVideo",
-            "pos": [
-                827.2566336087867,
-                -73.77449831827545
-            ],
-            "size": [
-                400,
-                346
-            ],
-            "flags": {},
-            "order": 4,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "image",
-                    "name": "image",
-                    "shape": 7,
-                    "type": "IMAGE",
-                    "link": 25
-                },
-                {
-                    "localized_name": "sampling_params",
-                    "name": "sampling_params",
-                    "shape": 7,
-                    "type": "SAMPLING_PARAMS",
-                    "link": 27
-                },
-                {
-                    "localized_name": "lora",
-                    "name": "lora",
-                    "shape": 7,
-                    "type": "REMOTE_LORA",
-                    "link": null
-                },
-                {
-                    "localized_name": "model_params",
-                    "name": "model_params",
-                    "shape": 7,
-                    "type": "VIDEO_PARAMS",
-                    "link": 28
-                },
-                {
-                    "localized_name": "url",
-                    "name": "url",
-                    "type": "STRING",
-                    "widget": {
-                        "name": "url"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "model",
-                    "name": "model",
-                    "type": "STRING",
-                    "widget": {
-                        "name": "model"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "prompt",
-                    "name": "prompt",
-                    "type": "STRING",
-                    "widget": {
-                        "name": "prompt"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "negative_prompt",
-                    "name": "negative_prompt",
-                    "type": "STRING",
-                    "widget": {
-                        "name": "negative_prompt"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "width",
-                    "name": "width",
-                    "type": "INT",
-                    "widget": {
-                        "name": "width"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "height",
-                    "name": "height",
-                    "type": "INT",
-                    "widget": {
-                        "name": "height"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "fps",
-                    "name": "fps",
-                    "type": "INT",
-                    "widget": {
-                        "name": "fps"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "num_frames",
-                    "name": "num_frames",
-                    "type": "INT",
-                    "widget": {
-                        "name": "num_frames"
-                    },
-                    "link": null
-                }
-            ],
-            "outputs": [
-                {
-                    "localized_name": "video",
-                    "name": "video",
-                    "type": "VIDEO",
-                    "links": [
-                        26
-                    ]
-                }
-            ],
-            "properties": {
-                "Node name for S&R": "VLLMOmniGenerateVideo"
-            },
-            "widgets_values": [
-                "http://localhost:8000/v1",
-                "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
-                "Make the cat in the attached video yawning. Eye closed and mouth wide open. Keep the original realistic style.",
-                "Cartoonish",
-                900,
-                1200,
-                12,
-                41
-            ]
-        },
-        {
-            "id": 13,
-            "type": "SaveVideo",
-            "pos": [
-                1257.5441369501657,
-                -44.12956394419863
-            ],
-            "size": [
-                374,
-                372.34896826553796
-            ],
-            "flags": {},
-            "order": 5,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "video",
-                    "name": "video",
-                    "type": "VIDEO",
-                    "link": 26
-                },
-                {
-                    "localized_name": "filename_prefix",
-                    "name": "filename_prefix",
-                    "type": "STRING",
-                    "widget": {
-                        "name": "filename_prefix"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "format",
-                    "name": "format",
-                    "type": "COMBO",
-                    "widget": {
-                        "name": "format"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "codec",
-                    "name": "codec",
-                    "type": "COMBO",
-                    "widget": {
-                        "name": "codec"
-                    },
-                    "link": null
-                }
-            ],
-            "outputs": [],
-            "properties": {},
-            "widgets_values": [
-                "video/ComfyUI",
-                "auto",
-                "auto"
-            ]
-        }
-    ],
-    "links": [
-        [
-            25,
-            4,
-            0,
-            12,
-            0,
-            "IMAGE"
-        ],
-        [
-            26,
-            12,
-            0,
-            13,
-            0,
-            "VIDEO"
-        ],
-        [
-            27,
-            8,
-            0,
-            12,
-            1,
-            "SAMPLING_PARAMS"
-        ],
-        [
-            28,
-            14,
-            0,
-            12,
-            3,
-            "VIDEO_PARAMS"
-        ]
-    ],
-    "groups": [
-        {
-            "id": 1,
-            "title": "Input",
-            "bounding": [
-                213.1706010087147,
-                -313.20095750667554,
-                560.8246471437027,
-                407.4246532472182
-            ],
-            "color": "#3f789e",
-            "font_size": 24,
-            "flags": {}
-        }
-    ],
-    "config": {},
-    "extra": {
-        "workflowRendererVersion": "LG",
-        "ds": {
-            "scale": 1.1469075819486894,
-            "offset": [
-                101.7632950847089,
-                492.29121889347806
-            ]
-        }
-    },
-    "version": 0.4
-}
+{"id":"3aea1d08-7de5-4997-ba71-b2f66ad72a21","revision":0,"last_node_id":24,"last_link_id":35,"nodes":[{"id":13,"type":"SaveVideo","pos":[1315.729153459605,-137.92038948394355],"size":[374,313.7692307692308],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"video","name":"video","type":"VIDEO","link":32},{"localized_name":"filename_prefix","name":"filename_prefix","type":"STRING","widget":{"name":"filename_prefix"},"link":null},{"localized_name":"format","name":"format","type":"COMBO","widget":{"name":"format"},"link":null},{"localized_name":"codec","name":"codec","type":"COMFY_DYNAMICCOMBO_V3","widget":{"name":"codec"},"link":null}],"outputs":[{"localized_name":"video","name":"video","type":"VIDEO","links":null}],"properties":{},"widgets_values":["video/ComfyUI","auto","auto"]},{"id":10,"type":"MarkdownNote","pos":[183.74636797766794,-571.2529158046766],"size":[240.3326478922474,136.3505820791881],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[],"title":"Note: Task and Input","properties":{},"widgets_values":["vLLM-Omni nodes are categorized based on the output modality. The \"Generate Video\" node supports both *text-to-video*, *frame-to-video*, and *reference-to-video* generation. The node will create corresponding payload depending on whether an input image is present or not."],"color":"#432","bgcolor":"#000"},{"id":4,"type":"LoadImage","pos":[452.07189605801125,-569.7242536425791],"size":[270,314],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[29]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["rap.png","image"]},{"id":23,"type":"LoadImage","pos":[9.206110246057657,-7.6955291509170785],"size":[270,314],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[34]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["white-cat.jpg","image"]},{"id":22,"type":"MarkdownNote","pos":[-9.783113411416666,-148.8790300038605],"size":[297.3012007349086,88],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[],"title":"Note: Reference to video (r2v)","properties":{},"widgets_values":["For r2v tasks, connect this Video Reference block to the main Generate Video block. Then disconnect the \"frame\" input above."],"color":"#432","bgcolor":"#000"},{"id":24,"type":"LoadAudio","pos":[317.9929742385805,-141.44821500065095],"size":[282.798828125,136],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[35]}],"properties":{"Node name for S&R":"LoadAudio"},"widgets_values":["nothing-beats-a-jet2-holiday.mp3",null,""]},{"id":21,"type":"VLLMOmniVideoReferences","pos":[376.6129172897595,105.41557277556292],"size":[156.9380859375,126],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"image_1","name":"image_1","shape":7,"type":"IMAGE","link":34},{"localized_name":"image_2","name":"image_2","shape":7,"type":"IMAGE","link":null},{"localized_name":"audio_1","name":"audio_1","shape":7,"type":"AUDIO","link":35},{"localized_name":"audio_2","name":"audio_2","shape":7,"type":"AUDIO","link":null},{"localized_name":"video_1","name":"video_1","shape":7,"type":"VIDEO","link":null},{"localized_name":"video_2","name":"video_2","shape":7,"type":"VIDEO","link":null}],"outputs":[{"localized_name":"references","name":"references","type":"VIDEO_REFERENCES","links":null}],"properties":{"Node name for S&R":"VLLMOmniVideoReferences"},"widgets_values":[]},{"id":8,"type":"VLLMOmniDiffusionSampling","pos":[83.24620208113915,422.08543477542605],"size":[284.205078125,226],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"n","name":"n","type":"INT","widget":{"name":"n"},"link":null},{"localized_name":"num_inference_steps","name":"num_inference_steps","type":"INT","widget":{"name":"num_inference_steps"},"link":null},{"localized_name":"guidance_scale","name":"guidance_scale","type":"FLOAT","widget":{"name":"guidance_scale"},"link":null},{"localized_name":"true_cfg_scale","name":"true_cfg_scale","type":"FLOAT","widget":{"name":"true_cfg_scLine truncated

docker/Dockerfile.npu

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.25.0
+ARG VLLM_ASCEND_TAG=v0.26.0
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
 
 # WORKDIR /vllm-workspace/vllm
@@ -26,6 +26,10 @@ RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi
     export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
     python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation
 
+RUN python3 -m pip install -v --no-deps \
+      "stepaudio2-minicpmo>=0.1.1" \
+      "step-audio2>=1.0.0"
+
 ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
 
 ENTRYPOINT []

docker/Dockerfile.npu.a3

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.25.0-a3
+ARG VLLM_ASCEND_TAG=v0.26.0-a3
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
 
 # WORKDIR /vllm-workspace/vllm
@@ -26,6 +26,10 @@ RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi
     export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
     python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation
 
+RUN python3 -m pip install -v --no-deps \
+      "stepaudio2-minicpmo>=0.1.1" \
+      "step-audio2>=1.0.0"
+
 ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
 
 ENTRYPOINT []

docker/Dockerfile.npu.ci

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.25.0
+ARG VLLM_ASCEND_TAG=v0.26.0
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
 
 ARG APP_DIR=/vllm-workspace/vllm-omni
@@ -35,6 +35,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
       "psutil>=7.2.0" \
       "soundfile>=0.13.1" \
       "pyttsx3>=2.99" \
+      "opencc==1.4.1" \
       "opencv-python>=4.12.0.88" \
       "imageio[ffmpeg]>=0.6.0" \
       "jiwer>=3.0.0" \

docker/Dockerfile.npu.ci.a3

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.25.0-a3
+ARG VLLM_ASCEND_TAG=v0.26.0-a3
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
 
 ARG APP_DIR=/vllm-workspace/vllm-omni
@@ -35,6 +35,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
       "psutil>=7.2.0" \
       "soundfile>=0.13.1" \
       "pyttsx3>=2.99" \
+      "opencc==1.4.1" \
       "opencv-python>=4.12.0.88" \
       "imageio[ffmpeg]>=0.6.0" \
       "jiwer>=3.0.0" \
@@ -44,6 +45,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
       "hyperpyyaml>=1.2.0" \
       "onnx>=1.16.0" \
       "librosa>=0.10.0" \
+      "torchmetrics>=1.9.0" \
       --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
       --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
       --default-timeout=60 && \

docs/.nav.yml

@@ -68,13 +68,12 @@ nav:
         - Sequence Parallel: user_guide/diffusion/parallelism/sequence_parallel.md
         - Tensor Parallel: user_guide/diffusion/parallelism/tensor_parallel.md
         - VAE Parallelism: user_guide/diffusion/parallelism/vae_parallelism.md
-      - CPU Offloading: user_guide/diffusion/cpu_offload_diffusion.md
+      - CPU Offloading: user_guide/diffusion/cpu_offload.md
       - LoRA: user_guide/diffusion/lora.md
       - Custom Pipeline: features/custom_pipeline.md
-      - Request-Level Batching: user_guide/diffusion/request_batching.md
-      - Step Execution: user_guide/diffusion/step_execution.md
+      - Execution Modes: user_guide/diffusion/execution_modes.md
     - Quantization:
-      - Overview: user_guide/quantization/overview.md
+      - Quantization: user_guide/quantization/overview.md
       - Online Quantization: user_guide/quantization/online.md
       - Quantized KV Cache: user_guide/quantization/quantized_kvcache.md
       - FP8 W8A8: user_guide/quantization/fp8.md
@@ -102,31 +101,49 @@ nav:
   - CI: contributing/ci
   - Design Documents:
     - design/index.md
-    - design/architecture_overview.md
+    - Architecture Documents:
+      - Architecture Overview: design/architecture_overview.md
     - Feature Design:
-      - design/feature/disaggregated_inference.md
-      - design/feature/ray_based_execution.md
-      - design/feature/omni_connectors/
-      - design/feature/prefix_caching.md
-      - design/feature/cfg_parallel.md
-      - design/feature/expert_parallel.md
-      - design/feature/sequence_parallel.md
-      - design/feature/tensor_parallel.md
-      - design/feature/vae_parallel.md
-      - design/feature/hsdp.md
-      - design/feature/cache_dit.md
-      - design/feature/teacache.md
-      - design/feature/skip_softmax.md
-      - design/feature/async_chunk.md
-      - design/feature/vae_parallel.md
-      - design/feature/diffusion_step_execution.md
-      - design/feature/diffusion_request_level_batching.md
-      - design/feature/diffusion_continuous_batching.md
+      - Runtime and Stage Execution:
+        - Disaggregated Inference: design/feature/disaggregated_inference.md
+        - Async Chunk: design/feature/async_chunk.md
+        - Async Diffusion Output: design/feature/async_diffusion_output.md
+        - Async Omni Output Materialization: design/feature/omni_async_output_materialization.md
+        - Automatic Prefix Caching: design/feature/prefix_caching.md
+      - Communication:
+        - OmniConnector Implementations:
+          - Mooncake Store Connector: design/feature/omni_connectors/mooncake_store_connector.md
+          - Mooncake Transfer Engine Connector: design/feature/omni_connectors/mooncake_transfer_engine_connector.md
+          - Mori Transfer Engine Connector: design/feature/omni_connectors/mori_transfer_engine_connector.md
+          - Shared Memory Connector: design/feature/omni_connectors/shared_memory_connector.md
+          - Yuanrong Store Connector: design/feature/omni_connectors/yuanrong_connector.md
+          - Yuanrong Transfer Engine Connector: design/feature/omni_connectors/yuanrong_transfer_engine_connector.md
+      - Diffusion Acceleration:
+        - Parallelism:
+          - CFG-Parallel: design/feature/cfg_parallel.md
+          - Expert Parallel: design/feature/expert_parallel.md
+          - Hybrid Sharded Data Parallel (HSDP): design/feature/hsdp.md
+          - Pipeline Parallel: design/feature/pipeline_parallel.md
+          - Sequence Parallel: design/feature/sequence_parallel.md
+          - Tensor Parallel: design/feature/tensor_parallel.md
+          - VAE Patch Parallelism: design/feature/vae_parallel.md
+        - Attention Backends:
+          - Skip-Softmax: design/feature/skip_softmax.md
+        - Quantization: design/feature/quantization.md
+        - Cache-DiT: design/feature/cache_dit.md
+        - TeaCache: design/feature/teacache.md
+        - Diffusion Continuous Batching: design/feature/diffusion_continuous_batching.md
+        - Distributed Layerwise Offload: design/feature/distributed_layerwise_offload.md
+    - Infrastructure and Performance:
+      - Prometheus Metrics: design/metrics.md
+      - Speech Generation Performance Optimizations: design/qwen3_omni_tts_performance_optimization.md
     - Module Design:
-      - design/module/ar_module.md
-      - design/module/dit_module.md
-      - design/module/entrypoint_module.md
-      - design/module/async_omni_architecture.md
+      - Runtime Modules:
+        - AR Module: design/module/ar_module.md
+        - DIT Module: design/module/dit_module.md
+      - Orchestration:
+        - Entrypoint Module: design/module/entrypoint_module.md
+        - AsyncOmni Architecture: design/module/async_omni_architecture.md
   - Docs Guide: contributing/DOCS_GUIDE.md
 - API Reference:
   - api/README.md

docs/README.md

@@ -52,12 +52,13 @@ vLLM-Omni is flexible and easy to use with:
 - Tensor, pipeline, data and expert parallelism support for distributed inference
 - Streaming outputs
 - OpenAI-compatible API server
+- Full-duplex realtime serving with streaming audio input and output (experimental)
 
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
-- Omni-modality models (e.g. Qwen3-Omni, Cosmos3, HunyuanImage, BAGEL)
+- Omni-modality models (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
 - TTS models (e.g. Qwen3-TTS, VoxCPM2, Ming-Omni-TTS, CosyVoice3)
-- Diffusion models — image, video, and audio generation (e.g. Qwen-Image, Wan2.2, FLUX)
+- Diffusion models — image, video, and audio generation (e.g. MiniMax H3, Qwen-Image, Wan2.2, FLUX)
 - Robot-policy and action models (e.g. GR00T-N1.7, DreamZero-DROID, InternVLA-A1, Cosmos3 action policy)
 
 For more information, checkout the following:

docs/cli/README.md

@@ -18,18 +18,12 @@ Specify the port:
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
 ```
 
-For a migrated model, load a custom deploy configuration with `--deploy-config`:
+Load a custom deploy configuration with `--deploy-config`:
 
 ```bash
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --deploy-config /path/to/deploy_config.yaml
 ```
 
-The deprecated `--stage-configs-path` flag is retained for models that still use the legacy `stage_args` schema:
-
-```bash
-vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --stage-configs-path /path/to/legacy_stage_config.yaml
-```
-
 ## bench
 
 Run benchmark tests for online serving throughput.

docs/cli/serve.md

@@ -8,7 +8,6 @@ The stage-based CLI is designed for deployments that require launching each pipe
 - For **migrated models** that utilize the bundled deployment YAML configurations located in
   `vllm_omni/deploy/`, the `--deploy-config` flag is only required to override the default configuration. By default, executing `vllm serve MODEL --omni ...`
   automatically loads the bundled deployment configuration.
-- For custom legacy `stage_args` YAMLs, pass the file with `--stage-configs-path`.
 
 Example: Initializing Stage 0 (Orchestrator and API Server):
 The commands below show a common device mapping where Stage 0 uses GPU 0 and
@@ -32,7 +31,7 @@ CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --omni-master-port 26000
 ```
 
-When utilizing a custom deployment YAML based on the new schema, append `--deploy-config /path/to/override.yaml` to each command execution. Conversely, for legacy models, substitute this parameter with `--stage-configs-path /path/to/stage_configs.yaml`.
+When utilizing a custom deployment YAML, append `--deploy-config /path/to/override.yaml` to each command execution.
 
 In the standard execution paradigm, the `--stage-overrides` argument is utilized to apply stage-specific configurations from a single CLI command.
 However, under the **stage-based CLI** paradigm, where each process strictly encapsulates a single stage, it is recommended to specify tuning parameters directly via discrete command-line flags for the respective stage, rather than constructing a composite `--stage-overrides` JSON string.

docs/community/governance.md

@@ -40,6 +40,7 @@ Committers have write access and merge rights. They typically have deep expertis
 - [@Isotr0py](https://github.com/Isotr0py): Diffusion and Quantization
 - [@linyueqian](https://github.com/linyueqian): TTS and Omni Support
 - [@lishunyang12](https://github.com/lishunyang12): Quantization and Configuration
+- [@NickCao](https://github.com/NickCao): CI, Tests, Entrypoint, and TTS Model Support
 - [@princepride](https://github.com/princepride): Diffusion and Omni Support
 - [@RuixiangMa](https://github.com/RuixiangMa): Diffusion models, parallel, cache, and docs
 - [@SamitHuang](https://github.com/SamitHuang): RL, Diffusion, and cache

docs/configuration/README.md

@@ -2,7 +2,7 @@
 
 This section lists the most common options for running vLLM-Omni.
 
-For options within a vLLM Engine. Please refer to [vLLM Configuration](https://docs.vllm.ai/en/v0.16.0/configuration/index.html)
+For options within a vLLM Engine, please refer to the [vLLM 0.26 configuration guide](https://docs.vllm.ai/en/v0.26.0/configuration/index.html).
 
 Currently, the main options are maintained by stage configs for each model.
 

docs/configuration/composable_parallel.md

@@ -34,12 +34,8 @@ Two additional `StrategySpec` slots — `layer_hook_specs` and `kernel_specs`
 | Flag | Description |
 |------|-------------|
 | `--strategy-config PATH` | Path to a `strategy.yaml`. Only takes effect on registry-based deploy paths (`--deploy-config` or the bundled default deploy YAML). Its derived sizing is overlaid onto the merged stages *before* CLI overrides. |
-| `--stage-configs-path PATH` | **Legacy.** Loads the old `stage_args` YAML schema for un-migrated models. Passing `--strategy-config` together with this path is silently inapplicable — vLLM-Omni emits a warning and the strategy is ignored (see `vllm_omni/entrypoints/utils.py`, lines 372–380). |
 | `--omni-lb-policy POLICY` | Orchestrator-wide load-balancer policy. Strategy-derived from any `stage_replica` axis; an explicit CLI value wins and must match the derived one or `AsyncOmniEngine` raises `Conflicting load-balancer policy` at construction. |
 
-!!! warning
-    `--strategy-config` cannot be used with `--stage-configs-path` (the legacy deploy path). If your model has not been migrated to the registry-based pipeline yet, the strategy is silently dropped with a warning. Migrate to a registry-based model or use `--deploy-config` to apply a strategy.
-
 ## Sub-schemas
 
 ### `axis`
@@ -216,17 +212,6 @@ Strategy keys MUST be `model_stage` *names* (strings), the same labels stage con
 !!! warning "Mixing CLI overrides with a strategy on the same axis"
     A strategy is meant to be the single writer for the axes it declares. If a CLI override (`--tensor-parallel-size 2`, `--stage-overrides '{"0": {"tensor_parallel_size": 2}}'`) sets the same axis, the **CLI value wins** and a warning is emitted from `_reconcile_strategy_with_cli`. The device-layout guard then re-runs against the effective layout, so a CLI override that breaks `tp * dp * pp * num_replicas` will still fail fast at config time.
 
-!!! warning "Passing `--strategy-config` against the legacy `--stage-configs-path`"
-    Composable parallel only applies on the registry-based deploy path. If a model still resolves through the legacy `stage_args` YAML (loaded via `--stage-configs-path`), `--strategy-config` is silently inapplicable and emits:
-
-    ```text
-    --strategy-config (<path>) was provided but model '<model>' resolves via the
-    legacy stage_configs YAML path, which does not support composable-parallel
-    strategies; the strategy is ignored. Use a registry-based model to apply it.
-    ```
-
-    Migrate the model to the new deploy schema (see `configuration/stage_configs.md`) before applying a strategy.
-
 !!! warning "Conflicting `omni_lb_policy` across stages"
     `omni_lb_policy` is a single pipeline-wide knob. If two `stage_replica` axes in different stages derive different policies, `apply_strategy_specs` raises:
 

docs/configuration/pd_disaggregation.md

@@ -158,7 +158,7 @@ runtime:
 
 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
-    --stage-configs-path /path/to/qwen3_omni_pd.yaml
+    --deploy-config /path/to/qwen3_omni_pd.yaml
 ```
 
 ## Operational Notes

docs/configuration/stage_configs.md

@@ -1,9 +1,9 @@
-# Deploy and legacy stage configurations
+# Pipeline and deploy configurations
 
-In vLLM-Omni, a model's `PipelineConfig` defines its fixed stage topology, while a deploy configuration controls how those stages run. Models that have not migrated to this split still use legacy stage configurations with a `stage_args` schema.
+In vLLM-Omni, a model's `PipelineConfig` defines its fixed stage topology, while a deploy configuration controls how those stages run.
 
 !!! note
-    Default deploy config YAMLs (for example, `vllm_omni/deploy/qwen2_5_omni.yaml`, `vllm_omni/deploy/qwen3_omni_moe.yaml`, and `vllm_omni/deploy/qwen3_tts.yaml`) are bundled and loaded automatically when neither `--stage-configs-path` nor `--deploy-config` is provided — the model registry resolves the right pipeline + deploy YAML by `model_type`. Custom legacy `stage_args` YAMLs are still accepted via `--stage-configs-path`.
+    Default deploy config YAMLs (for example, `vllm_omni/deploy/qwen2_5_omni.yaml`, `vllm_omni/deploy/qwen3_omni_moe.yaml`, and `vllm_omni/deploy/qwen3_tts.yaml`) are bundled and loaded automatically when `--deploy-config` is omitted. The model registry resolves the right pipeline and deploy YAML by `model_type`.
 
 ## Deploy configuration schema
 
@@ -86,10 +86,9 @@ stages:
 
 | Flag | Description |
 |------|-------------|
-| `--deploy-config PATH` | Load a new-schema deploy YAML. It is mutually exclusive with `--stage-configs-path`. **Optional** — when omitted, the bundled `vllm_omni/deploy/<model_type>.yaml` is auto-loaded by the model registry. |
+| `--deploy-config PATH` | Load a deploy YAML. **Optional** — when omitted, the bundled `vllm_omni/deploy/<model_type>.yaml` is auto-loaded by the model registry. |
 | `--stage-overrides JSON` | Per-stage JSON overrides, e.g. `'{"0":{"gpu_memory_utilization":0.5}}'`. Per-stage values always win over global flags. |
 | `--async-chunk` / `--no-async-chunk` | Flip the deploy YAML's `async_chunk:` bool. Unset (default) leaves the YAML value in force. |
-| `--stage-configs-path` | **Deprecated.** Load a legacy `stage_args` YAML. New-schema YAMLs passed to this flag are auto-detected for compatibility, but should use `--deploy-config`. It is mutually exclusive with `--deploy-config` and will be removed in a future release. |
 
 ### Stage-Based CLI Paradigm
 
@@ -119,8 +118,7 @@ CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --omni-master-port 26000
 ```
 
-When instantiating a custom deployment YAML conforming to the updated schema, append the `--deploy-config /path/to/override.yaml` directive
-to all node invocations. For legacy architectures (e.g., BAGEL) configured via deprecated `stage_args:` schemas, continue to specify the relevant configuration via `--stage-configs-path /path/to/config.yaml`.
+When instantiating a custom deployment YAML, append the `--deploy-config /path/to/override.yaml` directive to all node invocations.
 
 In the context of standard initialization architectures, utilizing the `--stage-overrides` parameter operates as the optimal methodology
 for delineating stage-specific tuning from the CLI interface:
@@ -220,8 +218,7 @@ Therefore, as a core part of vLLM-Omni, a model's pipeline and deployment config
 
 To override specific parameters, explicitly inject the customized configuration schema
 in both online and offline instantiation flows. Use the `--deploy-config` flag
-when loading a deploy configuration, reserving the `--stage-configs-path` parameter
-exclusively to maintain compatibility with legacy `stage_args` YAML constructs.
+when loading a deploy configuration.
 
 Examples:
 
@@ -236,11 +233,6 @@ For online serving:
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/deploy_config.yaml
 ```
 
-Legacy online serving:
-
-```bash
-vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
-```
 !!! important
     We are actively iterating on the definition of deployment configurations, and we welcome feedback from users and developers.
 
@@ -395,8 +387,8 @@ that support request-level batching with `step_execution` disabled.
 
 Use this together with `max_num_seqs > 1` for bursty serving traffic. `0`
 disables admission waiting and preserves the lowest first-request latency.
-For diffusion request-level batching tuning, see
-[Request-Level Batching](../user_guide/diffusion/request_batching.md).
+For diffusion execution and batching tuning, see
+[Diffusion Execution Modes](../user_guide/diffusion/execution_modes.md).
 
 Default: `0.0`
 

docs/contributing/ci/test_examples/l4_functionality_tests.inc.md

@@ -1,6 +1,6 @@
 **Scope**
 
-For diffusion models, the L4 functionality test covers all or common *diffusion features* that are supported by this model, including several [parallelism acceleration methods](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/parallelism_acceleration/), [CPU offloading](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/cpu_offload_diffusion/), [TeaCache](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/teacache/) and [Cache-DiT](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/cache_dit_acceleration/) cache backends, [quantization methods](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/quantization/overview/).
+For diffusion models, the L4 functionality test covers all or common *diffusion features* that are supported by this model, including several [parallelism acceleration methods](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/parallelism_acceleration/), [CPU offloading](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/cpu_offload/), [TeaCache](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/teacache/) and [Cache-DiT](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/cache_dit_acceleration/) cache backends, [quantization methods](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/quantization/overview/).
 
 **Test Case Design**
 

docs/contributing/ci/test_examples/l4_performance_tests.inc.md

@@ -44,11 +44,13 @@ Pass **`--test-config-file`** to run one JSON file as-is, or omit it for the bul
             "ignore_eos": true,
             "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
             "baseline": {
+                "H100": {
                     "mean_ttft_ms": [500, 800],
                     "mean_audio_ttfp_ms": [2000, 3500],
                     "mean_audio_rtf": [0.25, 0.35]
                 }
             }
+        }
     ]
 }
 ```
@@ -142,10 +144,6 @@ pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py \
   --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json
 pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \
   --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
-
-# Optional baseline assertion (default off)
-pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py --assert-baseline \
-  --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
 ```
 
 See also [Markers for Tests](#markers-for-tests) for registered hardware markers (`H100`, `L4`, `cuda`, `distributed_cuda`, …).
@@ -202,4 +200,4 @@ You can add any benchmark running parameters you need here. For all optional par
 | num_prompts     | int / array | Yes      | 10,[10, 20, 30] | Number of requests. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of qps or max_concurrency, e.g., [10,10,10]. If an array is used, its length must match the number of qps or max_concurrency. |
 | request_rate    | float / array | No  | 0.5, [0.5, 1, inf] | Queries per second. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of num_prompts, e.g., [1,1,1]. If an array is used, its length must match the number of num_prompts.                          |
 | max_concurrency | int / array | No       | 1, [1, 2, 3]    | Maximum concurrent in-flight requests. Same array / expansion rules as `request_rate` (mutually exclusive with QPS mode).                                                                                                                                                                                                             |
-| baseline        | object      | No       | see above       | Optional per-metric thresholds; keys must match benchmark output fields. Scalar, list (per sweep step), or object (keyed by concurrency or QPS string).  
+| baseline        | object      | No       | see below       | Optional thresholds. Prefer hardware-nested form: `{"H100": {"throughput_qps": …}, "A3": {…}}` (selected via runtime hardware label). Legacy flat `{metric: threshold}` still works. Per metric: scalar, list (sweep step), or object keyed by concurrency / QPS.

docs/contributing/ci/test_system_overview.md

@@ -56,7 +56,11 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
       <td>/</td>
       <td><code>core_model and cpu</code></td>
       <td rowspan="2">&lt;15min</td>
-      <td>/tests/{component_name}/test_xxx</td>
+      <td>
+        <code>tests/{component}/…</code> mirroring <code>vllm_omni/{component}/</code><br>
+        (e.g. <code>tests/diffusion/</code>, <code>tests/engine/</code>, <code>tests/entrypoints/</code>)<br>
+        Do <strong>not</strong> add new top-level <code>tests/</code> dirs unrelated to a component.
+      </td>
       <td>
         <a href="test_writing_guide.md#l1--l2-level-testing-unit-testing-and-basic-end-to-end-verification">L1 &amp; L2</a><br>
         Section 1 L1&amp;L2: Purpose, Test Content, Directory Location, Example
@@ -68,16 +72,18 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
       <td><strong>L2</strong><br>(E2E across models & GPU-required UT)</td>
       <td>Online (basic deployment scenarios):<br>dummy, normal inference function (output format, stream), some instance startup UT</td>
       <td>High-priority models + online basic scenarios; request success, non-empty output, format match (no Whisper/accuracy)</td>
-      <td>High-priority features (using random lightweight models)</td>
+      <td>High-priority features and component tests that require launching instances</td>
       <td>High-priority interfaces (using random lightweight models)</td>
       <td><code>core_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
       <td>
-        <strong>Model tests:</strong><br>
-        /tests/e2e/online_serving/test_{model_name}.py<br>
-        <strong>Feature tests:</strong><br>
-        /tests/{component_name}/test_xxx<br>
-        <strong>Interface tests:</strong><br>
-        /tests/entrypoints/test_xxx
+        <strong>Model E2E:</strong><br>
+        <code>tests/e2e/online_serving/test_{model}.py</code><br>
+        <code>tests/e2e/offline_inference/test_{model}.py</code><br>
+        <strong>Feature integration:</strong><br>
+        <code>tests/e2e/features/&lt;feature&gt;/</code><br>
+        (e.g. <code>fullduplex/</code>, <code>custom_pipeline/</code>, <code>rlhf_test/</code>)<br>
+        <strong>Component / interface Test (GPU):</strong><br>
+        <code>tests/{component}/…</code>, <code>tests/entrypoints/…</code>
       </td>
       <td>
         <a href="test_writing_guide.md#l1--l2-level-testing-unit-testing-and-basic-end-to-end-verification">L1 &amp; L2</a><br>
@@ -90,18 +96,19 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
       <td><strong>L3</strong><br>(Important Perf & Integration & Accuracy)</td>
       <td>Online & Offline (multiple deployment scenarios):<br>real model, normal inference function, normal accuracy</td>
       <td>High/medium-priority models + key online/offline scenarios; real weights, Whisper/similarity, preset voice gender, basic accuracy</td>
-      <td>Medium-priority features (using random lightweight models)</td>
+      <td>Medium-priority features and component tests that require launching instances</td>
       <td>Medium-priority interfaces (using random lightweight models)</td>
       <td><code>advanced_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
       <td>&lt;30min</td>
       <td>
-        <strong>Model tests:</strong><br>
-        /tests/e2e/online_serving/test_{model_name}.py<br>
-        /tests/e2e/offline_inference/test_{model_name}.py<br>
-        <strong>Feature tests:</strong><br>
-        /tests/{component_name}/test_xxx<br>
-        <strong>Interface tests:</strong><br>
-        /tests/entrypoints/test_xxx
+        <strong>Model E2E:</strong><br>
+        <code>tests/e2e/online_serving/test_{model}.py</code><br>
+        <code>tests/e2e/offline_inference/test_{model}.py</code><br>
+        <code>tests/e2e/accuracy/</code><br>
+        <strong>Feature integration:</strong><br>
+        <code>tests/e2e/features/&lt;feature&gt;/</code><br>
+        <strong>Component / interface Test:</strong><br>
+        <code>tests/{component}/…</code>, <code>tests/entrypoints/…</code>
       </td>
       <td>
         <a href="test_writing_guide.md#l3-level-testing-core-integration-performance-and-accuracy-verification">L3</a><br>
@@ -114,26 +121,23 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
       <td><strong>L4</strong><br>(Perf & Integration & Accuracy)</td>
       <td>Online: full functional scenarios + performance test + doc test + accuracy test</td>
       <td>High-priority models: function, performance, accuracy, and doc testing<br>Medium-priority models: function and doc testing</td>
-      <td>Low-priority features (using real weights)</td>
+      <td>Low-priority features and component tests that require launching instances</td>
       <td>Low-priority interfaces (using real weights)</td>
       <td><code>full_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
       <td>&lt;3 hour</td>
       <td>
-        <strong>Model tests:</strong><br>
-        /tests/e2e/online_serving/test_{model_name}_expansion.py<br>
-        <strong>Feature tests:</strong><br>
-        /tests/{component_name}/test_xxx<br>
-        <strong>Interface tests:</strong><br>
-        /tests/entrypoints/test_xxx<br>
+        <strong>Model E2E:</strong><br>
+        <code>tests/e2e/online_serving/test_{model}_expansion.py</code><br>
+        <code>tests/e2e/offline_inference/test_{model}_expansion.py</code><br>
+        <code>tests/e2e/accuracy/test_{model}.py</code><br>
+        <strong>Feature integration:</strong><br>
+        <code>tests/e2e/features/&lt;feature&gt;/</code><br>
+        <strong>Component / interface Test:</strong><br>
+        <code>tests/{component}/…</code>, <code>tests/entrypoints/…</code><br>
         <strong>Performance:</strong><br>
-        /tests/dfx/perf/tests/test_qwen3_omni_*.json (Omni), test_tts.json (TTS),<br>
-        test_voxcpm2.json, test_higgs_audio_v3.json, and<br>
-        /tests/dfx/perf/tests/test_{diffusion_model}_vllm_omni.json (Diffusion)<br>
-        <strong>Doc Test:</strong><br>
-        tests/examples/online_serving/test_{model_name}.py<br>
-        tests/examples/offline_inference/test_{model_name}.py<br>
-        <strong>Accuracy Test:</strong><br>
-        /tests/e2e/accuracy/test_{model_name}.py
+        <code>tests/dfx/perf/tests/</code><br>
+        <strong>Doc examples:</strong><br>
+        <code>tests/examples/online_serving/</code>, <code>tests/examples/offline_inference/</code>
       </td>
       <td>
         <a href="test_writing_guide.md#l4-level-testing-full-functionality-performance-and-documentation-testing">L4</a><br>
@@ -152,11 +156,9 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
       <td> Depends on reality </td>
       <td>
         <strong>Stability:</strong><br>
-        /tests/dfx/stability/tests/test_qwen3_omni.json<br>
-        /tests/dfx/stability/tests/test_wan22.json<br>
+        <code>tests/dfx/stability/tests/</code><br>
         <strong>Reliability:</strong><br>
-        tests/dfx/reliability/test_reliability_{model_key}.py<br>
-        (e.g. <code>test_reliability_qwen3_omni.py</code>, <code>test_reliability_wan22.py</code>, <code>test_reliability_hunyuan_image.py</code>, <code>test_reliability_voxcpm2.py</code>)
+        <code>tests/dfx/reliability/test_reliability_{model}.py</code>
       </td>
       <td>
         <a href="test_writing_guide.md#l5-level-testing-stability-and-reliability-testing">L5</a><br>
@@ -168,7 +170,9 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
   </tbody>
 </table>
 
-For per-level test authoring (directories, markers, examples), see [Test Writing Guide](./test_writing_guide.md).
+**Test Dir placement (summary):** component / unit under `tests/{component}/` mirroring `vllm_omni/`; model E2E under `tests/e2e/online_serving/`, `tests/e2e/offline_inference/`, and `tests/e2e/accuracy/`; feature integration under `tests/e2e/features/<feature>/`; doc example tests under `tests/examples/online_serving/` and `tests/examples/offline_inference/`; performance under `tests/dfx/perf/`; stability under `tests/dfx/stability/`; reliability under `tests/dfx/reliability/`. Do **not** add new top-level directories under `tests/` that are unrelated to a `vllm_omni` component (or to the established `e2e` / `dfx` / `helpers` / `examples` / `buildkite` layout).
+
+For per-level test authoring (markers, examples), see [Test Writing Guide](./test_writing_guide.md).
 
 ## Common Specifications
 

docs/contributing/ci/test_writing_guide.md

@@ -4,137 +4,20 @@ This document describes how to author tests for each CI level (L1–L5). For the
 
 ## Test Directory Structure
 
-Legend: `✅` = test exists, `⬜` = suggested to add.
-```
-vllm_omni/                                    tests/
-├── config/                             →     ├── config/
-│   ├── model.py                              │   └── test_model.py                    ⬜
-│   └── lora.py                               │   └── test_lora.py                      ⬜
-│
-├── core/                               →     ├── core/
-│   └── sched/                                 │   └── sched/
-│       ├── omni_ar_scheduler.py               │       ├── test_omni_ar_scheduler.py    ⬜
-│       ├── omni_generation_scheduler.py       │       ├── test_omni_generation_scheduler.py  ⬜
-│       └── output.py                          │       └── test_output.py               ✅ currently in entrypoints/test_omni_new_request_data.py (tests output.OmniNewRequestData)
-│
-├── diffusion/                          →     ├── diffusion/
-│   ├── diffusion_engine.py                    │   ├── test_diffusion_engine.py          ⬜
-│   ├── attention/                             │   ├── attention/
-│   │   ├── layer.py                            │   │   ├── test_attention_sp.py         ✅
-│   │   └── backends/                           │   │   └── test_flash_attn.py           ✅
-│   ├── distributed/                           │   ├── distributed/
-│   │   └── ...                                 │   │   ├── test_comm.py                 ✅
-│   │                                            │   │   ├── test_cfg_parallel.py        ✅
-│   │                                            │   │   └── test_sp_plan_hooks.py       ✅
-│   ├── lora/                                   │   ├── lora/
-│   │   └── ...                                 │   │   ├── test_base_linear.py          ✅
-│   │                                            │   │   └── test_lora_manager.py        ✅
-│   ├── models/                                 │   ├── models/
-│   │   ├── qwen_image/                         │   │   ├── qwen_image/                 (e2e coverage)
-│   │   ├── ovis_image/                         │   │   ├── ovis_image/
-│   │   │   └── ...                             │   │   │   └── test_ovis_image.py     ✅
-│   │   ├── z_image/                            │   │   └── z_image/
-│   │   └── ...                                 │   │       └── test_zimage_tp_constraints.py  ✅
-│   └── worker/                                 │   └── worker/
-│       ├── diffusion_worker.py                 │       └── test_diffusion_worker.py   ✅ file at diffusion/test_diffusion_worker.py
-│       └── diffusion_model_runner.py            │
-│
-├── distributed/                         →     ├── distributed/
-│   └── omni_connectors/                         │   └── omni_connectors/
-│       ├── adapter.py                           │       ├── test_adapter_and_flow.py   ✅
-│       ├── kv_transfer_manager.py               │       ├── test_basic_connectors.py   ✅
-│       ├── connectors/                           │       ├── test_kv_flow.py             ✅
-│       └── utils/                               │       └── test_omni_connector_configs.py  ✅
-│
-├── engine/                             →     ├── engine/
-│   ├── input_processor.py                      │   ├── test_input_processor.py         ⬜  (no processor.py in source)
-│   ├── output_processor.py                     │   └── test_output_processor.py         ⬜
-│   └── arg_utils.py                            │   └── test_arg_utils.py               ⬜
-│
-├── entrypoints/                        →     ├── entrypoints/
-│   ├── stage_utils.py                          │   ├── test_stage_utils.py            ✅
-│   ├── cli/                                     │   ├── cli/                           (benchmarks/test_serve_cli.py covers CLI serve)
-│   │   └── ...                                  │   │   └── test_*.py                  ⬜
-│   └── openai/                                  │   └── openai_api/                    # maps to entrypoints/openai/
-│       ├── api_server.py                        │       ├── test_api_server.py         ⬜  (e2e indirect coverage)
-│       ├── serving_chat.py                       │       ├── test_serving_chat_sampling_params.py  ✅
-│       ├── serving_speech.py                     │       ├── test_serving_speech.py     ✅
-│       └── image_api_utils.py                   │       └── test_image_server.py      ✅
-│
-├── inputs/                             →     ├── inputs/
-│   ├── data.py                                 │   ├── test_data.py                   ⬜
-│   ├── parse.py                                │   ├── test_parse.py                 ⬜
-│   └── preprocess.py                            │   └── test_preprocess.py            ✅ currently in entrypoints/test_omni_input_preprocessor.py
-│
-├── model_executor/                     →     ├── model_executor/
-│   ├── layers/                                  │   ├── layers/
-│   │   └── mrope.py                             │   │   └── test_mrope.py              ⬜
-│   ├── model_loader/                            │   ├── model_loader/
-│   │   └── weight_utils.py                      │   │   └── test_weight_utils.py      ⬜
-│   ├── models/                                  │   ├── models/
-│   │   ├── qwen2_5_omni/                         │   │   ├── qwen2_5_omni/
-│   │   │   ├── qwen2_5_omni_thinker.py           │   │   │   ├── test_audio_length.py  ✅
-│   │   │   ├── qwen2_5_omni_talker.py            │   │   │   ├── test_qwen2_5_omni_thinker.py  ⬜
-│   │   │   └── qwen2_5_omni_token2wav.py         │   │   │   ├── test_qwen2_5_omni_talker.py  ⬜
-│   │   └── qwen3_omni/                          │   │   │   └── test_qwen2_5_omni_token2wav.py  ⬜
-│   │       └── ...                               │   │   └── qwen3_omni/
-│   ├── stage_configs/                           │   │       └── test_*.py              ⬜
-│   │   └── *.yaml                               │   └── stage_configs/                 (used by e2e, test_*.py can be added)  ⬜
-│   └── stage_input_processors/                  │   └── stage_input_processors/
-│       └── ...                                  │       └── test_*.py                 ⬜
-│
-├── sample/                             →     ├── sample/
-│   └── __init__.py                             │   └── test_*.py                      ⬜
-│
-├── utils/                              →     ├── utils/
-│   └── __init__.py                             │   └── test_*.py                       ⬜  (no platform_utils.py currently)
-│
-├── worker/                             →     ├── worker/
-│   ├── gpu_ar_model_runner.py                  │   ├── test_gpu_ar_model_runner.py    ⬜
-│   ├── gpu_ar_worker.py                        │   ├── test_gpu_ar_worker.py           ⬜
-│   ├── gpu_generation_model_runner.py          │   ├── test_gpu_generation_model_runner.py  ✅
-│   ├── gpu_generation_worker.py                │   ├── test_gpu_generation_worker.py  ⬜
-│   ├── gpu_model_runner.py                     │   ├── test_omni_gpu_model_runner.py   ✅
-│   └── mixins.py                               │   └── (npu under platforms/npu/worker/)  # not worker/npu/
-│
-├── platforms/                          →     (no tests/platforms/, e2e and stage_configs provide indirect coverage)
-│   ├── cuda/
-│   ├── npu/worker/                             # NPU worker here, not vllm_omni/worker/npu/
-│   ├── rocm/
-│   └── xpu/worker/
-│
-├── outputs.py                          →     test_outputs.py                         ✅ (at tests root)
-├── (logger, patch, request, version)    →     (no corresponding unit test)
-│
-└── e2e (tests side only)               →     ├── e2e/
-                                               ├── online_serving/                     ✅ non-empty
-                                               │   ├── test_qwen2_5_omni.py
-                                               │   ├── test_async_omni.py
-                                               │   ├── test_qwen3_omni.py
-                                               │   ├── test_qwen3_omni_expansion.py
-                                               │   ├── test_mimo_audio.py
-                                               │   └── test_images_generations_lora.py
-                                               └── offline_inference/                  ✅
-                                                   ├── test_qwen2_5_omni.py
-                                                   ├── test_qwen3_omni.py
-                                                   ├── test_bagel_text2img.py
-                                                   ├── test_z_image.py
-                                                   ├── test_wan22.py
-                                                   ├── test_zimage_tensor_parallel.py
-                                                   ├── test_cache_dit.py
-                                                   ├── test_teacache.py
-                                                   ├── test_stable_audio_expansion.py
-                                                   ├── test_diffusion_cpu_offload.py
-                                                   ├── test_diffusion_layerwise_offload.py
-                                                   ├── test_diffusion_lora.py
-                                                   ├── test_sequence_parallel.py
-                                                   └── stage_configs/                  (legacy schema, still
-                                                       ├── bagel_*.yaml                 present for unmigrated
-                                                       └── npu/, rocm/, etc.            models)
+Canonical placement by CI level is in the **Test Dir** column of the level matrix in [Test System Overview](./test_system_overview.md).
 
-# Migrated models (qwen3_omni_moe, qwen2_5_omni, qwen3_tts) live under
-# vllm_omni/deploy/ instead — see docs/configuration/stage_configs.md.
-```
+Summary:
+
+- **Component / unit:** `tests/{component}/…` mirroring `vllm_omni/{component}/`
+- **Model E2E:** `tests/e2e/online_serving/`, `tests/e2e/offline_inference/`, `tests/e2e/accuracy/`
+- **Feature integration:** `tests/e2e/features/<feature>/`
+- **Doc example tests:** `tests/examples/online_serving/`, `tests/examples/offline_inference/`
+- **Performance:** `tests/dfx/perf/tests/` (JSON configs; runners under `tests/dfx/perf/scripts/`)
+- **Stability:** `tests/dfx/stability/tests/`
+- **Reliability:** `tests/dfx/reliability/`
+- Do **not** add new top-level `tests/` directories unrelated to a component (or to `e2e` / `dfx` / `helpers` / `examples` / `buildkite`)
+
+Deploy YAMLs: `vllm_omni/deploy/` via `tests.helpers.stage_config.get_deploy_config_path` (see [stage configs](../../configuration/stage_configs.md)).
 
 ## Markers for Tests
 
@@ -373,13 +256,13 @@ L2 level testing builds upon L1 by introducing GPU resources and verifying that
 
 #### 1.3 Test Directory and Execution Files
 
-A clear directory structure is key to managing test cases efficiently.
+A clear directory structure is key to managing test cases efficiently. See the **Test Dir** column in [Test System Overview](./test_system_overview.md) for the canonical placement rules.
 
--   ***L1 Test Directory***: `/tests/{component_name}/test_xxx.py`
--   -   Here, `{component_name}` corresponds to modules in the source code, such as `distributed`, `entrypoints`, etc., and `test_xxx.py` is the specific test file.
--   ***L2 Test Directory***:
--   -   Online Serving: `/tests/e2e/online_serving/test_{model_name}.py`
-    -   Offline Inference: `/tests/e2e/offline_inference/test_{model_name}.py`
+-   ***L1 (component / unit)***: `tests/{component_name}/…` mirroring `vllm_omni/{component_name}/` (e.g. `tests/diffusion/`, `tests/entrypoints/`). Do **not** invent new top-level `tests/` folders unrelated to a component.
+-   ***L2 (basic model E2E)***:
+    -   Online serving: `tests/e2e/online_serving/test_{model_name}.py`
+    -   Offline inference: `tests/e2e/offline_inference/test_{model_name}.py`
+-   ***Feature-level integration*** (cross-cutting, not a single-model smoke): `tests/e2e/features/<feature>/` (e.g. `fullduplex/`, `custom_pipeline/`, `rlhf_test/`).
 
 #### 1.4 Execution Method and Example
 
@@ -485,10 +368,12 @@ L3 level testing executes after code is merged into the main branch. Its core pu
 
 #### 2.3 Test Directory and Execution Files
 
--   ***Functional Testing***:
--   -   Online Serving: `/tests/e2e/online_serving/test_{model_name}_expansion.py`
-    -   Offline Inference: `/tests/e2e/offline_inference/test_{model_name}_expansion.py`
-    -   (Note: `_expansion.py` likely means it contains more comprehensive scenario cases compared to L2 tests).
+-   ***Functional / model E2E***:
+    -   Online serving: `tests/e2e/online_serving/test_{model_name}_expansion.py`
+    -   Offline inference: `tests/e2e/offline_inference/test_{model_name}_expansion.py`
+    -   Accuracy / similarity: `tests/e2e/accuracy/`
+    -   (`_expansion.py` holds broader scenario coverage than the L2 smoke file.)
+-   ***Feature-level integration***: `tests/e2e/features/<feature>/` (not under a new top-level `tests/` directory).
 
 #### 2.4 Execution Method and Example
 

docs/contributing/model/adding_diffusion_model.md

@@ -653,7 +653,7 @@ python end2end.py --model your-org/your-model-name --modality text2img --prompts
 Mirror BAGEL’s online serving setup:
 
 - Server launcher: `examples/online_serving/your_model_name/run_server.sh`
-  - Wrap `vllm serve ... --omni --port ...` (and `--stage-configs-path ...` if needed)
+  - Wrap `vllm serve ... --omni --port ...` (and `--deploy-config ...` if needed)
 - Client: `examples/online_serving/your_model_name/openai_chat_client.py`
   - Send requests to `POST /v1/chat/completions`
   - Support multimodal inputs (e.g., base64 image) if your model needs it
@@ -755,7 +755,8 @@ omni = Omni(model="your-model", ulysses_degree=2, ring_degree=2)
 
 ### Step Execution
 
-See detailed design guide: [How to add step execution support](../../design/feature/diffusion_step_execution.md)
+See the detailed
+[Diffusion Continuous Batching design guide](../../design/feature/diffusion_continuous_batching.md).
 
 Use this only when your pipeline can be split into stable request-scoped and
 step-scoped phases. The reference implementation is
@@ -769,9 +770,8 @@ step-scoped phases. The reference implementation is
 Do not enable `step_execution=True` until those four methods are implemented
 and validated against the request-level path.
 
-If you want the pipeline to work with the experimental batched step-wise path
-(`max_num_seqs > 1`), also see:
-[Continuous Batching for Step-Wise Diffusion](../../design/feature/diffusion_continuous_batching.md).
+The same design guide covers the experimental batched step-wise path used when
+`max_num_seqs > 1`.
 
 If you expose this in example scripts or recipes, keep it opt-in. Surface
 runtime features like `step_execution` as optional flags instead of silently
@@ -824,7 +824,7 @@ omni = Omni(model="your-model",
 
 ### CPU Offload
 
-See detailed guide: [CPU Offloading for Diffusion Models](../../user_guide/diffusion/cpu_offload_diffusion.md)
+See detailed guide: [CPU Offloading for Diffusion Models](../../user_guide/diffusion/cpu_offload.md)
 
 vLLM-Omni provides two offloading strategies to reduce GPU memory usage:
 

docs/contributing/model/adding_omni_model.md

@@ -48,8 +48,8 @@ vllm_omni/deploy/
 ```
 
 New in-tree models should register their `PipelineConfig` in
-`vllm_omni/config/pipeline_registry.py`; use `--stage-configs-path` only for
-custom legacy `stage_args` YAMLs.
+`vllm_omni/config/pipeline_registry.py`; use `--deploy-config` for custom
+deployment YAMLs.
 
 ## Step-by-Step Implementation
 

docs/contributing/model/adding_tts_model.md

@@ -449,9 +449,8 @@ Key configuration fields:
 !!! note
     New in-tree models should define frozen topology in `models/<model>/pipeline.py`,
     register it in `vllm_omni/config/pipeline_registry.py`, and put deployment
-    knobs in `vllm_omni/deploy/<model>.yaml`. Legacy `stage_args` YAMLs are still
-    accepted for custom configs via `--stage-configs-path`, but should not be used
-    for new bundled defaults.
+    knobs in `vllm_omni/deploy/<model>.yaml`. Use `--deploy-config` to load a
+    custom deployment YAML.
 
 ### Batch mode
 

docs/design/feature/async_diffusion_output.md

@@ -0,0 +1,174 @@
+# Async Diffusion Output
+
+## Table of Contents
+
+1. [Overview](#overview)
+2. [Performance](#performance)
+3. [Architecture](#architecture)
+4. [Model Coverage](#model-coverage)
+5. [Configuration](#configuration)
+6. [Related Files](#related-files)
+
+## Overview
+
+The async diffusion output feature moves the D2H (Device→Host) copy and SHM packing from the Worker main thread to a background daemon thread, so the GPU default stream can immediately start the next request's forward pass. This eliminates the GPU bubble caused by synchronous D2H/packing.
+
+**Core benefit**: the previous request's output D2H/packing overlaps with the next request's forward pass.
+
+**Automatically enabled** when `step_execution=False` (default, request-mode). No extra configuration needed. When `step_execution=True` (step-mode), the original synchronous path is used — pump thread and Worker background thread are not started.
+
+## Performance
+
+### HunyuanImage-3.0 (TP4)
+
+| Resolution | Async Off (QPS) | Async On (QPS) | Change |
+|---|---|---|---|
+| 1024×1024 | 0.4773 | 0.4802 | +0.60% |
+| 768×768 | 0.8370 | 0.8533 | +1.95% |
+
+**Analysis**: smaller resolution → larger QPS gain. D2H/packing is relatively fixed overhead; forward is variable (scales with resolution). Smaller images → shorter forward → D2H/packing occupies a larger fraction of total time → more benefit from overlapping it.
+
+## Architecture
+
+### Execution Timeline
+
+```
+Before (synchronous D2H):
+[forward req1] [D2H+SHM pack req1] [forward req2] [D2H+SHM pack req2]
+                                 ^^^^^^^^^^^^^^^^ GPU bubble
+
+After (async D2H):
+[forward req1] [forward req2]
+[D2H+SHM pack req1 (side stream)] [D2H+SHM pack req2 (side stream)]
+                                 ^^^^^^^^^^^^^^^^ bubble eliminated
+```
+
+### Data Flow
+
+```
+                    ┌─────────────────────────────────────────────────────┐
+                    │                 Worker Process                      │
+                    │                                                    │
+  execute_model ──> │ WorkerBusyLoop ──> forward() ──> DiffusionOutput   │
+                    │       │                              │ (GPU tensor)│
+                    │       │                    ┌─────────┴──────────┐  │
+                    │       │                    │  async output path │  │
+                    │       │                    │                    │  │
+                    │       │              compute_done ──> result_mq │  │
+                    │       │                    │                    │  │
+                    │       │                    ▼                    │  │
+                    │       │            AsyncOutputThread            │  │
+                    │       │         (side CUDA stream)             │  │
+                    │       │         wait_event(gpu_event)          │  │
+                    │       │         pack_diffusion_output_shm()    │  │
+                    │       │         D2H + SHM write                │  │
+                    │       │                    │                    │  │
+                    │       │              output_ready ──> result_mq │  │
+                    │       │                    │                    │  │
+                    │       ▼                    ▼                    │  │
+                    │  (dequeue next request)                        │  │
+                    └─────────────────────────────────────────────────────┘
+                                         │
+                                         ▼
+                    ┌─────────────────────────────────────────────────────┐
+                    │              Executor (main process)               │
+                    │                                                    │
+                    │  ResultPumpThread (sole reader of result_mq)       │
+                    │       │                                            │
+                    │       ├── AsyncDiffusionOutput(COMPUTE_DONE)       │
+                    │       │   → resolve _rpc_futures[rpc_id]           │
+                    │       │   → RunnerOutput(async_output_id=...)      │
+                    │       │                                            │
+                    │       ├── AsyncDiffusionOutput(OUTPUT_READY)       │
+                    │       │   → resolve _output_futures[async_output_id]│
+                    │       │   → or batch split via _batch_split_map   │
+                    │       │                                            │
+                    │       └── Non-async message                        │
+                    │           → _sync_result_buffer (for other RPCs)   │
+                    │                                                    │
+                    │  wait_output_ready(async_output_id) → Future       │
+                    └─────────────────────────────────────────────────────┘
+```
+
+### Key Components
+
+1. **AsyncDiffusionOutput** (`data.py`): Protocol envelope for `result_mq`. `kind` field routes messages:
+   - `COMPUTE_DONE` — forward finished, GPU can start next request
+   - `OUTPUT_READY` — D2H/SHM packing finished, final output available
+   - `RPC_RESULT` — ordinary RPC return (including error propagation)
+
+2. **Worker AsyncOutputThread** (`diffusion_worker.py`): Background daemon thread that:
+   - Waits for `gpu_event` (cross-stream sync ensuring forward wrote the tensor)
+   - Runs `pack_diffusion_output_shm()` on a side CUDA stream
+   - Enqueues `OUTPUT_READY` to `result_mq` when packing completes
+
+3. **ResultPumpThread** (`multiproc_executor.py`): Sole reader of `result_mq` in request-mode. Dispatches messages:
+   - `COMPUTE_DONE` / `RPC_RESULT` → resolves `_rpc_futures[rpc_id]`
+   - `OUTPUT_READY` → resolves `_output_futures[async_output_id]` (with batch split via `_batch_split_map`)
+   - Non-async messages → `_sync_result_buffer` for other RPCs
+
+4. **collective_rpc() two-path dispatch** (`multiproc_executor.py`):
+   - **Path 1**: `execute_model` / `execute_model_batch` → generates `rpc_id`, registers Future, waits for pump to deliver `compute_done`
+   - **Path 2**: All other RPCs → `_sync_result_buffer` (pump-fed) or `_result_mq` (step-mode, no pump)
+
+5. **record_device_event** (`platforms/`): Cross-platform GPU event recording for side-stream synchronization. Implemented for CUDA, NPU, ROCm, XPU, MUSA; base class returns `None` (safe no-op).
+
+6. **IPC side-stream D2H** (`ipc.py`): `pack_diffusion_output_shm()` and helpers accept `d2h_stream` parameter; use `pin_memory` + `copy_(non_blocking=True)` on the side stream instead of synchronous `.cpu()`.
+
+### Batch Split
+
+When `execute_model_batch` returns a `COMPUTE_DONE` with a single `async_output_id` for the entire batch, the executor splits it into per-request `async_output_id`s (format `{batch_id}/{request_id}`) via `_batch_split_map`. When `OUTPUT_READY` arrives, the pump extracts per-request results from the batch output and resolves each request's output future independently.
+
+## Model Coverage
+
+### Currently Supported (request-mode, `step_execution=False`)
+
+All models running in request-mode (`step_execution=False`, the default) automatically use async output.
+
+**Verified models**:
+
+| Model | Type | `supports_request_batch` |
+|---|---|---|
+| **HunyuanImage-3.0** | Image | `False` |
+| **Qwen-Image** | Image | `True` |
+
+Other models with `step_execution=False` are also supported but not yet verified.
+
+### Not Yet Supported (step-mode, `step_execution=True`)
+
+When `step_execution=True` (or `streaming_output=True`, which auto-enables step-mode), models use `execute_stepwise` instead of `execute_model`, which is not in the async Path 1 whitelist. Async output is not applicable.
+
+**Verified**: Helios (`step_execution=True`, does not benefit from this feature).
+
+Other models with `step_execution=True` are also not applicable.
+
+Adapting step-mode requires additional design because:
+
+- Each step's `RunnerOutput` must be available synchronously for the scheduler to advance
+- Intermediate latent tensors may not need D2H (only the final step needs D2H + postprocess)
+- Steps are serial (step N+1 depends on step N output), so the overlap benefit is limited to "final step's D2H overlaps with the next request's forward"
+
+**Planned approach**: first cover only the final step's async D2H, keep intermediate steps synchronous. This matches the request-mode benefit pattern.
+
+## Configuration
+
+No configuration needed. Async output is automatically enabled when `step_execution=False` (default).
+
+| `step_execution` | Mode | Async Output | Pump Thread | Worker BG Thread |
+|---|---|---|---|---|
+| `False` (default) | request-mode | ✅ enabled | ✅ started | ✅ started |
+| `True` | step-mode | ❌ disabled | ❌ not started | ❌ not started |
+
+## Related Files
+
+- `vllm_omni/diffusion/data.py`: `AsyncDiffusionOutput`, `AsyncOutputKind`, `DiffusionOutput.async_output_id`
+- `vllm_omni/diffusion/worker/diffusion_worker.py`: `WorkerProc._return_result()`, `_async_output_loop()`, `_generate_async_output_id()`
+- `vllm_omni/diffusion/executor/multiproc_executor.py`: `ResultPumpThread`, `collective_rpc()` two-path dispatch, `wait_output_ready()`, `_batch_split_map`
+- `vllm_omni/diffusion/diffusion_engine.py`: `step()` / `step_streaming()` / `add_req_and_wait_for_response()` async output waiting
+- `vllm_omni/diffusion/sched/request_scheduler.py`: `update_from_output()` — `async_output_id` → `FINISHED_COMPLETED`
+- `vllm_omni/diffusion/ipc.py`: `pack_diffusion_output_shm()` side-stream D2H path
+- `vllm_omni/platforms/interface.py`: `OmniPlatform.record_device_event()` base class
+- `vllm_omni/platforms/cuda/platform.py`: `CudaOmniPlatform.record_device_event()`
+- `vllm_omni/platforms/npu/platform.py`: `NPUOmniPlatform.record_device_event()`
+- `vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py`: `get_hunyuan_image3_post_process_func()`, postprocess removal
+- `vllm_omni/diffusion/registry.py`: `_DIFFUSION_POST_PROCESS_FUNCS` postprocess registry

docs/design/feature/diffusion_continuous_batching.md

@@ -1,165 +1,232 @@
-# Continuous Batching for Step-Wise Diffusion
+# Diffusion Continuous Batching
 
-!!! warning "Experimental Feature"
-    This feature is experimental. It currently applies only to native
-    diffusion pipelines running with `step_execution=True`.
-
-This document describes the batching extension built on top of
-[Diffusion Step Execution](diffusion_step_execution.md). The base
-step-execution contract is unchanged. The batching work is mainly in the
-scheduler and runner layers.
-
-## Why It Helps
-
-Step-wise execution breaks a long denoise loop into scheduler-visible units.
-That gives the runtime a place to admit other compatible requests between
-steps instead of waiting for an entire request to finish.
-
-This matters most in low-MFU or bursty serving scenarios:
-
-- one request's denoise step may not fully saturate the GPU
-- several compatible requests can share the same denoise forward pass
-- throughput and device utilization can improve without changing request-local
-  scheduler state
-
-This is **not** a guaranteed single-request latency win. The main benefit is
-usually higher utilization and better throughput when the workload contains
-multiple in-flight compatible requests.
+This document describes the unified diffusion execution architecture, including
+request execution, request-level batching, step execution, continuous batching,
+and output delivery. For user-facing configuration and CLI examples, see
+[Diffusion Execution Modes](../../user_guide/diffusion/execution_modes.md).
 
 ## Overview
 
-With continuous batching enabled:
+`DiffusionEngine` selects an execution mode from `step_execution` and uses
+`max_num_seqs` as that mode's scheduler capacity:
 
-- the scheduler may keep multiple compatible requests active at the same time
-- the runner packs request-local step state into one `InputBatch`
-- `denoise_step()` runs on that batch
-- `step_scheduler()` and `post_decode()` still run per request
+| Configuration | Engine mode | Scheduler | Execution |
+|---|---|---|---|
+| `step_execution=False`, `max_num_seqs=1` | `REQUEST_BATCH` | `RequestScheduler` | One complete request-level `forward()` |
+| `step_execution=False`, `max_num_seqs>1` | `REQUEST_BATCH` | `RequestScheduler` | One fused `forward()` over compatible requests |
+| `step_execution=True`, `max_num_seqs=1` | `STEP_BATCH` | `StepScheduler` | One request advanced one denoise step per scheduler tick |
+| `step_execution=True`, `max_num_seqs>1` | `STEP_BATCH` | `StepScheduler` | Compatible requests advanced together in step waves |
 
-The current implementation is conservative:
+Serial request execution is handled inside `REQUEST_BATCH`; it is not a
+separate engine mode. Similarly, continuous batching is the multi-request
+configuration of `STEP_BATCH`, not an independent execution mode.
 
-- only compatible requests are batched together
-- per-request progress and completion remain independent
+The engine performs the following initialization:
 
-Here, "continuous batching" means the step-wise path enabled by
-`step_execution=True`. Request-mode `DiffusionRequestBatch` is static
-request-level batching for one full pipeline `forward()` call; it does not
-admit or remove requests between denoise steps.
+1. Resolve model-specific pre- and post-processing hooks.
+2. Select `REQUEST_BATCH` or `STEP_BATCH`.
+3. Construct the configured `DiffusionExecutor`.
+4. Construct `RequestScheduler` or `StepScheduler`, unless a scheduler was
+   explicitly injected.
+5. Bind `execute_batch` or `execute_step`.
 
-## Enablement
+`DiffusionEngine.make_engine()` owns startup warmup after engine construction.
+This keeps construction, backend selection, and warmup failure cleanup in one
+lifecycle.
 
-Use `--step-execution` as the feature gate, then increase `--max-num-seqs`
-above `1` if you want batching:
+## Unified Output Stream
 
-```bash
-vllm serve Qwen/Qwen-Image --omni \
-  --port 8091 \
-  --step-execution \
-  --max-num-seqs 8
+All asynchronous diffusion requests use the same per-request output stream,
+regardless of whether the caller exposes streaming results to the user.
+
+```python
+async for output in engine.step_streaming(request):
+    ...
 ```
 
-`--max-num-seqs 1` keeps the step-wise path without enabling batching.
+When a request is admitted, the engine creates an
+`asyncio.Queue[DiffusionOutput]` keyed by its request ID. The busy loop sends
+both intermediate chunks and final results through that queue:
 
-For a reproducible replay flow using the bundled serving benchmark, see the
-Qwen-Image replay commands in
-[`benchmarks/diffusion/README.md`](gh-file:benchmarks/diffusion/README.md)
-and
-[`benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md`](gh-file:benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md).
+```text
+request
+  -> scheduler.add_request()
+  -> scheduler.schedule()
+  -> execute_batch() or execute_step()
+  -> scheduler.update_from_output()
+  -> per-request output queue
+  -> step_streaming()
+```
 
-## Scheduler
+Streaming callers forward every yielded output. Non-streaming callers consume
+the same generator to completion and return only its final output. This avoids
+maintaining separate future-based and queue-based result paths.
 
-The scheduler derives its batch capacity from `max_num_seqs` through
-`max_num_running_reqs`.
+`DiffusionEngine.step()` and
+`async_add_req_and_wait_for_response()` remain deprecated compatibility
+wrappers. New integrations should consume `step_streaming()` or
+`async_add_req_and_stream_response()`.
 
-Batch admission is gated by
-[`StepBatchSamplingParamsKey`](gh-file:vllm_omni/diffusion/sched/interface.py),
-which is built from shape-sensitive and CFG-sensitive sampling fields. This is
-the core correctness rule for batching: requests are only co-batched when they
-share the same denoise tensor contract.
+### Completion and Cancellation
 
-There are three important details:
+Output delivery and scheduler lifecycle are related but separate:
 
-- `num_inference_steps` is not part of the key, so requests with different
-  total step counts can still share a batch
-- requests also do not need to be at the same current denoise progress; active
-  requests can continue batching even when their current step indices diverge
-- admission is still FIFO, so an incompatible request at the head of the
-  waiting queue blocks later compatible requests
+- A terminal scheduler result always finalizes request state.
+- A disconnected or cancelled consumer removes only its delivery queue.
+- The scheduler may still finish and clean up the request after its queue has
+  been removed.
+- Engine shutdown sends an error output to all remaining streams.
 
-Today that compatibility rule is still shape-sensitive. `height`, `width`,
-`num_frames`, and CFG-related fields remain part of the key, so different
-resolutions or incompatible guidance settings do **not** co-batch yet. The
-key also covers LoRA identity (`lora_int_id`, `lora_scale`), so requests
-targeting different adapters or scales run in separate batches and the
-worker can activate exactly one adapter per step.
+This separation prevents cancelled consumers from leaving active request IDs in
+the scheduler.
 
-The scheduler batching unit is one logical `OmniDiffusionRequest`. In the
-step-wise path, runtime tensor batching is represented as `StepInputBatch`. For
-request-mode prompt semantics, see
-[Request-Level Batching](../../user_guide/diffusion/request_batching.md).
+## Request-Batch Execution
 
-## Runner
+Request-batch execution runs the complete diffusion pipeline for each scheduler
+wave. Each `OmniDiffusionRequest` remains an independent logical request with
+its own prompt, sampling parameters, seed, request ID, output, error, and abort
+state.
 
-The runner keeps persistent per-request execution state in
-[`StepRequestState`](gh-file:vllm_omni/diffusion/worker/utils.py),
-while the scheduler owns a separate lightweight
-[`SchedulerRequestState`](gh-file:vllm_omni/diffusion/sched/interface.py)
-for queueing and lifecycle tracking.
+With `max_num_seqs=1`, `execute_batch()` handles the serial request path. With a
+larger value, a pipeline can opt into fused execution by declaring:
 
-For each step, the runner builds an
-[`InputBatch`](gh-file:vllm_omni/diffusion/worker/input_batch.py) from the
-active request states:
+```python
+supports_request_batch = True
+```
 
-- prompt embeddings and masks are normalized and padded
-- dynamic tensors such as `latents` and `timesteps` are gathered each step
-- buffers are reused when batch composition stays the same
+Its `forward()` method must accept `DiffusionRequestBatch` and return one
+`DiffusionOutput` per request. The runner validates the result count and maps
+outputs back to their original request IDs with `BatchRunnerOutput`.
 
-The step-wise batched path is:
+Current in-tree request-batch implementations include the Flux, LTX-2, SD3, and
+Qwen-Image pipelines. Treat the pipeline capability flag in the source as
+authoritative because support continues to expand.
 
-1. Run `prepare_encode()` for newly admitted requests.
+### Request-Batch Data Flow
+
+- `DiffusionSchedulerOutput` carries scheduled request IDs and payloads.
+- `DiffusionRequestBatch` provides the pipeline-facing static batch.
+- `BatchRunnerOutput` carries per-request results back to the engine.
+
+Request-local setup, such as seed handling and output/error mapping, remains
+per request. Cache refresh, homogeneous LoRA activation, and
+`pipeline.forward(req_batch)` can be shared by the batch.
+
+### Compatibility and Admission
+
+`RequestBatchSamplingParamsKey` contains fields that affect the tensor contract,
+including shape, guidance, output count, and LoRA identity. Only compatible
+requests can share a fused forward call.
+
+Admission is FIFO and conservative. An incompatible request at the front of the
+waiting queue can prevent later compatible requests from joining the current
+wave.
+
+`request_batch_max_wait_ms` optionally delays the first schedule operation of a
+new wave so bursty arrivals can accumulate. It applies only when:
+
+- the pipeline supports request batching;
+- step execution is disabled;
+- no requests are currently running; and
+- the configured wait is greater than zero.
+
+The wait ends when the queue reaches capacity, stabilizes briefly, reaches its
+deadline, or the engine stops.
+
+## Step-Batch Execution
+
+Step execution exposes denoising progress to the scheduler. A supporting
+pipeline implements four stateful operations:
+
+| Operation | Responsibility |
+|---|---|
+| `prepare_encode(state)` | Validate input, encode prompts, initialize latents and timesteps, and create request-local scheduler state |
+| `denoise_step(input_batch, *, states=...)` | Run one denoise forward for the scheduler-provided request states |
+| `step_scheduler(state, noise_pred)` | Update latents and advance request progress |
+| `post_decode(state)` | Decode and postprocess a completed request or output boundary |
+
+Persistent request state lives in `StepRequestState`. Pipeline-specific fields
+that do not belong in the shared contract should be stored in `state.extra`.
+Queueing and lifecycle metadata remain in the scheduler's request state.
+
+Current native pipelines that explicitly enable step execution include
+Qwen-Image, HunyuanImage3, and Helios. Step execution alone does not imply
+continuous-batching support: Qwen-Image accepts batched step states.
+HunyuanImage3 accepts batched step states only when its resolved self-attention
+backend is `TORCH_SDPA`; otherwise it rejects groups larger than one request.
+Configure `DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA` or
+`diffusion_attention_config.default.backend=TORCH_SDPA` when
+`max_num_seqs>1`. See the
+[HunyuanImage-3.0 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/HunyuanImage-3.0-Instruct.md)
+for its validated configuration. Helios supports only a single active step
+request and must use `max_num_seqs=1`.
+
+### Continuous Batching
+
+When `max_num_seqs>1`, `StepScheduler` can keep multiple compatible requests
+active. The runner gathers their state into `InputBatch`, performs one batched
+denoise forward, then applies scheduler updates per request:
+
+1. Receive transferred KV payloads, then run `prepare_encode()` for newly
+   admitted requests.
 2. Build or refresh `InputBatch`.
-3. Run one batched `denoise_step(input_batch)`.
-4. Slice the batched `noise_pred` back per request.
-5. Run per-request `step_scheduler()`.
-6. Run `post_decode()` only for requests that finished denoising.
-7. Scatter updated latents back into persistent request state with
-   [`scatter_latents()`](gh-file:vllm_omni/diffusion/worker/input_batch.py).
+3. Run one batched `denoise_step(input_batch, states=states)`.
+4. Slice noise predictions back to each request.
+5. Run each request's `step_scheduler()`.
+6. Run `post_decode()` at a chunk boundary or request completion.
+7. Scatter updated latents back into persistent request state.
 
-This keeps the shared work limited to the denoise forward pass while preserving
-request-local scheduler state and outputs.
+`StepBatchSamplingParamsKey` protects the batched tensor contract. Requests can
+have different total step counts and current step indices, but shape-sensitive,
+CFG-sensitive, and LoRA fields must be compatible. FIFO head-of-line blocking
+also applies here.
 
-## Engine
+Chunk-capable pipelines may emit intermediate results. Final-only step
+pipelines emit when the request finishes even if `streaming_output=True`.
 
-[`DiffusionEngine`](gh-file:vllm_omni/diffusion/diffusion_engine.py) provides
-the background loop and async add-request path needed for multiple requests to
-accumulate in the scheduler.
+## Model Author Guidelines
 
-When `step_execution=True`, the engine routes work through the step-wise
-executor path. The continuous batching behavior is defined by scheduler-side
-compatibility gating and runner-side `StepInputBatch` packing.
+When converting a request-level pipeline:
 
-## Current Limitations
+- Reuse the same helpers as `forward()` to avoid behavior drift.
+- Copy request-scoped scheduler state into `state.scheduler`.
+- Advance `state.step_index` only in `step_scheduler()`.
+- Keep model forward work in `denoise_step()`.
+- Keep latent mutation in `step_scheduler()`.
+- Keep final decoding equivalent to the tail of `forward()`.
+- Store masks, condition latents, and other request-local tensors in the
+  request state rather than on the shared pipeline.
 
-- Experimental feature; use `max_num_seqs=1` for the conservative single-request
-  step path.
-- Only native pipelines that already support `step_execution=True`.
-- Only homogeneous batches keyed by `StepBatchSamplingParamsKey` are supported.
-- `cache_backend`, KV transfer, and other request-mode extras are not wired
-  into the batched step-wise path yet.
-- Future work can relax the current same-shape restriction with richer
-  heterogeneous batching policies such as bucketing or padded execution for
-  different resolutions.
+Before setting `supports_step_execution = True`, validate output parity for the
+same seed and sampling parameters, scheduler isolation across concurrent
+requests, abort cleanup, progress reporting, and all supported CFG paths.
 
-## Related Files
+Before setting `supports_request_batch = True`, validate the batch input
+contract, one-output-per-request result shape, request identity and error
+mapping, seeded concurrency, LoRA compatibility, and tensor IPC.
 
-- Scheduler base:
-  [`vllm_omni/diffusion/sched/base_scheduler.py`](gh-file:vllm_omni/diffusion/sched/base_scheduler.py)
-- Scheduler interface:
-  [`vllm_omni/diffusion/sched/interface.py`](gh-file:vllm_omni/diffusion/sched/interface.py)
-- Step scheduler:
-  [`vllm_omni/diffusion/sched/step_scheduler.py`](gh-file:vllm_omni/diffusion/sched/step_scheduler.py)
-- Runner:
-  [`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py)
-- Input batch:
-  [`vllm_omni/diffusion/worker/input_batch.py`](gh-file:vllm_omni/diffusion/worker/input_batch.py)
-- Tests:
-  [`tests/diffusion/test_diffusion_scheduler.py`](gh-file:tests/diffusion/test_diffusion_scheduler.py)
+## Limitations
+
+- Request batching requires an explicit pipeline capability when
+  `max_num_seqs>1`.
+- Step execution requires the four-method stateful pipeline contract.
+- Both batching paths currently require homogeneous compatibility keys.
+- FIFO scheduling can reduce batching opportunities.
+- `request_batch_max_wait_ms` trades first-request latency for burst
+  coalescing.
+- All diffusion cache backends are currently unsupported in step mode.
+- KV transfer is supported for newly admitted step requests; some other
+  request-mode extras remain unsupported.
+- Step continuous batching remains experimental; use `max_num_seqs=1` for the
+  conservative step path.
+
+## Related Implementation
+
+- Engine: [`vllm_omni/diffusion/diffusion_engine.py`](gh-file:vllm_omni/diffusion/diffusion_engine.py)
+- Model contract: [`vllm_omni/diffusion/models/interface.py`](gh-file:vllm_omni/diffusion/models/interface.py)
+- Scheduler interface: [`vllm_omni/diffusion/sched/interface.py`](gh-file:vllm_omni/diffusion/sched/interface.py)
+- Request scheduler: [`vllm_omni/diffusion/sched/request_scheduler.py`](gh-file:vllm_omni/diffusion/sched/request_scheduler.py)
+- Step scheduler: [`vllm_omni/diffusion/sched/step_scheduler.py`](gh-file:vllm_omni/diffusion/sched/step_scheduler.py)
+- Runner: [`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py)
+- Request batch: [`vllm_omni/diffusion/worker/request_batch.py`](gh-file:vllm_omni/diffusion/worker/request_batch.py)
+- Step input batch: [`vllm_omni/diffusion/worker/input_batch.py`](gh-file:vllm_omni/diffusion/worker/input_batch.py)

docs/design/feature/diffusion_request_level_batching.md

@@ -1,175 +0,0 @@
-# Request-Level Batching for Diffusion
-
-This document describes the request-mode batching path for diffusion pipelines.
-For end-user enablement and tuning, see
-[Request-Level Batching](../../user_guide/diffusion/request_batching.md).
-
-This is separate from
-[Continuous Batching for Step-Wise Diffusion](diffusion_continuous_batching.md).
-Request-level batching runs one full pipeline `forward()` over a static batch of
-compatible requests. Step-wise continuous batching admits work between denoise
-steps when `step_execution=True`.
-
-## Why It Helps
-
-The request-level design avoids coupling several logical requests to one request
-object. This keeps request identity, abort/error handling, and per-request
-metadata unambiguous while still allowing one fused pipeline forward pass for
-bursty or concurrent traffic.
-
-## Overview
-
-With request-level batching enabled:
-
-- each `OmniDiffusionRequest` contains one `prompt` and one `request_id`
-- the scheduler groups compatible waiting requests into one scheduler wave
-- `DiffusionRequestBatch` wraps the scheduled requests for pipeline `forward()`
-- batch-capable pipelines return `list[DiffusionOutput]`, one output per
-  request
-- `BatchRunnerOutput` maps each result back to its original `request_id`
-
-Pipelines opt in with `supports_request_batch = True` and a `forward()` method
-that accepts `DiffusionRequestBatch` and returns `list[DiffusionOutput]`.
-Pipelines that do not opt in keep the existing per-request execution path.
-
-## Enablement
-
-Request-level batching is the request-mode path, so `step_execution` must remain
-disabled. Increase `max_num_seqs` above `1` to let the scheduler keep multiple
-compatible requests active:
-
-```bash
-vllm serve Qwen/Qwen-Image --omni \
-  --port 8091 \
-  --max-num-seqs 4
-```
-
-For bursty online ingress, `request_batch_max_wait_ms` can add a bounded
-admission wait before the first `schedule()` of a scheduler wave:
-
-```bash
-vllm serve Qwen/Qwen-Image --omni \
-  --port 8091 \
-  --max-num-seqs 4 \
-  --request-batch-max-wait-ms 20
-```
-
-`request_batch_max_wait_ms=0` disables this wait and is the default.
-
-## Request Contract
-
-`OmniDiffusionRequest` represents one logical request. It owns one prompt,
-sampling parameters, request id, and request-local metadata. Runtime batches are
-formed by the scheduler and represented separately from the request payload.
-
-Runtime batching is represented by:
-
-- [`DiffusionSchedulerOutput`](gh-file:vllm_omni/diffusion/sched/interface.py)
-  for scheduled request ids and request payloads
-- [`DiffusionRequestBatch`](gh-file:vllm_omni/diffusion/worker/request_batch.py)
-  for the pipeline-facing request batch
-- [`BatchRunnerOutput`](gh-file:vllm_omni/diffusion/worker/utils.py) for
-  per-request results
-
-`DiffusionRequestBatch` intentionally exposes compatibility properties such as
-`prompts`, `sampling_params`, `request_id`, and `kv_sender_info` so migrated
-pipelines can stay close to upstream code while using a batch-aware contract.
-
-## Scheduler
-
-The scheduler derives its capacity from `max_num_seqs` through
-`max_num_running_reqs`. It exposes waiting/running queue counters so the engine
-can decide whether admission wait is useful before scheduling a new wave.
-
-Batch compatibility is controlled by
-[`RequestBatchSamplingParamsKey`](gh-file:vllm_omni/diffusion/sched/interface.py). The key
-contains shape-sensitive and guidance-sensitive fields, including output count
-and LoRA identity. Requests with incompatible shapes, CFG settings, output
-counts, LoRA adapters, or LoRA scales are kept in separate batches.
-
-Admission is conservative:
-
-- the scheduler only batches compatible requests
-- FIFO ordering is preserved
-- an incompatible request at the head of the waiting queue blocks later
-  compatible requests
-
-## Engine
-
-[`DiffusionEngine`](gh-file:vllm_omni/diffusion/diffusion_engine.py) resolves
-request-batch capability during initialization from the configured pipeline
-class, including custom pipeline classes.
-
-The capability check uses the pipeline class attribute
-`supports_request_batch = True`. Pipelines that set this attribute must implement
-a request-batch-compatible `forward()` contract and return one
-`DiffusionOutput` per request; the runner validates that return shape at runtime.
-
-When the selected pipeline is batch-capable and `step_execution=False`, request
-mode routes scheduler waves through the batch executor path. Otherwise it keeps
-the per-request executor path.
-
-The optional admission wait runs only when:
-
-- request batching is supported
-- `step_execution=False`
-- `request_batch_max_wait_ms > 0`
-- no requests are currently running
-
-The wait exits early when the waiting queue reaches capacity, when the queue is
-stable for a short window, when the deadline expires, or when the engine stops.
-
-## Executor And Runner
-
-The executor exposes two request-mode entries:
-
-- `execute_request`: one worker call per scheduled request
-- `execute_batch`: one worker call for the whole `DiffusionSchedulerOutput`
-
-On the batch path, the worker builds a `DiffusionRequestBatch` and runs the
-pipeline once. Request-local setup remains per request:
-
-- KV transfer metadata
-- random generator and seed handling
-- request output/error/abort mapping
-
-Shared batch setup happens once per batch when possible:
-
-- cache refresh
-- LoRA activation for the homogeneous adapter key
-- pipeline `forward(req_batch)`
-
-Large tensor IPC still uses the shared-memory packing path. The packer traverses
-both normal `RunnerOutput.result` wrappers and nested batch results so batched
-outputs do not fall back to pickle IPC for tensor payloads.
-
-## Current Limitations
-
-- Only pipelines that declare the request-batch contract use fused batch
-  execution.
-- Batches are homogeneous under `RequestBatchSamplingParamsKey`; heterogeneous resolution or
-  incompatible guidance settings do not co-batch yet.
-- FIFO scheduling can reduce batching opportunities when an incompatible
-  request is at the front of the queue.
-- `request_batch_max_wait_ms` improves burst coalescing but can add latency to
-  the first request in a scheduler wave. Keep it small for latency-sensitive
-  serving.
-- Step-wise continuous batching is documented separately and only applies when
-  `step_execution=True`.
-
-## Related Files
-
-- Request object and request batch:
-  [`vllm_omni/diffusion/request.py`](gh-file:vllm_omni/diffusion/request.py)
-- Scheduler interface:
-  [`vllm_omni/diffusion/sched/interface.py`](gh-file:vllm_omni/diffusion/sched/interface.py)
-- Scheduler base:
-  [`vllm_omni/diffusion/sched/base_scheduler.py`](gh-file:vllm_omni/diffusion/sched/base_scheduler.py)
-- Engine:
-  [`vllm_omni/diffusion/diffusion_engine.py`](gh-file:vllm_omni/diffusion/diffusion_engine.py)
-- Worker runner:
-  [`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py)
-- Executor interface:
-  [`vllm_omni/diffusion/executor/abstract.py`](gh-file:vllm_omni/diffusion/executor/abstract.py)
-- Tests:
-  [`tests/diffusion/test_diffusion_engine.py`](gh-file:tests/diffusion/test_diffusion_engine.py)

docs/design/feature/diffusion_step_execution.md

@@ -1,131 +0,0 @@
-# Adding Step Execution Support for Diffusion Pipelines
-
-This guide documents vLLM-Omni's stepwise diffusion contract for model authors
-and contributors implementing `step_execution=True` support for a diffusion
-pipeline.
-
-For end-user enablement, supported models, and current limitations, see
-[Step Execution](../../user_guide/diffusion/step_execution.md).
-
-This document describes the base step-execution contract only. For the
-experimental batching policy layered on top of the step-wise path, see
-[Continuous Batching for Step-Wise Diffusion](diffusion_continuous_batching.md).
-
-## Current Support Scope
-
-`step_execution` is **not** a generic diffusion toggle. It only works for
-pipelines that implement the segmented stateful contract in
-[`vllm_omni/diffusion/models/interface.py`](gh-file:vllm_omni/diffusion/models/interface.py).
-
-This page is intentionally author-facing. Treat runtime enablement
-(`step_execution=True` in Python or `--step-execution` in serving) as an
-opt-in user knob layered on top of the implementation contract below.
-
-Current in-tree support:
-
-| Pipeline | Example models | Step execution |
-|----------|----------------|----------------|
-| `QwenImagePipeline` | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes |
-| All other diffusion pipelines | `QwenImageEditPipeline`, `QwenImageEditPlusPipeline`, `QwenImageLayeredPipeline`, GLM-Image, Wan, Flux, etc. | No |
-
-Current engine/runtime limitations:
-
-- Continuous batching with `max_num_seqs > 1` is experimental and documented in
-  [Continuous Batching for Step-Wise Diffusion](diffusion_continuous_batching.md).
-  Keep `max_num_seqs=1` for the conservative single-request step path.
-- `cache_backend` is not supported in step mode.
-- Request-mode extras such as KV transfer are not wired into step mode yet.
-- Unsupported pipelines now fail early during model loading instead of failing on the first request.
-
-## Execution Contract
-
-Step mode is driven by four pipeline methods plus the shared mutable request
-state object:
-
-- `prepare_encode(state)`: one-time request preparation.
-- `denoise_step(state)`: compute the noise prediction for the current step.
-- `step_scheduler(state, noise_pred)`: mutate latents and advance step state.
-- `post_decode(state)`: decode the final output after denoising is complete.
-
-The state lives in
-[`vllm_omni/diffusion/worker/utils.py`](gh-file:vllm_omni/diffusion/worker/utils.py)
-as `StepRequestState`. Store request-scoped tensors there, or use
-`state.extra` for model-specific fields that do not justify extending the core
-dataclass.
-
-The worker-side step loop lives in
-[`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py):
-
-1. `prepare_encode()` runs once for a new request.
-2. `denoise_step()` runs every scheduler tick.
-3. `step_scheduler()` mutates `state.latents` and advances `state.step_index`.
-4. `post_decode()` runs exactly once after `state.denoise_completed` becomes true.
-
-## Recommended Split
-
-When converting an existing request-level `forward()` pipeline, keep the split
-strict and mechanical:
-
-| Request-level phase | Stepwise method | What belongs there |
-|---------------------|-----------------|--------------------|
-| Input validation, prompt encoding, latent init, timestep prep, per-request scheduler creation | `prepare_encode()` | Anything that should happen once per request |
-| Transformer forward / noise prediction | `denoise_step()` | Pure denoise computation for the current timestep |
-| `scheduler.step(...)` and `step_index += 1` | `step_scheduler()` | Only latent/state mutation for one step |
-| VAE decode / postprocess | `post_decode()` | Final decode only |
-
-Keep the stepwise path reusing the same helpers as the request-level path
-whenever possible. Reimplementing the denoise loop from scratch is the easiest
-way to introduce behavioral drift.
-
-## Qwen-Image Reference
-
-[`pipeline_qwen_image.py`](gh-file:vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py)
-is the reference implementation and is split correctly for the current
-contract:
-
-- `prepare_encode()` reuses `_prepare_generation_context()` so prompt encoding,
-  latent init, timestep creation, CFG setup, and shape bookkeeping stay aligned
-  with `forward()`.
-- `prepare_encode()` deep-copies `self.scheduler` **after**
-  `prepare_timesteps()` so request-specific scheduler state is isolated.
-- `denoise_step()` reuses `_build_denoise_kwargs()` plus
-  `predict_noise_maybe_with_cfg()`, so sequential CFG, CFG-parallel, and
-  non-CFG behavior stay identical to the request-level path.
-- `step_scheduler()` only calls
-  `scheduler_step_maybe_with_cfg(..., per_request_scheduler=state.scheduler)`
-  and increments `state.step_index`.
-- `post_decode()` reuses `_decode_latents()`, so the final image decode matches
-  the normal `forward()` path.
-
-That decomposition is the target pattern for future models.
-
-## Rules For New Pipelines
-
-- Do not keep request-scoped scheduler state on `self.scheduler`. Copy it into
-  `state.scheduler` during `prepare_encode()`.
-- Do not mutate `state.step_index` inside `denoise_step()`. Only
-  `step_scheduler()` should advance the step.
-- Do not decode partial outputs in `denoise_step()` or `step_scheduler()`.
-- If the request-level pipeline has condition latents, masks, or edit-specific
-  tensors, store them in `state` or `state.extra`, not in global pipeline
-  attributes.
-- Preserve CFG behavior by sharing the same helper path used by `forward()`.
-- Keep `post_decode()` equivalent to the tail of `forward()`.
-
-## Validation Checklist
-
-Before marking a pipeline as `supports_step_execution = True`, verify:
-
-- Stepwise output matches request-level output for the same seed and sampling params.
-- Per-request scheduler state is isolated across concurrent requests.
-- Abort during denoise does not leak cached state.
-- `step_index` reported by `RunnerOutput` matches the scheduler progress.
-- CFG-parallel and non-CFG paths both work if the request-level pipeline supports them.
-
-## Related Files
-
-- Contract: [`vllm_omni/diffusion/models/interface.py`](gh-file:vllm_omni/diffusion/models/interface.py)
-- State: [`vllm_omni/diffusion/worker/utils.py`](gh-file:vllm_omni/diffusion/worker/utils.py)
-- Runner loop: [`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py)
-- Scheduler transport: [`vllm_omni/diffusion/sched/interface.py`](gh-file:vllm_omni/diffusion/sched/interface.py)
-- Reference pipeline: [`vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py`](gh-file:vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py)

docs/design/feature/distributed_layerwise_offload.md

@@ -0,0 +1,156 @@
+# Distributed Layerwise Offload
+
+This document describes distributed layerwise offload (DLO) for diffusion
+models. DLO keeps only a small number of DiT blocks on the accelerator and
+streams the remaining blocks from host memory. The distributed backend can
+either shard those host-side weights across an existing parallel group or keep
+the standard loader's rank-local weights and avoid an additional collective.
+
+For user-facing commands, see the
+[distributed layerwise offloading guide](../../user_guide/diffusion/cpu_offload.md)
+and the [Cosmos3 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-DistOffload.md).
+
+## Status
+
+DLO is implemented for multi-device diffusion execution. The default
+AllGather path is the primary path for DP and SP deployments. The
+`--dlo-no-use-allgather` path is a rank-local compatibility mode: it is useful
+for standard-loader sharding, workstation bring-up, and systems where an
+additional DLO collective is undesirable, but it does not reduce host weight
+storage across ranks.
+
+The compatibility matrix below describes the current implementation. The
+unit-level guards are covered, but not every parallelism combination has a
+full model-and-hardware end-to-end test.
+
+## Design
+
+### DLO consumes the existing parallel topology
+
+DLO does not create a new DP, TP, or SP topology. It reads the configured
+`DiffusionParallelConfig` and attaches offload hooks to the DiT blocks after the
+standard distributed groups have been initialized.
+
+The DLO weight-sharding group is selected as follows:
+
+1. Use the existing DP group when `data_parallel_size > 1`.
+2. When DP is one and SP is greater than one, use the SP group.
+3. Otherwise, run rank-locally without a DLO process group.
+
+TP is deliberately not used as DLO's AllGather group. HSDP has its own
+parameter-sharding lifecycle and is not allowed to be sharded a second time by
+DLO's AllGather path.
+
+### AllGather path
+
+With the default `dlo_use_allgather=True`, each rank stores approximately
+`1 / group_size` of each streamable block in pinned host memory. The next
+block's shard is copied to a device buffer and reconstructed with
+`all_gather_into_tensor` on a communication stream while the current block is
+executing.
+
+```text
+Compute:    [Block N]             [Block N+1]          [Block N+2]
+H2D:                      [shard N+1]           [shard N+2]
+AllGather:                [full N+1]             [full N+2]
+Buffers:    [current slot]       [prefetch slot]       [current slot]
+```
+
+![DLO double-buffer prefetch pipeline](../figures/dlo/dlo_pipeline.gif)
+
+The backend uses two shared device buffers, so accelerator weight residency is
+bounded by the largest streamed blocks rather than the complete model.
+
+When DP is greater than one, the engine can process one request per DP rank in
+the same denoising wave. Because AllGather is a collective, all participating
+requests must take the same execution path at every denoising step.
+
+### Rank-local path without DLO AllGather
+
+With `--dlo-no-use-allgather`, DLO forces its internal offload shard size to
+one. The regular model loader remains responsible for preparing each rank's
+weights, including TP-local tensors or HSDP-managed parameters. DLO then
+streams those rank-local tensors block by block using H2D copies only.
+
+This mode means:
+
+- DP still provides independent replicas, but DLO does not shard weights
+  across DP ranks.
+- SP still performs its normal activation/attention collectives, but DLO does
+  not shard weights across SP ranks.
+- TP/HSDP/SP collectives, if configured, are not disabled by this flag; only
+  DLO's additional weight AllGather is disabled.
+- Pure DP deployments keep a full host-side model copy per rank, subject to
+  shared OS page-cache behavior.
+- The scheduler does not require a synchronized DP request wave for DLO.
+
+This path is intentionally not implemented by reusing the AllGather mmap
+loader. It relies on the standard loader so model-specific TP/HSDP transforms
+remain intact.
+
+## Parallelism compatibility
+
+| Parallelism | DLO + AllGather | DLO without AllGather |
+|---|---|---|
+| **DP** | Supported primary path. DLO shards host weights across the DP group and can run DP multi-concurrency. | Supported rank-local path. DP replicas remain independent; no DLO weight sharding or cross-DP DLO collective. |
+| **SP** | Supported in the implementation. With DP=1, DLO uses the SP group for host-weight sharding; SP still shards sequence/activation work. | SP remains active, but DLO keeps standard-loader rank-local weights and adds no SP weight collective. |
+| **TP > 1** | Unsupported in the DLO mmap path. TP-aware loader callbacks are bypassed, so the backend rejects this configuration when it enters that path. | Standard loading is retained and TP-local tensors can be streamed. This is the intended compatibility path, but it still needs broader model and hardware validation. |
+| **HSDP** | Rejected. HSDP has already sharded parameters, so DLO AllGather would double-shard them. | Accepted by configuration. HSDP owns parameter sharding and its own gathers; DLO only stages rank-local parameters. End-to-end coverage is limited. |
+
+### Combined dimensions
+
+- **DP + SP:** DLO uses the DP group for weight sharding when DP is greater
+  than one; SP continues to use its own sequence-parallel group. If DP is one,
+  the SP group becomes DLO's sharding group in AllGather mode.
+- **DP + TP/SP without AllGather:** standard model loading defines the
+  rank-local tensor layout. DLO adds no cross-DP, cross-TP, or cross-SP weight
+  collective.
+- **HSDP + SP:** the general parallel configuration permits HSDP over SP, but
+  DLO must use `--dlo-no-use-allgather`. HSDP remains responsible for weight
+  materialization and synchronization.
+- **HSDP + DP or TP:** rejected independently by the diffusion parallel
+  configuration.
+
+## Request and loading constraints
+
+AllGather DP multi-concurrency requires:
+
+- explicit `num_inference_steps`;
+- the same `num_inference_steps` for all requests in a wave; and
+- identical request arguments that affect the collective execution path.
+
+The no-AllGather path does not impose these DLO-specific synchronized-wave
+requirements.
+
+The mmap loader is used only by the supported DLO+AllGather path when the
+model has a compatible checkpoint layout. Online quantization is incompatible
+with that sharded mmap path; use `--dlo-no-use-allgather` or disable online
+quantization. Models that do not meet the mmap requirements use the regular
+loader path.
+
+## Validation coverage
+
+Current source-level validation includes:
+
+- HSDP + DLO + AllGather rejection;
+- HSDP + DLO without AllGather acceptance at configuration level;
+- TP rejection in the DLO+AllGather mmap path;
+- resident-layer requests requiring no-AllGather;
+- DP request-wave validation for denoising-step compatibility;
+- sharding, double-buffer, AllGather-size, and heterogeneous-block regression
+  tests.
+
+The highest-value missing coverage is end-to-end numerical comparison against
+ordinary layerwise offload for DP+SP, TP+no-AllGather, and HSDP+SP+no-AllGather
+on the target CUDA/NCCL or CANN/HCCL hardware.
+
+## Recommendations
+
+- Use **DP + DLO AllGather** for the supported throughput and host-memory
+  scaling path.
+- Use **SP + DLO AllGather** for long-sequence workloads when DP concurrency is
+  not the goal.
+- Use **no-AllGather** to bring up TP or workstation PCIe configurations, with
+  the expectation of higher host-memory use and lower validation confidence.
+- Prefer **HSDP alone** for production HSDP deployments until the combined
+  HSDP + DLO no-AllGather path has broader end-to-end coverage.

docs/design/feature/omni_async_output_materialization.md

@@ -0,0 +1,386 @@
+# Async Omni Output Materialization
+
+## Table of Contents
+
+1. [Overview](#overview)
+2. [Performance](#performance)
+3. [Architecture](#architecture)
+4. [Enabling the Feature](#enabling-the-feature)
+5. [Compatibility and Fallbacks](#compatibility-and-fallbacks)
+6. [Related Files](#related-files)
+
+## Overview
+
+Async Omni output materialization moves CPU-side construction of
+`OmniModelRunnerOutput` out of the autoregressive (AR) decode critical path.
+It complements [Async Chunk](async_chunk.md): async chunk pipelines partial
+outputs between stages, while async output materialization prevents the
+construction of those partial outputs from blocking the next decode step.
+
+An AR stage must return sampled token IDs quickly because the scheduler needs
+them for the next decode step. However, an Omni stage can also produce hidden
+states, multimodal tensors, streaming inter-stage payloads, and connector
+metadata. Building these payloads can require device-to-host (D2H) copies,
+tensor slicing, flattening, and Python object construction.
+
+Without this feature, all of that work runs inline in `sample_tokens()`:
+
+```text
+sample tokens
+  -> update decode state
+  -> copy hidden and multimodal outputs to CPU
+  -> build per-request payloads
+  -> build the streaming wire payload
+  -> collect connector signals
+  -> return sampled tokens
+  -> launch the next decode step
+```
+
+With async output materialization, only work required by the next decode step
+stays on the critical path:
+
+```text
+sample tokens
+  -> update decode state
+  -> snapshot output state and start asynchronous D2H copies
+  -> register sampled tokens
+  -> launch the next decode step
+
+background output path
+  -> wait for the payload snapshot
+  -> build per-request payloads
+  -> build the streaming wire payload
+  -> collect connector signals
+  -> construct OmniModelRunnerOutput
+```
+
+This allows payload construction for step `N` to overlap with GPU decode work
+for step `N + 1`. It does not change model computation or generated output.
+
+!!! note "Platform and validation scope"
+    The performance and correctness validation in this document covers CUDA
+    and ROCm. This is a validation scope, not a runtime platform guard: XPU
+    inherits `GPUARModelRunner`, and MUSA selects `GPUARWorker`, so those
+    backends may enter the asynchronous path when its other guards pass. XPU
+    and MUSA have not been validated for this optimization. Ascend NPU uses a
+    separate `NPUARModelRunner` path that still constructs
+    `OmniModelRunnerOutput` and drains connector output synchronously before
+    wrapping the result as an asynchronous runner output.
+
+<p align="center">
+  <img
+    alt="Decode step gap before and after async output materialization"
+    src="../../../source/architecture/qwen3-omni-async-output-step-gap.svg"
+    width="100%"
+  >
+</p>
+
+<p align="center">
+  <em>
+    Figure 1: Moving payload construction off the decode path reduces the
+    observed gap between consecutive Talker steps from about 2.8 ms to 41 µs.
+    Adapted from the
+    <a href="https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization">
+      Qwen3-Omni optimization blog
+    </a>.
+  </em>
+</p>
+
+For Qwen3-Omni, the feature is enabled for both AR stages:
+
+- **Thinker**: Defers hidden-state and multimodal payload materialization.
+- **Talker**: Runs its decode-state postprocess eagerly, then defers codec
+  payload materialization. Because Code2Wav consumes codec codes rather than
+  Talker hidden states, the Talker also skips unnecessary hidden-state D2H.
+- **Code2Wav**: Does not use this path because it is a generation stage rather
+  than an AR stage.
+
+Qwen3-TTS uses the same mechanism in its AR Talker stage. The Talker updates
+the decode state it needs for the next step before returning, then constructs
+the codec payload for Code2Wav in the background.
+
+## Performance
+
+In the controlled Qwen3-Omni optimization sweep, enabling async output
+materialization on top of CUDA Graph and async chunk produced the following
+results at concurrency 64:
+
+| Configuration | Request throughput | Mean audio TTFP | Mean audio RTF |
+|---|---:|---:|---:|
+| CUDA Graph + async chunk | 9.3 req/s | 655 ms | 0.63 |
+| + async output materialization | 11.3 req/s | 631 ms | 0.47 |
+| Change | **+22%** | **-4%** | **-25%** |
+
+The sweep used `Qwen/Qwen3-Omni-30B-A3B-Instruct`, 640 Seed-TTS English
+prompts, and one replica each for Thinker, Talker, and Code2Wav. Async output
+materialization recovers throughput by removing CPU payload construction from
+the decode critical path without regressing time to first audio packet (TTFP).
+See the
+[Qwen3-Omni optimization blog](https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization)
+for the complete benchmark methodology and results.
+
+The implementation was also validated against the synchronous safe-copy path.
+The text and audio output hashes matched, confirming that overlap does not
+change the generated result. See
+[PR #4476](https://github.com/vllm-project/vllm-omni/pull/4476) for the
+correctness and profiling details.
+
+## Architecture
+
+### Output Lifecycle
+
+`OmniAsyncGPUModelRunnerOutput` extends vLLM's
+`AsyncGPUModelRunnerOutput`. It preserves the existing asynchronous
+sampled-token feedback while adding a background builder for the complete Omni
+output.
+
+The output lifecycle is:
+
+1. `GPUARModelRunner.sample_tokens()` samples tokens and performs the
+   bookkeeping required by the next decode step.
+2. The runner snapshots step-local metadata such as request IDs, token spans,
+   scheduler output, and query start locations.
+3. CUDA payload tensors are cloned before reusable model or CUDA Graph buffers
+   can be overwritten.
+4. The cloned payload is copied to pinned CPU memory on a dedicated CUDA
+   stream. A CUDA event records when the copy is ready.
+5. `OmniAsyncGPUModelRunnerOutput` starts a background thread that waits for
+   the payload event and builds `OmniModelRunnerOutput`.
+6. The AR runner immediately registers the sampled-token CPU copy with the
+   input batch and returns, allowing async scheduling to advance.
+7. When the engine calls `get_output()`, it joins the background builder,
+   propagates any builder exception, and finalizes sampled tokens and logprobs
+   through the upstream output implementation.
+
+```text
+GPU / decode path                         Background output path
+
+forward + sample
+      |
+      +-- clone output tensors
+      +-- enqueue D2H copy --------------> wait for D2H event
+      +-- register sampled tokens               |
+      +-- return async output                    +-- slice per request
+      |                                         +-- build multimodal payloads
+next decode step                                +-- partition streaming payloads
+                                                +-- build the wire payload
+                                                +-- drain connector signals
+                                                +-- build OmniModelRunnerOutput
+```
+
+### Safe State Snapshots
+
+The next scheduler step can mutate runner state while the background builder is
+still active. Most step-local builder inputs are therefore detached snapshots.
+The snapshotted state includes:
+
+- scheduler token counts and speculative-token metadata
+- request IDs and request-to-batch-index mappings
+- sampled token IDs, logprobs, and prompt logprobs
+- query start locations and scheduled-token spans
+- hidden states and multimodal output tensors
+- KV-connector and encoder-cache outputs captured for the step
+
+CUDA tensor cloning is required because CUDA Graph and model output buffers can
+be reused by the next forward pass. The dedicated copy stream and pinned host
+buffers make the D2H transfer asynchronous; the snapshot retains its cloned
+CUDA sources until the transfer event completes.
+
+### Live Runner State and Ownership
+
+The complete builder input is not a detached snapshot. Connector output state
+remains runner-owned: `get_omni_connector_output()` drains live, per-cycle
+connector signals after the streaming payload is built. Correctness therefore
+depends on that builder being the only consumer that drains connector signals
+for its output cycle. Receive-side connector state written by the background
+receiver is coordinated by the connector mixin's `_lock`.
+`OmniAsyncGPUModelRunnerOutput.get_output()` joins the builder and is the
+completion and exception boundary before the materialized output is consumed.
+
+The shared output builder also contains a full-payload accumulation branch
+that reads `self.requests`. That branch is not reachable when async Omni output
+materialization is enabled: this feature requires `async_chunk`, while
+`should_accumulate_full_payload_output()` returns `False` whenever
+`async_chunk` is enabled. The feature path partitions each step's payload into
+streaming inter-stage and client handoffs instead.
+
+### Output Builder
+
+The background builder performs the work that previously ran inline:
+
+- resolves which requests need downstream payloads
+- converts hidden and multimodal tensors into per-request CPU payloads
+- applies model-specific payload processing
+- partitions per-step payloads into inter-stage and client streams
+- creates the tensor-only `multimodal_outputs` wire payload
+- drains live connector readiness signals after building the streaming payload
+- constructs the final `OmniModelRunnerOutput`
+
+Input-side connector operations remain synchronous with model execution. In
+particular, receiving inputs and flushing previously completed connector
+outputs are not deferred because they affect scheduler-visible request state.
+
+### Qwen3-Omni Stage Behavior
+
+| Stage | Async output behavior | Reason |
+|---|---|---|
+| Thinker | Snapshots hidden states and multimodal outputs; builds the downstream payload in the background | The Talker needs the payload, but the next Thinker decode step only needs sampled-token feedback |
+| Talker | Runs lightweight postprocess eagerly; snapshots codec outputs; omits hidden states from the downstream payload | `hidden_states.last` is needed by the next Talker step, while Code2Wav only needs codec codes |
+| Code2Wav | Uses the normal generation-stage output path | Code2Wav is not executed by `GPUARModelRunner` |
+
+The Qwen3-TTS Talker follows the same pattern as the Qwen3-Omni Talker:
+postprocess runs eagerly because the next decode step needs its updated state,
+while codec payload construction runs asynchronously. Its Code2Wav stage also
+uses the normal generation-stage path.
+
+## Enabling the Feature
+
+There is no separate `async_omni_output` command-line or YAML option. The
+feature is selected automatically for model stages that opt in when the runtime
+conditions are safe. On the validated CUDA and ROCm configurations, using the
+bundled deployment profile is enough to turn it on for the supported
+Qwen3-Omni and Qwen3-TTS pipelines. The selection logic has no CUDA/ROCm-only
+guard; see [Compatibility and Fallbacks](#compatibility-and-fallbacks) for the
+other platform paths. The same profile does not enable background Omni
+materialization on Ascend NPU.
+
+### Example 1: Qwen3-Omni
+
+Start the full Qwen3-Omni pipeline:
+
+```bash
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \
+  --omni \
+  --port 8091
+```
+
+The model registry automatically loads
+`vllm_omni/deploy/qwen3_omni_moe.yaml`. Its relevant settings are:
+
+```yaml
+async_chunk: true
+
+stages:
+  - stage_id: 0  # Thinker
+    enable_prefix_caching: false
+    # async_scheduling defaults to true for AR stages
+
+  - stage_id: 1  # Talker
+    enable_prefix_caching: false
+    # async_scheduling defaults to true for AR stages
+
+  - stage_id: 2  # Code2Wav
+    enable_prefix_caching: false
+    async_scheduling: false
+```
+
+With this profile, async output materialization activates automatically for:
+
+- **Stage 0, Thinker**: Defers hidden-state and multimodal payload
+  construction.
+- **Stage 1, Talker**: Defers codec payload construction after eagerly
+  updating its decode state.
+
+Stage 2 does not use this feature because Code2Wav is not an AR stage.
+
+### Example 2: Qwen3-TTS
+
+Start Qwen3-TTS with any checkpoint that uses the Qwen3-TTS Talker pipeline.
+For example:
+
+```bash
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
+  --omni \
+  --port 8091
+```
+
+The model registry automatically loads `vllm_omni/deploy/qwen3_tts.yaml`. Its
+relevant settings are:
+
+```yaml
+async_chunk: true
+
+stages:
+  - stage_id: 0  # Talker
+    async_scheduling: true
+    enable_prefix_caching: false
+
+  - stage_id: 1  # Code2Wav
+    enable_prefix_caching: false
+```
+
+Async output materialization activates only for Stage 0, the AR Talker. The
+Talker keeps the state required by its next decode step on the GPU, skips an
+unnecessary hidden-state D2H copy, and builds the codec payload for Code2Wav in
+the background. Stage 1 uses the generation-stage output path.
+
+The same behavior applies to the Base and VoiceDesign Qwen3-TTS checkpoints
+because they use the same Talker implementation.
+
+!!! warning
+    Do not pass `--no-async-chunk` or enable prefix caching when you want this
+    optimization. Either change causes the runner to fall back to synchronous
+    output construction. Enabling async chunk alone cannot activate the feature
+    for a model that has not opted into async Omni output.
+
+!!! note "Prefix cache compatibility"
+    Async Omni output materialization and Omni prefix caching cannot currently
+    run together. `_should_use_async_omni_output()` returns `False` whenever
+    `self.omni_prefix_cache` is present, so prefix caching selects synchronous
+    output materialization. Supporting both features requires snapshotting or
+    otherwise synchronizing prefix-cache merge and update state before the
+    background output path can consume it safely.
+
+## Compatibility and Fallbacks
+
+`GPUARModelRunner` uses the async output path only when all of the following
+runtime conditions hold:
+
+| Requirement | Reason |
+|---|---|
+| AR async scheduling is enabled | The optimization relies on the scheduler advancing while the prior output is materialized |
+| `async_chunk` is enabled | The feature targets incremental downstream Omni payloads |
+| The model stage opts in with `use_async_omni_output` | Models must declare that their output lifecycle is safe to defer |
+| Omni prefix cache is disabled | Prefix-cache merge and update ordering currently requires synchronous materialization |
+| Speculative decoding is disabled | Speculative output state is not included in this deferred path |
+| Routed-expert output is disabled | Routed-expert extraction currently requires the synchronous path |
+| Postprocess is absent or explicitly runs eagerly | State needed by the next decode step must be updated before the runner returns |
+
+These checks are evaluated per stage on runners based on `GPUARModelRunner`.
+An unsupported combination does not prevent serving; it only falls back to
+synchronous output construction for that stage. CUDA and ROCm are the validated
+platforms. XPU inherits `GPUARModelRunner`, and MUSA selects `GPUARWorker`, so
+they may select the async path when these checks pass, but this optimization is
+not yet validated on either backend.
+
+Ascend NPU does not use this selection path. `NPUARModelRunner.sample_tokens()`
+fully constructs `OmniModelRunnerOutput`, calls
+`get_omni_connector_output()`, and only then creates the asynchronous wrapper,
+so Omni payload materialization remains synchronous.
+
+!!! note
+    `use_async_omni_output`,
+    `eager_omni_postprocess_before_async_output`, and
+    `omni_pooler_payload_include_hidden` are model implementation contracts,
+    not user-facing configuration fields.
+
+## Related Files
+
+- `vllm_omni/worker/gpu_ar_model_runner.py`: Async output object, safe tensor
+  snapshots, compatibility guards, and deferred Omni output builder.
+- `vllm_omni/platforms/npu/worker/npu_ar_model_runner.py`: Separate Ascend NPU
+  runner, where Omni output materialization remains synchronous.
+- `vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py`: Thinker and
+  Talker opt-in behavior.
+- `vllm_omni/deploy/qwen3_omni_moe.yaml`: Default Qwen3-Omni deployment
+  settings.
+- `vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py`: Qwen3-TTS
+  Talker opt-in and eager postprocess behavior.
+- `vllm_omni/deploy/qwen3_tts.yaml`: Default Qwen3-TTS deployment settings.
+- `tests/worker/test_gpu_ar_model_runner.py`: Snapshot, guard, connector
+  ordering, and background error-propagation tests.
+- [Async Chunk](async_chunk.md): Inter-stage chunking and scheduling design.
+- [Qwen3-Omni optimization blog](https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization):
+  Optimization context and controlled performance results.
+- [PR #4476](https://github.com/vllm-project/vllm-omni/pull/4476): Feature
+  implementation, validation, and review history.

docs/design/feature/quantization.md

@@ -0,0 +1,249 @@
+# Quantization
+
+This document describes the quantization architecture in vLLM-Omni and the
+extension points for contributors. For user-facing configuration, supported
+hardware, and method-specific instructions, see the
+[quantization user guide](../../user_guide/quantization/overview.md).
+
+## Goals and scope
+
+The design has four goals:
+
+1. Provide one `quantization_config` entry point for diffusion-only,
+   multi-stage omni/TTS, and multi-stage diffusion models.
+2. Reuse vLLM's quantization registry and `QuantizationConfig` contract where
+   a vLLM backend already provides the required behavior.
+3. Allow different components of one pipeline to use different quantization
+   configurations.
+4. Preserve the distinction between online quantization of a BF16/FP16
+   checkpoint and loading a pre-quantized checkpoint with serialized scales.
+
+This document covers configuration resolution, component routing, model
+integration, backend extension, and validation. Kernel implementations and
+user-facing support matrices are outside its scope.
+
+## Architecture
+
+The quantization path is split across the following layers:
+
+| Layer | Responsibility | Source |
+| --- | --- | --- |
+| Public API | Exposes `build_quant_config`, component routing, and backend registration | `vllm_omni/quantization/__init__.py` |
+| Factory | Normalizes method names, selects Omni overrides or vLLM registry backends, detects ModelOpt configs, and reconciles checkpoint metadata | `vllm_omni/quantization/factory.py` |
+| Component router | Resolves a quantization config from a layer prefix using longest-prefix matching | `vllm_omni/quantization/component_config.py` |
+| Method configs | Implement `QuantizationConfig` and select the platform-specific linear method | `vllm_omni/quantization/*_config.py` |
+| Diffusion configuration | Stores the canonical runtime config and propagates checkpoint configuration | `vllm_omni/diffusion/data.py` |
+| Worker and model integration | Passes the resolved config into vLLM's model configuration and model layers | `vllm_omni/diffusion/worker/diffusion_worker.py` and `vllm_omni/diffusion/models/` |
+
+The runtime contract remains vLLM's `QuantizationConfig`:
+
+```text
+user/CLI or model config
+          |
+          v
+OmniDiffusionConfig.quantization_config
+          |
+          +--> build_quant_config()
+          |        |
+          |        +--> Omni override
+          |        +--> vLLM registry backend
+          |        +--> ComponentQuantizationConfig
+          |
+          +--> checkpoint metadata reconciliation
+          |
+          +--> diffusion worker / vLLM config
+          |
+          +--> model layers call get_quant_method(layer, prefix)
+```
+
+The model layer receives either a method implementation or `None`. Returning
+`None` leaves that layer unquantized.
+
+## Configuration resolution
+
+`build_quant_config()` accepts the following forms:
+
+| Input | Resolution |
+| --- | --- |
+| `None` or `"none"` | Disable quantization. |
+| Method string, such as `"fp8"` | Resolve an Omni override first, then the vLLM quantization registry. |
+| Flat dictionary with `method` or `quant_method` | Normalize the method and construct its `QuantizationConfig`. |
+| Per-component dictionary | Construct a `ComponentQuantizationConfig`. |
+| Existing `QuantizationConfig` | Pass through without rebuilding it. |
+
+Method aliases are normalized case-insensitively, with `-` and `_` treated as
+equivalent. `auto-round` and `auto_round` use the Omni INC/AutoRound adapter.
+ModelOpt checkpoint metadata is detected from its method, producer metadata, or
+`quant_algo`, then mapped to the corresponding vLLM ModelOpt configuration.
+
+The factory uses Omni-specific overrides for methods whose diffusion behavior
+or platform dispatch is not provided by the vLLM registry. The current override
+set includes INT8, BitsAndBytes, MXFP8, MXFP4, MXFP4 dual-scale, and INC/
+AutoRound. Other methods, such as FP8, GGUF, and ModelOpt, are resolved through
+vLLM's registry when their configuration is compatible.
+
+## Per-component routing
+
+Multi-stage models do not necessarily want one quantization policy for every
+module. A component map expresses that policy without requiring each pipeline
+to construct separate vLLM configurations:
+
+```python
+quantization_config = {
+    "transformer": {"method": "fp8"},
+    "vae": None,
+    "default": None,
+}
+```
+
+`ComponentQuantizationConfig` applies the following rules:
+
+1. Component keys are interpreted as runtime layer-prefixes.
+2. The longest matching prefix wins.
+3. `None` disables quantization for the matching component.
+4. `default`, when present, handles prefixes with no explicit match.
+5. A matching child config delegates to its normal
+   `get_quant_method(layer, prefix)` implementation.
+
+Prefix matching happens after the model's weight-name mapping. New model
+integrations must verify the actual runtime prefixes, especially when a
+`WeightsMapper` changes checkpoint names. A prefix mismatch can silently route
+a layer to the default configuration, so component-routing tests should cover
+both positive matches and fall-through behavior.
+
+## Online and pre-quantized checkpoints
+
+The same method name can represent two different loading paths:
+
+- **Online**: the source checkpoint contains BF16/FP16 weights; the method
+  creates quantized weights or scales while the model is loaded.
+- **Pre-quantized**: the checkpoint contains quantized weights and the metadata
+  required by the method; the loader consumes those serialized tensors.
+
+Method configuration owns the mode flag, for example
+`is_checkpoint_int8_serialized` or `is_checkpoint_mxfp8_serialized`. The
+configuration's `get_quant_method()` selects the corresponding linear method.
+
+For pipelines with transformer-specific `config.json` files, such as cascade
+models, `resolve_quant_config_from_disk()` reconciles the active configuration
+with each transformer's checkpoint metadata. Its invariants are:
+
+- If no active configuration exists, valid checkpoint metadata can be used for
+  auto-detection.
+- A method mismatch is an error rather than an implicit conversion.
+- Serialized-checkpoint flags rebuild an online configuration into the
+  matching offline configuration.
+- Checkpoint-specific `ignored_layers` are preserved by rebuilding the active
+  configuration when they differ.
+- AutoRound MXFP8 metadata (`data_type="mx_fp"`) selects its offline path.
+
+This prevents a model from interpreting serialized weights with an online
+loader or silently applying the wrong method to one transformer in a
+multi-transformer pipeline.
+
+## Model and layer integration
+
+Diffusion model constructors receive the resolved configuration and pass it to
+quantizable vLLM layers. Each method configuration is responsible for:
+
+- identifying supported layers in `get_quant_method()`;
+- creating the correct parameter shapes and loaders;
+- processing weights after loading when conversion is required; and
+- applying the platform-specific quantized operation at runtime.
+
+Model code must keep precision-sensitive or unsupported modules unquantized.
+The shared `safe_quant_config()` helper is used by model integrations for
+normalization and modulation layers where a generic FP8 configuration would be
+unsafe. Text encoders, audio/vision encoders, VAEs, schedulers, and tokenizers
+are not implicitly quantized just because the main transformer is quantized;
+they require explicit component routing and method support.
+
+Weight quantization and quantized KV cache are separate paths. The former is
+resolved through `quantization_config`; runtime attention/KV-cache quantization
+is configured by the diffusion attention configuration and must be validated
+independently.
+
+## Platform and parallelism boundaries
+
+Platform checks belong in the method implementation rather than in the common
+factory. This keeps configuration parsing portable while allowing CUDA, NPU,
+and XPU methods to select different kernels or reject unsupported execution at
+model construction time. The user guide is the source of truth for the current
+hardware support matrix.
+
+Quantization methods must also respect the weight layout produced by the
+configured parallelism strategy:
+
+- Tensor-parallel layers may receive partitioned weight shapes and loaders.
+- Sequence-parallel execution changes activation movement but does not by
+  itself change the quantization configuration.
+- Data-parallel replicas build their own method state; quantization metadata
+  must be available consistently on every rank.
+- `WeightsMapper` transformations must be applied before matching ignored-layer
+  lists or component prefixes.
+
+Distributed layerwise offload has an additional boundary: its sharded mmap
+AllGather path is not compatible with online quantization because the path
+expects already materialized checkpoint weights. Use the rank-local
+`--dlo-no-use-allgather` path or a compatible pre-quantized checkpoint when
+combining these features. See the [DLO design document](distributed_layerwise_offload.md)
+for the full compatibility matrix.
+
+## Adding a quantization backend
+
+Contributors should follow this sequence:
+
+1. Check whether an existing vLLM backend can provide the required weight
+   format and kernel behavior. Prefer reuse over a duplicate Omni backend.
+2. If Omni-specific behavior is required, implement a `QuantizationConfig` and
+   its `QuantizeMethodBase`/linear method in `vllm_omni/quantization/`.
+3. Implement `get_name()`, supported activation dtypes, minimum capability,
+   checkpoint parsing, ignored-layer handling, and `get_quant_method()`.
+4. Add a lazy factory override in `factory.py`. Keep optional platform
+   dependencies out of package import paths and normalize aliases to one
+   canonical method name.
+5. If the method is pre-quantized, define the checkpoint metadata and the
+   serialized loading path before adding an online path.
+6. Add model-specific component routing only when the model contains stages
+   that must use different policies.
+7. Document user-visible configuration in the quantization user guide and keep
+   implementation rationale here.
+
+The public factory registration hook,
+`register_quantization_override(method, builder)`, is intended for integrations
+that need to register an Omni-compatible builder without changing the vLLM
+registry.
+
+## Validation
+
+Validation should cover the complete path, not only whether a config object can
+be instantiated:
+
+| Area | Required checks |
+| --- | --- |
+| Factory | Strings, aliases, flat dictionaries, invalid inputs, passthrough, and component maps |
+| Method config | Online/offline selection, ignored layers, platform guards, and weight loading |
+| Checkpoint metadata | Auto-detection, method mismatch errors, serialized flags, and per-transformer metadata |
+| Model integration | Correct quantization scope, unquantized components, and parallel weight layouts |
+| Numerical quality | Same-seed BF16 versus quantized output comparison at representative resolutions and step counts |
+| Performance | Peak device memory, generation latency, and any backend-specific throughput metric |
+
+The unit tests live under `tests/diffusion/quantization/`. The trajectory
+similarity tool in
+`vllm_omni/quantization/tools/compare_diffusion_trajectory_similarity.py`
+provides a repeatable quality and memory comparison for diffusion models.
+Method support and feature combinations remain model- and hardware-dependent;
+passing factory tests alone does not establish production support.
+
+## Design invariants
+
+- `quantization_config` is the single runtime source of truth after config
+  normalization.
+- A checkpoint method mismatch fails loudly instead of risking weight
+  corruption.
+- Components without an explicit supported quantization path remain in their
+  original precision.
+- Optional backend dependencies are imported lazily.
+- Method implementations own platform-specific capability checks and kernels.
+- User-facing support claims require model, hardware, numerical, and memory
+  validation in addition to configuration-level tests.

docs/design/feature/ray_based_execution.md

@@ -1,59 +0,0 @@
-# Distributed utils
-
-This directory (vllm_omni/distributed/ray_utils) contains utilities for distributed execution in vllm-omni, supporting both **Ray** and **Multiprocessing** backends.
-## 1. Installation
-```bash
-pip install "ray[default]"
-```
-## 2. Ray Utils
-
-The `ray_utils` module provides helper functions for managing Ray clusters and actors, which is essential for:
-*   **Multi-node deployment**: Running pipeline stages across different physical machines.
-*   **Resource management**: Efficient GPU/CPU allocation.
-
-### 2.1 Basic Usage
-
-To use the Ray backend, specify `worker_backend="ray"` when initializing the engine.
-
-**Command Line Example:**
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B \
-  --omni \
-  --port 8091 \
-  --worker-backend ray \
-  --ray-address auto
-```
-
-### 2.2 Cluster Setup
-
-**Step 1: Start Head Node**
-Run this on your primary machine:
-```bash
-ray start --head --port=6399
-```
-
-**Step 2: Connect Worker Nodes**
-Run this on each worker machine:
-```bash
-ray start --address=<HEAD_NODE_IP>:6399
-```
-
-> **Tip**: For a complete cluster setup script, refer to the vLLM example:
-> [run_cluster.sh](https://github.com/vllm-project/vllm/blob/main/examples/online_serving/run_cluster.sh)
-
-### 2.3 Distributed Connector Support
-
-When running on Ray, the system automatically adapts its communication strategy:
-
-*   **Cross-Node**: Recommended to use `MooncakeTransferEngineConnector` (RDMA, fastest) or `MooncakeStoreConnector` (TCP fallback).
-*   **Same-Node**: Can still use `SharedMemoryConnector` for efficiency, or Ray's native object store (plasma).
-*   **SHM payload routing**: `SharedMemoryConnector` always stores payloads in `/dev/shm`.
-
-### 2.4 Internal Helpers
-
-*   **`initialize_ray_cluster`**: Connects to an existing Ray cluster or starts a local one.
-
-## 3. Troubleshooting
-
-*   **Connection Issues**: Ensure the Ray head node is accessible and ports (default 6399 in this example) are open.
-*   **Version Mismatch**: Ensure all nodes run the same version of Ray and Python.

docs/design/feature/tensor_parallel.md

@@ -264,7 +264,7 @@ Complete examples in the codebase:
 | **Z-Image** | `vllm_omni/diffusion/models/z_image/z_image_transformer.py` | Standard TP | Full implementation with validation |
 | **FLUX** | `vllm_omni/diffusion/models/flux/flux_transformer.py` | Dual-stream | Image + text streams |
 | **Qwen-Image** | `vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py` | Standard TP | With RoPE |
-| **TP Tests** | `tests/e2e/offline_inference/test_zimage_parallelism.py` | E2E testing | TP correctness and performance |
+| **TP Tests** | `tests/diffusion/distributed/test_tensor_parallel.py` | E2E testing | TP correctness and performance |
 | **Constraint Tests** | `tests/diffusion/models/z_image/test_zimage_tp_constraints.py` | Unit testing | Validation logic |
 
 ---

docs/design/index.md

@@ -1,6 +1,8 @@
 # Design Documents
 
-This section contains design documents and architecture specifications for vLLM-Omni.
+This section contains design documents and architecture specifications for
+vLLM-Omni. The sidebar groups documents by the system concern they describe,
+while the source paths remain stable so existing links continue to work.
 
 ## Architecture Documents
 
@@ -8,18 +10,63 @@ This section contains design documents and architecture specifications for vLLM-
 
 ## Feature Design Documents
 
-- [Disaggregated Inference](feature/disaggregated_inference.md)
-- [Ray-based Execution](feature/ray_based_execution.md)
-- [Adding Step Execution Support for Diffusion Pipelines](feature/diffusion_step_execution.md)
-- [Request-Level Batching for Diffusion](feature/diffusion_request_level_batching.md)
-- [Continuous Batching for Step-Wise Diffusion](feature/diffusion_continuous_batching.md)
+### Runtime and stage execution
 
-## Infrastructure Design Documents
+- [Disaggregated Inference](feature/disaggregated_inference.md)
+- [Async Chunk](feature/async_chunk.md)
+- [Async Diffusion Output](feature/async_diffusion_output.md)
+- [Async Omni Output Materialization](feature/omni_async_output_materialization.md)
+- [Automatic Prefix Caching in Omni Models](feature/prefix_caching.md)
+
+### Communication
+
+#### OmniConnector implementations
+
+- [Mooncake Store Connector](feature/omni_connectors/mooncake_store_connector.md)
+- [Mooncake Transfer Engine Connector](feature/omni_connectors/mooncake_transfer_engine_connector.md)
+- [Mori Transfer Engine Connector](feature/omni_connectors/mori_transfer_engine_connector.md)
+- [Shared Memory Connector](feature/omni_connectors/shared_memory_connector.md)
+- [Yuanrong Store Connector](feature/omni_connectors/yuanrong_connector.md)
+- [Yuanrong Transfer Engine Connector](feature/omni_connectors/yuanrong_transfer_engine_connector.md)
+
+### Diffusion acceleration
+
+#### Parallelism
+
+- [CFG-Parallel](feature/cfg_parallel.md)
+- [Expert Parallel](feature/expert_parallel.md)
+- [Hybrid Sharded Data Parallel (HSDP)](feature/hsdp.md)
+- [Pipeline Parallel](feature/pipeline_parallel.md)
+- [Sequence Parallel](feature/sequence_parallel.md)
+- [Tensor Parallel](feature/tensor_parallel.md)
+- [VAE Patch Parallelism](feature/vae_parallel.md)
+
+#### Attention Backends
+
+- [Skip-Softmax](feature/skip_softmax.md)
+
+#### Quantization
+
+- [Quantization](feature/quantization.md)
+
+- [Cache-DiT](feature/cache_dit.md)
+- [TeaCache](feature/teacache.md)
+- [Diffusion Continuous Batching](feature/diffusion_continuous_batching.md)
+- [Distributed Layerwise Offload](feature/distributed_layerwise_offload.md)
+
+## Infrastructure and Performance
 
 - [Prometheus Metrics](metrics.md)
+- [Speech Generation Performance Optimizations](qwen3_omni_tts_performance_optimization.md)
 
 ## Module Design Documents
 
+### Runtime Modules
+
 - [AR Module](module/ar_module.md)
 - [DIT Module](module/dit_module.md)
+
+### Orchestration
+
 - [Entrypoint Module](module/entrypoint_module.md)
+- [AsyncOmni Architecture (Qwen3-Omni Example)](module/async_omni_architecture.md)

docs/design/metrics.md

@@ -148,7 +148,13 @@ Upstream vLLM's `Scheduler.make_stats()` runs on every AR generation step, retur
 
 `OmniSchedulerMixin.make_stats()` (in `vllm_omni/core/sched/omni_scheduler_mixin.py`) throttles stats emission to at most once per second **per scheduler** — i.e. per `(stage, replica)` since each replica owns its own scheduler instance. Between intervals it returns `None`, which the engine core skips serializing. This keeps gauges fresh enough for Prometheus scrapes (typically 15-30s intervals) while eliminating the per-step overhead.
 
-The orchestrator side does not add its own throttle on top: the per-replica recording loop gates only on `raw_outputs.scheduler_stats is not None` (i.e. this replica's scheduler passed its own 1 Hz gate). A previous global `_last_stats_ts` on the orchestrator was removed because it starved every replica other than the first to emit within each second.
+The orchestrator side does not add its own throttle on top. Scheduler gauges are
+recorded when `raw_outputs.scheduler_stats is not None` (i.e. this replica's
+scheduler passed its own 1 Hz gate), while output-batch request/token metrics
+are recorded from `IterationStats` whenever a non-empty output batch is
+processed. A previous global `_last_stats_ts` on the orchestrator was removed
+because it starved every replica other than the first to emit within each
+second.
 
 ## Metric Definitions
 

docs/design/module/dit_module.md

@@ -252,8 +252,7 @@ while True:
   batch of compatible independent requests for one full pipeline forward when
   the pipeline supports request-level batching; the step-wise path can also
   admit or remove compatible requests between denoise steps. See
-  [Request-Level Batching](../feature/diffusion_request_level_batching.md) and
-  [Continuous Batching for Step-Wise Diffusion](../feature/diffusion_continuous_batching.md).
+  [Diffusion Continuous Batching](../feature/diffusion_continuous_batching.md).
 
 ---
 

docs/design/qwen3_omni_tts_performance_optimization.md

@@ -419,8 +419,7 @@ Notes:
 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \
   --omni \
-  --port 8091 \
-  --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml
+  --port 8091
 ```
 
 #### 3) Key config knobs
@@ -494,7 +493,7 @@ The default config (`qwen3_tts.yaml`) enables the full optimization stack:
 vllm-omni serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
   --omni \
   --port 8000 \
-  --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts_no_async_chunk.yaml
+  --no-async-chunk
 ```
 
 #### 3) Key config knobs

docs/getting_started/installation/README.md

@@ -1,5 +1,8 @@
 # Installation
 
+!!! important
+    vLLM-Omni is released against the matching upstream vLLM major/minor version. vLLM-Omni 0.26.x requires vLLM 0.26.x; the stable v0.26.0 instructions pin vLLM 0.26.0.
+
 vLLM-Omni supports the following hardware platforms:
 
 - [GPU](gpu.md)

docs/getting_started/installation/gpu/cuda.inc.md

@@ -5,7 +5,7 @@
 # --8<-- [end:requirements]
 # --8<-- [start:set-up-using-python]
 
-vLLM-Omni depends vLLM. So please follow instructions below mainly for vLLM.
+vLLM-Omni depends on the matching major/minor release of vLLM. The vLLM-Omni 0.26.x release line uses vLLM 0.26.x.
 
 !!! note
     PyTorch installed via `conda` will statically link `NCCL` library, which can cause issues when vLLM tries to use `NCCL`. See <gh-issue:8420> for more details.
@@ -124,7 +124,7 @@ If you want to specify the base vLLM version:
 ```bash
 DOCKER_BUILDKIT=1 docker build \
   -f docker/Dockerfile.cuda \
-  --build-arg BASE_IMAGE=vllm/vllm-openai:v0.22.1 \
+  --build-arg BASE_IMAGE=vllm/vllm-openai:v0.26.0 \
   -t vllm-omni-cuda .
 ```
 

docs/getting_started/installation/npu/npu.inc.md

@@ -9,8 +9,8 @@ The recommended way to use vLLM-Omni on NPU is through the vLLM-Ascend pre-built
 
 ```bash
 # vLLM-Ascend image
-# Atlas A2: quay.io/atlas-ci/vllm-ascend:v0.25.0
-# Atlas A3: quay.io/atlas-ci/vllm-ascend:v0.25.0-a3
+# Atlas A2: quay.io/atlas-ci/vllm-ascend:v0.26.0
+# Atlas A3: quay.io/atlas-ci/vllm-ascend:v0.26.0-a3
 docker run --rm \
     --name vllm-omni-npu \
     --shm-size=1g \
@@ -28,11 +28,11 @@ docker run --rm \
     -v /etc/ascend_install.info:/etc/ascend_install.info \
     -v /root/.cache:/root/.cache \
     -p 8000:8000 \
-    -it quay.io/atlas-ci/vllm-ascend:0.25.0-a3 bash
+    -it quay.io/atlas-ci/vllm-ascend:v0.26.0-a3 bash
 
 # Inside the container, install vLLM-Omni from source
 cd /vllm-workspace
-git clone -b v0.25.0 https://github.com/vllm-project/vllm-omni.git
+git clone -b v0.26.0 https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
 pip install -v -e . --no-build-isolation
 # or VLLM_OMNI_TARGET_DEVICE=npu pip install -v -e .
@@ -53,16 +53,20 @@ We are keeping [issue #886](https://github.com/vllm-project/vllm-omni/issues/886
 You can also build vLLM-Omni from the latest main branch if you want to use the latest features or bug fixes. (But sometimes it will break for a while. You can check [issue #886](https://github.com/vllm-project/vllm-omni/issues/886) for the status of the latest commit of vLLM-Omni main branch on NPU.)
 
 ```bash
-# Pin vLLM version to 0.18.0
-git clone -b v0.18.0 https://github.com/vllm-project/vllm.git
+# Pin vLLM and vLLM-Ascend to the v0.26 release line
+git clone -b v0.26.0 https://github.com/vllm-project/vllm.git
+cd vllm
 VLLM_TARGET_DEVICE=empty pip install -v -e .
+cd ..
 
-git clone -b v0.18.0rc1 https://github.com/vllm-project/vllm-ascend.git
+git clone -b releases/v0.26.0rc https://github.com/vllm-project/vllm-ascend.git
+cd vllm-ascend
 pip install -v -e .
+cd ..
 
 # Install vLLM-Omni from the latest main branch
 git clone https://github.com/vllm-project/vllm-omni.git
-cd /vllm-workspace/vllm-omni
+cd vllm-omni
 pip install -v -e . --no-build-isolation
 # or VLLM_OMNI_TARGET_DEVICE=npu pip install -v -e .
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
@@ -84,8 +88,8 @@ Supported images as following.
 Here's an example deployment command that has been verified on 4 x NPUs:
 
 ```bash
-# Atlas A2: quay.io/ascend/vllm-omni:v0.25.0
-# Atlas A3: quay.io/ascend/vllm-omni:v0.25.0-a3
+# Atlas A2: quay.io/ascend/vllm-omni:v0.26.0
+# Atlas A3: quay.io/ascend/vllm-omni:v0.26.0-a3
 docker run --rm \
     --name vllm-omni-a3 \
     --shm-size=4g \
@@ -110,7 +114,7 @@ docker run --rm \
     -v /etc/ascend_install.info:/etc/ascend_install.info \
     -v ~/.cache:/root/.cache \
     -p 8091:8091 \
-    -it quay.io/ascend/vllm-omni:v0.25.0-a3 bash
+    -it quay.io/ascend/vllm-omni:v0.26.0-a3 bash
 ```
 
 !!! tip
@@ -122,9 +126,9 @@ Or build IMAGE from **source code**:
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
 # A2
-docker build -t vllm-omni-dev-image:latest -f ./docker/Dockerfile.npu .
+# docker build -t vllm-omni-dev-image:latest -f ./docker/Dockerfile.npu .
 # A3
-# docker build -t vllm-omni-dev-image:latest -f ./docker/Dockerfile.npu.a3 .
+docker build -t vllm-omni-dev-image:latest -f ./docker/Dockerfile.npu.a3 .
 ```
 
 # --8<-- [end:pre-built-images]

docs/getting_started/quickstart.md

@@ -87,9 +87,8 @@ if __name__ == "__main__":
 
 !!! info
 
-    For diffusion request-level batching controls such as `max_num_seqs` and
-    `request_batch_max_wait_ms`, see
-    [Request-Level Batching](../user_guide/diffusion/request_batching.md).
+    For diffusion request batching, step execution, and streaming controls, see
+    [Diffusion Execution Modes](../user_guide/diffusion/execution_modes.md).
 
 For more usages, please refer to [offline inference](../user_guide/examples/offline_inference/qwen2_5_omni.md)
 

docs/models/supported_models.md

@@ -44,6 +44,7 @@ th {
 | `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
 | `LTX2DistilledPipeline` | LTX-2 distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
 | `LingBotVideoPipeline` | LingBot-Video dense and MoE T2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | |
+| `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | | | |
 | `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | |
 | `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | |

docs/serving/speech_api.md

@@ -276,20 +276,42 @@ Client -> Server:
 
 | Message | Description |
 |---------|-------------|
-| `{"type": "session.config", ...}` | Session configuration (sent once, first message) |
+| `{"type": "session.config", ...}` | Session configuration (first message; may be resent between utterances to change it) |
 | `{"type": "input.text", "text": "..."}` | Text chunk |
-| `{"type": "input.done"}` | End of input, flushes remaining buffer |
+| `{"type": "input.done"}` | End of utterance: flushes the buffer and keeps the connection open |
+| `{"type": "session.close"}` | End of connection |
 
 Server -> Client:
 
 | Message | Description |
 |---------|-------------|
-| `{"type": "audio.start", "sentence_index": 0, "sentence_text": "...", "format": "pcm", "sample_rate": 24000}` | Audio generation starting for the buffered input |
+| `{"type": "audio.start", "utterance_index": 0, "sentence_index": 0, "sentence_text": "...", "format": "pcm", "sample_rate": 24000}` | Audio generation starting for the buffered input |
 | Binary frame | Raw audio bytes (one or more PCM chunks when `stream_audio=true`) |
-| `{"type": "audio.done", "sentence_index": 0, "total_bytes": 96000, "error": false}` | Audio complete for the buffered input |
-| `{"type": "session.done", "total_sentences": N}` | Session complete |
+| `{"type": "audio.done", "utterance_index": 0, "sentence_index": 0, "total_bytes": 96000, "error": false}` | Audio complete for the buffered input |
+| `{"type": "session.done", "utterance_index": 0, "total_sentences": N}` | Flushed utterance complete |
 | `{"type": "error", "message": "..."}` | Non-fatal error |
 
+### Flushing vs. Closing
+
+`input.done` is a flush, not a disconnect. The server synthesizes the buffered
+text, emits `session.done`, and then waits on the same connection for the next
+utterance, so a client that speaks repeatedly (for example one driven by an
+upstream LLM) pays the WebSocket handshake once instead of once per utterance.
+
+* The session config is sticky. Send `input.text` again straight after
+  `session.done` to reuse it, or send another `session.config` first to change
+  voice, format, or reference audio. A `session.config` sent while text is
+  still buffered is rejected so no pending input is silently dropped.
+* An utterance is the flush unit, not a linguistic one: it is whatever text was
+  buffered when `input.done` arrived, of any length, synthesized as one request.
+  `utterance_index` counts those flushes across the connection, so it tells you
+  which `input.done` a frame belongs to. `sentence_index` counts within one
+  flush and so pairs with `total_sentences`, which means every utterance reports
+  `sentence_index: 0` of `total_sentences: 1` (or `0` for an empty buffer).
+* End the connection with `session.close`, or by closing the socket. An idle
+  connection is still closed after the server's idle timeout, which now also
+  applies to the gap between utterances.
+
 ### Session Config Parameters
 
 All REST API parameters are supported, plus:

docs/source/architecture/qwen3-omni-async-output-step-gap.svg

@@ -0,0 +1,150 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="1024" height="600" viewBox="0 0 1024 600" role="img" aria-labelledby="title desc">
+  <title id="title">Qwen3-Omni Talker step gap before and after asynchronous output materialization</title>
+  <desc id="desc">Before the optimization, synchronous D2H copy, per-request payload construction, streaming payload handoff, and connector-signal drain separate consecutive Talker forward and sample steps by about 2.8 milliseconds. After the optimization, that Omni output work runs on a background builder while the next Talker step executes, leaving a boundary of about 41 microseconds for snapshot, D2H enqueue, state update, and sampled-token feedback.</desc>
+  <defs>
+    <marker id="time-arrow" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto">
+      <path d="M 0 0 L 10 5 L 0 10 z" fill="#64748b"/>
+    </marker>
+    <marker id="red-end" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="5" markerHeight="5" orient="auto">
+      <path d="M 0 0 L 10 5 L 0 10 z" fill="#b91c1c"/>
+    </marker>
+    <marker id="red-start" viewBox="0 0 10 10" refX="2" refY="5" markerWidth="5" markerHeight="5" orient="auto">
+      <path d="M 10 0 L 0 5 L 10 10 z" fill="#b91c1c"/>
+    </marker>
+    <marker id="blue-end" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="6" markerHeight="6" orient="auto">
+      <path d="M 0 0 L 10 5 L 0 10 z" fill="#2563eb"/>
+    </marker>
+    <style>
+      .canvas { fill: #ffffff; stroke: #e5e7eb; }
+      .title { font: 700 22px system-ui, -apple-system, Segoe UI, sans-serif; fill: #111827; }
+      .subtitle { font: 500 13px system-ui, -apple-system, Segoe UI, sans-serif; fill: #64748b; }
+      .phase { font: 700 16px system-ui, -apple-system, Segoe UI, sans-serif; fill: #1f2937; }
+      .phase-sub { font: 500 12px system-ui, -apple-system, Segoe UI, sans-serif; fill: #64748b; }
+      .lane { font: 600 12px system-ui, -apple-system, Segoe UI, sans-serif; fill: #374151; }
+      .block-title { font: 700 10px system-ui, -apple-system, Segoe UI, sans-serif; letter-spacing: .25px; fill: #ffffff; }
+      .block-op { font: 500 9.5px system-ui, -apple-system, Segoe UI, sans-serif; fill: #ffffff; }
+      .axis { font: 500 11px system-ui, -apple-system, Segoe UI, sans-serif; fill: #94a3b8; }
+      .measure { font: 700 12px system-ui, -apple-system, Segoe UI, sans-serif; fill: #b91c1c; }
+      .inset-title { font: 700 11px system-ui, -apple-system, Segoe UI, sans-serif; fill: #475569; }
+      .inset-step { font: 600 10px system-ui, -apple-system, Segoe UI, sans-serif; fill: #ffffff; }
+      .note-title { font: 700 12px system-ui, -apple-system, Segoe UI, sans-serif; fill: #1e3a8a; }
+      .note-text { font: 500 10.5px system-ui, -apple-system, Segoe UI, sans-serif; fill: #475569; }
+      .stat-cap { font: 600 11px system-ui, -apple-system, Segoe UI, sans-serif; fill: #64748b; }
+      .stat-big { font: 800 16px system-ui, -apple-system, Segoe UI, sans-serif; fill: #111827; }
+      .stat-sub { font: 500 11px system-ui, -apple-system, Segoe UI, sans-serif; fill: #64748b; }
+      .before-card { fill: #eef4ff; stroke: #c7dbf6; }
+      .after-card { fill: #eafaf0; stroke: #bfe7cf; }
+      .talker { fill: #16a34a; }
+      .sync-output { fill: #ea580c; }
+      .async-output { fill: #2563eb; }
+      .zoom-card { fill: #fff7f7; stroke: #f3caca; }
+      .note-card { fill: #eff6ff; stroke: #bfdbfe; }
+      .stat-blue { fill: #eef4ff; stroke: #c7dbf6; }
+      .stat-green { fill: #eafaf0; stroke: #bfe7cf; }
+      .stat-gray { fill: #f4f5f7; stroke: #e2e5ea; }
+      .guide { stroke: #e2e8f0; stroke-width: 1; }
+      .time-line { stroke: #64748b; stroke-width: 1.3; fill: none; marker-end: url(#time-arrow); }
+      .measure-line { stroke: #b91c1c; stroke-width: 1.4; marker-start: url(#red-start); marker-end: url(#red-end); }
+      .zoom-lead { stroke: #dc2626; stroke-width: 1.15; stroke-dasharray: 4 3; fill: none; }
+      .handoff { stroke: #2563eb; stroke-width: 1.3; stroke-dasharray: 3 2; fill: none; marker-end: url(#blue-end); }
+    </style>
+  </defs>
+
+  <rect x="14" y="14" width="996" height="572" rx="14" class="canvas"/>
+
+  <text x="44" y="52" class="title">Async output: what moves off the Talker decode path</text>
+  <text x="44" y="74" class="subtitle">Qwen3-Omni Talker stage &#183; consecutive decode steps in a fixed profiler window</text>
+
+  <!-- ===================== BEFORE ===================== -->
+  <rect x="44" y="94" width="144" height="110" rx="10" class="before-card"/>
+  <text x="60" y="120" class="phase">Before</text>
+  <text x="60" y="139" class="phase-sub">Omni output built</text>
+  <text x="60" y="156" class="phase-sub">synchronously on the</text>
+  <text x="60" y="173" class="phase-sub">AR critical path</text>
+
+  <text x="206" y="122" class="lane">AR path</text>
+  <line x1="300" y1="88" x2="300" y2="210" class="guide"/>
+
+  <rect x="300" y="104" width="190" height="54" rx="5" class="talker"/>
+  <text x="395" y="124" text-anchor="middle" class="block-title">TALKER STAGE &#183; STEP N</text>
+  <text x="395" y="145" text-anchor="middle" class="block-op">forward &#8594; sample</text>
+
+  <rect x="490" y="104" width="250" height="54" rx="5" class="sync-output"/>
+  <text x="615" y="120" text-anchor="middle" class="block-title">SYNC OMNI OUTPUT &#183; STEP N</text>
+  <text x="615" y="138" text-anchor="middle" class="block-op">D2H &#8594; per-request slice / payload build</text>
+  <text x="615" y="152" text-anchor="middle" class="block-op">streaming handoff &#8594; connector-signal drain</text>
+
+  <rect x="740" y="104" width="220" height="54" rx="5" class="talker"/>
+  <text x="850" y="124" text-anchor="middle" class="block-title">TALKER STAGE &#183; STEP N+1</text>
+  <text x="850" y="145" text-anchor="middle" class="block-op">forward &#8594; sample</text>
+
+  <line x1="490" y1="174" x2="740" y2="174" class="measure-line"/>
+  <text x="615" y="194" text-anchor="middle" class="measure">~2.8 ms GPU gap: output work is inline</text>
+
+  <!-- ===================== AFTER ===================== -->
+  <rect x="44" y="218" width="144" height="140" rx="10" class="after-card"/>
+  <text x="60" y="244" class="phase">After</text>
+  <text x="60" y="263" class="phase-sub">Only step feedback</text>
+  <text x="60" y="280" class="phase-sub">stays inline; payload</text>
+  <text x="60" y="297" class="phase-sub">work runs in the</text>
+  <text x="60" y="314" class="phase-sub">background</text>
+
+  <text x="206" y="254" class="lane">Talker GPU</text>
+  <text x="206" y="326" class="lane">Output builder</text>
+  <line x1="300" y1="212" x2="300" y2="370" class="guide"/>
+
+  <rect x="300" y="226" width="235" height="54" rx="5" class="talker"/>
+  <text x="418" y="246" text-anchor="middle" class="block-title">TALKER STAGE &#183; STEP N</text>
+  <text x="418" y="267" text-anchor="middle" class="block-op">forward &#8594; sample</text>
+
+  <rect x="540" y="226" width="235" height="54" rx="5" class="talker"/>
+  <text x="658" y="246" text-anchor="middle" class="block-title">TALKER STAGE &#183; STEP N+1</text>
+  <text x="658" y="267" text-anchor="middle" class="block-op">forward &#8594; sample</text>
+
+  <path d="M 535 280 L 548 298" class="handoff"/>
+  <rect x="540" y="298" width="360" height="58" rx="5" class="async-output"/>
+  <text x="720" y="315" text-anchor="middle" class="block-title">BACKGROUND OMNI OUTPUT &#183; STEP N</text>
+  <text x="720" y="334" text-anchor="middle" class="block-op">wait for D2H &#8594; per-request slice / payload build</text>
+  <text x="720" y="349" text-anchor="middle" class="block-op">streaming handoff &#8594; connector-signal drain</text>
+
+  <line x1="300" y1="366" x2="930" y2="366" class="time-line"/>
+  <text x="922" y="382" text-anchor="end" class="axis">time</text>
+
+  <!-- leads from the real 41 us boundary to the magnified inset -->
+  <line x1="535" y1="280" x2="485" y2="424" class="zoom-lead"/>
+  <line x1="540" y1="280" x2="525" y2="424" class="zoom-lead"/>
+
+  <!-- ===================== 41 µs BOUNDARY ===================== -->
+  <rect x="300" y="386" width="400" height="92" rx="10" class="zoom-card"/>
+  <text x="316" y="404" class="inset-title">Zoom &#183; Talker step boundary (magnified)</text>
+
+  <rect x="320" y="416" width="165" height="26" rx="4" class="talker"/>
+  <rect x="525" y="416" width="155" height="26" rx="4" class="talker"/>
+  <text x="403" y="433" text-anchor="middle" class="inset-step">Talker step N</text>
+  <text x="603" y="433" text-anchor="middle" class="inset-step">Talker step N+1</text>
+
+  <line x1="485" y1="452" x2="525" y2="452" class="measure-line"/>
+  <text x="505" y="469" text-anchor="middle" class="measure">~41 &#181;s</text>
+
+  <rect x="720" y="386" width="240" height="92" rx="10" class="note-card"/>
+  <text x="736" y="405" class="note-title">Inline at the ~41 &#181;s boundary</text>
+  <text x="736" y="424" class="note-text">&#8226; eager Talker state update</text>
+  <text x="736" y="441" class="note-text">&#8226; output snapshot + D2H enqueue</text>
+  <text x="736" y="458" class="note-text">&#8226; sampled-token feedback</text>
+
+  <!-- ===================== SUMMARY ===================== -->
+  <rect x="44" y="494" width="296" height="70" rx="9" class="stat-blue"/>
+  <text x="60" y="514" class="stat-cap">Step-to-step gap</text>
+  <text x="60" y="538" class="stat-big">2.8 ms &#8594; 41 &#181;s</text>
+  <text x="60" y="555" class="stat-sub">~68&#215; tighter</text>
+
+  <rect x="356" y="494" width="296" height="70" rx="9" class="stat-green"/>
+  <text x="372" y="514" class="stat-cap">Talker GPU</text>
+  <text x="372" y="538" class="stat-big">next step starts immediately</text>
+  <text x="372" y="555" class="stat-sub">decode workers stay busy</text>
+
+  <rect x="668" y="494" width="312" height="70" rx="9" class="stat-gray"/>
+  <text x="684" y="514" class="stat-cap">Background builder</text>
+  <text x="684" y="538" class="stat-big">materializes Omni output</text>
+  <text x="684" y="555" class="stat-sub">payload work overlaps the next Talker step</text>
+</svg>

docs/user_guide/diffusion/attention_backends.md

@@ -15,7 +15,7 @@ The full set of backends and their platform defaults is in the **Backend Options
 | Value | Notes |
 |---|---|
 | `TRTLLM_ATTN` | FlashInfer's trtllm-gen FMHA (TensorRT-LLM's generated kernels, vendored by FlashInfer). BF16, GQA native, `head_dim=128`. Datacenter Blackwell only (sm_100 / sm_103). Supports **Skip-Softmax** sparse attention — see below. Requires `flashinfer`. |
-| `FLASH_ATTN` | Wraps FlashAttention 2. Default on Hopper / Ada / Ampere when `flash-attn` is installed. |
+| `FLASH_ATTN` | Wraps FlashAttention 4 on Blackwell when `flash-attn-4` is installed, then falls back to FlashAttention 3/2. Default on Hopper / Ada / Ampere when a compatible FlashAttention package is installed. |
 | `CUDNN_ATTN` | Pins `sdpa_kernel([CUDNN_ATTENTION])`. Default on Blackwell (sm_10x / sm_12x) with cuDNN ≥ 9.5. Wins on mask-heavy DiTs (HunyuanVideo-1.5: 2× e2e vs SDPA). |
 | `FLASHINFER_ATTN` | Calls FlashInfer's dense `single_prefill_with_kv_cache` directly with `custom_mask` for non-causal masked attention. Used as Blackwell fallback when cuDNN is unavailable. Requires `flashinfer`. |
 | `TORCH_SDPA` | PyTorch `scaled_dot_product_attention` with the default backend dispatcher. Most conservative; always available. |
@@ -167,6 +167,48 @@ sequence length, since Skip-Softmax only accelerates attention. Dense BF16 is `T
 Requires datacenter Blackwell with `head_dim == 128`; elsewhere, or without FlashInfer, selecting
 `TRTLLM_ATTN` raises rather than silently degrading.
 
+## TRTLLM_ATTN SAGE Quantization
+
+On top of dense/Skip-Softmax, `TRTLLM_ATTN` can run **SAGE** (SageAttention) — the trtllm-gen
+FMHA kernel with per-block quantized Q/K and per-channel fp8 V, so the two attention matmuls run
+in low precision. Enable it through the typed `quant` block on the attention spec:
+
+| Key | Valid values | Meaning |
+|---|---|---|
+| `dtype_qk` | `int8`, `fp8_e4m3` | Quantization dtype for Q and K. Absent ⇒ dense (SAGE off). |
+| `q_block_size` | `1`, `4`, `16` | Per-token block size for Q scales (default `1`). |
+| `k_block_size` | `1`, `4`, `16` | Per-token block size for K scales (default `16`). |
+
+V is always quantized per-channel to fp8_e4m3 and K-smoothing is applied inside the routine, so
+neither is a user knob. Only block sizes `1`, `4`, `16` have compiled kernels.
+
+The `quant` block is a shared `AttnQuantSpec` consumed by more than one backend; each reads the
+fields that apply and rejects the rest. `TRTLLM_ATTN` (SAGE) reads `dtype_qk` (`int8` / `fp8_e4m3`),
+`q_block_size`, and `k_block_size`; `FLASHINFER_ATTN` reads `dtype_qk` (`float16` / `bfloat16`),
+`dtype_vo`, and `flashinfer_backend` (see its section). A `dtype_qk` from the wrong set raises.
+
+```bash
+vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \
+  --diffusion-attention-config '{"default": {"backend": "TRTLLM_ATTN",
+      "quant": {"dtype_qk": "fp8_e4m3", "q_block_size": 1, "k_block_size": 16}}}'
+```
+
+```python
+from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, AttnQuantSpec
+
+AttentionConfig(
+    default=AttentionSpec(
+        backend="TRTLLM_ATTN",
+        quant=AttnQuantSpec(dtype_qk="fp8_e4m3", q_block_size=1, k_block_size=16),
+    ),
+)
+```
+
+
+**Requirements.** Needs FlashInfer ≥ 0.6.16rc1. Kernel
+availability is arch-dependent: `fp8_e4m3` QK has kernels on both **SM100** (B200) and **SM103**
+(B300); `int8` QK kernels are compiled for **SM100 only**.
+
 ## End-to-End Benchmark (BF16, sm_120 RTX Pro 6000 Blackwell)
 
 Same prompt and seed across runs. `Total generation time` from `text_to_video.py` / `text_to_image.py`.
@@ -193,9 +235,17 @@ DIFFUSION_ATTENTION_BACKEND=FLASHINFER_ATTN python examples/offline_inference/te
     --model Lightricks/LTX-2 ...
 ```
 
-### FA4 not yet integrated
+### FlashAttention-4 on Blackwell
 
-FlashAttention-4 (released March 2026) targets Blackwell natively and reportedly beats cuDNN by ~20% on B200. As of this writing the `flash-attn-4 4.0.0b10` wheel crashes with `AttributeError: 'NoneType' object has no attribute '_trait'` during JIT on sm_120. Not yet wired into vLLM-Omni; revisit when stable lands.
+Install the optional CUDA 13 extra to use the CuTe-based FA4 path:
+
+```bash
+pip install 'vllm-omni[fa4]'
+```
+
+`FLASH_ATTN` prefers `flash_attn.cute` on Blackwell and falls back to FA3/FA2
+when it is unavailable. Version `4.0.0b18` is required; earlier beta
+wheels had known JIT failures on Blackwell.
 
 ## Choosing a Backend Manually
 
@@ -353,64 +403,3 @@ DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN_3 python examples/offline_inference/text_t
     --tensor-parallel-size 2 \
     --output outputs/hv15_sage3.mp4
 ```
-
-### Mixed backends across roles
-
-Use `FLASH_ATTN` for self-attention and `TORCH_SDPA` for cross-attention:
-
-```bash
-python examples/offline_inference/text_to_video/text_to_video.py \
-    --model Wan-AI/Wan2.2-TI2V-5B-Diffusers \
-    --prompt "A dog running across a field of golden wheat." \
-    --diffusion-attention-config.per_role.self.backend FLASH_ATTN \
-    --diffusion-attention-config.per_role.cross.backend TORCH_SDPA \
-    --tensor-parallel-size 2 \
-    --output outputs/wan22_mixed.mp4
-```
-
-### Compare against FlashAttention
-
-Unset the backend override, or explicitly use `FLASH_ATTN`:
-
-```bash
-python examples/offline_inference/text_to_video/text_to_video.py \
-    --model Wan-AI/Wan2.2-TI2V-5B-Diffusers \
-    --prompt "A dog running across a field of golden wheat." \
-    --height 704 --width 1280 --num-frames 49 \
-    --num-inference-steps 30 --seed 42 --guidance-scale 5.0 \
-    --tensor-parallel-size 2 \
-    --output outputs/wan22_fa3.mp4
-```
-
-## Validation Guidance
-
-Don't assume a faster attention backend is numerically interchangeable with `TORCH_SDPA`.
-
-Always compare:
-
-- End-to-end runtime
-- Diffusion-stage runtime (`add_req_and_wait` line in DiffusionEngine.step breakdown)
-- Output quality against a known-good baseline (CLIP similarity, frame-level diff, or visual review)
-
-At minimum, keep the same:
-
-- model
-- prompt
-- seed
-- resolution
-- frame count / step count
-- parallel config (TP / CFG-parallel / Ulysses degrees)
-
-## Reproducing the Benchmark Table
-
-The end-to-end numbers above were collected by running `text_to_video.py` /
-`text_to_image.py` with the same prompt and seed while varying
-`DIFFUSION_ATTENTION_BACKEND`. For a quick kernel-level comparison of the
-backends without loading a model:
-
-```bash
-python benchmarks/diffusion/bench_attention_backends.py --preset hv15
-```
-
-It runs all three BF16 backends on representative DiT attention shapes and
-prints a ranking table at the end.

docs/user_guide/diffusion/cpu_offload.md

@@ -179,7 +179,145 @@ class Flux2Transformer2DModel(nn.Module):
 - Support single GPU only for now
 
 
-### Implementation Notes
+## Distributed Layerwise Offloading
+
+### How It Works
+
+Distributed layerwise offloading extends single-GPU layerwise offloading to
+multi-device deployments. In the default mode, each rank stores only
+**1/dp_size** of the model weights in host memory; full layer weights are
+reconstructed at runtime via **AllGather** on a dedicated communication stream,
+overlapped with computation via a fixed double-buffer scheme. The
+`--dlo-no-use-allgather` mode instead streams standard-loader rank-local
+weights independently and does not shard weights across ranks.
+
+For the implementation design and compatibility matrix, see
+[Distributed Layerwise Offload](../../design/feature/distributed_layerwise_offload.md).
+
+**Key features:**
+- **Weight sharding + AllGather**: each rank stores 1/dp_size of weights,
+  reconstructed per-layer via `all_gather_into_tensor`
+- **Fixed double-buffer**: exactly 2 transformer blocks on each device at any
+  time, regardless of model size
+- **DP multi-concurrency**: N concurrent requests processed in parallel
+  (AllGather only gathers weight shards, which are request-independent)
+- **mmap weight loading**: weights loaded as mmap views pointing to shared OS
+  page cache, eliminating O(dp_size × model_size) RSS during model creation
+- **Platform-agnostic**: works on NVIDIA GPU (CUDA/NCCL) and Ascend NPU
+  (CANN/HCCL) via vLLM-Omni's platform abstraction
+
+**Execution Flow (per device):**
+
+```
+Compute Stream:  [Layer N]          [Layer N+1]          [Layer N+2]
+H2D Stream:      [H2D Shard N+1]   [H2D Shard N+2]
+AllGather:       [AG N+1]          [AG N+2]
+Slot Usage:      Slot0: Layer N    Slot1: Layer N+1
+```
+
+### Usage
+
+**CLI:**
+```bash
+# 4× GPU/NPU with AllGather (recommended)
+vllm serve /path/to/model --omni \
+  --enable-distributed-layerwise-offload \
+  --data-parallel-size 4
+
+# Without DLO AllGather (each rank streams standard-loader rank-local weights)
+vllm serve /path/to/model --omni \
+  --enable-distributed-layerwise-offload \
+  --data-parallel-size 4 \
+  --dlo-no-use-allgather
+
+# With SP instead of DP (long sequences)
+vllm serve /path/to/model --omni \
+  --enable-distributed-layerwise-offload \
+  --usp 4
+```
+
+**Python API:**
+```python
+from vllm_omni import Omni
+
+m = Omni(
+    model="/path/to/model",
+    enable_distributed_layerwise_offload=True,
+    dlo_use_allgather=True,  # default
+)
+```
+
+### CLI Flags
+
+| Flag | Description | Default |
+|------|-------------|---------|
+| `--enable-distributed-layerwise-offload` | Enable DLO | `false` |
+| `--data-parallel-size N` | Number of DP ranks; DLO weight-sharding group when AllGather is enabled | `1` |
+| `--dlo-use-allgather` | Shard + AllGather (saves CPU, requires concurrent requests) | `true` |
+| `--dlo-no-use-allgather` | Use standard-loader rank-local weights; no DLO sharding or AllGather | `false` |
+
+### How model weights are loaded (mmap path)
+
+When DLO + AllGather is active, the offloader:
+
+1. **Saves non-persistent buffers** (e.g. RoPE `inv_freq`, timestep `freqs`)
+   from the normally-created transformer
+2. **Converts to meta device** via `to_empty(device="meta")`, releasing random
+   initialization weights
+3. **Loads checkpoint weights as mmap views** via `safe_open().get_tensor()`,
+   which return views into the OS page cache (shared across all ranks, 0 RSS)
+4. **Calls `post_load_weights()`** to apply model-specific dtype conversions
+   (e.g. Cosmos3's `time_embedder` → FP32)
+5. **Restores non-persistent buffers** from saved copies
+
+This approach requires **zero model-specific code changes** — no pipeline or
+transformer modifications are needed.
+
+### OffloadPlan (declarative topology metadata)
+
+Models can optionally declare an `OffloadPlan` class variable to provide
+topology metadata (block attribute names, submodules to offload) without
+any offload-specific logic:
+
+```python
+from vllm_omni.diffusion.offloader import OffloadPlan
+
+class MyPipeline(nn.Module):
+    _dit_modules = ["transformer"]
+    _offload_plan = OffloadPlan(
+        block_attrs={"transformer": ("blocks",)},
+        offload_submodules={"context_encoder": "layers"},
+    )
+```
+
+When not declared, the offloader falls back to `_layerwise_offload_blocks_attrs`
+and heuristic attribute search.  This is backward-compatible — existing models
+work without any changes.
+
+### DP Multi-concurrency
+
+When `--data-parallel-size > 1` and AllGather is enabled, the scheduler batches
+up to `dp_size` requests per denoise step.  Each DP rank processes a different
+request while AllGather synchronizes only weight shards (request-independent).
+
+**Requirements for concurrent requests:**
+- `num_inference_steps` must be specified explicitly (None is not allowed)
+- All concurrent requests must have the same `num_inference_steps` value
+- These constraints exist because AllGather is a collective that requires
+  every rank to participate at each denoise step
+
+### Limitations
+- Online quantization (FP8) is incompatible with mmap loading — falls back to
+  regular loading only when the model is outside the DLO+AllGather mmap path;
+  online quantization with DLO+AllGather is rejected. Use
+  `--dlo-no-use-allgather` or disable online quantization.
+- Tensor Parallel > 1 is rejected in the DLO+AllGather mmap path because TP-aware
+  weight-loader callbacks are bypassed. The no-AllGather path retains the
+  standard TP loader and is experimental.
+- HSDP + AllGather is rejected because it would double-shard parameters. HSDP
+  with no-AllGather is accepted at configuration level, but has limited
+  end-to-end validation.
+- `num_inference_steps=None` is not allowed in DP multi-concurrency mode
 
 **Module Discovery**
 
@@ -208,7 +346,10 @@ OffloadBackend (base class)
 ├── ModelLevelOffloadBackend → uses SequentialOffloadHook (.to() swap)
 │                              (delegates to a pipeline's enable_omni_model_cpu_offload
 │                               for split models like Cosmos3)
-└── LayerWiseOffloadBackend → uses LayerwiseOffloadHook
+├── LayerWiseOffloadBackend → uses LayerwiseOffloadHook
+│                          (single-GPU, full weights on host)
+└── DistributedLayerwiseOffloadBackend → uses DistributedLayerwiseOffloadHook
+                                         (multi-GPU, sharded AllGather or rank-local mode)
 ```
 
 Factory function `get_offload_backend()` selects the appropriate backend based on
@@ -221,20 +362,28 @@ reasoner/generator components inside the model forward pass.
 
 ## Supported Models
 
-| Architecture | Example Models | DiT Class | Model-Level Offload | Layerwise Offload | Blocks Attrs (Layerwise specific) |
-|--------------|----------------|-----------|---------------------|-------------------|-----------------------------------|
-| Flux2Pipeline | `black-forest-labs/FLUX.2-dev` | `Flux2Transformer2DModel` | ✓ | ✓ | `"transformer_blocks"`, `"single_transformer_blocks"` |
-| LongCatImagePipeline | `meituan-longcat/LongCat-Image` | `LongCatImageTransformer2DModel` | - | ✓ | `"transformer_blocks"`, `"single_transformer_blocks"` |
-| NextStep11Pipeline | `stepfun-ai/NextStep-1.1` | `NextStepModel` | - | ✓ | `"layers"` |
-| OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | - | ✓ | `"transformer"` |
-| QwenImagePipeline | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | ✓ | ✓ | `"transformer_blocks"` |
-| StableDiffusionXLPipeline | `stabilityai/stable-diffusion-xl-base-1.0` | `SDXLUNet2DConditionModel` | ✓ | ✓ | `"down_blocks"`, `"up_blocks"` |
-| StableDiffusion3Pipeline | `stabilityai/stable-diffusion-3.5-medium` | `SD3Transformer2DModel` | - | ✓ | `"transformer_blocks"` |
-| Wan22I2VPipeline | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | `"blocks"` |
-| Wan22Pipeline | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | `"blocks"` |
-| SoulXSingerPipeline / SoulXSingerSVCPipeline | `Soul-AILab/SoulX-Singer` | `DiffLlama` (`cfm_decoder.model.diff_estimator`) | ✓ | ✓ | `"layers"` |
-| BagelPipeline | `ByteDance-Seed/BAGEL-7B-MoT` | `Qwen2MoTModel` | - | ✓ | `"layers"`, `"customized modules"` |
+| Architecture | Example Models | DiT Class | Model-Level Offload | Layerwise Offload | Distributed Layerwise Offload | Blocks Attrs (Layerwise specific) |
+|--------------|----------------|-----------|---------------------|-------------------|-------------------------------|-----------------------------------|
+| Flux2Pipeline | `black-forest-labs/FLUX.2-dev` | `Flux2Transformer2DModel` | ✓ | ✓ | - | `"transformer_blocks"`, `"single_transformer_blocks"` |
+| LongCatImagePipeline | `meituan-longcat/LongCat-Image` | `LongCatImageTransformer2DModel` | - | ✓ | - | `"transformer_blocks"`, `"single_transformer_blocks"` |
+| NextStep11Pipeline | `stepfun-ai/NextStep-1.1` | `NextStepModel` | - | ✓ | - | `"layers"` |
+| OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | - | ✓ | - | `"transformer"` |
+| QwenImagePipeline | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | ✓ | ✓ | - | `"transformer_blocks"` |
+| StableDiffusionXLPipeline | `stabilityai/stable-diffusion-xl-base-1.0` | `SDXLUNet2DConditionModel` | ✓ | ✓ | - | `"down_blocks"`, `"up_blocks"` |
+| StableDiffusion3Pipeline | `stabilityai/stable-diffusion-3.5-medium` | `SD3Transformer2DModel` | - | ✓ | - | `"transformer_blocks"` |
+| Wan22I2VPipeline | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | - | `"blocks"` |
+| Wan22Pipeline | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | - | `"blocks"` |
+| SoulXSingerPipeline / SoulXSingerSVCPipeline | `Soul-AILab/SoulX-Singer` | `DiffLlama` (`cfm_decoder.model.diff_estimator`) | ✓ | ✓ | - | `"layers"` |
+| BagelPipeline | `ByteDance-Seed/BAGEL-7B-MoT` | `Qwen2MoTModel` | - | ✓ | - | `"layers"`, `"customized modules"` |
+| Cosmos3OmniDiffusersPipeline | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | `Cosmos3VFMTransformer`, `Cosmos3LanguageModel` | ✓ | ✓ | ✓ | `"layers"`, `"gen_layers"` |
 
 **Notes:**
 - Model-Level Offloading is expected to be supported by all common diffusion models (DiT and encoders) naturally
 - Layerwise Offloading requires DiT class to define `_layerwise_offload_blocks_attrs` pointing to transformer blocks
+- Distributed Layerwise Offloading reuses the layerwise block topology, but the
+  model and parallelism combination still needs validation. AllGather mode
+  additionally requires a compatible checkpoint/mmap layout; no-AllGather mode
+  retains the standard loader's rank-local tensors. See the
+  [DLO design](../../design/feature/distributed_layerwise_offload.md) and
+  [Cosmos3 DistOffload recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-DistOffload.md)
+  for usage examples.

docs/user_guide/diffusion/execution_modes.md

@@ -0,0 +1,252 @@
+# Diffusion Execution Modes
+
+vLLM-Omni supports complete-request execution and step-wise diffusion
+execution. Both modes use the same asynchronous engine output stream; the
+configuration controls how work is scheduled and whether intermediate outputs
+are exposed.
+
+## Choose a Mode
+
+| Goal | CLI configuration |
+|---|---|
+| Serial request execution | `--max-num-seqs 1` |
+| Fused request-level batching | `--max-num-seqs N` |
+| Single-request step execution | `--step-execution --max-num-seqs 1` |
+| Step-wise continuous batching | `--step-execution --max-num-seqs N` |
+| Chunked diffusion output | `--diffusion-streaming-output` |
+
+`N` must be greater than `1` to allow batching. The selected pipeline must
+support request-level batching or batched-step execution as appropriate;
+single-request step support alone is not sufficient for step-wise continuous
+batching.
+
+## Request Execution
+
+Request execution is the default when `--step-execution` is omitted. The
+pipeline performs a complete `forward()` for each scheduler wave.
+
+### Serial Requests
+
+Use the conservative serial path with any request-mode pipeline:
+
+```bash
+vllm serve MODEL --omni \
+  --port 8091 \
+  --max-num-seqs 1
+```
+
+### Request-Level Batching
+
+Set `max_num_seqs` above one to combine compatible independent requests into a
+single pipeline forward:
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --max-num-seqs 4
+```
+
+For bursty traffic, add a small admission window:
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --max-num-seqs 4 \
+  --request-batch-max-wait-ms 20
+```
+
+`--request-batch-max-wait-ms 0` is the default. A nonzero value can improve
+batch formation but adds up to that much latency before a new scheduler wave.
+It has no effect in step mode.
+
+Each prompt remains a separate logical request. Do not submit a top-level list
+as one packed prompt. Concurrent serving requests are batched internally when
+their shapes, guidance settings, output counts, and LoRA settings are
+compatible.
+
+Only pipelines that declare request-batch support accept
+`max_num_seqs>1` in request mode. Unsupported pipelines fail during engine
+initialization; use `max_num_seqs=1` instead.
+
+## Step Execution
+
+Step execution lets the scheduler advance and abort requests between denoise
+steps:
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --step-execution \
+  --max-num-seqs 1
+```
+
+Set a larger capacity to allow compatible requests to share denoise waves:
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --step-execution \
+  --max-num-seqs 8
+```
+
+Step continuous batching is experimental. Start with `max_num_seqs=1` when
+validating a model or debugging correctness, then increase it for
+multi-request throughput.
+
+Step execution is capability-based, not a generic switch for every diffusion
+model. Qwen-Image supports step-wise continuous batching. HunyuanImage3 also
+supports it, but only when its resolved self-attention backend is `TORCH_SDPA`;
+set `DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA` or configure
+`diffusion_attention_config.default.backend=TORCH_SDPA` before using
+`--max-num-seqs >1`. See the
+[HunyuanImage-3.0 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/HunyuanImage-3.0-Instruct.md)
+for its validated configuration. Helios supports single-request step execution only: use
+`--step-execution --max-num-seqs 1` for Helios. Consult the selected pipeline's
+documentation and source for the latest support status.
+
+## Streaming Output
+
+Use `--diffusion-streaming-output` for a pipeline that can produce intermediate
+diffusion outputs:
+
+```bash
+vllm serve BestWishYsh/Helios-Distilled --omni \
+  --port 8000 \
+  --diffusion-streaming-output
+```
+
+Streaming output requires step execution. If
+`--diffusion-streaming-output` is set without `--step-execution`, the engine
+enables step execution automatically. Model initialization fails if the
+pipeline does not implement step execution.
+
+Chunk-capable pipelines emit intermediate and final outputs through the same
+request stream. Final-only step pipelines emit only the final result even when
+streaming output is enabled.
+
+The normal non-streaming serving path uses the same internal stream but drains
+it and returns only the final output. Users do not need to enable
+`--diffusion-streaming-output` to benefit from the unified engine lifecycle.
+
+## Send Requests
+
+Execution-mode flags configure the server. They do not change the
+OpenAI-compatible client request format. For example, after starting any of
+the non-streaming configurations above:
+
+```bash
+curl -X POST http://localhost:8091/v1/images/generations \
+  -H "Content-Type: application/json" \
+  -d '{
+    "prompt": "a cat sitting on a windowsill",
+    "size": "1024x1024",
+    "num_inference_steps": 50,
+    "seed": 42
+  }' | jq -r '.data[0].b64_json' | base64 -d > cat.png
+```
+
+To create a request-level batch, send multiple independent requests
+concurrently. Do not put several prompts into one `prompt` field:
+
+```bash
+for prompt in "a red fox" "a blue bird" "a green frog"; do
+  curl -s -X POST http://localhost:8091/v1/images/generations \
+    -H "Content-Type: application/json" \
+    -d "{\"prompt\":\"${prompt}\",\"size\":\"1024x1024\",\"seed\":42}" \
+    > /dev/null &
+done
+wait
+```
+
+The scheduler may batch these requests when their sampling parameters are
+compatible. See the
+[Image Generation API](../../serving/image_generation_api.md) for response
+formats and additional client examples. Streaming models can use
+model-specific streaming endpoints documented by their serving guide.
+
+## Python API
+
+The Python arguments mirror the CLI flags:
+
+```python
+from vllm_omni import Omni
+
+omni = Omni(
+    model="Qwen/Qwen-Image",
+    max_num_seqs=4,
+    request_batch_max_wait_ms=20.0,
+)
+
+outputs = omni.generate(
+    [
+        "a cup of coffee on a table",
+        "a toy dinosaur on a sandy beach",
+        "a fox waking up in bed and yawning",
+    ]
+)
+```
+
+Each list item passed to `Omni.generate()` becomes an independent logical
+request that the scheduler may batch with compatible items.
+
+For step execution:
+
+```python
+from vllm_omni import Omni
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+
+omni = Omni(
+    model="Qwen/Qwen-Image",
+    step_execution=True,
+    max_num_seqs=1,
+)
+
+outputs = omni.generate(
+    "A cat sitting on a windowsill",
+    OmniDiffusionSamplingParams(num_inference_steps=50),
+)
+```
+
+## Stage Configuration
+
+Use the equivalent engine arguments in a deployment YAML:
+
+```yaml
+stage_args:
+  - stage_id: 0
+    stage_type: diffusion
+    engine_args:
+      step_execution: false
+      max_num_seqs: 4
+      request_batch_max_wait_ms: 20
+```
+
+For step execution, set `step_execution: true` and remove
+`request_batch_max_wait_ms`.
+
+## CLI Reference
+
+| Flag | Default | Effect |
+|---|---:|---|
+| `--step-execution` | disabled | Select step-wise scheduling |
+| `--max-num-seqs` | `1` for diffusion stages | Set request- or step-scheduler capacity |
+| `--request-batch-max-wait-ms` | `0` | Wait for burst coalescing in request mode |
+| `--diffusion-streaming-output` | disabled | Expose supported intermediate diffusion outputs and require step execution |
+
+## Limitations and Troubleshooting
+
+- Requests batch only when their compatibility-sensitive parameters match.
+- Different LoRA adapters or scales run in separate batches.
+- FIFO scheduling can cause an incompatible request to block later compatible
+  requests.
+- All diffusion cache backends are unsupported in step mode. KV transfer is
+  supported for newly admitted step requests; some other request-mode extras
+  remain unsupported.
+- If request-mode startup reports that the pipeline does not support batching,
+  use `--max-num-seqs 1`.
+- If step-mode startup mentions `prepare_encode()`, `denoise_step()`,
+  `step_scheduler()`, or `post_decode()`, the pipeline does not implement the
+  required step contract.
+
+For implementation details and model-author guidance, see
+[Diffusion Continuous Batching](../../design/feature/diffusion_continuous_batching.md).

docs/user_guide/diffusion/request_batching.md

@@ -1,119 +0,0 @@
-# Request-Level Batching
-
-Request-level batching lets diffusion serving combine multiple compatible
-logical requests into one pipeline `forward()` call. Each prompt remains a
-separate request with its own `request_id`, sampling parameters, seed, output,
-error, and abort state. The scheduler decides which requests can run together.
-
-!!! warning "Prompt List Semantics"
-
-    Diffusion request-level batching does not support a top-level packed
-    list-prompt request. Submit multiple prompts as independent requests and let
-    the scheduler batch compatible in-flight requests. Multimodal payloads stay
-    inside a single prompt dict, for example
-    `{"prompt": "...", "multi_modal_data": {"image": image}}`.
-
-## Enablement
-
-Request-level execution is the default diffusion request mode when
-`step_execution=False`. `max_num_seqs` controls the maximum number of compatible
-requests in one scheduler wave:
-
-- `max_num_seqs=1` is the serial request path.
-- `max_num_seqs>1` allows fused request-level batching for pipelines that
-  declare request-batch support.
-
-Increase `max_num_seqs` above `1` to allow the request scheduler to keep more
-than one compatible request active:
-
-```bash
-vllm serve Qwen/Qwen-Image --omni \
-  --port 8091 \
-  --max-num-seqs 4
-```
-
-For bursty online traffic, you can also set a small admission wait window. This
-lets the engine wait briefly before the first `schedule()` of a new wave so
-nearby compatible requests can arrive and share the same fused forward pass:
-
-```bash
-vllm serve Qwen/Qwen-Image --omni \
-  --port 8091 \
-  --max-num-seqs 4 \
-  --request-batch-max-wait-ms 20
-```
-
-`--request-batch-max-wait-ms 0` is the default and disables admission waiting,
-so there is no added wait latency.
-
-For deploy YAMLs, configure the diffusion stage engine args:
-
-```yaml
-stage_args:
-  - stage_id: 0
-    stage_type: diffusion
-    engine_args:
-      max_num_seqs: 4
-      request_batch_max_wait_ms: 20
-```
-
-## Compatibility
-
-Only pipelines that declare request-batch support can use the fused
-multi-request `pipeline.forward(batch)` path. Pipelines that do not support this
-contract can still use request-level execution with `max_num_seqs=1`, which is
-the serial request path. Setting `max_num_seqs>1` for a pipeline without
-request-batch support fails early during engine initialization.
-
-The scheduler batches only compatible requests. Compatibility is based on
-shape-sensitive and guidance-sensitive sampling fields, including resolution,
-frame count, CFG settings, output count, and LoRA identity. Requests with
-different LoRA adapters or scales are kept in separate batches so the worker
-activates one adapter per batch.
-
-Request-level batching applies when `step_execution=False`. For the separate
-step-wise runtime, see [Step Execution](step_execution.md).
-
-## Tuning
-
-- `max_num_seqs` caps the number of active compatible requests in one scheduler
-  wave. `1` means serial request execution; values above `1` enable fused
-  request-level batching when the pipeline supports it.
-- `request_batch_max_wait_ms` is an upper bound on extra admission wait before a
-  new wave starts. Keep it small for latency-sensitive serving; values such as
-  `10` to `50` ms are a practical starting range for bursty HTTP ingress.
-- `0` disables admission waiting and preserves the lowest first-request latency.
-- FIFO ordering is conservative: an incompatible request at the front of the
-  waiting queue can block later compatible requests from joining the current
-  batch.
-
-## Python API
-
-When constructing `Omni`, pass the same engine arguments:
-
-```python
-from vllm_omni.entrypoints.omni import Omni
-
-omni = Omni(
-    model="Qwen/Qwen-Image",
-    max_num_seqs=4,
-    request_batch_max_wait_ms=20.0,
-)
-
-outputs = omni.generate(
-    [
-        "a cup of coffee on a table",
-        "a toy dinosaur on a sandy beach",
-        "a fox waking up in bed and yawning",
-    ]
-)
-```
-
-`Omni.generate([...])` submits each list item as its own logical diffusion
-request. The runtime may batch those requests internally when their sampling
-parameters are compatible.
-
-## For Contributors
-
-For implementation details and model-author guidance, see
-[Request-Level Batching for Diffusion](../../design/feature/diffusion_request_level_batching.md).

docs/user_guide/diffusion/step_execution.md

@@ -1,100 +0,0 @@
-# Step Execution
-
-Step execution is an opt-in diffusion execution mode enabled with
-`step_execution=True` when constructing `Omni`.
-
-It is not a generic diffusion toggle for every pipeline. Only pipelines that
-implement the stepwise contract support it today.
-
-## Quick Start
-
-### Python API
-
-```python
-from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-
-omni = Omni(
-    model="Qwen/Qwen-Image",
-    step_execution=True,
-)
-
-outputs = omni.generate(
-    "A cat sitting on a windowsill",
-    OmniDiffusionSamplingParams(
-        num_inference_steps=50,
-    ),
-)
-```
-
-### Serving
-
-```bash
-vllm serve Qwen/Qwen-Image --omni \
-  --port 8091 \
-  --step-execution \
-  --max-num-seqs 8
-```
-
-For serving, `--step-execution` enables the step-wise runtime. Continuous
-batching only becomes relevant when `--max-num-seqs > 1`.
-
-`max_num_seqs` controls the step scheduler capacity:
-
-- `max_num_seqs=1` runs one request through the step-wise state machine.
-- `max_num_seqs>1` allows compatible active requests to share step waves.
-
-## Supported Pipelines
-
-| Pipeline | Example models | Step execution |
-|----------|----------------|----------------|
-| `QwenImagePipeline` | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes |
-| All other diffusion pipelines | `QwenImageEditPipeline`, `QwenImageEditPlusPipeline`, `QwenImageLayeredPipeline`, GLM-Image, Wan, Flux, etc. | No |
-
-!!! warning "Experimental continuous batching"
-    When `--step-execution` is enabled and `max_num_seqs > 1` is configured,
-    the step-wise path can batch
-    compatible requests together. This is experimental. Requests with
-    incompatible sampling parameters are intentionally kept in separate batches,
-    and `max_num_seqs=1` remains the conservative single-request step path.
-
-## Current Limitations
-
-- Continuous batching under `step_execution` is experimental and only batches
-  compatible requests.
-- `cache_backend` is not supported together with step execution.
-- Unsupported pipelines fail early during model loading.
-- Request-mode extras such as KV transfer are not wired into step mode yet.
-- LoRA is supported in step mode, but each batch must use a single adapter:
-  requests with different `lora_request` or `lora_scale` are scheduled into
-  separate batches.
-
-## When To Use It
-
-Use step execution only when you specifically need the pipeline to run through
-its stepwise request state machine. For normal diffusion inference, leave it
-disabled unless your workflow depends on this mode.
-
-For Qwen-Image online serving, the usual progression is:
-
-- start with `--step-execution --max-num-seqs 1` if you only need the step-wise path
-- increase `--max-num-seqs` after that if you want the experimental compatible-request batching behavior
-
-If you are looking for general diffusion speedups, see
-[Diffusion Features Overview](../diffusion_features.md).
-
-## Troubleshooting
-
-If model loading fails with a message mentioning `prepare_encode()`,
-`denoise_step()`, `step_scheduler()`, and `post_decode()`, the selected
-pipeline does not support step execution.
-
-## For Model Authors
-
-If you want to add step execution support to a new diffusion pipeline, see the
-implementation guide:
-[Diffusion Step Execution Design](../../design/feature/diffusion_step_execution.md).
-
-If you also want that pipeline to participate in the experimental batched
-step-wise path, see:
-[Continuous Batching for Step-Wise Diffusion](../../design/feature/diffusion_continuous_batching.md).

docs/user_guide/diffusion_features.md

@@ -59,7 +59,7 @@ Memory optimization methods help reduce GPU memory usage, enabling inference on
 
 | Method | Description | Best For |
 |--------|-------------|----------|
-| **[CPU Offload](diffusion/cpu_offload_diffusion.md)** | Offloads model components to CPU memory | Limited VRAM, large models on consumer GPUs |
+| **[CPU Offload](diffusion/cpu_offload.md)** | Offloads model components to CPU memory | Limited VRAM, large models on consumer GPUs |
 | **[Quantization](quantization/overview.md)** | Reduces transformer stages from BF16 to FP8/INT8/etc. | Limited VRAM, minimal accuracy loss    |
 | **[VAE Parallelism](diffusion/parallelism/vae_parallelism.md)** | Distributes VAE decode work across GPUs | High-resolution generation with reduced VAE memory peak |
 
@@ -80,12 +80,11 @@ denoise steps.
 
 | Method | Description | Best For |
 |--------|-------------|----------|
-| **[Request-Level Batching](diffusion/request_batching.md)** | Scheduler batches compatible independent diffusion requests into one pipeline forward pass | Bursty online serving and multi-request throughput |
-| **[Step Execution](diffusion/step_execution.md)** | Per-step denoise execution with mid-request abort support | Request cancellation between denoise steps, fine-grained execution control |
+| **[Diffusion Execution Modes](diffusion/execution_modes.md)** | Configures serial requests, request batching, step execution, continuous batching, and streaming output | Matching latency, throughput, cancellation, and output-delivery requirements |
 
-**Note:** Request-level batching is available for pipelines that declare the
-request-batch forward contract. Step execution is currently supported by
-QwenImagePipeline only. See [Supported Models](#supported-models) for details.
+**Note:** Request-level batching and step execution are capability-based.
+Consult the execution guide and the selected pipeline's documentation for
+current support.
 
 ### Quantization Methods
 
@@ -105,8 +104,8 @@ The following tables show which models support each feature:
 
 > Notes:
 
-> 1. CPU Offload has two methods: Module-wise (default for models with DiT + text encoder) and Layerwise. The tables below show **Layerwise support** only. Split models like Cosmos3 (no separate text encoder) swap their reasoner/generator components for module-wise offload; see the [CPU Offload Guide](diffusion/cpu_offload_diffusion.md).
-> 2. The **💾Quantization** column is collapsed for readability. See [Quantization Overview](quantization/overview.md) for per-method and per-model support details.
+> 1. CPU Offload has two methods: Module-wise (default for models with DiT + text encoder) and Layerwise. The tables below show **Layerwise support** only. Split models like Cosmos3 (no separate text encoder) swap their reasoner/generator components for module-wise offload; see the [CPU Offload Guide](diffusion/cpu_offload.md).
+> 2. The **💾Quantization** column is collapsed for readability. See [Quantization](quantization/overview.md) for per-method and per-model support details.
 
 ### ImageGen
 
@@ -160,7 +159,7 @@ The following tables show which models support each feature:
 | **HunyuanVideo-1.5 T2V I2V** |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
 | **DreamID-Omni**             |     ❌     |     ❌      |           ❌           |       ✅        |         ❌         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **Cosmos3**                  |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
-
+| **MiniMax-H3**               |     ❌     |     ✅      |           ✅           |       ❌        |       ✅ (DiT/TE)  |         ❌         |   ✅    |             ✅             |       ✅ (tile)       |      ✅ (DiT)      |        ❌         |
 
 **Frame Interpolation Support**
 
@@ -206,7 +205,7 @@ The following tables show which models support each feature:
     3. CPU Offloading (Layerwise) and CPU Offloading (Module-wise) are not compatible.
     4. CPU Offloading (Layerwise) supports single-card for now.
     5. Using FP8-Quant as an example of qunatization methods.
-    6. Step Execution is not compatible with cache backends (TeaCache, Cache-DiT). LoRA is supported, but each scheduled batch must use a single adapter (requests with different `lora_request` or `lora_scale` are kept in separate batches).
+    6. Step Execution is not compatible with any diffusion cache backend. LoRA is supported, but each scheduled batch must use a single adapter (requests with different `lora_request` or `lora_scale` are kept in separate batches).
 
 
 ## Multi-Thread Weight Loading
@@ -275,9 +274,9 @@ Measured on NVIDIA H800:
 
 **Memory Optimization:**
 
-- **[CPU Offload Guide](diffusion/cpu_offload_diffusion.md)** - Offload model components to CPU, reduce GPU memory usage
+- **[CPU Offload Guide](diffusion/cpu_offload.md)** - Offload model components to CPU, reduce GPU memory usage
 - **[VAE Parallelism Guide](diffusion/parallelism/vae_parallelism.md)** - Distribute VAE decode work across GPUs for high-resolution images and videos
-- **[Quantization Overview](quantization/overview.md)** - Overview of quantization methods for diffusion, multi-stage omni/TTS, and multi-stage diffusion models
+- **[Quantization](quantization/overview.md)** - Quantization methods for diffusion, multi-stage omni/TTS, and multi-stage diffusion models
 
 **Extensions:**
 
@@ -286,7 +285,7 @@ Measured on NVIDIA H800:
 
 **Execution Modes:**
 
-- **[Step Execution Guide](diffusion/step_execution.md)** - Per-step denoise execution with mid-request abort support
+- **[Diffusion Execution Modes](diffusion/execution_modes.md)** - Configure request batching, step execution, continuous batching, and streaming output
 
 **Startup Optimization:**
 

docs/user_guide/examples/offline_inference/text_to_image.md

@@ -184,7 +184,7 @@ if __name__ == "__main__":
 !!! info
 
     For diffusion request-level batching controls such as `max_num_seqs`, see
-    [Request-Level Batching](../../diffusion/request_batching.md).
+    [Diffusion Execution Modes](../../diffusion/execution_modes.md).
 
 ### Negative Prompts
 

docs/user_guide/examples/online_serving/bagel.md

@@ -40,7 +40,7 @@ bash run_server_stage_cli.sh --stage 0
 bash run_server_stage_cli.sh --stage 1
 ```
 
-To use a custom deploy YAML (note: `--stage-configs-path` is deprecated in favor of `--deploy-config`):
+To use a custom deploy YAML:
 
 ```bash
 vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 \

docs/user_guide/examples/online_serving/glm_image.md

@@ -11,8 +11,8 @@ memory, sampling params) live in `vllm_omni/deploy/glm_image.yaml`.
 vllm serve zai-org/GLM-Image --omni --port 8091
 ```
 
-The config system auto-detects the pipeline from the model's `model_index.json` — no
-manual `--stage-configs-path` or `--deploy-config` needed.
+The config system auto-detects the pipeline from the model's `model_index.json`, so
+no manual `--deploy-config` is needed.
 
 By default, stage 0 (AR) runs on GPU 0 and stage 1 (Diffusion) on GPU 1. To colocate
 both stages on a single GPU, override per stage:

docs/user_guide/examples/online_serving/qwen2_5_omni.md

@@ -15,9 +15,9 @@ Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
 ```
 
-If you have custom stage configs file, launch the server with command below
+To use a custom deploy config, launch the server with the command below:
 ```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
+vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/deploy_config.yaml
 ```
 
 ### Send Multi-modal Request
@@ -182,7 +182,7 @@ The script supports the following arguments:
 - `--model`: Model name/path (default: Qwen/Qwen2.5-Omni-7B)
 - `--server-port`: Port for vLLM server (default: 8091)
 - `--gradio-port`: Port for Gradio demo (default: 7861)
-- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
+- `--deploy-config`: Path to a custom deploy config YAML file (optional)
 - `--server-host`: Host for vLLM server (default: 0.0.0.0)
 - `--gradio-ip`: IP for Gradio demo (default: 127.0.0.1)
 - `--share`: Share Gradio demo publicly (creates a public link)
@@ -195,9 +195,9 @@ The script supports the following arguments:
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
 ```
 
-If you have custom stage configs file:
+To use a custom deploy config:
 ```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
+vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/deploy_config.yaml
 ```
 
 **Step 2: Run the Gradio demo**

docs/user_guide/examples/online_serving/qwen3_omni.md

@@ -306,7 +306,7 @@ The gradio script supports the following arguments:
     ``````
 ??? abstract "qwen3_omni_moe_thinking.yaml"
     ``````yaml
-    --8<-- "examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml"
+    --8<-- "vllm_omni/deploy/qwen3_omni_moe_thinking.yaml"
     ``````
 ??? abstract "run_curl_multimodal_generation.sh"
     ``````sh

docs/user_guide/examples/online_serving/text_to_speech.md

@@ -422,12 +422,19 @@ curl -X POST http://localhost:8091/v1/audio/speech \
 Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming.
 
 ### Streaming WebSocket
-The `/v1/audio/speech/stream` endpoint accepts text incrementally, splits it at sentence boundaries, and emits one PCM stream per sentence:
+The `/v1/audio/speech/stream` endpoint accepts text incrementally and synthesizes the buffered text as one continuous request on `input.done`:
 ```bash
 python qwen3_tts/streaming_speech_client.py --text "Hello world. How are you? I am fine."
 python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-delay 0.1
 ```
 
+`input.done` flushes without closing, so repeating `--text` synthesizes several utterances over one connection:
+```bash
+python qwen3_tts/streaming_speech_client.py \
+    --text "First utterance." \
+    --text "Second utterance, same connection."
+```
+
 ### Gradio demos
 ```bash
 ./qwen3_tts/run_gradio_demo.sh                              # CustomVoice (default)

docs/user_guide/fault_injection_reliability_matrix.md

@@ -69,19 +69,19 @@ The behaviors and conclusions above are summarized from current fault injection
 
 ### Example: `SIGTERM` fault injection log (Qwen3-Omni)
 
-The following full log is from a real run where a `SIGTERM` signal was sent to the serve root process.
+The following log is based on a real run where a `SIGTERM` signal was sent to the serve root process. Its command line has been normalized to the current `--deploy-config` option.
 
 ```text
-Launching OmniServer with: /workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --stage-configs-path vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
+Launching OmniServer with: /workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --deploy-config vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
 Server ready on 127.0.0.1:60675 (OmniServer startup took 206.023s)
 OmniServer started successfully
-[reliability][process-kill] current_server_proc pid=556855 name=python3 cmdline=/workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --stage-configs-path vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
+[reliability][process-kill] current_server_proc pid=556855 name=python3 cmdline=/workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --deploy-config vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
 [reliability][process-kill] current_server_proc pid=557118 name=python3 cmdline=/workspace/.venv/bin/python3 -c from multiprocessing.resource_tracker import main;main(57)
 [reliability][process-kill] current_server_proc pid=557119 name=VLLM::StageEngineCoreProc_noid_replica0_DP0 cmdline=VLLM::StageEngineCoreProc_noid_replica0_DP0
 [reliability][process-kill] current_server_proc pid=557122 name=VLLM::StageEngineCoreProc_noid_replica0_DP0 cmdline=VLLM::StageEngineCoreProc_noid_replica0_DP0
 [reliability][process-kill] current_server_proc pid=558201 name=VLLM::StageEngineCoreProc_noid_replica0_DP0 cmdline=VLLM::StageEngineCoreProc_noid_replica0_DP0
 
-[reliability][process-kill] root-kill pid=556855 name=python3 signal=SIGTERM cmdline=/workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --stage-configs-path vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
+[reliability][process-kill] root-kill pid=556855 name=python3 signal=SIGTERM cmdline=/workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --deploy-config vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
 [0;36m(APIServer pid=556855)[0;0m INFO 05-19 12:24:51 [omni_base.py:463] [AsyncOmni] Shutting down
 [0;36m(APIServer pid=556855)[0;0m INFO 05-19 12:24:51 [async_omni_engine.py:2397] [AsyncOmniEngine] Shutting down Orchestrator
 [0;36m(APIServer pid=556855)[0;0m INFO 05-19 12:24:51 [orchestrator.py:351] [Orchestrator] Received shutdown signal

docs/user_guide/quantization/fp8.md

@@ -93,6 +93,7 @@ warmup_quack_fp8([(14040, 2048, 6144), (14040, 2048, 2048)])
 | HunyuanImage-3.0 | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | Yes | Yes | All layers; use the Hunyuan stage config for multi-stage runs | None | |
 | HunyuanVideo-1.5 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `720p_t2v`, `480p_i2v` | Yes | Yes | All layers | None | |
 | Cosmos3 | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | Yes | Not validated | All layers | None | |
+| MiniMax H3 | `MiniMaxAI/MiniMax-H3` | Yes | Not validated | DiT linears except patch, timestep, and final projections; not compatible with layerwise offload | None | |
 
 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
 

docs/user_guide/quantization/overview.md

@@ -5,6 +5,9 @@ path. The same configuration entrypoint is used across diffusion-only models,
 multi-stage omni/TTS models, and multi-stage diffusion models, but each model
 type has a different quantization scope.
 
+For the internal architecture and backend extension points, see the
+[quantization design document](../../design/feature/quantization.md).
+
 ## Quantization Modes
 
 | Mode | Guide | Description | Methods |

examples/model_display_names.yml

@@ -8,5 +8,4 @@ mimo_audio: MiMo-Audio
 ming_flash_omni: Ming-flash-omni 2.0
 qwen2_5_omni: Qwen2.5-Omni
 qwen3_omni: Qwen3-Omni
-sensenova_u1: SenseNova-U1
 step_audio2: Step-Audio2

examples/offline_inference/image_to_video/image_to_video.py

@@ -334,6 +334,7 @@ def main():
     is_ltx23 = "ltx23" in model_class_name_lower or "ltx-2.3" in model_name
     is_ltx2 = is_ltx2_distilled or is_ltx23 or "ltx2" in model_class_name_lower or "ltx-2" in model_name
     is_cosmos = "cosmos" in model_name or (model_class_name is not None and "cosmos" in model_class_name.lower())
+    is_cosmos_edge = is_cosmos and ("edge" in model_name or "edge" in model_class_name_lower)
 
     image = PIL.Image.open(args.image).convert("RGB") if args.image else None
     last_image = PIL.Image.open(args.last_image).convert("RGB") if args.last_image else None
@@ -347,7 +348,19 @@ def main():
 
     # Per-model generation defaults, applied only when the matching flag is omitted.
     # Cosmos3 would otherwise silently inherit the Wan2.2 defaults (wrong size/steps/shift).
-    if is_cosmos:
+    if is_cosmos_edge:
+        # Cosmos3-Edge native defaults: 480x832, gs 5.0, flow_shift 3.0 — NOT the Nano/Super
+        # 720p / gs 6.0 / flow_shift 10.0 below, which produce degenerate output on Edge.
+        d_fps, d_guidance, d_num_frames, d_steps, d_flow_shift, d_max_area, d_mod = (
+            24,
+            5.0,
+            189,
+            35,
+            3.0,
+            480 * 832,
+            16,
+        )
+    elif is_cosmos:
         d_fps, d_guidance, d_num_frames, d_steps, d_flow_shift, d_max_area, d_mod = (
             24,
             6.0,

examples/offline_inference/personaplex/personaplex_offline.py

@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Offline PersonaPlex inference through the vllm-omni fullduplex backend.
+
+This is the runnable end-to-end proof for the integration: it drives a user WAV
+through :class:`PersonaPlexEngine` + :class:`PersonaPlexSession` (our wrappers
+around the Moshi/Mimi lockstep loop) and writes the agent's audio + inner-monologue
+text. It mirrors ``python -m moshi.offline`` but goes through our ``FrameStepper``
+seam, so a green run validates that the session/adapter plumbing is faithful.
+
+Prereqs: HF access to ``nvidia/personaplex-7b-v1`` (``export HF_TOKEN=...``). The
+stack is moshi-free (native temporal + depformer + streaming Mimi via
+``transformers``); decoding is greedy.
+
+Example:
+    python examples/offline_inference/personaplex/personaplex_offline.py \\
+        --input-wav input_assistant.wav --output-wav out.wav \\
+        --voice-prompt NATF2.pt --persona "You are a wise and friendly teacher."
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+
+import numpy as np
+
+from vllm_omni.experimental.fullduplex.personaplex import (
+    PersonaPlexConfig,
+    PersonaPlexEngine,
+    PersonaPlexSession,
+)
+from vllm_omni.experimental.fullduplex.personaplex.config import DEFAULT_PERSONA
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--input-wav", required=True, help="User-side input WAV")
+    parser.add_argument("--output-wav", required=True, help="Agent audio output WAV")
+    parser.add_argument("--output-text", default=None, help="Optional JSON for agent text")
+    parser.add_argument("--voice-prompt", default="NATF2.pt", help="Voice clone (bundled .pt or .wav)")
+    parser.add_argument("--persona", default=DEFAULT_PERSONA, help="System role text")
+    parser.add_argument("--hf-repo", default="nvidia/personaplex-7b-v1")
+    parser.add_argument("--device", default="cuda")
+    parser.add_argument("--cpu-offload", action="store_true")
+    args = parser.parse_args()
+
+    import sphn  # local import: only the runnable driver needs the audio I/O dep
+
+    config = PersonaPlexConfig(
+        hf_repo=args.hf_repo,
+        voice_prompt=args.voice_prompt,
+        persona=args.persona,
+        device=args.device,
+        cpu_offload=args.cpu_offload,
+    )
+
+    engine = PersonaPlexEngine(config).load()
+    session = PersonaPlexSession(engine, config)
+    session.open()
+
+    # Load exactly like the reference (moshi.lm.load_audio): native read then
+    # explicit resample, so greedy runs stay bit-faithful to `python -m moshi.offline`.
+    user_pcm, src_sr = sphn.read(args.input_wav)
+    user_pcm = sphn.resample(user_pcm, src_sample_rate=src_sr, dst_sample_rate=config.sample_rate)
+    audio, text = session.run(np.asarray(user_pcm[0], dtype=np.float32))
+
+    sphn.write_wav(args.output_wav, audio, config.sample_rate)
+    print(f"wrote {audio.shape[0] / config.sample_rate:.2f}s of agent audio to {args.output_wav}")
+    print(f"agent text: {text!r}")
+    if args.output_text:
+        with open(args.output_text, "w") as f:
+            json.dump({"text": text}, f, ensure_ascii=False)
+
+
+if __name__ == "__main__":
+    main()

examples/offline_inference/text_to_image/README.md

@@ -260,7 +260,7 @@ if __name__ == "__main__":
     For diffusion pipelines, the input list is sliced into single-item requests
     before feeding into the diffusion pipeline. For request-level batching
     controls such as `max_num_seqs`, see
-    [Request-Level Batching](../../../docs/user_guide/diffusion/request_batching.md).
+    [Diffusion Execution Modes](../../../docs/user_guide/diffusion/execution_modes.md).
 
 ### Negative Prompts
 

examples/offline_inference/text_to_speech/cosyvoice3/end2end.py

@@ -182,10 +182,13 @@ def run_e2e():
                 print(f"Multimodal output keys: {mm.keys()}")
                 if "audio" in mm:
                     audio_out = mm["audio"]
+                    # The model emits its own rate (24 kHz for CosyVoice3); take
+                    # it from the payload so the file is not mislabeled.
+                    sample_rate = int(mm["sr"]) if "sr" in mm.keys() else 24000
                     print(f"Generated Audio Shape: {audio_out.shape}")
                     out_path = f"output_{i}.wav"
-                    sf.write(out_path, audio_out.cpu().numpy().squeeze(), 22050)
-                    print(f"Saved audio to {out_path}")
+                    sf.write(out_path, audio_out.cpu().numpy().squeeze(), sample_rate)
+                    print(f"Saved audio to {out_path} (sr={sample_rate})")
             else:
                 print("No multimodal output found.")
         except Exception as e:

examples/offline_inference/text_to_video/text_to_video.py

@@ -56,6 +56,19 @@ _MODEL_PRESETS = {
         "flow_shift": 10.0,
         "output": "cosmos3_t2v_output.mp4",
     },
+    # Cosmos3-Edge is a compact checkpoint with its own native resolution / guidance /
+    # flow_shift. It must NOT inherit the Nano/Super "cosmos" numbers above (720p / gs 6.0 /
+    # flow_shift 10.0), which produce degenerate output on Edge.
+    "cosmos3_edge": {
+        "height": 480,
+        "width": 832,
+        "num_frames": 189,
+        "num_inference_steps": 35,
+        "guidance_scale": 5.0,
+        "fps": 24,
+        "flow_shift": 3.0,
+        "output": "cosmos3_edge_t2v_output.mp4",
+    },
     "helios": {
         "height": 384,
         "width": 640,
@@ -103,6 +116,10 @@ def _detect_preset(model: str, model_class_name: str | None = None) -> dict:
         return _MODEL_PRESETS["ltx2"]
     if "vace" in model_lower:
         return _MODEL_PRESETS["vace"]
+    # Edge must be matched before the generic cosmos branch (its "cosmos" substring would
+    # otherwise pick up the Nano/Super 720p / gs 6.0 / flow_shift 10.0 preset).
+    if ("cosmos" in model_lower or "cosmos" in class_lower) and ("edge" in model_lower or "edge" in class_lower):
+        return _MODEL_PRESETS["cosmos3_edge"]
     if "cosmos" in model_lower:
         return _MODEL_PRESETS["cosmos"]
     if "hunyuan" in model_lower:

examples/online_serving/audex/run_server.sh

@@ -53,5 +53,5 @@ vllm-omni serve "$MODEL" \
     --host 0.0.0.0 \
     --port "$PORT" \
     --trust-remote-code \
-    --stage-configs-path "$DEPLOY_YAML" \
+    --deploy-config "$DEPLOY_YAML" \
     --omni

examples/online_serving/chart-helm/templates/_helpers.tpl

@@ -28,8 +28,11 @@ If image.command is set, uses that as a full override.
 - {{ .Values.omniArgs.numGpus | quote }}
 {{- end }}
 {{- if .Values.omniArgs.stageConfigsPath }}
-- "--stage-configs-path"
-- {{ .Values.omniArgs.stageConfigsPath | quote }}
+{{- fail "omniArgs.stageConfigsPath has been removed; use omniArgs.deployConfigPath" }}
+{{- end }}
+{{- if .Values.omniArgs.deployConfigPath }}
+- "--deploy-config"
+- {{ .Values.omniArgs.deployConfigPath | quote }}
 {{- end }}
 {{- if and .Values.omniArgs.cacheBackend (ne .Values.omniArgs.cacheBackend "none") }}
 - "--cache-backend"

examples/online_serving/chart-helm/tests/deployment_test.yaml

@@ -78,6 +78,26 @@ tests:
           path: spec.template.spec.containers[0].command
           content: "--enable-layerwise-offload"
 
+  - it: should include deploy config in command
+    set:
+      omniArgs:
+        deployConfigPath: "/configs/deploy.yaml"
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].command[8]
+          value: "--deploy-config"
+      - equal:
+          path: spec.template.spec.containers[0].command[9]
+          value: "/configs/deploy.yaml"
+
+  - it: should reject removed stage config value
+    set:
+      omniArgs:
+        stageConfigsPath: "/configs/legacy.yaml"
+    asserts:
+      - failedTemplate:
+          errorMessage: "omniArgs.stageConfigsPath has been removed; use omniArgs.deployConfigPath"
+
   - it: should use full command override when image.command is set
     set:
       image:

examples/online_serving/chart-helm/values.yaml

@@ -67,8 +67,8 @@ omniArgs:
   enableCpuOffload: false
   # -- Number of GPUs for diffusion inference (empty = auto)
   numGpus:
-  # -- Path to stage configs file (empty = auto-detected from model)
-  stageConfigsPath:
+  # -- Path to deploy config file (empty = auto-detected from model)
+  deployConfigPath:
   # -- Cache backend: none, tea_cache, or cache_dit
   cacheBackend: "none"
   # -- Default sampling params as a JSON string

examples/online_serving/minicpmo/README.md

@@ -23,17 +23,18 @@ including `librosa`.
 
 The deploy config auto-loads via `--omni`.
 The default `vllm_omni/deploy/minicpmo_4_5.yaml` keeps all three stages on
-logical device 0 with memory budgets of 55%, 22%, and 22%. The profile admits
-at most four sequences per stage and bounds startup video profiling to 32
-frames per video. For throughput,`minicpmo_4_5_batching.yaml` gives the Thinker
+logical device 0 with memory budgets of 55%, 15%, and 15%, leaving headroom
+for runtime kernels such as the HiFi-GAN vocoder's cuDNN workspace. The
+profile admits at most four sequences per stage and bounds startup video
+profiling to 32 frames per video. For throughput,
+`minicpmo_4_5_2gpu.yaml` gives the Thinker
 GPU 0 (90%) and colocates the Talker (55%) and Code2Wav (35%) on GPU 1. That
 profile admits at most four concurrent sequences per stage.
 
 | deploy config | GPUs | Notes |
 |---|---|---|
 | `minicpmo_4_5.yaml` (default) | 1 | Memory-constrained compatibility layout. |
-| `minicpmo_4_5_batching.yaml` | 2 | Recommended continuous-batching layout; Talker and Code2Wav share GPU 1. |
-| `minicpmo_4_5_2gpu.yaml` | 2 | Backward-compatible alias for `minicpmo_4_5_batching.yaml`. |
+| `minicpmo_4_5_2gpu.yaml` | 2 | Recommended continuous-batching layout; Talker and Code2Wav share GPU 1. |
 | `minicpmo_4_5_3gpu.yaml` | 3 | One GPU per stage. |
 | `minicpmo_4_5_8x4090.yaml` | 8 | Full 8x4090 layout. |
 | `minicpmo_4_5_duplex.yaml` | 1 | Experimental native full-duplex overlay. |
@@ -89,17 +90,18 @@ python openai_chat_completion_client_for_multimodal_generation.py \
     --prompt "Briefly introduce yourself."
 ```
 
-Streaming text + audio:
+Streaming text + audio (use `--stream`):
 
 ```bash
-python streaming_chat_completion.py \
-    --base-url http://localhost:8099/v1 \
-    --output minicpmo_stream.wav
+python openai_chat_completion_client_for_multimodal_generation.py \
+    --query-type text \
+    --prompt "Briefly introduce yourself." \
+    --port 8099 \
+    --stream
 ```
 
-The example prints text deltas immediately and joins the independently encoded
-24 kHz WAV audio deltas into one valid WAV file. Add `--text-only` to skip
-Talker and Code2Wav.
+The client prints text deltas as they arrive and saves streamed audio chunks
+to WAV files.
 
 Shared helpers also work if you pass MiniCPM defaults yourself:
 

examples/online_serving/minicpmo/realtime_web/app/static/playback_worklet.js

@@ -6,6 +6,7 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
     this.playedFrames = 0;
     this.underrunFrames = 0;
     this.drain = null;
+    this.terminalFadeFrames = 0;
     this.started = false;
     this.activeResponseId = null;
     this.initialBufferFrames = Math.round(sampleRate * 0.2);
@@ -31,6 +32,7 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
       }
     } else if (message.type === 'drain') {
       this.drain = { responseId: message.responseId || null };
+      this.terminalFadeFrames = Math.min(this.fadeFrames, this.bufferedFrames());
       if (!this.started && this.bufferedFrames() > 0) {
         this.bufferWaitFrames = 0;
         this.startPlayback();
@@ -42,6 +44,7 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
       this.playedFrames = 0;
       this.underrunFrames = 0;
       this.drain = null;
+      this.terminalFadeFrames = 0;
       this.started = false;
       this.activeResponseId = null;
       this.bufferWaitFrames = this.initialBufferFrames;
@@ -76,6 +79,7 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
     this.playedFrames = 0;
     this.underrunFrames = 0;
     this.drain = null;
+    this.terminalFadeFrames = 0;
     this.started = false;
     this.activeResponseId = null;
     this.bufferWaitFrames = this.initialBufferFrames;
@@ -116,6 +120,7 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
     while (target < output.length && this.queue.length > 0) {
       const pcm = this.queue[0];
       const count = Math.min(output.length - target, pcm.length - this.offset);
+      const remainingBeforeChunk = this.bufferedFrames();
       for (let index = 0; index < count; index += 1) {
         let sample = pcm[this.offset + index] / 32768;
         if (this.fadeInFrames > 0) {
@@ -123,6 +128,16 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
           sample *= elapsed / this.fadeFrames;
           this.fadeInFrames -= 1;
         }
+        const remainingFrames = remainingBeforeChunk - index;
+        if (
+          this.drain
+          && this.terminalFadeFrames > 0
+          && remainingFrames <= this.terminalFadeFrames
+        ) {
+          sample *= this.terminalFadeFrames === 1
+            ? 0
+            : (remainingFrames - 1) / (this.terminalFadeFrames - 1);
+        }
         output[target + index] = sample;
       }
       target += count;

examples/online_serving/minicpmo/run_gradio_demo.sh

@@ -3,7 +3,7 @@
 #
 # Prereq:
 #   Start a vllm-omni OpenAI server for MiniCPM-o 4.5 on :8099 (see the
-#   8x4090 stage config under vllm_omni/model_executor/stage_configs).
+#   8x4090 deploy config at vllm_omni/deploy/minicpmo_4_5_8x4090.yaml).
 set -e
 
 HERE="$(cd "$(dirname "$0")" && pwd)"

examples/online_serving/minicpmo/streaming_chat_completion.py

@@ -1,71 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Stream MiniCPM-o 4.5 text and audio from /v1/chat/completions."""
-
-from __future__ import annotations
-
-import argparse
-import base64
-import io
-import wave
-from pathlib import Path
-
-from openai import OpenAI
-
-
-def append_wav_chunk(encoded: str, pcm_parts: list[bytes]) -> tuple[int, int, int]:
-    """Decode one base64 WAV delta and append its PCM frames."""
-    with wave.open(io.BytesIO(base64.b64decode(encoded)), "rb") as wav:
-        channels = wav.getnchannels()
-        sample_width = wav.getsampwidth()
-        sample_rate = wav.getframerate()
-        pcm_parts.append(wav.readframes(wav.getnframes()))
-    return channels, sample_width, sample_rate
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--base-url", default="http://localhost:8099/v1")
-    parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
-    parser.add_argument("--prompt", default="Say hello, then introduce vLLM in one sentence.")
-    parser.add_argument("--output", type=Path, default=Path("minicpmo_stream.wav"))
-    parser.add_argument("--text-only", action="store_true")
-    args = parser.parse_args()
-
-    modalities = ["text"] if args.text_only else ["text", "audio"]
-    stream = OpenAI(base_url=args.base_url, api_key="EMPTY").chat.completions.create(
-        model=args.model,
-        messages=[{"role": "user", "content": args.prompt}],
-        modalities=modalities,
-        stream=True,
-        extra_body={"chat_template_kwargs": {"use_tts_template": not args.text_only}},
-    )
-
-    pcm_parts: list[bytes] = []
-    audio_format: tuple[int, int, int] | None = None
-    for chunk in stream:
-        for choice in chunk.choices:
-            content = getattr(choice.delta, "content", None)
-            if not content:
-                continue
-            if getattr(chunk, "modality", None) == "audio":
-                current_format = append_wav_chunk(content, pcm_parts)
-                if audio_format is not None and current_format != audio_format:
-                    raise ValueError(f"Audio format changed: {audio_format} -> {current_format}")
-                audio_format = current_format
-            elif getattr(chunk, "modality", None) == "text":
-                print(content, end="", flush=True)
-
-    print()
-    if pcm_parts and audio_format is not None:
-        channels, sample_width, sample_rate = audio_format
-        with wave.open(str(args.output), "wb") as wav:
-            wav.setnchannels(channels)
-            wav.setsampwidth(sample_width)
-            wav.setframerate(sample_rate)
-            wav.writeframes(b"".join(pcm_parts))
-        print(f"Audio saved to {args.output} ({sample_rate} Hz)")
-
-
-if __name__ == "__main__":
-    main()

examples/online_serving/personaplex/README.md

@@ -0,0 +1,84 @@
+# PersonaPlex full-duplex online serving
+
+Serve [`nvidia/personaplex-7b-v1`](https://huggingface.co/nvidia/personaplex-7b-v1)
+(a Moshi-based full-duplex speech-to-speech model) with the native vLLM-Omni engine.
+
+The server hosts the **official PersonaPlex web client** at `/` (auto-downloaded from
+the model repo) and implements its WebSocket protocol at `/api/chat`, so you talk to the
+model live in the browser — mic in, agent speech + inner-monologue text out.
+
+> Experimental. Requires a GPU and Hugging Face access to the gated repo
+> (`HF_TOKEN` with access to `nvidia/personaplex-7b-v1`).
+
+## Start the server
+
+```bash
+HF_TOKEN=... CUDA_VISIBLE_DEVICES=0 bash run_server.sh
+# or directly:
+HF_TOKEN=... CUDA_VISIBLE_DEVICES=0 python -m \
+    vllm_omni.experimental.fullduplex.personaplex.serving.server --port 8124
+```
+
+## Talk to it
+
+- **Browser (recommended):** open `http://localhost:8124/` and allow the microphone.
+  Use headphones so the agent does not hear itself.
+- **Headless:** stream a 24 kHz mono WAV and save the reply:
+
+  ```bash
+  python duplex_client.py --url ws://localhost:8124/api/chat --input user.wav --out reply.wav
+  ```
+
+## Unified Realtime API
+
+The default `vllm_omni/deploy/personaplex.yaml` also enables main's engine-owned
+full-duplex control plane:
+
+```bash
+CUDA_VISIBLE_DEVICES=0 python -m vllm_omni.entrypoints.cli.main serve \
+  /path/to/personaplex-7b-v1 \
+  --omni \
+  --stage-configs-path vllm_omni/deploy/personaplex.yaml
+```
+
+Validate the actual `/v1/realtime?duplex=1` scheduler path with paced 24 kHz
+PCM, two concurrent sessions, overflow admission, per-session slot recycling,
+and non-silent output:
+
+```bash
+python tests/e2e/online_serving/personaplex_realtime_duplex.py \
+  --model /path/to/personaplex-7b-v1 \
+  --input-wav /path/to/speech.wav \
+  --output-dir /tmp/personaplex-realtime-duplex
+```
+
+This is distinct from the standalone `/api/chat` compatibility server above.
+The unified endpoint advertises `supports_barge_in=false`: overlapping speech
+is native model behavior, but destructive output interruption and model-state
+rewind have not been validated for PersonaPlex.
+
+## Protocol (`/api/chat`)
+
+Binary WebSocket messages, first byte is a tag (identical to `moshi.server`):
+
+| Direction | Tag | Payload |
+| --- | --- | --- |
+| server → client | `\x00` | ready handshake (after the system-prompt prefill) |
+| client → server | `\x01` | Opus-encoded mic audio (24 kHz) |
+| server → client | `\x01` | Opus-encoded agent audio |
+| server → client | `\x02` | UTF-8 inner-monologue text |
+
+Connect with query params `text_prompt` (persona) and `voice_prompt` (voice file).
+A raw-PCM endpoint (`/v1/audio/duplex`, JSON `open` + float32 frames) is also available
+for clients that do not want an Opus dependency.
+
+## Notes
+
+- **Run the client near the server.** Real-time 80 ms audio is sensitive to network
+  latency/jitter; over a high-latency remote link playback can stutter regardless of
+  engine speed. On localhost it is smooth.
+- Greedy decoding by default. One conversation per server by default (a second
+  concurrent connection is rejected with WS close 1013 until the first ends);
+  pass `--batch-size N` to host N concurrent conversations on one engine
+  (elastic batching; connections beyond N are rejected until a slot frees).
+- Full runbook: [`recipes/NVIDIA/PersonaPlex.md`](../../../recipes/NVIDIA/PersonaPlex.md).

examples/online_serving/personaplex/duplex_client.py

@@ -0,0 +1,114 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Headless client for the PersonaPlex duplex server (official Moshi protocol).
+
+Streams a 24 kHz mono WAV to ``/api/chat`` exactly as the browser client does
+(Opus over WebSocket, tags ``\\x00`` ready / ``\\x01`` audio / ``\\x02`` text),
+collects the agent's reply audio + inner-monologue text, and writes a WAV.
+
+The browser UI at ``http://<host>:<port>/`` is the primary demo; this script is
+for headless testing / scripting.
+
+    python duplex_client.py --url ws://localhost:8124/api/chat \
+        --input user.wav --out reply.wav
+
+Deps: ``websockets``, ``sphn``, ``soundfile``, ``numpy``.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+
+import numpy as np
+import soundfile as sf
+import sphn
+
+SR = 24000  # PersonaPlex / Mimi codec rate
+
+
+async def converse(url: str, pcm: np.ndarray) -> tuple[np.ndarray, str]:
+    import websockets
+
+    reader = sphn.OpusStreamReader(SR)  # decode agent audio
+    writer = sphn.OpusStreamWriter(SR)  # encode our mic audio
+    text: list[str] = []
+    ready = asyncio.Event()
+    last = {"t": 0.0}
+
+    async with websockets.connect(url, max_size=None) as ws:
+
+        async def receive() -> None:
+            loop = asyncio.get_event_loop()
+            while True:
+                try:
+                    msg = await ws.recv()
+                except Exception:
+                    return
+                if not isinstance(msg, (bytes, bytearray)) or not msg:
+                    continue
+                tag, payload = msg[0], msg[1:]
+                if tag == 0:
+                    ready.set()
+                elif tag == 1:
+                    reader.append_bytes(bytes(payload))
+                    last["t"] = loop.time()
+                elif tag == 2:
+                    text.append(payload.decode("utf8", errors="replace"))
+
+        recv_task = asyncio.create_task(receive())
+        await asyncio.wait_for(ready.wait(), timeout=120)  # waits for system-prompt prefill
+
+        frame = 1920  # 80 ms
+        for i in range(0, len(pcm), frame):
+            chunk = pcm[i : i + frame]
+            if len(chunk) < frame:
+                chunk = np.concatenate([chunk, np.zeros(frame - len(chunk), dtype=np.float32)])
+            writer.append_pcm(chunk)
+            data = writer.read_bytes()
+            if data:
+                await ws.send(b"\x01" + data)
+            await asyncio.sleep(frame / SR)  # pace at real time
+
+        loop = asyncio.get_event_loop()
+        last["t"] = loop.time()
+        while loop.time() - last["t"] < 2.5:  # drain until the agent stops talking
+            await asyncio.sleep(0.1)
+        recv_task.cancel()
+
+    chunks: list[np.ndarray] = []
+    while True:
+        out = reader.read_pcm()
+        if out is None or out.shape[-1] == 0:
+            break
+        chunks.append(np.asarray(out, dtype=np.float32))
+    audio = np.concatenate(chunks) if chunks else np.zeros(0, dtype=np.float32)
+    return audio, "".join(text)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--url", default="ws://localhost:8124/api/chat")
+    ap.add_argument("--input", required=True, help="24 kHz mono WAV of the user turn")
+    ap.add_argument("--out", default="personaplex_reply.wav")
+    ap.add_argument("--tail", type=float, default=10.0, help="trailing silence (s) for the agent reply")
+    ap.add_argument("--persona", default="")
+    ap.add_argument("--voice", default="NATF2.pt")
+    args = ap.parse_args()
+
+    wav, sr = sf.read(args.input, dtype="float32")
+    if wav.ndim > 1:
+        wav = wav.mean(axis=1)
+    if sr != SR:
+        raise SystemExit(f"input must be {SR} Hz mono; got {sr} Hz. Resample it first.")
+    wav = np.concatenate([wav, np.zeros(int(args.tail * SR), dtype=np.float32)])
+
+    url = f"{args.url}?text_prompt={args.persona}&voice_prompt={args.voice}"
+    audio, text = asyncio.run(converse(url, wav))
+    sf.write(args.out, audio if audio.size else np.zeros(SR, dtype=np.float32), SR)
+    print(f"agent reply: {audio.size / SR:.2f}s -> {args.out}")
+    print(f"inner-monologue: {text.strip()!r}")
+
+
+if __name__ == "__main__":
+    main()

examples/online_serving/personaplex/run_server.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# PersonaPlex full-duplex online serving.
+#
+# Serves the OFFICIAL PersonaPlex web client at / (auto-downloaded from
+# nvidia/personaplex-7b-v1) and the Moshi WebSocket protocol at /api/chat,
+# backed by the native vLLM-Omni engine. Requires HF access to the gated repo.
+
+HOST="${HOST:-0.0.0.0}"
+PORT="${PORT:-8124}"
+
+echo "Starting PersonaPlex duplex server on ${HOST}:${PORT}..."
+echo "Open http://localhost:${PORT}/ for the official web client (run it near the server)."
+
+python -m vllm_omni.experimental.fullduplex.personaplex.serving.server \
+    --host "$HOST" --port "$PORT"

examples/online_serving/qwen2_5_omni/README.md

@@ -12,9 +12,9 @@ Please refer to [README.md](../../../README.md)
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
 ```
 
-If you have custom stage configs file, launch the server with command below
+To use a custom deploy config, launch the server with the command below:
 ```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
+vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/deploy_config.yaml
 ```
 
 ### Send Multi-modal Request
@@ -173,7 +173,7 @@ The script supports the following arguments:
 - `--model`: Model name/path (default: Qwen/Qwen2.5-Omni-7B)
 - `--server-port`: Port for vLLM server (default: 8091)
 - `--gradio-port`: Port for Gradio demo (default: 7861)
-- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
+- `--deploy-config`: Path to a custom deploy config YAML file (optional)
 - `--server-host`: Host for vLLM server (default: 0.0.0.0)
 - `--gradio-ip`: IP for Gradio demo (default: 127.0.0.1)
 - `--share`: Share Gradio demo publicly (creates a public link)
@@ -186,9 +186,9 @@ The script supports the following arguments:
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
 ```
 
-If you have custom stage configs file:
+To use a custom deploy config:
 ```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
+vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/deploy_config.yaml
 ```
 
 **Step 2: Run the Gradio demo**

examples/online_serving/qwen2_5_omni/run_gradio_demo.sh

@@ -13,7 +13,7 @@ set -e
 MODEL="Qwen/Qwen2.5-Omni-7B"
 SERVER_PORT=8091
 GRADIO_PORT=7861
-STAGE_CONFIGS_PATH=""
+DEPLOY_CONFIG_PATH=""
 SERVER_HOST="0.0.0.0"
 GRADIO_IP="127.0.0.1"
 GRADIO_SHARE=false
@@ -33,8 +33,8 @@ while [[ $# -gt 0 ]]; do
             GRADIO_PORT="$2"
             shift 2
             ;;
-        --stage-configs-path)
-            STAGE_CONFIGS_PATH="$2"
+        --deploy-config)
+            DEPLOY_CONFIG_PATH="$2"
             shift 2
             ;;
         --server-host)
@@ -56,7 +56,7 @@ while [[ $# -gt 0 ]]; do
             echo "  --model MODEL                 Model name/path (default: Qwen/Qwen2.5-Omni-7B)"
             echo "  --server-port PORT            Port for vLLM server (default: 8091)"
             echo "  --gradio-port PORT            Port for Gradio demo (default: 7861)"
-            echo "  --stage-configs-path PATH     Path to custom stage configs YAML file (optional)"
+            echo "  --deploy-config PATH          Path to custom deploy config YAML file (optional)"
             echo "  --server-host HOST            Host for vLLM server (default: 0.0.0.0)"
             echo "  --gradio-ip IP                IP for Gradio demo (default: 127.0.0.1)"
             echo "  --share                       Share Gradio demo publicly"
@@ -87,8 +87,8 @@ echo "=========================================="
 
 # Build vLLM server command
 SERVER_CMD=("vllm" "serve" "$MODEL" "--omni" "--port" "$SERVER_PORT" "--host" "$SERVER_HOST")
-if [ -n "$STAGE_CONFIGS_PATH" ]; then
-    SERVER_CMD+=("--stage-configs-path" "$STAGE_CONFIGS_PATH")
+if [ -n "$DEPLOY_CONFIG_PATH" ]; then
+    SERVER_CMD+=("--deploy-config" "$DEPLOY_CONFIG_PATH")
 fi
 
 # Function to cleanup on exit

examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml

@@ -1,36 +0,0 @@
-# Stage config for running Qwen3-Omni-MoE-Thinking (text-only output)
-# This config is for models like Qwen3-Omni-30B-A3B-Thinking that only have the
-# thinker component and do not support audio output.
-#
-# Single stage: Thinker (multimodal understanding + text generation)
-
-# The following config has been verified on 2x H100-80G GPUs.
-stage_args:
-  - stage_id: 0
-    runtime:
-      devices: "0,1"
-    engine_args:
-      model_stage: thinker
-      max_num_seqs: 1
-      model_arch: Qwen3OmniMoeForConditionalGeneration
-      worker_type: ar
-      scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
-      gpu_memory_utilization: 0.9
-      enforce_eager: true
-      trust_remote_code: true
-      engine_output_type: text
-      distributed_executor_backend: "mp"
-      enable_prefix_caching: true
-      hf_config_name: thinker_config
-      tensor_parallel_size: 2
-    final_output: true
-    final_output_type: text
-    is_comprehension: true
-    default_sampling_params:
-      temperature: 0.4
-      top_p: 0.9
-      top_k: 1
-      max_tokens: 2048
-      seed: 42
-      detokenize: True
-      repetition_penalty: 1.05

examples/online_serving/qwen3_omni/run_gradio_demo.sh

@@ -13,7 +13,7 @@ set -e
 MODEL="Qwen/Qwen3-Omni-30B-A3B-Instruct"
 SERVER_PORT=8091
 GRADIO_PORT=7861
-STAGE_CONFIGS_PATH=""
+DEPLOY_CONFIG=""
 SERVER_HOST="0.0.0.0"
 GRADIO_IP="127.0.0.1"
 GRADIO_SHARE=false
@@ -33,8 +33,8 @@ while [[ $# -gt 0 ]]; do
             GRADIO_PORT="$2"
             shift 2
             ;;
-        --stage-configs-path)
-            STAGE_CONFIGS_PATH="$2"
+        --deploy-config)
+            DEPLOY_CONFIG="$2"
             shift 2
             ;;
         --server-host)
@@ -56,7 +56,7 @@ while [[ $# -gt 0 ]]; do
             echo "  --model MODEL                 Model name/path (default: Qwen/Qwen3-Omni-30B-A3B-Instruct)"
             echo "  --server-port PORT            Port for vLLM server (default: 8091)"
             echo "  --gradio-port PORT            Port for Gradio demo (default: 7861)"
-            echo "  --stage-configs-path PATH     Path to custom stage configs YAML file (optional)"
+            echo "  --deploy-config PATH          Path to custom deploy config YAML file (optional)"
             echo "  --server-host HOST            Host for vLLM server (default: 0.0.0.0)"
             echo "  --gradio-ip IP                IP for Gradio demo (default: 127.0.0.1)"
             echo "  --share                       Share Gradio demo publicly"
@@ -87,8 +87,8 @@ echo "=========================================="
 
 # Build vLLM server command
 SERVER_CMD=("vllm" "serve" "$MODEL" "--omni" "--port" "$SERVER_PORT" "--host" "$SERVER_HOST")
-if [ -n "$STAGE_CONFIGS_PATH" ]; then
-    SERVER_CMD+=("--stage-configs-path" "$STAGE_CONFIGS_PATH")
+if [ -n "$DEPLOY_CONFIG" ]; then
+    SERVER_CMD+=("--deploy-config" "$DEPLOY_CONFIG")
 fi
 
 # Function to cleanup on exit

examples/online_serving/text_to_speech/README.md

@@ -336,7 +336,7 @@ vllm serve OpenMOSS-Team/MOSS-TTS-Nano --omni --port 8091
 # or:
 ./moss_tts_nano/run_server.sh
 ```
-The deploy config at `vllm_omni/deploy/moss_tts_nano.yaml` auto-loads; no `--stage-configs-path`, `--trust-remote-code`, or `--enforce-eager` flags are needed.
+The deploy config at `vllm_omni/deploy/moss_tts_nano.yaml` auto-loads; no `--deploy-config`, `--trust-remote-code`, or `--enforce-eager` flags are needed.
 
 ### Sending requests
 ```bash
@@ -537,12 +537,19 @@ curl -X POST http://localhost:8091/v1/audio/speech \
 Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming.
 
 ### Streaming WebSocket
-The `/v1/audio/speech/stream` endpoint accepts text incrementally, splits it at sentence boundaries, and emits one PCM stream per sentence:
+The `/v1/audio/speech/stream` endpoint accepts text incrementally and synthesizes the buffered text as one continuous request on `input.done`:
 ```bash
 python qwen3_tts/streaming_speech_client.py --text "Hello world. How are you? I am fine."
 python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-delay 0.1
 ```
 
+`input.done` flushes without closing, so repeating `--text` synthesizes several utterances over one connection:
+```bash
+python qwen3_tts/streaming_speech_client.py \
+    --text "First utterance." \
+    --text "Second utterance, same connection."
+```
+
 To receive word-level timestamps, launch the server with a forced aligner:
 ```bash
 vllm-omni serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \

examples/online_serving/text_to_speech/qwen3_tts/streaming_speech_client.py

@@ -7,6 +7,12 @@ Usage:
     # Send full text at once
     python streaming_speech_client.py --text "Hello world. How are you? I am fine."
 
+    # Several utterances over one connection: input.done flushes each of them
+    # and the connection stays open, so only the first pays the handshake
+    python streaming_speech_client.py \
+        --text "First utterance." \
+        --text "Second utterance, same connection."
+
     # Pick a built-in speaker for CustomVoice models
     python streaming_speech_client.py \
         --text "打开电子枪，关闭扫描，切换到低倍模式。" \
@@ -64,18 +70,27 @@ except ImportError:
     raise SystemExit(1)
 
 
-def save_audio_file(output_dir: str, sentence_index: int, response_format: str, chunks: list[bytes]) -> str:
+def frame_basename(msg: dict) -> str:
+    """Name a file after the utterance and sentence a frame belongs to.
+
+    Every utterance is sentence 0, so the utterance index is what keeps the
+    files of one connection from overwriting each other.
+    """
+    return f"utterance_{msg['utterance_index']:03d}_sentence_{msg['sentence_index']:03d}"
+
+
+def save_audio_file(output_dir: str, basename: str, response_format: str, chunks: list[bytes]) -> str:
     audio_bytes = b"".join(chunks)
     if response_format == "pcm":
-        raw_path = os.path.join(output_dir, f"sentence_{sentence_index:03d}.pcm")
+        raw_path = os.path.join(output_dir, f"{basename}.pcm")
         with open(raw_path, "wb") as f:
             f.write(audio_bytes)
 
-        wav_path = os.path.join(output_dir, f"sentence_{sentence_index:03d}.wav")
+        wav_path = os.path.join(output_dir, f"{basename}.wav")
         save_pcm_wav(wav_path, audio_bytes)
         return wav_path
 
-    filename = os.path.join(output_dir, f"sentence_{sentence_index:03d}.{response_format}")
+    filename = os.path.join(output_dir, f"{basename}.{response_format}")
     with open(filename, "wb") as f:
         f.write(audio_bytes)
     return filename
@@ -95,26 +110,8 @@ def timestamp_words_text(timestamps: list[dict] | None) -> str:
     return " ".join(str(item.get("word", "")) for item in timestamps if item.get("word"))
 
 
-async def stream_tts(
-    url: str,
-    text: str,
-    config: dict,
-    output_dir: str,
-    simulate_stt: bool = False,
-    stt_delay: float = 0.1,
-    save_chunks: bool = False,
-) -> None:
-    """Connect to the streaming TTS endpoint and process audio responses."""
-    os.makedirs(output_dir, exist_ok=True)
-
-    async with websockets.connect(url) as ws:
-        # 1. Send session config
-        config_msg = {"type": "session.config", **config}
-        await ws.send(json.dumps(config_msg))
-        print(f"Sent session config: {config}")
-
-        # 2. Send text (either all at once or word-by-word)
-        async def send_text():
+async def send_utterance(ws, text: str, simulate_stt: bool, stt_delay: float) -> None:
+    """Send one utterance's text, then input.done to flush it."""
     if simulate_stt:
         words = text.split(" ")
         for i, word in enumerate(words):
@@ -140,16 +137,19 @@ async def stream_tts(
         )
         print(f"Sent full text: {text!r}")
 
-            # 3. Signal end of input
     await ws.send(json.dumps({"type": "input.done"}))
     print("Sent input.done")
 
-        # Run sender and receiver concurrently
-        sender_task = asyncio.create_task(send_text())
 
-        response_format = config.get("response_format", "wav")
-        word_timestamps = bool(config.get("word_timestamps", False))
-        current_sentence_index = 0
+async def receive_utterance(
+    ws,
+    output_dir: str,
+    response_format: str,
+    word_timestamps: bool,
+    save_chunks: bool,
+) -> None:
+    """Consume frames until session.done marks the flushed utterance complete."""
+    current_label = "utterance 0"
     current_sentence_text = ""
     current_chunks: list[bytes] = []
     current_timestamps: list[dict] = []
@@ -157,25 +157,24 @@ async def stream_tts(
     # the sentence is a failure only if no frame ever carried timestamps.
     alignment_frame_seen = False
 
-        try:
     while True:
         message = await ws.recv()
 
         if isinstance(message, bytes):
             current_chunks.append(message)
-                    print(f"  Received audio chunk for sentence {current_sentence_index}: {len(message)} bytes")
+            print(f"  Received audio chunk for {current_label}: {len(message)} bytes")
         else:
             # JSON frame
             msg = json.loads(message)
             msg_type = msg.get("type")
 
             if msg_type == "audio.start":
-                        current_sentence_index = msg["sentence_index"]
+                current_label = f"utterance {msg['utterance_index']}"
                 current_sentence_text = msg.get("sentence_text", "")
                 current_chunks = []
                 current_timestamps = []
                 alignment_frame_seen = False
-                        print(f"  [sentence {msg['sentence_index']}] Generating: {current_sentence_text!r}")
+                print(f"  [{current_label}] Generating: {current_sentence_text!r}")
             elif msg_type == "audio.chunk":
                 audio = base64.b64decode(msg["audio_b64"])
                 current_chunks.append(audio)
@@ -185,7 +184,7 @@ async def stream_tts(
                 if save_chunks and audio:
                     chunk_base = os.path.join(
                         output_dir,
-                                f"sentence_{msg['sentence_index']:03d}_chunk_{msg['chunk_id']:03d}",
+                        f"{frame_basename(msg)}_chunk_{msg['chunk_id']:03d}",
                     )
                     chunk_pcm = f"{chunk_base}.pcm"
                     chunk_wav = f"{chunk_base}.wav"
@@ -196,6 +195,7 @@ async def stream_tts(
                     with open(chunk_json, "w", encoding="utf-8") as f:
                         json.dump(
                             {
+                                "utterance_index": msg["utterance_index"],
                                 "sentence_index": msg["sentence_index"],
                                 "chunk_id": msg["chunk_id"],
                                 "chunk_start_ms": msg.get("chunk_start_ms"),
@@ -209,15 +209,12 @@ async def stream_tts(
                             indent=2,
                         )
                 if chunk_timestamps is None:
-                            print(
-                                f"  [sentence {msg['sentence_index']}] chunk {msg['chunk_id']}: "
-                                f"{len(audio)} bytes, timestamps unavailable"
-                            )
+                    print(f"  [{current_label}] chunk {msg['chunk_id']}: {len(audio)} bytes, timestamps unavailable")
                 else:
                     alignment_frame_seen = True
                     current_timestamps.extend(chunk_timestamps)
                     print(
-                                f"  [sentence {msg['sentence_index']}] chunk {msg['chunk_id']}: "
+                        f"  [{current_label}] chunk {msg['chunk_id']}: "
                         f"{len(audio)} bytes, {len(chunk_timestamps)} timestamp(s)"
                     )
                 if save_chunks and audio:
@@ -225,7 +222,7 @@ async def stream_tts(
             elif msg_type == "audio.done":
                 filename = save_audio_file(
                     output_dir,
-                            msg["sentence_index"],
+                    frame_basename(msg),
                     response_format,
                     current_chunks,
                 )
@@ -234,11 +231,12 @@ async def stream_tts(
                     sentence_timestamps = current_timestamps if alignment_frame_seen else None
                     ts_filename = os.path.join(
                         output_dir,
-                                f"sentence_{msg['sentence_index']:03d}_timestamps.json",
+                        f"{frame_basename(msg)}_timestamps.json",
                     )
                     with open(ts_filename, "w", encoding="utf-8") as f:
                         json.dump(
                             {
+                                "utterance_index": msg["utterance_index"],
                                 "sentence_index": msg["sentence_index"],
                                 "sentence_text": current_sentence_text,
                                 "covered_text": timestamp_words_text(sentence_timestamps),
@@ -249,23 +247,60 @@ async def stream_tts(
                             indent=2,
                         )
                 print(
-                            f"  [sentence {msg['sentence_index']}] Done"
+                    f"  [{current_label}] Done"
                     f" bytes={msg.get('total_bytes', len(b''.join(current_chunks)))}"
                     f" error={msg.get('error', False)}"
                     f" -> {filename}"
                 )
                 if word_timestamps:
-                            print(f"  [sentence {msg['sentence_index']}] Timestamps -> {ts_filename}")
+                    print(f"  [{current_label}] Timestamps -> {ts_filename}")
                 current_chunks = []
                 current_timestamps = []
                 alignment_frame_seen = False
             elif msg_type == "session.done":
-                        print(f"\nSession complete: {msg['total_sentences']} sentence(s) generated")
-                        break
+                print(f"\nUtterance {msg['utterance_index']} complete: {msg['total_sentences']} sentence(s) generated")
+                return
             elif msg_type == "error":
                 print(f"  ERROR: {msg['message']}")
             else:
                 print(f"  Unknown message: {msg}")
+
+
+async def stream_tts(
+    url: str,
+    texts: list[str],
+    config: dict,
+    output_dir: str,
+    simulate_stt: bool = False,
+    stt_delay: float = 0.1,
+    save_chunks: bool = False,
+) -> None:
+    """Connect to the streaming TTS endpoint and process audio responses.
+
+    Every text is synthesized over the same connection: input.done flushes an
+    utterance without closing, so only the first one pays the handshake.
+    """
+    os.makedirs(output_dir, exist_ok=True)
+    response_format = config.get("response_format", "wav")
+    word_timestamps = bool(config.get("word_timestamps", False))
+
+    async with websockets.connect(url) as ws:
+        # 1. Send session config; it stays in effect for every utterance below.
+        config_msg = {"type": "session.config", **config}
+        await ws.send(json.dumps(config_msg))
+        print(f"Sent session config: {config}")
+
+        for text in texts:
+            # 2. Send text (either all at once or word-by-word), then flush.
+            sender_task = asyncio.create_task(send_utterance(ws, text, simulate_stt, stt_delay))
+            try:
+                await receive_utterance(
+                    ws,
+                    output_dir=output_dir,
+                    response_format=response_format,
+                    word_timestamps=word_timestamps,
+                    save_chunks=save_chunks,
+                )
             finally:
                 sender_task.cancel()
                 try:
@@ -273,6 +308,10 @@ async def stream_tts(
                 except asyncio.CancelledError:
                     pass  # Task cancellation is expected during shutdown
 
+        # 3. Release the connection once there is nothing left to say.
+        await ws.send(json.dumps({"type": "session.close"}))
+        print("Sent session.close")
+
     print(f"\nAudio files saved to: {output_dir}/")
 
 
@@ -286,7 +325,8 @@ def main():
     parser.add_argument(
         "--text",
         required=True,
-        help="Text to synthesize",
+        action="append",
+        help="Text to synthesize; repeat to send several utterances over one connection",
     )
     parser.add_argument(
         "--output-dir",
@@ -385,7 +425,7 @@ def main():
     asyncio.run(
         stream_tts(
             url=args.url,
-            text=args.text,
+            texts=args.text,
             config=config,
             output_dir=args.output_dir,
             simulate_stt=args.simulate_stt,

mkdocs.yml

@@ -71,6 +71,14 @@ plugins:
   - search
   - autorefs
   - awesome-nav
+  - redirects:
+      redirect_maps:
+        user_guide/diffusion/request_batching.md: user_guide/diffusion/execution_modes.md
+        user_guide/diffusion/step_execution.md: user_guide/diffusion/execution_modes.md
+        user_guide/diffusion/cpu_offload_diffusion.md: user_guide/diffusion/cpu_offload.md
+        user_guide/quantization/index.md: user_guide/quantization/overview.md
+        design/feature/diffusion_request_level_batching.md: design/feature/diffusion_continuous_batching.md
+        design/feature/diffusion_step_execution.md: design/feature/diffusion_continuous_batching.md
   - glightbox
   - git-revision-date-localized:
       # exclude files

pyproject.toml

@@ -70,11 +70,11 @@ dev = [
     "FlagEmbedding==1.4.0", # for hunyuan_image3 tests
     "kernels==0.14.1", # for accuracy tests comparison with diffusers output (0.15.0 incompatible with diffusers 0.38.0)
     "s3tokenizer==0.3.0", # for tts test
-    "step-audio2==1.0.0", # Step-Audio2 Token2Wav (flashcosyvoice / hyperpyyaml)
-    "stepaudio2-minicpmo", # for MiniCPM-o 4.5 talker vocoder (stepaudio2-minicpmo Token2wav)
+    "step-audio2==1.0.0", # Step-Audio2 / MiniCPM-o 4.5 Token2Wav (flashcosyvoice / hyperpyyaml)
 ]
 
 demo = [
+    "gradio>=6.7.0",
     "opencv-python>=4.12.0.88",  # cli/stream_client frame reading
     "requests>=2.28.0",  # cli/stream_client HTTP
     "cn2an>=0.5.22",  # Chinese number normalization for TTS (e.g. "1次" -> "一次")
@@ -95,6 +95,12 @@ quack = [
     "quack-kernels>=0.3.11",
 ]
 
+# Optional Blackwell-native FlashAttention-4 backend. Keep this opt-in because
+# the CuTe/CUTLASS dependency is CUDA 13 and Blackwell specific.
+fa4 = [
+    "flash-attn-4[cu13]==4.0.0b18",
+]
+
 docs = [
     "mkdocs>=1.5.0",
     "mkdocs-api-autonav",
@@ -102,6 +108,7 @@ docs = [
     "mkdocstrings-python",
     "mkdocs-gen-files",
     "mkdocs-awesome-nav",
+    "mkdocs-redirects",
     "mkdocs-glightbox",
     "mkdocs-git-revision-date-localized-plugin",
     "mkdocs-minify-plugin",